From bbe22ea96e6aef091862d3fe67631a7800597f5f Mon Sep 17 00:00:00 2001 From: Aashish Dhakal <85501584+dhakalaashish@users.noreply.github.com> Date: Fri, 5 Apr 2024 16:05:49 +0545 Subject: [PATCH 1/5] Create GOES16 Geostationary Satellite Provider --- apps/server/package.json | 1 + .../server/src/Interfaces/GeoEventProvider.ts | 4 +- .../src/Services/GeoEvent/GeoEventHandler.ts | 5 +- .../GeoEventProviderRegistry.ts | 2 + .../GOES16GeoEventProviderClass.ts | 194 ++++++++++++++ .../NasaGeoEventProviderClass.ts | 1 + .../src/Services/SiteAlert/CreateSiteAlert.ts | 252 ++++++++++++------ .../src/pages/api/cron/geo-event-fetcher.ts | 14 +- apps/server/src/server/api/routers/user.ts | 1 - .../src/server/api/zodSchemas/site.schema.ts | 2 +- apps/server/src/utils/geometry.ts | 27 ++ yarn.lock | 242 ++++++++++++++++- 12 files changed, 649 insertions(+), 96 deletions(-) create mode 100644 apps/server/src/Services/GeoEventProvider/ProviderClass/GOES16GeoEventProviderClass.ts create mode 100644 apps/server/src/utils/geometry.ts diff --git a/apps/server/package.json b/apps/server/package.json index 901ec793e..769c89fc1 100644 --- a/apps/server/package.json +++ b/apps/server/package.json @@ -20,6 +20,7 @@ "db:generate": "dotenv -e ../../.env npx prisma generate" }, "dependencies": { + "@google/earthengine": "^0.1.396", "@logtail/node": "^0.4.0", "@planet-sdk/common": "^0.1.11", "@prisma/client": "^5.0.0", diff --git a/apps/server/src/Interfaces/GeoEventProvider.ts b/apps/server/src/Interfaces/GeoEventProvider.ts index 89b49e2c0..b1815ad6b 100644 --- a/apps/server/src/Interfaces/GeoEventProvider.ts +++ b/apps/server/src/Interfaces/GeoEventProvider.ts @@ -7,16 +7,17 @@ export enum GeoEventProviderClientId { VIIRS_NOAA20_NRT = 'VIIRS_NOAA20_NRT', VIIRS_SNPP_NRT = 'VIIRS_SNPP_NRT', VIIRS_SNPP_SP = 'VIIRS_SNPP_SP', + GEOSTATIONARY = 'GEOSTATIONARY' } export enum GeoEventProviderClient { FIRMS = 'FIRMS', + GOES16 = 'GOES-16' } export interface GeoEventProviderConfig { bbox: string; slice: string; - apiUrl: string; client: GeoEventProviderClient; //'FIRMS' } @@ -28,6 +29,7 @@ export interface GeoEventProviderClass { geoEventProviderId: string, slice: string, clientApiKey: string, + lastRun: Date | null ) => Promise; } diff --git a/apps/server/src/Services/GeoEvent/GeoEventHandler.ts b/apps/server/src/Services/GeoEvent/GeoEventHandler.ts index 70d73e839..b09cd3a14 100644 --- a/apps/server/src/Services/GeoEvent/GeoEventHandler.ts +++ b/apps/server/src/Services/GeoEvent/GeoEventHandler.ts @@ -3,9 +3,8 @@ import { AlertType } from "../../Interfaces/SiteAlert"; import { type geoEventInterface as GeoEvent } from "../../Interfaces/GeoEvent"; import { createXXHash3 } from "hash-wasm"; import { prisma } from '../../server/db'; -import { logger } from "../../../src/server/logger"; -const processGeoEvents = async (breadcrumbPrefix: string, geoEventProviderClientId: GeoEventProviderClientId, geoEventProviderId: string, slice: string, geoEvents: GeoEvent[]) => { +const processGeoEvents = async (geoEventProviderClientId: GeoEventProviderClientId, geoEventProviderId: string, slice: string, geoEvents: GeoEvent[]) => { const hasher = await createXXHash3(); // Create the hasher outside the function const buildChecksum = (geoEvent: GeoEvent): string => { hasher.init(); // Reset the hasher @@ -85,7 +84,7 @@ const processGeoEvents = async (breadcrumbPrefix: string, geoEventProviderClient geoEventProviderClientId: geoEventProviderClientId, geoEventProviderId: geoEventProviderId, radius: geoEvent.radius ? geoEvent.radius : 0, - slice: slice, + slice: geoEventProviderClientId === 'GEOSTATIONARY' ? geoEvent.slice : slice, data: geoEvent.data, })) diff --git a/apps/server/src/Services/GeoEventProvider/GeoEventProviderRegistry.ts b/apps/server/src/Services/GeoEventProvider/GeoEventProviderRegistry.ts index 11e6b9232..71bd0da2f 100644 --- a/apps/server/src/Services/GeoEventProvider/GeoEventProviderRegistry.ts +++ b/apps/server/src/Services/GeoEventProvider/GeoEventProviderRegistry.ts @@ -1,5 +1,6 @@ import {type GeoEventProviderClass} from '../../Interfaces/GeoEventProvider'; import NasaGeoEventProvider from './ProviderClass/NasaGeoEventProviderClass'; +import GOES16GeoEventProvider from './ProviderClass/GOES16GeoEventProviderClass'; // import additional GeoEvent provider implementations below const createGeoEventProviderClassRegistry = function ( @@ -31,6 +32,7 @@ const createGeoEventProviderClassRegistry = function ( const GeoEventProviderClassRegistry = createGeoEventProviderClassRegistry([ new NasaGeoEventProvider(), + new GOES16GeoEventProvider() // add new alert providers here ]); diff --git a/apps/server/src/Services/GeoEventProvider/ProviderClass/GOES16GeoEventProviderClass.ts b/apps/server/src/Services/GeoEventProvider/ProviderClass/GOES16GeoEventProviderClass.ts new file mode 100644 index 000000000..f3c1fbeff --- /dev/null +++ b/apps/server/src/Services/GeoEventProvider/ProviderClass/GOES16GeoEventProviderClass.ts @@ -0,0 +1,194 @@ +import { + type GeoEventProviderConfig, + type GeoEventProviderClientId, + type GeoEventProviderConfigGeneral, + type GeoEventProviderClass +} from '../../../Interfaces/GeoEventProvider'; +import { type geoEventInterface as GeoEvent } from "../../../Interfaces/GeoEvent" +import {Confidence} from '../../../Interfaces/GeoEvent'; +import {determineSlice} from "../../../utils/geometry" +import ee from '@google/earthengine' + +type FireDataEntry = [number, number, Date]; +type AllFireData = FireDataEntry[]; +interface PrivateKeyJson { + "type": string; + "project_id": string; + "private_key_id": string; + "private_key": string; + "client_email": string; + "client_id": string; + "auth_uri": string; + "token_uri": string; + "auth_provider_x509_cert_url": string; + "client_x509_cert_url": string; + "universe_domain": string; +} + +interface GOES16GeoEventProviderConfig extends GeoEventProviderConfig { + // Add any additional config properties if needed + privateKey: PrivateKeyJson; +} + +class GOES16GeoEventProviderClass implements GeoEventProviderClass { + private config: GeoEventProviderConfigGeneral | undefined; + + constructor() { + this.getLatestGeoEvents = this.getLatestGeoEvents.bind(this); + } + + getKey(): string { + return 'GOES-16'; + } + + initialize(config?: GeoEventProviderConfigGeneral): void { + this.config = config; + } + + async authenticateEarthEngine(): Promise { + const {privateKey} = this.getConfig() + const private_key = JSON.parse(JSON.stringify(privateKey)) + return new Promise((resolve, reject) => { + ee.data.authenticateViaPrivateKey( + private_key, + () => { + ee.initialize( + null, + null, + () => { + console.log('Google Earth Engine authentication successful'); + resolve(); + }, + (err) => { + console.error('Google Earth Engine initialization error', err); + reject(err); + } + ); + }, + (err) => { + console.error('Google Earth Engine authentication error', err); + reject(err); + } + ); + }); + } + + async getLatestGeoEvents(geoEventProviderClientId: string, geoEventProviderId: string, slice: string, clientApiKey: string, lastRun: Date | null): Promise { + return new Promise(async (resolve, reject) => { + try { + // Ensure Earth Engine is authenticated and initialized before fetching data + await this.authenticateEarthEngine(); + let allFireData: AllFireData = []; + + // If lastRun is more than 2 hours ago or null, then start from 2 hours ago, else start from lastRunDate + const currentDateTime = new Date(); + const lastRunDate = lastRun ? new Date(lastRun) : null; + const twoHoursAgo = new Date(currentDateTime.getTime() - 2 * 3600 * 1000); + const fromDateTime = (!lastRunDate || (currentDateTime.getTime() - lastRunDate.getTime()) > 2 * 3600 * 1000) ? twoHoursAgo : lastRunDate; + + const images = ee.ImageCollection("NOAA/GOES/16/FDCF").filterDate('2024-04-18T08:51:15Z', '2024-04-18T09:51:15Z'); + + // Fetch and process images here... + // The process includes fetching image IDs, processing them to extract fire data, etc. + // This is a simplified outline; integrate the logic from your initial example here. + const getImagesId = () => { + return new Promise((resolve, reject) => { + images.evaluate((imageCollection) => { + if (imageCollection && imageCollection.features) { + const imagesData = imageCollection.features.map(feature => feature.id); + resolve(imagesData); + } else { + reject(new Error("No features found")); + } + }); + }); + }; + try { + const array_imagesId = await getImagesId() as string[]; + for (const imageId of array_imagesId) { + const image = ee.Image(`${imageId}`) + // Get the datetime information from the image metadata + const datetimeInfo = await ee.Date(image.get('system:time_start')).getInfo(); + const datetime = new Date(datetimeInfo.value); + + + const temperatureImage = image.select('Temp'); + const xMin = -142; // On station as GOES-E + const xMax = xMin + 135; + const geometry = ee.Geometry.Rectangle([xMin, -65, xMax, 65], null, true); + var temperatureVector = temperatureImage.reduceToVectors({ + geometry: geometry, + scale: 2000, + geometryType: 'centroid', + labelProperty: 'temp', + maxPixels: 1e10, + }); + const fireData = await new Promise((resolve, reject) => { + temperatureVector.evaluate((featureCollection) => { + if (featureCollection && featureCollection.features) { + // Map each feature to include datetime in its data + // [long, lat, eventDate] + const fireDataWithTime = featureCollection.features.map(feature => [...feature.geometry.coordinates, datetime]); + resolve(fireDataWithTime); + } else { + reject(new Error("No features found")); + } + }); + }) as FireDataEntry; + + // Concatenate the current image's fire data with the master array + allFireData = allFireData.concat(fireData); + }; + } catch (error) { + console.error("Error fetching fire data:", error); + } + + // Normalize the fire data into GeoEvent format + const geoEventsData: GeoEvent[] = allFireData.map((fireData: FireDataEntry) => ({ + type: 'fire', + latitude: fireData[1], + longitude: fireData[0], + eventDate: new Date(fireData[2]), + confidence: Confidence.High, + isProcessed: false, + geoEventProviderClientId: geoEventProviderClientId as GeoEventProviderClientId, + geoEventProviderId: geoEventProviderId, + slice: determineSlice(fireData[1], fireData[0]), + data: {'satellite': clientApiKey, 'slice': slice} + })); + + resolve(geoEventsData); + } catch (error) { + console.error('Failed to fetch or process GOES-16 data', error); + reject(error); + } + }); + } + + getConfig(): GOES16GeoEventProviderConfig { + if (typeof this.config === 'undefined') { + throw new Error(`Invalid or incomplete GOES-16 event provider configuration`); + } + const config = this.config + if (typeof config.client === "undefined") { + throw new Error(`Missing property 'client' in alert provider configuration`); + } + if (typeof config.bbox === "undefined") { + throw new Error(`Missing property 'bbox' in alert provider configuration`); + } + if (typeof config.slice === "undefined") { + throw new Error(`Missing property 'slice' in alert provider configuration`); + } + if (typeof config.privateKey === "undefined") { + throw new Error(`Missing property 'satelliteType' in alert provider configuration`); + } + return { + client: config.client, + bbox: config.bbox, + slice: config.slice, + privateKey: config.privateKey + }; + } +} + +export default GOES16GeoEventProviderClass; diff --git a/apps/server/src/Services/GeoEventProvider/ProviderClass/NasaGeoEventProviderClass.ts b/apps/server/src/Services/GeoEventProvider/ProviderClass/NasaGeoEventProviderClass.ts index f1291c2c7..45465946b 100644 --- a/apps/server/src/Services/GeoEventProvider/ProviderClass/NasaGeoEventProviderClass.ts +++ b/apps/server/src/Services/GeoEventProvider/ProviderClass/NasaGeoEventProviderClass.ts @@ -11,6 +11,7 @@ import type DataRecord from '../../../Interfaces/DataRecord'; import { Confidence } from '../../../Interfaces/GeoEvent'; interface NasaGeoEventProviderConfig extends GeoEventProviderConfig { + apiUrl: string; } class NasaGeoEventProviderClass implements GeoEventProviderClass { diff --git a/apps/server/src/Services/SiteAlert/CreateSiteAlert.ts b/apps/server/src/Services/SiteAlert/CreateSiteAlert.ts index 309ae176a..8d4bedb8b 100644 --- a/apps/server/src/Services/SiteAlert/CreateSiteAlert.ts +++ b/apps/server/src/Services/SiteAlert/CreateSiteAlert.ts @@ -9,89 +9,175 @@ const createSiteAlerts = async ( slice: string, ) => { let siteAlertsCreatedCount = 0; - try { - const siteAlertCreationQuery = Prisma.sql`INSERT INTO "SiteAlert" (id, TYPE, "isProcessed", "eventDate", "detectedBy", confidence, latitude, longitude, "siteId", "data", "distance") - - SELECT - gen_random_uuid(), - e.type, - FALSE, - e."eventDate", - ${geoEventProviderClientId}, - e.confidence, - e.latitude, - e.longitude, - s.id AS SiteId, - e.data, - ST_Distance(ST_SetSRID(ST_MakePoint(e.longitude, e.latitude), 4326), ST_GeomFromEWKB(decode(dg_elem, 'hex'))) AS distance - FROM - "GeoEvent" e - CROSS JOIN - "Site" s, - jsonb_array_elements_text(s."geometry"->'properties'->'detection_geometry') AS dg_elem - WHERE - s."type" = 'MultiPolygon' - AND s."deletedAt" IS NULL - AND s."isMonitored" = TRUE - AND (s."stopAlertUntil" IS NULL OR s."stopAlertUntil" < CURRENT_TIMESTAMP) - AND e."isProcessed" = FALSE - AND e. "geoEventProviderId" = ${geoEventProviderId} - AND s.slices @> ('["' || ${slice} || '"]')::jsonb - AND ST_Within(ST_SetSRID(ST_MakePoint(e.longitude, e.latitude), 4326), ST_GeomFromEWKB(decode(dg_elem, 'hex'))) - AND NOT EXISTS ( - SELECT 1 - FROM "SiteAlert" - WHERE "SiteAlert".longitude = e.longitude - AND "SiteAlert".latitude = e.latitude - AND "SiteAlert"."eventDate" = e."eventDate" - AND "SiteAlert"."siteId" = s.id - ) - - UNION - - SELECT - gen_random_uuid(), - e.type, - FALSE, - e."eventDate", - ${geoEventProviderClientId}, - e.confidence, - e.latitude, - e.longitude, - s.id AS SiteId, - e.data, - ST_Distance(ST_SetSRID(e.geometry, 4326), s."detectionGeometry") AS distance - FROM - "GeoEvent" e - INNER JOIN "Site" s ON ST_Within(ST_SetSRID(e.geometry, 4326), s."detectionGeometry") - AND s."deletedAt" IS NULL - AND s."isMonitored" = TRUE - AND (s."stopAlertUntil" IS NULL OR s."stopAlertUntil" < CURRENT_TIMESTAMP) - AND e."isProcessed" = FALSE - AND e. "geoEventProviderId" = ${geoEventProviderId} - AND s.slices @> ('["' || ${slice} || '"]')::jsonb - AND (s.type = 'Polygon' OR s.type = 'Point') - AND NOT EXISTS ( - SELECT 1 - FROM "SiteAlert" - WHERE "SiteAlert".longitude = e.longitude - AND "SiteAlert".latitude = e.latitude - AND "SiteAlert"."eventDate" = e."eventDate" - AND "SiteAlert"."siteId" = s.id - ); - `; - - const updateGeoEventIsProcessedToTrue = Prisma.sql`UPDATE "GeoEvent" SET "isProcessed" = true WHERE "isProcessed" = false AND "geoEventProviderId" = ${geoEventProviderId} AND "slice" = ${slice}`; - - // Create SiteAlerts by joining New GeoEvents and Sites that have the event's location in their proximity - // And, Set all GeoEvents as processed - const results = await prisma.$transaction([ - prisma.$executeRaw(siteAlertCreationQuery), - prisma.$executeRaw(updateGeoEventIsProcessedToTrue), - ]); - siteAlertsCreatedCount = results[0]; - } catch (error) { - logger(`Failed to create SiteAlerts. Error: ${error}`, 'error'); + if(geoEventProviderClientId === 'GEOSTATIONARY'){ + try { + const siteAlertCreationQuery = Prisma.sql`INSERT INTO "SiteAlert" (id, TYPE, "isProcessed", "eventDate", "detectedBy", confidence, latitude, longitude, "siteId", "data", "distance") + + SELECT + gen_random_uuid(), + e.type, + FALSE, + e."eventDate", + ${geoEventProviderClientId}, + e.confidence, + e.latitude, + e.longitude, + s.id AS SiteId, + e.data, + ST_Distance(ST_SetSRID(ST_MakePoint(e.longitude, e.latitude), 4326), ST_GeomFromEWKB(decode(dg_elem, 'hex'))) AS distance + FROM + "GeoEvent" e + CROSS JOIN + "Site" s, + jsonb_array_elements_text(s."geometry"->'properties'->'detection_geometry') AS dg_elem + WHERE + s."type" = 'MultiPolygon' + AND s."deletedAt" IS NULL + AND s."isMonitored" = TRUE + AND (s."stopAlertUntil" IS NULL OR s."stopAlertUntil" < CURRENT_TIMESTAMP) + AND e."isProcessed" = FALSE + AND e. "geoEventProviderId" = ${geoEventProviderId} + AND ST_Within(ST_SetSRID(ST_MakePoint(e.longitude, e.latitude), 4326), ST_GeomFromEWKB(decode(dg_elem, 'hex'))) + AND NOT EXISTS ( + SELECT 1 + FROM "SiteAlert" + WHERE "SiteAlert".longitude = e.longitude + AND "SiteAlert".latitude = e.latitude + AND "SiteAlert"."eventDate" = e."eventDate" + AND "SiteAlert"."siteId" = s.id + ) + + UNION + + SELECT + gen_random_uuid(), + e.type, + FALSE, + e."eventDate", + ${geoEventProviderClientId}, + e.confidence, + e.latitude, + e.longitude, + s.id AS SiteId, + e.data, + ST_Distance(ST_SetSRID(e.geometry, 4326), s."detectionGeometry") AS distance + FROM + "GeoEvent" e + INNER JOIN "Site" s ON ST_Within(ST_SetSRID(e.geometry, 4326), s."detectionGeometry") + AND s."deletedAt" IS NULL + AND s."isMonitored" = TRUE + AND (s."stopAlertUntil" IS NULL OR s."stopAlertUntil" < CURRENT_TIMESTAMP) + AND e."isProcessed" = FALSE + AND e. "geoEventProviderId" = ${geoEventProviderId} + AND (s.type = 'Polygon' OR s.type = 'Point') + AND NOT EXISTS ( + SELECT 1 + FROM "SiteAlert" + WHERE "SiteAlert".longitude = e.longitude + AND "SiteAlert".latitude = e.latitude + AND "SiteAlert"."eventDate" = e."eventDate" + AND "SiteAlert"."siteId" = s.id + ); + `; + + const updateGeoEventIsProcessedToTrue = Prisma.sql`UPDATE "GeoEvent" SET "isProcessed" = true WHERE "isProcessed" = false AND "geoEventProviderId" = ${geoEventProviderId}`; + + // Create SiteAlerts by joining New GeoEvents and Sites that have the event's location in their proximity + // And, Set all GeoEvents as processed + const results = await prisma.$transaction([ + prisma.$executeRaw(siteAlertCreationQuery), + prisma.$executeRaw(updateGeoEventIsProcessedToTrue), + ]); + siteAlertsCreatedCount = results[0]; + } catch (error) { + logger(`Failed to create SiteAlerts. Error: ${error}`, 'error'); + } + }else { + // Non-geostationary satellites do not belong to a specific site + try { + const siteAlertCreationQuery = Prisma.sql`INSERT INTO "SiteAlert" (id, TYPE, "isProcessed", "eventDate", "detectedBy", confidence, latitude, longitude, "siteId", "data", "distance") + + SELECT + gen_random_uuid(), + e.type, + FALSE, + e."eventDate", + ${geoEventProviderClientId}, + e.confidence, + e.latitude, + e.longitude, + s.id AS SiteId, + e.data, + ST_Distance(ST_SetSRID(ST_MakePoint(e.longitude, e.latitude), 4326), ST_GeomFromEWKB(decode(dg_elem, 'hex'))) AS distance + FROM + "GeoEvent" e + CROSS JOIN + "Site" s, + jsonb_array_elements_text(s."geometry"->'properties'->'detection_geometry') AS dg_elem + WHERE + s."type" = 'MultiPolygon' + AND s."deletedAt" IS NULL + AND s."isMonitored" = TRUE + AND (s."stopAlertUntil" IS NULL OR s."stopAlertUntil" < CURRENT_TIMESTAMP) + AND e."isProcessed" = FALSE + AND e. "geoEventProviderId" = ${geoEventProviderId} + AND s.slices @> ('["' || ${slice} || '"]')::jsonb + AND ST_Within(ST_SetSRID(ST_MakePoint(e.longitude, e.latitude), 4326), ST_GeomFromEWKB(decode(dg_elem, 'hex'))) + AND NOT EXISTS ( + SELECT 1 + FROM "SiteAlert" + WHERE "SiteAlert".longitude = e.longitude + AND "SiteAlert".latitude = e.latitude + AND "SiteAlert"."eventDate" = e."eventDate" + AND "SiteAlert"."siteId" = s.id + ) + + UNION + + SELECT + gen_random_uuid(), + e.type, + FALSE, + e."eventDate", + ${geoEventProviderClientId}, + e.confidence, + e.latitude, + e.longitude, + s.id AS SiteId, + e.data, + ST_Distance(ST_SetSRID(e.geometry, 4326), s."detectionGeometry") AS distance + FROM + "GeoEvent" e + INNER JOIN "Site" s ON ST_Within(ST_SetSRID(e.geometry, 4326), s."detectionGeometry") + AND s."deletedAt" IS NULL + AND s."isMonitored" = TRUE + AND (s."stopAlertUntil" IS NULL OR s."stopAlertUntil" < CURRENT_TIMESTAMP) + AND e."isProcessed" = FALSE + AND e. "geoEventProviderId" = ${geoEventProviderId} + AND s.slices @> ('["' || ${slice} || '"]')::jsonb + AND (s.type = 'Polygon' OR s.type = 'Point') + AND NOT EXISTS ( + SELECT 1 + FROM "SiteAlert" + WHERE "SiteAlert".longitude = e.longitude + AND "SiteAlert".latitude = e.latitude + AND "SiteAlert"."eventDate" = e."eventDate" + AND "SiteAlert"."siteId" = s.id + ); + `; + + const updateGeoEventIsProcessedToTrue = Prisma.sql`UPDATE "GeoEvent" SET "isProcessed" = true WHERE "isProcessed" = false AND "geoEventProviderId" = ${geoEventProviderId} AND "slice" = ${slice}`; + + // Create SiteAlerts by joining New GeoEvents and Sites that have the event's location in their proximity + // And, Set all GeoEvents as processed + const results = await prisma.$transaction([ + prisma.$executeRaw(siteAlertCreationQuery), + prisma.$executeRaw(updateGeoEventIsProcessedToTrue), + ]); + siteAlertsCreatedCount = results[0]; + } catch (error) { + logger(`Failed to create SiteAlerts. Error: ${error}`, 'error'); + } } return siteAlertsCreatedCount; }; diff --git a/apps/server/src/pages/api/cron/geo-event-fetcher.ts b/apps/server/src/pages/api/cron/geo-event-fetcher.ts index c093ebda3..9db1872a0 100644 --- a/apps/server/src/pages/api/cron/geo-event-fetcher.ts +++ b/apps/server/src/pages/api/cron/geo-event-fetcher.ts @@ -90,17 +90,21 @@ export default async function alertFetcher(req: NextApiRequest, res: NextApiResp // Loop for each active provider and fetch geoEvents const promises = activeProviders.map(async (provider) => { - const { config, id: geoEventProviderId, clientId: geoEventProviderClientId, clientApiKey } = provider + const { config, id: geoEventProviderId, clientId: geoEventProviderClientId, clientApiKey, lastRun } = provider + // For GOES-16, geoEventProviderId is 55, geoEventProviderClientId is GEOSTATIONARY, and clientApiKey is GOES-16 const parsedConfig: GeoEventProviderConfig = JSON.parse(JSON.stringify(config)) - const client = parsedConfig.client + const client = parsedConfig.client // For GOES-16 = GOES-16 const geoEventProvider = GeoEventProviderClassRegistry.get(client); geoEventProvider.initialize(parsedConfig); const slice = parsedConfig.slice; - const breadcrumbPrefix = `${geoEventProviderClientId} Slice ${slice}:` + let breadcrumbPrefix = `${geoEventProviderClientId} Slice ${slice}:` + if(geoEventProviderClientId === 'GEOSTATIONARY'){ + breadcrumbPrefix = `Geostationary Satellite ${clientApiKey}:` + } // First fetch all geoEvents from the provider - return await geoEventProvider.getLatestGeoEvents(geoEventProviderClientId, geoEventProviderId, slice, clientApiKey) + return await geoEventProvider.getLatestGeoEvents(geoEventProviderClientId, geoEventProviderId, slice, clientApiKey, lastRun) .then(async (geoEvents) => { // If there are geoEvents, emit an event to find duplicates and persist them logger(`${breadcrumbPrefix} Fetched ${geoEvents.length} geoEvents`, "info"); @@ -115,7 +119,7 @@ export default async function alertFetcher(req: NextApiRequest, res: NextApiResp // Process each chunk sequentially for (const geoEventChunk of geoEventChunks) { - const processedGeoEvent = await processGeoEvents(breadcrumbPrefix, geoEventProviderClientId as GeoEventProviderClientId, geoEventProviderId, slice, geoEventChunk); + const processedGeoEvent = await processGeoEvents(geoEventProviderClientId as GeoEventProviderClientId, geoEventProviderId, slice, geoEventChunk); totalEventCount += processedGeoEvent.geoEventCount; totalNewGeoEvent += processedGeoEvent.newGeoEventCount; } diff --git a/apps/server/src/server/api/routers/user.ts b/apps/server/src/server/api/routers/user.ts index 477d4b729..ad408d0c2 100644 --- a/apps/server/src/server/api/routers/user.ts +++ b/apps/server/src/server/api/routers/user.ts @@ -11,7 +11,6 @@ export const userRouter = createTRPCRouter({ // profile procedure signs in only clients, but cannot sign in an admin. // However, it logs both client and admin (admin with or without impersonatedUser headers ) profile: protectedProcedure .query(async ({ctx}) => { - debugger; try { // If ctx.user is null, then sign in a new user. if (ctx.user === null) { diff --git a/apps/server/src/server/api/zodSchemas/site.schema.ts b/apps/server/src/server/api/zodSchemas/site.schema.ts index 420bb934d..ef9398530 100644 --- a/apps/server/src/server/api/zodSchemas/site.schema.ts +++ b/apps/server/src/server/api/zodSchemas/site.schema.ts @@ -1,6 +1,6 @@ import {z} from 'zod'; import {nameSchema} from './user.schema'; - +// All coordinates are in [longitude, latitude] format const PointSchema = z.object({ type: z.literal("Point"), coordinates: z.tuple([z.number(), z.number()]), diff --git a/apps/server/src/utils/geometry.ts b/apps/server/src/utils/geometry.ts new file mode 100644 index 000000000..29d1252e8 --- /dev/null +++ b/apps/server/src/utils/geometry.ts @@ -0,0 +1,27 @@ +import slices from "../utils/slices" +interface Slice { + name: string; + bbox: string; + coordinates: number[][]; + description: string; +} +export interface Slices { + [key: string]: Slice; +} +// Given that the slices are defined by their bounding boxes (bbox) which simplifies the geometry to rectangular areas, +// we can determine slice using a quick containment check. + +// This method assumes that the slices do not overlap and are well-defined. + +export function determineSlice(latitude: number, longitude: number): string { + for (const [sliceKey, slice] of Object.entries(slices)) { + const bbox = slice.bbox.split(',').map(Number); + const [minLon, minLat, maxLon, maxLat] = bbox; + + // Check if the point is within the bbox of the slice + if (longitude >= minLon && longitude <= maxLon && latitude >= minLat && latitude <= maxLat) { + return sliceKey; // Return the slice name if the point is within the bbox + } + } + return '0'; // Return '0' if the point doesn't fall into any slice +} \ No newline at end of file diff --git a/yarn.lock b/yarn.lock index a93b2c9cf..085477d39 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1209,6 +1209,14 @@ resolved "https://registry.yarnpkg.com/@eslint/js/-/js-8.43.0.tgz#559ca3d9ddbd6bf907ad524320a0d14b85586af0" integrity sha512-s2UHCoiXfxMvmfzqoN+vrQ84ahUSYde9qNO1MdxmoEhyHWsfmwOpFlwYV+ePJEVc7gFnATGUi376WowX1N7tFg== +"@google/earthengine@^0.1.396": + version "0.1.396" + resolved "https://registry.yarnpkg.com/@google/earthengine/-/earthengine-0.1.396.tgz#9968175634747dc142ea4d9d9523f426b231cbed" + integrity sha512-GWHukKcqhYgiBHY73c+Z+NXsKI9olRrvlmrPwDPmk2kqGUJ3Odz6EvC8shO34DWjAzCJ7hbIGAuze649JMjH8g== + dependencies: + googleapis "^92.0.0" + xmlhttprequest "^1.8.0" + "@hapi/hoek@^9.0.0": version "9.3.0" resolved "https://registry.yarnpkg.com/@hapi/hoek/-/hoek-9.3.0.tgz#8368869dcb735be2e7f5cb7647de78e167a251fb" @@ -3347,6 +3355,11 @@ arraybuffer.prototype.slice@^1.0.2: is-array-buffer "^3.0.2" is-shared-array-buffer "^1.0.2" +arrify@^2.0.0: + version "2.0.1" + resolved "https://registry.yarnpkg.com/arrify/-/arrify-2.0.1.tgz#c9655e9331e0abcd588d2a7cad7e9956f66701fa" + integrity sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug== + asap@^2.0.0, asap@~2.0.6: version "2.0.6" resolved "https://registry.yarnpkg.com/asap/-/asap-2.0.6.tgz#e50347611d7e690943208bbdafebcbc2fb866d46" @@ -3617,7 +3630,7 @@ base-64@^0.1.0: resolved "https://registry.yarnpkg.com/base-64/-/base-64-0.1.0.tgz#780a99c84e7d600260361511c4877613bf24f6bb" integrity sha512-Y5gU45svrR5tI2Vt/X9GPd3L0HNIKzGu202EjxrXMpuc2V2CiKgemAbUUsqYmZJvPtCXoUKjNZwBJzsNScUbXA== -base64-js@^1.1.2, base64-js@^1.3.1: +base64-js@^1.1.2, base64-js@^1.3.0, base64-js@^1.3.1: version "1.5.1" resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.5.1.tgz#1b1b440160a5bf7ad40b650f095963481903930a" integrity sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA== @@ -3635,6 +3648,11 @@ base@^0.11.1: mixin-deep "^1.2.0" pascalcase "^0.1.1" +bignumber.js@^9.0.0: + version "9.1.2" + resolved "https://registry.yarnpkg.com/bignumber.js/-/bignumber.js-9.1.2.tgz#b7c4242259c008903b13707983b5f4bbd31eda0c" + integrity sha512-2/mKyZH9K85bzOEfhXDBFZTGd1CTs+5IHpeFQo9luiBG7hghdC851Pj2WAhb6E3R6b9tZj/XKhbg4fum+Kepug== + bl@^4.1.0: version "4.1.0" resolved "https://registry.yarnpkg.com/bl/-/bl-4.1.0.tgz#451535264182bec2fbbc83a62ab98cf11d9f7b3a" @@ -3795,6 +3813,17 @@ call-bind@^1.0.0, call-bind@^1.0.2: function-bind "^1.1.1" get-intrinsic "^1.0.2" +call-bind@^1.0.7: + version "1.0.7" + resolved "https://registry.yarnpkg.com/call-bind/-/call-bind-1.0.7.tgz#06016599c40c56498c18769d2730be242b6fa3b9" + integrity sha512-GHTSNSYICQ7scH7sZ+M2rFopRoLh8t2bLSW6BbgrtLsahOIB5iyAVJf9GjWK3cYTDaMj4XdBpM1cA6pIS0Kv2w== + dependencies: + es-define-property "^1.0.0" + es-errors "^1.3.0" + function-bind "^1.1.2" + get-intrinsic "^1.2.4" + set-function-length "^1.2.1" + caller-callsite@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/caller-callsite/-/caller-callsite-2.0.0.tgz#847e0fce0a223750a9a027c54b33731ad3154134" @@ -4374,6 +4403,15 @@ define-data-property@^1.0.1: gopd "^1.0.1" has-property-descriptors "^1.0.0" +define-data-property@^1.1.4: + version "1.1.4" + resolved "https://registry.yarnpkg.com/define-data-property/-/define-data-property-1.1.4.tgz#894dc141bb7d3060ae4366f6a0107e68fbe48c5e" + integrity sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A== + dependencies: + es-define-property "^1.0.0" + es-errors "^1.3.0" + gopd "^1.0.1" + define-properties@^1.1.3, define-properties@^1.1.4, define-properties@^1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/define-properties/-/define-properties-1.2.0.tgz#52988570670c9eacedd8064f4a990f2405849bd5" @@ -4572,7 +4610,7 @@ eastasianwidth@^0.2.0: resolved "https://registry.yarnpkg.com/eastasianwidth/-/eastasianwidth-0.2.0.tgz#696ce2ec0aa0e6ea93a397ffcf24aa7840c827cb" integrity sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA== -ecdsa-sig-formatter@1.0.11: +ecdsa-sig-formatter@1.0.11, ecdsa-sig-formatter@^1.0.11: version "1.0.11" resolved "https://registry.yarnpkg.com/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz#ae0f0fa2d85045ef14a817daa3ce9acd0489e5bf" integrity sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ== @@ -4758,6 +4796,18 @@ es-abstract@^1.22.1: unbox-primitive "^1.0.2" which-typed-array "^1.1.11" +es-define-property@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/es-define-property/-/es-define-property-1.0.0.tgz#c7faefbdff8b2696cf5f46921edfb77cc4ba3845" + integrity sha512-jxayLKShrEqqzJ0eumQbVhTYQM27CfT1T35+gCgDFoL82JLsXqTJ76zv6A0YLOgEnLUMvLzsDsGIrl8NFpT2gQ== + dependencies: + get-intrinsic "^1.2.4" + +es-errors@^1.3.0: + version "1.3.0" + resolved "https://registry.yarnpkg.com/es-errors/-/es-errors-1.3.0.tgz#05f75a25dab98e4fb1dcd5e1472c0546d5057c8f" + integrity sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw== + es-iterator-helpers@^1.0.12: version "1.0.15" resolved "https://registry.yarnpkg.com/es-iterator-helpers/-/es-iterator-helpers-1.0.15.tgz#bd81d275ac766431d19305923707c3efd9f1ae40" @@ -5219,6 +5269,11 @@ extend-shallow@^3.0.0, extend-shallow@^3.0.2: assign-symbols "^1.0.0" is-extendable "^1.0.1" +extend@^3.0.2: + version "3.0.2" + resolved "https://registry.yarnpkg.com/extend/-/extend-3.0.2.tgz#f8b1136b4071fbd8eb140aff858b1019ec2915fa" + integrity sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g== + extglob@^2.0.4: version "2.0.4" resolved "https://registry.yarnpkg.com/extglob/-/extglob-2.0.4.tgz#ad00fe4dc612a9232e8718711dc5cb5ab0285543" @@ -5280,6 +5335,11 @@ fast-safe-stringify@^2.1.1: resolved "https://registry.yarnpkg.com/fast-safe-stringify/-/fast-safe-stringify-2.1.1.tgz#c406a83b6e70d9e35ce3b30a81141df30aeba884" integrity sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA== +fast-text-encoding@^1.0.0: + version "1.0.6" + resolved "https://registry.yarnpkg.com/fast-text-encoding/-/fast-text-encoding-1.0.6.tgz#0aa25f7f638222e3396d72bf936afcf1d42d6867" + integrity sha512-VhXlQgj9ioXCqGstD37E/HBeqEGV/qOD/kmbVG8h5xKBYvM1L3lR1Zn4555cQ8GkYbJa8aJSipLPndE1k6zK2w== + fast-xml-parser@^4.0.12: version "4.2.5" resolved "https://registry.yarnpkg.com/fast-xml-parser/-/fast-xml-parser-4.2.5.tgz#a6747a09296a6cb34f2ae634019bf1738f3b421f" @@ -5516,6 +5576,11 @@ function-bind@^1.1.1: resolved "https://registry.yarnpkg.com/function-bind/-/function-bind-1.1.1.tgz#a56899d3ea3c9bab874bb9773b7c5ede92f4895d" integrity sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A== +function-bind@^1.1.2: + version "1.1.2" + resolved "https://registry.yarnpkg.com/function-bind/-/function-bind-1.1.2.tgz#2c02d864d97f3ea6c8830c464cbd11ab6eab7a1c" + integrity sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA== + function.prototype.name@^1.1.5: version "1.1.5" resolved "https://registry.yarnpkg.com/function.prototype.name/-/function.prototype.name-1.1.5.tgz#cce0505fe1ffb80503e6f9e46cc64e46a12a9621" @@ -5560,6 +5625,25 @@ gauge@^4.0.3: strip-ansi "^6.0.1" wide-align "^1.1.5" +gaxios@^4.0.0: + version "4.3.3" + resolved "https://registry.yarnpkg.com/gaxios/-/gaxios-4.3.3.tgz#d44bdefe52d34b6435cc41214fdb160b64abfc22" + integrity sha512-gSaYYIO1Y3wUtdfHmjDUZ8LWaxJQpiavzbF5Kq53akSzvmVg0RfyOcFDbO1KJ/KCGRFz2qG+lS81F0nkr7cRJA== + dependencies: + abort-controller "^3.0.0" + extend "^3.0.2" + https-proxy-agent "^5.0.0" + is-stream "^2.0.0" + node-fetch "^2.6.7" + +gcp-metadata@^4.2.0: + version "4.3.1" + resolved "https://registry.yarnpkg.com/gcp-metadata/-/gcp-metadata-4.3.1.tgz#fb205fe6a90fef2fd9c85e6ba06e5559ee1eefa9" + integrity sha512-x850LS5N7V1F3UcV7PoupzGsyD6iVwTVvsh3tbXfkctZnBnjW5yu5z1/3k3SehF7TyoTIe78rJs02GMMy+LF+A== + dependencies: + gaxios "^4.0.0" + json-bigint "^1.0.0" + gensync@^1.0.0-beta.2: version "1.0.0-beta.2" resolved "https://registry.yarnpkg.com/gensync/-/gensync-1.0.0-beta.2.tgz#32a6ee76c3d7f52d46b2b1ae5d93fea8580a25e0" @@ -5620,6 +5704,17 @@ get-intrinsic@^1.0.2, get-intrinsic@^1.1.1, get-intrinsic@^1.1.3, get-intrinsic@ has-proto "^1.0.1" has-symbols "^1.0.3" +get-intrinsic@^1.2.4: + version "1.2.4" + resolved "https://registry.yarnpkg.com/get-intrinsic/-/get-intrinsic-1.2.4.tgz#e385f5a4b5227d449c3eabbad05494ef0abbeadd" + integrity sha512-5uYhsJH8VJBTv7oslg4BznJYhDoRI6waYCxMmCdnTrcCrHA/fCFKoTFz2JKKE0HdDFUF7/oQuhzumXJK7paBRQ== + dependencies: + es-errors "^1.3.0" + function-bind "^1.1.2" + has-proto "^1.0.1" + has-symbols "^1.0.3" + hasown "^2.0.0" + get-package-type@^0.1.0: version "0.1.0" resolved "https://registry.yarnpkg.com/get-package-type/-/get-package-type-0.1.0.tgz#8de2d803cff44df3bc6c456e6668b36c3926e11a" @@ -5767,11 +5862,53 @@ globby@^11.1.0: merge2 "^1.4.1" slash "^3.0.0" +google-auth-library@^7.0.2, google-auth-library@^7.14.0: + version "7.14.1" + resolved "https://registry.yarnpkg.com/google-auth-library/-/google-auth-library-7.14.1.tgz#e3483034162f24cc71b95c8a55a210008826213c" + integrity sha512-5Rk7iLNDFhFeBYc3s8l1CqzbEBcdhwR193RlD4vSNFajIcINKI8W8P0JLmBpwymHqqWbX34pJDQu39cSy/6RsA== + dependencies: + arrify "^2.0.0" + base64-js "^1.3.0" + ecdsa-sig-formatter "^1.0.11" + fast-text-encoding "^1.0.0" + gaxios "^4.0.0" + gcp-metadata "^4.2.0" + gtoken "^5.0.4" + jws "^4.0.0" + lru-cache "^6.0.0" + google-libphonenumber@^3.2.10: version "3.2.32" resolved "https://registry.yarnpkg.com/google-libphonenumber/-/google-libphonenumber-3.2.32.tgz#63c48a9c247b64a3bc2eec21bdf3fcfbf2e148c0" integrity sha512-mcNgakausov/B/eTgVeX8qc8IwWjRrupk9UzZZ/QDEvdh5fAjE7Aa211bkZpZj42zKkeS6MTT8avHUwjcLxuGQ== +google-p12-pem@^3.1.3: + version "3.1.4" + resolved "https://registry.yarnpkg.com/google-p12-pem/-/google-p12-pem-3.1.4.tgz#123f7b40da204de4ed1fbf2fd5be12c047fc8b3b" + integrity sha512-HHuHmkLgwjdmVRngf5+gSmpkyaRI6QmOg77J8tkNBHhNEI62sGHyw4/+UkgyZEI7h84NbWprXDJ+sa3xOYFvTg== + dependencies: + node-forge "^1.3.1" + +googleapis-common@^5.0.2: + version "5.1.0" + resolved "https://registry.yarnpkg.com/googleapis-common/-/googleapis-common-5.1.0.tgz#845a79471c787e522e03c50d415467140e9e356a" + integrity sha512-RXrif+Gzhq1QAzfjxulbGvAY3FPj8zq/CYcvgjzDbaBNCD6bUl+86I7mUs4DKWHGruuK26ijjR/eDpWIDgNROA== + dependencies: + extend "^3.0.2" + gaxios "^4.0.0" + google-auth-library "^7.14.0" + qs "^6.7.0" + url-template "^2.0.8" + uuid "^8.0.0" + +googleapis@^92.0.0: + version "92.0.0" + resolved "https://registry.yarnpkg.com/googleapis/-/googleapis-92.0.0.tgz#291b9826a5a4509a9e9a6974ef942328857bfe18" + integrity sha512-5HgJg7XvqEEJ+GO+2gvnzd5cAcDuSS/VB6nW7thoyj2GMq9nH4VvJwncSevinjLCnv06a+VSxrXNiL5vePHojA== + dependencies: + google-auth-library "^7.0.2" + googleapis-common "^5.0.2" + gopd@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/gopd/-/gopd-1.0.1.tgz#29ff76de69dac7489b7c0918a5788e56477c332c" @@ -5799,6 +5936,15 @@ grid-index@^1.1.0: resolved "https://registry.yarnpkg.com/grid-index/-/grid-index-1.1.0.tgz#97f8221edec1026c8377b86446a7c71e79522ea7" integrity sha512-HZRwumpOGUrHyxO5bqKZL0B0GlUpwtCAzZ42sgxUPniu33R1LSFH5yrIcBCHjkctCAh3mtWKcKd9J4vDDdeVHA== +gtoken@^5.0.4: + version "5.3.2" + resolved "https://registry.yarnpkg.com/gtoken/-/gtoken-5.3.2.tgz#deb7dc876abe002178e0515e383382ea9446d58f" + integrity sha512-gkvEKREW7dXWF8NV8pVrKfW7WqReAmjjkMBh6lNCCGOM4ucS0r0YyXXl0r/9Yj8wcW/32ISkfc8h5mPTDbtifQ== + dependencies: + gaxios "^4.0.0" + google-p12-pem "^3.1.3" + jws "^4.0.0" + has-bigints@^1.0.1, has-bigints@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/has-bigints/-/has-bigints-1.0.2.tgz#0871bd3e3d51626f6ca0966668ba35d5602d6eaa" @@ -5821,6 +5967,13 @@ has-property-descriptors@^1.0.0: dependencies: get-intrinsic "^1.1.1" +has-property-descriptors@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/has-property-descriptors/-/has-property-descriptors-1.0.2.tgz#963ed7d071dc7bf5f084c5bfbe0d1b6222586854" + integrity sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg== + dependencies: + es-define-property "^1.0.0" + has-proto@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/has-proto/-/has-proto-1.0.1.tgz#1885c1305538958aff469fef37937c22795408e0" @@ -5886,6 +6039,13 @@ hash-wasm@^4.9.0: resolved "https://registry.yarnpkg.com/hash-wasm/-/hash-wasm-4.9.0.tgz#7e9dcc9f7d6bd0cc802f2a58f24edce999744206" integrity sha512-7SW7ejyfnRxuOc7ptQHSf4LDoZaWOivfzqw+5rpcQku0nHfmicPKE51ra9BiRLAmT8+gGLestr1XroUkqdjL6w== +hasown@^2.0.0: + version "2.0.2" + resolved "https://registry.yarnpkg.com/hasown/-/hasown-2.0.2.tgz#003eaf91be7adc372e84ec59dc37252cedb80003" + integrity sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ== + dependencies: + function-bind "^1.1.2" + hermes-estree@0.8.0: version "0.8.0" resolved "https://registry.yarnpkg.com/hermes-estree/-/hermes-estree-0.8.0.tgz#530be27243ca49f008381c1f3e8b18fb26bf9ec0" @@ -7021,6 +7181,13 @@ jsesc@~0.5.0: resolved "https://registry.yarnpkg.com/jsesc/-/jsesc-0.5.0.tgz#e7dee66e35d6fc16f710fe91d5cf69f70f08911d" integrity sha512-uZz5UnB7u4T9LvwmFqXii7pZSouaRPorGs5who1Ip7VO0wxanFvBL7GkM6dTHlgX+jhBApRetaWpnDabOeTcnA== +json-bigint@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/json-bigint/-/json-bigint-1.0.0.tgz#ae547823ac0cad8398667f8cd9ef4730f5b01ff1" + integrity sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ== + dependencies: + bignumber.js "^9.0.0" + json-parse-better-errors@^1.0.1: version "1.0.2" resolved "https://registry.yarnpkg.com/json-parse-better-errors/-/json-parse-better-errors-1.0.2.tgz#bb867cfb3450e69107c131d1c514bab3dc8bcaa9" @@ -7102,6 +7269,15 @@ jwa@^1.4.1: ecdsa-sig-formatter "1.0.11" safe-buffer "^5.0.1" +jwa@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/jwa/-/jwa-2.0.0.tgz#a7e9c3f29dae94027ebcaf49975c9345593410fc" + integrity sha512-jrZ2Qx916EA+fq9cEAeCROWPTfCwi1IVHqT2tapuqLEVVDKFDENFw1oL+MwrTvH6msKxsd1YTDVw6uKEcsrLEA== + dependencies: + buffer-equal-constant-time "1.0.1" + ecdsa-sig-formatter "1.0.11" + safe-buffer "^5.0.1" + jwks-rsa@^3.0.1: version "3.0.1" resolved "https://registry.yarnpkg.com/jwks-rsa/-/jwks-rsa-3.0.1.tgz#ba79ddca7ee7520f7bb26b942ef1aee91df8d7e4" @@ -7122,6 +7298,14 @@ jws@^3.2.2: jwa "^1.4.1" safe-buffer "^5.0.1" +jws@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/jws/-/jws-4.0.0.tgz#2d4e8cf6a318ffaa12615e9dec7e86e6c97310f4" + integrity sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg== + dependencies: + jwa "^2.0.0" + safe-buffer "^5.0.1" + jwt-decode@^3.1.2: version "3.1.2" resolved "https://registry.yarnpkg.com/jwt-decode/-/jwt-decode-3.1.2.tgz#3fb319f3675a2df0c2895c8f5e9fa4b67b04ed59" @@ -8210,6 +8394,11 @@ node-fetch@^2.2.0, node-fetch@^2.6.0, node-fetch@^2.6.11, node-fetch@^2.6.7: dependencies: whatwg-url "^5.0.0" +node-forge@^1.3.1: + version "1.3.1" + resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-1.3.1.tgz#be8da2af243b2417d5f646a770663a92b7e9ded3" + integrity sha512-dPEtOeMvF9VMcYV/1Wb8CPoVAXtp6MKMlcbAt4ddqmGqUJ6fQZFXkNZNkNlfevtNkGtaSoXf/vNNNSvgrdXwtA== + node-gyp@^9.3.0: version "9.4.0" resolved "https://registry.yarnpkg.com/node-gyp/-/node-gyp-9.4.0.tgz#2a7a91c7cba4eccfd95e949369f27c9ba704f369" @@ -8324,6 +8513,11 @@ object-inspect@^1.12.3, object-inspect@^1.9.0: resolved "https://registry.yarnpkg.com/object-inspect/-/object-inspect-1.12.3.tgz#ba62dffd67ee256c8c086dfae69e016cd1f198b9" integrity sha512-geUvdk7c+eizMNUDkRpW1wJwgfOiOeHbxBR/hLXK1aT6zmVSO0jsQcs7fj6MGw89jC/cjGfLcNOrtMYtGqm81g== +object-inspect@^1.13.1: + version "1.13.1" + resolved "https://registry.yarnpkg.com/object-inspect/-/object-inspect-1.13.1.tgz#b96c6109324ccfef6b12216a956ca4dc2ff94bc2" + integrity sha512-5qoj1RUiKOMsCCNLV1CBiPYE10sziTsnmNxkAI/rZhiD63CF7IqdFGC/XzjWjpSgLf0LxXX3bDFIh0E18f6UhQ== + object-keys@^1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/object-keys/-/object-keys-1.1.1.tgz#1c47f272df277f3b1daf061677d9c82e2322c60e" @@ -8952,6 +9146,13 @@ qs@^6.10.3, qs@^6.11.0, qs@^6.9.4: dependencies: side-channel "^1.0.4" +qs@^6.7.0: + version "6.12.0" + resolved "https://registry.yarnpkg.com/qs/-/qs-6.12.0.tgz#edd40c3b823995946a8a0b1f208669c7a200db77" + integrity sha512-trVZiI6RMOkO476zLGaBIzszOdFPnCCXHPG9kn0yuS1uz6xdVxPfZdB3vUig9pxPFDM9BRAgz/YUIVQ1/vuiUg== + dependencies: + side-channel "^1.0.6" + query-string@^7.1.3: version "7.1.3" resolved "https://registry.yarnpkg.com/query-string/-/query-string-7.1.3.tgz#a1cf90e994abb113a325804a972d98276fe02328" @@ -9718,6 +9919,18 @@ set-blocking@^2.0.0: resolved "https://registry.yarnpkg.com/set-blocking/-/set-blocking-2.0.0.tgz#045f9782d011ae9a6803ddd382b24392b3d890f7" integrity sha512-KiKBS8AnWGEyLzofFfmvKwpdPzqiy16LvQfK3yv/fVH7Bj13/wl3JSR1J+rfgRE9q7xUJK4qvgS8raSOeLUehw== +set-function-length@^1.2.1: + version "1.2.2" + resolved "https://registry.yarnpkg.com/set-function-length/-/set-function-length-1.2.2.tgz#aac72314198eaed975cf77b2c3b6b880695e5449" + integrity sha512-pgRc4hJ4/sNjWCSS9AmnS40x3bNMDTknHgL5UaMBTMyJnU90EgWh1Rz+MC9eFu4BuN/UwZjKQuY/1v3rM7HMfg== + dependencies: + define-data-property "^1.1.4" + es-errors "^1.3.0" + function-bind "^1.1.2" + get-intrinsic "^1.2.4" + gopd "^1.0.1" + has-property-descriptors "^1.0.2" + set-function-name@^2.0.0, set-function-name@^2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/set-function-name/-/set-function-name-2.0.1.tgz#12ce38b7954310b9f61faa12701620a0c882793a" @@ -9799,6 +10012,16 @@ side-channel@^1.0.4: get-intrinsic "^1.0.2" object-inspect "^1.9.0" +side-channel@^1.0.6: + version "1.0.6" + resolved "https://registry.yarnpkg.com/side-channel/-/side-channel-1.0.6.tgz#abd25fb7cd24baf45466406b1096b7831c9215f2" + integrity sha512-fDW/EZ6Q9RiO8eFG8Hj+7u/oW+XrPTIChwCOM2+th2A6OblDtYYIpve9m+KvI9Z4C9qSEXlaGR6bTEYHReuglA== + dependencies: + call-bind "^1.0.7" + es-errors "^1.3.0" + get-intrinsic "^1.2.4" + object-inspect "^1.13.1" + signal-exit@^3.0.0, signal-exit@^3.0.2, signal-exit@^3.0.3, signal-exit@^3.0.7: version "3.0.7" resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.7.tgz#a9a1767f8af84155114eaabd73f99273c8f59ad9" @@ -10716,6 +10939,11 @@ url-parse@^1.5.9: querystringify "^2.1.1" requires-port "^1.0.0" +url-template@^2.0.8: + version "2.0.8" + resolved "https://registry.yarnpkg.com/url-template/-/url-template-2.0.8.tgz#fc565a3cccbff7730c775f5641f9555791439f21" + integrity sha512-XdVKMF4SJ0nP/O7XIPB0JwAEuT9lDIYnNsK8yGVe43y0AWoKeJNdv3ZNWh7ksJ6KqQFjOO6ox/VEitLnaVNufw== + url@^0.11.0: version "0.11.1" resolved "https://registry.yarnpkg.com/url/-/url-0.11.1.tgz#26f90f615427eca1b9f4d6a28288c147e2302a32" @@ -10754,6 +10982,11 @@ utils-merge@1.0.1: resolved "https://registry.yarnpkg.com/utils-merge/-/utils-merge-1.0.1.tgz#9f95710f50a267947b2ccc124741c1028427e713" integrity sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA== +uuid@^8.0.0: + version "8.3.2" + resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.3.2.tgz#80d5b5ced271bb9af6c445f21a1a04c606cefbe2" + integrity sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg== + uuid@^9.0.0: version "9.0.0" resolved "https://registry.yarnpkg.com/uuid/-/uuid-9.0.0.tgz#592f550650024a38ceb0c562f2f6aa435761efb5" @@ -11008,6 +11241,11 @@ xmldom@^0.6.0: resolved "https://registry.yarnpkg.com/xmldom/-/xmldom-0.6.0.tgz#43a96ecb8beece991cef382c08397d82d4d0c46f" integrity sha512-iAcin401y58LckRZ0TkI4k0VSM1Qg0KGSc3i8rU+xrxe19A/BN1zHyVSJY7uoutVlaTSzYyk/v5AmkewAP7jtg== +xmlhttprequest@^1.8.0: + version "1.8.0" + resolved "https://registry.yarnpkg.com/xmlhttprequest/-/xmlhttprequest-1.8.0.tgz#67fe075c5c24fef39f9d65f5f7b7fe75171968fc" + integrity sha512-58Im/U0mlVBLM38NdZjHyhuMtCqa61469k2YP/AaPbvCoV9aQGUpbJBj1QRm2ytRiVQBD/fsw7L2bJGDVQswBA== + xtend@^4.0.0, xtend@~4.0.1: version "4.0.2" resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.2.tgz#bb72779f5fa465186b1f438f674fa347fdb5db54" From 2ee347f741b0231fcdeb980317361cad9fac741f Mon Sep 17 00:00:00 2001 From: Aashish Dhakal <85501584+dhakalaashish@users.noreply.github.com> Date: Thu, 18 Apr 2024 17:05:19 +0545 Subject: [PATCH 2/5] Automatically process geostationary siteAlerts --- apps/server/src/Services/SiteAlert/CreateSiteAlert.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/server/src/Services/SiteAlert/CreateSiteAlert.ts b/apps/server/src/Services/SiteAlert/CreateSiteAlert.ts index 8d4bedb8b..3d4b03027 100644 --- a/apps/server/src/Services/SiteAlert/CreateSiteAlert.ts +++ b/apps/server/src/Services/SiteAlert/CreateSiteAlert.ts @@ -81,13 +81,15 @@ const createSiteAlerts = async ( `; const updateGeoEventIsProcessedToTrue = Prisma.sql`UPDATE "GeoEvent" SET "isProcessed" = true WHERE "isProcessed" = false AND "geoEventProviderId" = ${geoEventProviderId}`; - + // REMOVE after 2nd release + const updateGeostationarySiteAlertIsProcessedToTrue = Prisma.sql`UPDATE "SiteAlert" SET "isProcessed" = true WHERE "isProcessed" = false AND "detectedBy" = ${geoEventProviderClientId}`; // Create SiteAlerts by joining New GeoEvents and Sites that have the event's location in their proximity // And, Set all GeoEvents as processed const results = await prisma.$transaction([ prisma.$executeRaw(siteAlertCreationQuery), prisma.$executeRaw(updateGeoEventIsProcessedToTrue), ]); + await prisma.$executeRaw(updateGeostationarySiteAlertIsProcessedToTrue) siteAlertsCreatedCount = results[0]; } catch (error) { logger(`Failed to create SiteAlerts. Error: ${error}`, 'error'); From b01dc0b8b8b4f8846ec4d2ea5ce1ddb0934d79cb Mon Sep 17 00:00:00 2001 From: Aashish Dhakal <85501584+dhakalaashish@users.noreply.github.com> Date: Mon, 22 Apr 2024 15:51:33 +0545 Subject: [PATCH 3/5] Fix alert fetching time from static to dynamic --- .../ProviderClass/GOES16GeoEventProviderClass.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/server/src/Services/GeoEventProvider/ProviderClass/GOES16GeoEventProviderClass.ts b/apps/server/src/Services/GeoEventProvider/ProviderClass/GOES16GeoEventProviderClass.ts index f3c1fbeff..cffc57b31 100644 --- a/apps/server/src/Services/GeoEventProvider/ProviderClass/GOES16GeoEventProviderClass.ts +++ b/apps/server/src/Services/GeoEventProvider/ProviderClass/GOES16GeoEventProviderClass.ts @@ -86,7 +86,7 @@ class GOES16GeoEventProviderClass implements GeoEventProviderClass { const twoHoursAgo = new Date(currentDateTime.getTime() - 2 * 3600 * 1000); const fromDateTime = (!lastRunDate || (currentDateTime.getTime() - lastRunDate.getTime()) > 2 * 3600 * 1000) ? twoHoursAgo : lastRunDate; - const images = ee.ImageCollection("NOAA/GOES/16/FDCF").filterDate('2024-04-18T08:51:15Z', '2024-04-18T09:51:15Z'); + const images = ee.ImageCollection("NOAA/GOES/16/FDCF").filterDate(fromDateTime, currentDateTime); // Fetch and process images here... // The process includes fetching image IDs, processing them to extract fire data, etc. From 2b8e19370c34820dd5f19a9fd160dab95c3024e8 Mon Sep 17 00:00:00 2001 From: Aashish Dhakal <85501584+dhakalaashish@users.noreply.github.com> Date: Tue, 23 Apr 2024 10:54:48 +0545 Subject: [PATCH 4/5] Optimize site filtering by slice during SiteAlert creation for geostationary satellites. --- apps/server/src/Services/GeoEvent/GeoEventHandler.ts | 5 +++++ apps/server/src/Services/SiteAlert/CreateSiteAlert.ts | 10 ++++++++++ 2 files changed, 15 insertions(+) diff --git a/apps/server/src/Services/GeoEvent/GeoEventHandler.ts b/apps/server/src/Services/GeoEvent/GeoEventHandler.ts index b09cd3a14..e0d63cf15 100644 --- a/apps/server/src/Services/GeoEvent/GeoEventHandler.ts +++ b/apps/server/src/Services/GeoEvent/GeoEventHandler.ts @@ -45,6 +45,11 @@ const processGeoEvents = async (geoEventProviderClientId: GeoEventProviderClient // having the provided providerKey const geoEvents = await prisma.geoEvent.findMany({ select: { id: true }, + // IMPROVEMENT: this code does not identify duplicate between providers, + // It only identifies duplicate within a provider + // To identify duplicates between providers, remove geoEventProviderId from the where clause + // However, that would increase memory usage, and possibly freeze the process + // Identify ways to test for duplication against the entire database. where: { geoEventProviderId: geoEventProviderId, eventDate: { gt: new Date(Date.now() - 30 * 60 * 60 * 1000) } }, }); // Only compare with data from last 26 hrs diff --git a/apps/server/src/Services/SiteAlert/CreateSiteAlert.ts b/apps/server/src/Services/SiteAlert/CreateSiteAlert.ts index 3d4b03027..4a2abea24 100644 --- a/apps/server/src/Services/SiteAlert/CreateSiteAlert.ts +++ b/apps/server/src/Services/SiteAlert/CreateSiteAlert.ts @@ -37,6 +37,11 @@ const createSiteAlerts = async ( AND (s."stopAlertUntil" IS NULL OR s."stopAlertUntil" < CURRENT_TIMESTAMP) AND e."isProcessed" = FALSE AND e. "geoEventProviderId" = ${geoEventProviderId} + AND EXISTS ( + SELECT 1 + FROM jsonb_array_elements_text(s.slices) AS slice_element + WHERE slice_element = ANY(string_to_array(${Prisma.raw(`'${slice}'`)}, ',')::text[]) + ) AND ST_Within(ST_SetSRID(ST_MakePoint(e.longitude, e.latitude), 4326), ST_GeomFromEWKB(decode(dg_elem, 'hex'))) AND NOT EXISTS ( SELECT 1 @@ -70,6 +75,11 @@ const createSiteAlerts = async ( AND e."isProcessed" = FALSE AND e. "geoEventProviderId" = ${geoEventProviderId} AND (s.type = 'Polygon' OR s.type = 'Point') + AND EXISTS ( + SELECT 1 + FROM jsonb_array_elements_text(s.slices) AS slice_element + WHERE slice_element = ANY(string_to_array(${Prisma.raw(`'${slice}'`)}, ',')::text[]) + ) AND NOT EXISTS ( SELECT 1 FROM "SiteAlert" From d8cb1de8fbdc5fc12fbbbc941d5d20dbd192a6fb Mon Sep 17 00:00:00 2001 From: Aashish Dhakal <85501584+dhakalaashish@users.noreply.github.com> Date: Tue, 23 Apr 2024 11:23:15 +0545 Subject: [PATCH 5/5] Prioritize Most-Overdue Providers --- apps/server/src/pages/api/cron/geo-event-fetcher.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/apps/server/src/pages/api/cron/geo-event-fetcher.ts b/apps/server/src/pages/api/cron/geo-event-fetcher.ts index 9db1872a0..28b975b46 100644 --- a/apps/server/src/pages/api/cron/geo-event-fetcher.ts +++ b/apps/server/src/pages/api/cron/geo-event-fetcher.ts @@ -55,7 +55,6 @@ export default async function alertFetcher(req: NextApiRequest, res: NextApiResp let newSiteAlertCount = 0; let processedProviders = 0; - // while (processedProviders <= limit) { const activeProviders: GeoEventProvider[] = await prisma.$queryRaw` SELECT * @@ -63,8 +62,8 @@ export default async function alertFetcher(req: NextApiRequest, res: NextApiResp WHERE "isActive" = true AND "fetchFrequency" IS NOT NULL AND ("lastRun" + ("fetchFrequency" || ' minutes')::INTERVAL) < (current_timestamp AT TIME ZONE 'UTC') + ORDER BY (current_timestamp AT TIME ZONE 'UTC' - "lastRun") DESC LIMIT ${limit}; - `; // Filter out those active providers whose last (run date + fetchFrequency (in minutes) > current time // Break the loop if there are no active providers