Skip to content

Commit

Permalink
replace with xxhash3
Browse files Browse the repository at this point in the history
  • Loading branch information
sagararyal committed Jul 2, 2023
1 parent 760a833 commit e3fef23
Showing 1 changed file with 13 additions and 10 deletions.
23 changes: 13 additions & 10 deletions apps/server/src/Services/GeoEvent/GeoEventHandler.ts
Original file line number Diff line number Diff line change
@@ -1,35 +1,38 @@
import { GeoEventProviderClientId } from "../../Interfaces/GeoEventProvider";
import { type GeoEventProviderClientId } from "../../Interfaces/GeoEventProvider";
import { AlertType } from "../../Interfaces/SiteAlert";
import { type geoEventInterface as GeoEvent } from "../../Interfaces/GeoEvent";
import md5 from "md5";
import { createXXHash3 } from "hash-wasm";
import { prisma } from '../../server/db';
import { logger } from "../../../src/server/logger";

const processGeoEvents = async (breadcrumbPrefix: string, geoEventProviderClientId: GeoEventProviderClientId, geoEventProviderId: string, slice: string, geoEvents: GeoEvent[]) => {
const buildChecksum = (geoEvent: GeoEvent): string => {
return md5(
const buildChecksum = async (geoEvent: GeoEvent): Promise<string> => {
const hasher = await createXXHash3();
return hasher.update(
geoEvent.type +
geoEvent.latitude.toString() +
geoEvent.longitude.toString() +
geoEvent.eventDate.toISOString()
);
).digest('hex');
};
// events from multiple sources but same satellite with the same geoEventProviderId will be considered duplicates

// Check whether the fetchId already exists in the database and returns only the ones that are not in the database in the variable newGeoEvents
const compareIds = (dbEventIds: string[], fetchedEvents: GeoEvent[]): GeoEvent[] => {
const compareIds = async (dbEventIds: string[], fetchedEvents: GeoEvent[]): Promise<GeoEvent[]> => {
const newGeoEvents: GeoEvent[] = [];
const fetchedIds: string[] = [];

// Identify new hashes
fetchedEvents.forEach((fetchedEvent: GeoEvent) => {
const id = buildChecksum(fetchedEvent);
for (const fetchedEvent of fetchedEvents) {
// await keyword added before buildChecksum
const id = await buildChecksum(fetchedEvent);
fetchedIds.push(id);
if (!dbEventIds.includes(id)) {
fetchedEvent.id = id;
newGeoEvents.push(fetchedEvent);
}
});
}

return newGeoEvents;
};

Expand All @@ -45,7 +48,7 @@ const processGeoEvents = async (breadcrumbPrefix: string, geoEventProviderClient
return geoEvents.map(geoEvent => geoEvent.id);
};

const newGeoEvents = compareIds(await fetchDbEventIds(geoEventProviderId), geoEvents);
const newGeoEvents = await compareIds(await fetchDbEventIds(geoEventProviderId), geoEvents);

const filterDuplicateEvents = (newGeoEvents: GeoEvent[]): GeoEvent[] => {
const filteredNewGeoEvents: GeoEvent[] = [];
Expand Down

1 comment on commit e3fef23

@vercel
Copy link

@vercel vercel bot commented on e3fef23 Jul 2, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please sign in to comment.