parse-data.js

const fs = require('fs');
const {
  parseEventsData,
  parseSkuProductMappingData,
  formatEvents,
} = require('./helpers/data-process.js');
const { deleteFile } = require('./helpers/file-manipulation.js');

// We load into memory the csv file generated by dbeaver
const eventsData = fs.readFileSync('data/DB_ALL2.csv', { encoding: 'utf8', flag: 'r' });
// const eventsData = fs.readFileSync('data/STAGING.csv', { encoding: 'utf8', flag: 'r' });
const EVENT_TYPES_TO_KEEP_INBOUND = ['SKU_INBOUNDED', 'SKU_INCREMENTED', 'SKU_FOUND'];
const EVENT_TYPES_TO_KEEP_OUTBOUND = ['SKU_OUTBOUNDED', 'SKU_DECREMENTED', 'SKU_MISSING'];

// We parse the data into Objects
const events = parseEventsData(eventsData);
const WMSevents = events.filter((event) => event.source === 'WMS');
const relevantInboundEvents = WMSevents.filter((event) =>
  EVENT_TYPES_TO_KEEP_INBOUND.includes(event.payload.type),
);
const relevantOutboundEvents = WMSevents.filter((event) =>
  EVENT_TYPES_TO_KEEP_OUTBOUND.includes(event.payload.type),
);

// const skuMappingData = fs.readFileSync('data/STAGING_SKU_MAPPING.csv', {
const skuMappingData = fs.readFileSync('data/sku-mapping2.csv', {
  encoding: 'utf8',
  flag: 'r',
});
const skuMapping = parseSkuProductMappingData(skuMappingData);

// We format the events and keep only the relevant information
const { events: eventsInboundFormatted, results: resultsInbound } = formatEvents(
  relevantInboundEvents,
  skuMapping,
);
const { events: eventsOutBoundFormatted, results: resultsOutbound } = formatEvents(
  relevantOutboundEvents,
  skuMapping,
);
deleteFile('data/relevantInboundEvents.csv');
deleteFile('data/relevantOutboundEvents.csv');

// We save the events to a file
const eventsInboundWriter = fs.createWriteStream('data/relevantInboundEvents.csv', { flags: 'a' });
const eventsOutboundWriter = fs.createWriteStream('data/relevantOutboundEvents.csv', {
  flags: 'a',
});

eventsInboundFormatted.forEach((event) => {
  eventsInboundWriter.write(`${Object.values(event).join()}\n`);
});
eventsOutBoundFormatted.forEach((event) => {
  eventsOutboundWriter.write(`${Object.values(event).join()}\n`);
});

console.log(`Inbound Events written to file: ${eventsInboundFormatted.length} events`);
console.log(resultsInbound);
console.log(`Outbound Events written to file: ${eventsOutBoundFormatted.length} events`);
console.log(resultsOutbound);