Enable to import vac as CSV

.changeset/gorgeous-dolls-shake.md

@@ -0,0 +1,5 @@
+---
+"@frameless/overige-objecten-api": minor
+---
+
+Maak het mogelijk om VAC als een CSV-bestand te importeren via de API.

apps/overige-objecten-api/package.json

@@ -20,40 +20,42 @@
     "test:watch": "OVERIGE_OBJECTEN_API_PORT=3000 jest --watch"
   },
   "dependencies": {
+    "@frameless/ui": "0.1.0",
+    "@utrecht/component-library-react": "7.3.5",
+    "@utrecht/web-component-library-react": "3.0.0",
     "cors": "2.8.5",
+    "csv-parser": "3.0.0",
+    "dompurify": "3.2.1",
     "dotenv": "16.4.5",
     "express": "4.21.0",
     "express-openapi-validator": "5.3.7",
     "js-yaml": "4.1.0",
-    "swagger-ui-express": "5.0.1",
+    "lodash.memoize": "4.1.2",
     "lodash.merge": "4.6.2",
     "lodash.snakecase": "4.1.1",
-    "lodash.memoize": "4.1.2",
     "react": "18.2.0",
-    "@utrecht/web-component-library-react": "3.0.0",
-    "@utrecht/component-library-react": "7.3.5",
-    "@frameless/ui": "0.1.0",
-    "dompurify": "3.2.1"
+    "swagger-ui-express": "5.0.1",
+    "p-limit": "3.0.0"
   },
   "devDependencies": {
     "@types/cors": "2.8.17",
     "@types/dompurify": "3.2.0",
     "@types/jest": "29.5.12",
+    "@types/lodash.memoize": "4.1.9",
+    "@types/lodash.merge": "4.6.9",
+    "@types/lodash.snakecase": "4.1.9",
     "@types/supertest": "6.0.2",
     "@types/swagger-ui-express": "4.1.6",
     "@types/yamljs": "0.2.34",
     "jest": "29.7.0",
+    "jest-fetch-mock": "3.0.3",
     "nodemon": "3.1.7",
     "openapi-typescript": "7.4.1",
     "rimraf": "6.0.1",
     "supertest": "7.0.0",
     "ts-jest": "29.2.3",
     "ts-node": "10.9.2",
-    "typescript": "5.0.4",
-    "jest-fetch-mock": "3.0.3",
-    "@types/lodash.merge": "4.6.9",
-    "@types/lodash.snakecase": "4.1.9",
-    "@types/lodash.memoize": "4.1.9"
+    "typescript": "5.0.4"
   },
   "repository": {
     "type": "git+ssh",

apps/overige-objecten-api/src/components/Markdown.tsx

@@ -1,23 +1,14 @@
 import { Markdown as ReactMarkdown } from '@frameless/ui';
-import DOMPurify from 'dompurify';
-import { JSDOM } from 'jsdom';
-import memoize from 'lodash.memoize';
 import React from 'react';
 import type { Price } from '../strapi-product-type';
+import { sanitizeHTML } from '../utils';
 
 export interface MarkdownProps {
   children: string;
   priceData?: Price[];
 }
 
-const createDOMPurify = memoize(() => {
-  const { window } = new JSDOM();
-  return DOMPurify(window);
-});
-
 export const Markdown = ({ children: html, priceData }: MarkdownProps) => {
-  const domPurify = createDOMPurify();
-  const sanitizeHTML = memoize((html) => domPurify.sanitize(html));
   const DOMPurifyHTML = sanitizeHTML(html);
 
   return html ? (

apps/overige-objecten-api/src/controllers/import/index.ts

@@ -0,0 +1,63 @@
+import type { NextFunction, Request, Response } from 'express';
+import fs from 'node:fs';
+import pLimit from 'p-limit';
+import { CREATE_VAC } from '../../queries';
+import { CreateVacResponse } from '../../strapi-product-type';
+import { fetchData, processCsvFile } from '../../utils';
+
+const limit = pLimit(5); // Limit the number of concurrent file uploads
+export const importController = async (req: Request, res: Response, next: NextFunction) => {
+  const type = req.body.type;
+  if (req.file) {
+    const filePath = req.file.path;
+    const requiredColumns = ['vraag', 'antwoord'];
+
+    try {
+      // Process the CSV file and sanitize results
+      const isAuthHasToken = req.headers?.authorization?.startsWith('Token');
+      const tokenAuth = isAuthHasToken ? req.headers?.authorization?.split(' ')[1] : req.headers?.authorization;
+      const graphqlURL = new URL('/graphql', process.env.STRAPI_PRIVATE_URL);
+      const sanitizedResults = await processCsvFile(filePath, requiredColumns);
+      const locale = req.query?.locale || 'nl';
+
+      if (type === 'vac') {
+        // Loop through the sanitized results and create entries one by one
+        const results = await Promise.all(
+          sanitizedResults.map((entry) =>
+            limit(async () => {
+              try {
+                const { data: responseData } = await fetchData<CreateVacResponse>({
+                  url: graphqlURL.href,
+                  query: CREATE_VAC,
+                  variables: { locale, data: entry },
+                  headers: {
+                    Authorization: `Bearer ${tokenAuth}`,
+                  },
+                });
+                return responseData;
+              } catch (error: any) {
+                next(error);
+                // eslint-disable-next-line no-console
+                console.error('Error processing entry:', error);
+                return { error: error.message, entry };
+              }
+            }),
+          ),
+        );
+        // Delete temporary file after processing
+        fs.unlinkSync(filePath);
+        res.json({ message: 'CSV converted to JSON', data: results });
+      } else {
+        res.status(400).send('Invalid import type.');
+      }
+    } catch (error) {
+      fs.unlinkSync(filePath); // Delete the temporary file in case of error
+      // Forward any errors to the error handler middleware
+      next(error);
+      return null;
+    }
+  } else {
+    res.status(400).send('No file uploaded.');
+  }
+  return null;
+};

apps/overige-objecten-api/src/controllers/index.ts

@@ -3,3 +3,4 @@ export { createVacController } from './objects/create';
 export { updateVacController } from './objects/update';
 export { openAPIController } from './openapi';
 export { objecttypesController } from './objecttypes';
+export { importController } from './import';

apps/overige-objecten-api/src/controllers/objects/create.ts

@@ -3,7 +3,7 @@ import snakeCase from 'lodash.snakecase';
 import slugify from 'slugify';
 import { v4 } from 'uuid';
 import { CREATE_INTERNAL_FIELD, CREATE_KENNISARTIKEL, CREATE_VAC } from '../../queries';
-import type { CreateInternalField, CreateProduct, DataVacItem } from '../../strapi-product-type';
+import type { CreateInternalField, CreateProduct, CreateVacResponse } from '../../strapi-product-type';
 import type { components } from '../../types/openapi';
 import {
   concatenateFieldValues,
@@ -13,13 +13,6 @@ import {
   getTheServerURL,
   mapContentByCategory,
 } from '../../utils';
-type VACData = {
-  data: {
-    createVac: {
-      data: DataVacItem;
-    };
-  };
-};
 
 const categoryToKeyMap: { [key: string]: string } = {
   bewijs: 'bewijs',
@@ -66,7 +59,7 @@ export const createVacController: RequestHandler = async (req, res, next) => {
           trefwoorden: vac?.trefwoorden,
         },
       };
-      const { data: responseData } = await fetchData<VACData>({
+      const { data: responseData } = await fetchData<CreateVacResponse>({
         url: graphqlURL.href,
         query: CREATE_VAC,
         variables: { locale, data: vacPayload },

apps/overige-objecten-api/src/server.ts

@@ -6,9 +6,11 @@ import express from 'express';
 import { NextFunction, Request, Response } from 'express';
 import * as OpenApiValidator from 'express-openapi-validator';
 import yaml from 'js-yaml';
+import multer from 'multer';
 import fs from 'node:fs';
 import path from 'node:path';
 import swaggerUi from 'swagger-ui-express';
+import { importController } from './controllers';
 import { objects, objecttypes, openapi } from './routers';
 import { envAvailability, ErrorHandler } from './utils';
 config();
@@ -46,7 +48,12 @@ const corsOption: CorsOptions = {
 };
 const apiSpec = path.join(__dirname, './docs/openapi.yaml');
 const app = express();
+// Multer file upload middleware.
+// The order is important, so this should be before the express.json() middleware to parse the file.
+const upload = multer({ dest: 'tmp/uploads/' });
+app.post('/api/v2/import', upload.single('file'), importController);
 app.use(express.json());
+app.use(express.urlencoded({ extended: true }));
 
 const port = process.env.OVERIGE_OBJECTEN_API_PORT;
 // Centralized error handler middleware

apps/overige-objecten-api/src/strapi-product-type.ts

@@ -177,3 +177,11 @@ export interface Section {
   component?: string;
   internal_field: InternalField;
 }
+
+export type CreateVacResponse = {
+  data: {
+    createVac: {
+      data: DataVacItem;
+    };
+  };
+};

apps/overige-objecten-api/src/utils/index.ts

@@ -18,3 +18,5 @@ export { processData } from './processData';
 export { convertSpotlightToHTML } from './convertSpotlightToHTML';
 export { convertMultiColumnsButtonToHTML } from './convertMultiColumnsButtonToHTML';
 export { createHTMLFiles } from './createHTMLFiles';
+export { processCsvFile } from './processCsvFile';
+export { sanitizeHTML } from './sanitizeHTML';

apps/overige-objecten-api/src/utils/processCsvFile.ts

@@ -0,0 +1,54 @@
+import csvParser from 'csv-parser';
+import fs from 'node:fs';
+import { v4 } from 'uuid';
+import { sanitizeHTML } from './sanitizeHTML';
+
+export const processCsvFile = (filePath: string, requiredColumns: string[]) => {
+  const results: Record<string, string>[] = [];
+  let hasRequiredColumns = true;
+
+  // Create read stream for CSV file
+  return new Promise<any[]>((resolve, reject) => {
+    fs.createReadStream(filePath)
+      .pipe(csvParser())
+      .on('headers', (headers) => {
+        // Check if required columns exist in the CSV file headers
+        const missingColumns = requiredColumns.filter((col) => !headers.includes(col));
+
+        if (missingColumns.length > 0) {
+          // If any required columns are missing, reject with error
+          hasRequiredColumns = false;
+          reject({ error: 'Missing required columns', missingColumns });
+        }
+      })
+      .on('data', (data) => {
+        // If the columns are valid, push the data into results
+        if (hasRequiredColumns) {
+          results.push({ vraag: data.vraag, antwoord: data.antwoord });
+        }
+      })
+      .on('end', () => {
+        if (hasRequiredColumns) {
+          const sanitizedResults = results.map((result) => {
+            const DOMPurifyHTML = sanitizeHTML(result?.antwoord);
+            return {
+              vac: {
+                vraag: result?.vraag,
+                antwoord: {
+                  content: DOMPurifyHTML,
+                },
+                doelgroep: null,
+                uuid: v4(),
+              },
+            };
+          });
+          resolve(sanitizedResults);
+        } else {
+          reject({ error: 'CSV failed validation' });
+        }
+      })
+      .on('error', (err) => {
+        reject({ error: 'Failed to parse CSV', details: err.message });
+      });
+  });
+};

apps/overige-objecten-api/src/utils/sanitizeHTML.ts

@@ -0,0 +1,9 @@
+import DOMPurify from 'dompurify';
+import { JSDOM } from 'jsdom';
+import { memoize } from 'lodash'; // For memoization of sanitize function
+const createDOMPurify = memoize(() => {
+  const { window } = new JSDOM();
+  return DOMPurify(window);
+});
+const domPurify = createDOMPurify();
+export const sanitizeHTML = memoize((html: string) => domPurify.sanitize(html, { FORBID_ATTR: ['style'] }));

yarn.lock

@@ -11759,6 +11759,13 @@ csstype@^3.0.2, csstype@^3.1.2:
   resolved "https://registry.yarnpkg.com/csstype/-/csstype-3.1.3.tgz#d80ff294d114fb0e6ac500fbf85b60137d7eff81"
   integrity sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw==
 
+csv-parser@3.0.0:
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/csv-parser/-/csv-parser-3.0.0.tgz#b88a6256d79e090a97a1b56451f9327b01d710e7"
+  integrity sha512-s6OYSXAK3IdKqYO33y09jhypG/bSDHPuyCme/IdEHfWpLf/jKcpitVFyOC6UemgGk8v7Q5u2XE0vvwmanxhGlQ==
+  dependencies:
+    minimist "^1.2.0"
+
 csvtojson@2.0.10:
   version "2.0.10"
   resolved "https://registry.yarnpkg.com/csvtojson/-/csvtojson-2.0.10.tgz#11e7242cc630da54efce7958a45f443210357574"
@@ -21395,6 +21402,13 @@ p-is-promise@^3.0.0:
   resolved "https://registry.yarnpkg.com/p-is-promise/-/p-is-promise-3.0.0.tgz#58e78c7dfe2e163cf2a04ff869e7c1dba64a5971"
   integrity sha512-Wo8VsW4IRQSKVXsJCn7TomUaVtyfjVDn3nUP7kE967BQk0CwFpdbZs0X0uk5sW9mkBa9eNM7hCMaG93WUAwxYQ==
 
+p-limit@3.0.0:
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/p-limit/-/p-limit-3.0.0.tgz#8a9da09ee359017af6a3aa6b8ede13f5894224ec"
+  integrity sha512-2FnzNu8nBx8Se231yrvScYw34Is5J5MtvKOQt7Lii+DGpM89xnCT7kIH/HJwniNkQpjB7zy/O3LckEfMVqYvFg==
+  dependencies:
+    p-try "^2.0.0"
+
 p-limit@3.1.0, p-limit@^3.0.2, p-limit@^3.1.0:
   version "3.1.0"
   resolved "https://registry.yarnpkg.com/p-limit/-/p-limit-3.1.0.tgz#e1daccbe78d0d1388ca18c64fea38e3e57e3706b"