diff --git a/.changeset/gorgeous-dolls-shake.md b/.changeset/gorgeous-dolls-shake.md new file mode 100644 index 00000000..841bcf51 --- /dev/null +++ b/.changeset/gorgeous-dolls-shake.md @@ -0,0 +1,5 @@ +--- +"@frameless/overige-objecten-api": minor +--- + +Maak het mogelijk om VAC als een CSV-bestand te importeren via de API. diff --git a/apps/overige-objecten-api/package.json b/apps/overige-objecten-api/package.json index 83429eba..262129bc 100644 --- a/apps/overige-objecten-api/package.json +++ b/apps/overige-objecten-api/package.json @@ -20,40 +20,42 @@ "test:watch": "OVERIGE_OBJECTEN_API_PORT=3000 jest --watch" }, "dependencies": { + "@frameless/ui": "0.1.0", + "@utrecht/component-library-react": "7.3.5", + "@utrecht/web-component-library-react": "3.0.0", "cors": "2.8.5", + "csv-parser": "3.0.0", + "dompurify": "3.2.1", "dotenv": "16.4.5", "express": "4.21.0", "express-openapi-validator": "5.3.7", "js-yaml": "4.1.0", - "swagger-ui-express": "5.0.1", + "lodash.memoize": "4.1.2", "lodash.merge": "4.6.2", "lodash.snakecase": "4.1.1", - "lodash.memoize": "4.1.2", "react": "18.2.0", - "@utrecht/web-component-library-react": "3.0.0", - "@utrecht/component-library-react": "7.3.5", - "@frameless/ui": "0.1.0", - "dompurify": "3.2.1" + "swagger-ui-express": "5.0.1", + "p-limit": "3.0.0" }, "devDependencies": { "@types/cors": "2.8.17", "@types/dompurify": "3.2.0", "@types/jest": "29.5.12", + "@types/lodash.memoize": "4.1.9", + "@types/lodash.merge": "4.6.9", + "@types/lodash.snakecase": "4.1.9", "@types/supertest": "6.0.2", "@types/swagger-ui-express": "4.1.6", "@types/yamljs": "0.2.34", "jest": "29.7.0", + "jest-fetch-mock": "3.0.3", "nodemon": "3.1.7", "openapi-typescript": "7.4.1", "rimraf": "6.0.1", "supertest": "7.0.0", "ts-jest": "29.2.3", "ts-node": "10.9.2", - "typescript": "5.0.4", - "jest-fetch-mock": "3.0.3", - "@types/lodash.merge": "4.6.9", - "@types/lodash.snakecase": "4.1.9", - "@types/lodash.memoize": "4.1.9" + "typescript": "5.0.4" }, "repository": { "type": "git+ssh", diff --git a/apps/overige-objecten-api/src/components/Markdown.tsx b/apps/overige-objecten-api/src/components/Markdown.tsx index 8f6829b6..2c21b75c 100644 --- a/apps/overige-objecten-api/src/components/Markdown.tsx +++ b/apps/overige-objecten-api/src/components/Markdown.tsx @@ -1,23 +1,14 @@ import { Markdown as ReactMarkdown } from '@frameless/ui'; -import DOMPurify from 'dompurify'; -import { JSDOM } from 'jsdom'; -import memoize from 'lodash.memoize'; import React from 'react'; import type { Price } from '../strapi-product-type'; +import { sanitizeHTML } from '../utils'; export interface MarkdownProps { children: string; priceData?: Price[]; } -const createDOMPurify = memoize(() => { - const { window } = new JSDOM(); - return DOMPurify(window); -}); - export const Markdown = ({ children: html, priceData }: MarkdownProps) => { - const domPurify = createDOMPurify(); - const sanitizeHTML = memoize((html) => domPurify.sanitize(html)); const DOMPurifyHTML = sanitizeHTML(html); return html ? ( diff --git a/apps/overige-objecten-api/src/controllers/import/index.ts b/apps/overige-objecten-api/src/controllers/import/index.ts new file mode 100644 index 00000000..d2816612 --- /dev/null +++ b/apps/overige-objecten-api/src/controllers/import/index.ts @@ -0,0 +1,63 @@ +import type { NextFunction, Request, Response } from 'express'; +import fs from 'node:fs'; +import pLimit from 'p-limit'; +import { CREATE_VAC } from '../../queries'; +import { CreateVacResponse } from '../../strapi-product-type'; +import { fetchData, processCsvFile } from '../../utils'; + +const limit = pLimit(5); // Limit the number of concurrent file uploads +export const importController = async (req: Request, res: Response, next: NextFunction) => { + const type = req.body.type; + if (req.file) { + const filePath = req.file.path; + const requiredColumns = ['vraag', 'antwoord']; + + try { + // Process the CSV file and sanitize results + const isAuthHasToken = req.headers?.authorization?.startsWith('Token'); + const tokenAuth = isAuthHasToken ? req.headers?.authorization?.split(' ')[1] : req.headers?.authorization; + const graphqlURL = new URL('/graphql', process.env.STRAPI_PRIVATE_URL); + const sanitizedResults = await processCsvFile(filePath, requiredColumns); + const locale = req.query?.locale || 'nl'; + + if (type === 'vac') { + // Loop through the sanitized results and create entries one by one + const results = await Promise.all( + sanitizedResults.map((entry) => + limit(async () => { + try { + const { data: responseData } = await fetchData({ + url: graphqlURL.href, + query: CREATE_VAC, + variables: { locale, data: entry }, + headers: { + Authorization: `Bearer ${tokenAuth}`, + }, + }); + return responseData; + } catch (error: any) { + next(error); + // eslint-disable-next-line no-console + console.error('Error processing entry:', error); + return { error: error.message, entry }; + } + }), + ), + ); + // Delete temporary file after processing + fs.unlinkSync(filePath); + res.json({ message: 'CSV converted to JSON', data: results }); + } else { + res.status(400).send('Invalid import type.'); + } + } catch (error) { + fs.unlinkSync(filePath); // Delete the temporary file in case of error + // Forward any errors to the error handler middleware + next(error); + return null; + } + } else { + res.status(400).send('No file uploaded.'); + } + return null; +}; diff --git a/apps/overige-objecten-api/src/controllers/index.ts b/apps/overige-objecten-api/src/controllers/index.ts index 618aa30b..4a59353a 100644 --- a/apps/overige-objecten-api/src/controllers/index.ts +++ b/apps/overige-objecten-api/src/controllers/index.ts @@ -3,3 +3,4 @@ export { createVacController } from './objects/create'; export { updateVacController } from './objects/update'; export { openAPIController } from './openapi'; export { objecttypesController } from './objecttypes'; +export { importController } from './import'; diff --git a/apps/overige-objecten-api/src/controllers/objects/create.ts b/apps/overige-objecten-api/src/controllers/objects/create.ts index 25b6a732..cc4f48c5 100644 --- a/apps/overige-objecten-api/src/controllers/objects/create.ts +++ b/apps/overige-objecten-api/src/controllers/objects/create.ts @@ -3,7 +3,7 @@ import snakeCase from 'lodash.snakecase'; import slugify from 'slugify'; import { v4 } from 'uuid'; import { CREATE_INTERNAL_FIELD, CREATE_KENNISARTIKEL, CREATE_VAC } from '../../queries'; -import type { CreateInternalField, CreateProduct, DataVacItem } from '../../strapi-product-type'; +import type { CreateInternalField, CreateProduct, CreateVacResponse } from '../../strapi-product-type'; import type { components } from '../../types/openapi'; import { concatenateFieldValues, @@ -13,13 +13,6 @@ import { getTheServerURL, mapContentByCategory, } from '../../utils'; -type VACData = { - data: { - createVac: { - data: DataVacItem; - }; - }; -}; const categoryToKeyMap: { [key: string]: string } = { bewijs: 'bewijs', @@ -66,7 +59,7 @@ export const createVacController: RequestHandler = async (req, res, next) => { trefwoorden: vac?.trefwoorden, }, }; - const { data: responseData } = await fetchData({ + const { data: responseData } = await fetchData({ url: graphqlURL.href, query: CREATE_VAC, variables: { locale, data: vacPayload }, diff --git a/apps/overige-objecten-api/src/server.ts b/apps/overige-objecten-api/src/server.ts index 6fc5c09b..51fb3cee 100644 --- a/apps/overige-objecten-api/src/server.ts +++ b/apps/overige-objecten-api/src/server.ts @@ -6,9 +6,11 @@ import express from 'express'; import { NextFunction, Request, Response } from 'express'; import * as OpenApiValidator from 'express-openapi-validator'; import yaml from 'js-yaml'; +import multer from 'multer'; import fs from 'node:fs'; import path from 'node:path'; import swaggerUi from 'swagger-ui-express'; +import { importController } from './controllers'; import { objects, objecttypes, openapi } from './routers'; import { envAvailability, ErrorHandler } from './utils'; config(); @@ -46,7 +48,12 @@ const corsOption: CorsOptions = { }; const apiSpec = path.join(__dirname, './docs/openapi.yaml'); const app = express(); +// Multer file upload middleware. +// The order is important, so this should be before the express.json() middleware to parse the file. +const upload = multer({ dest: 'tmp/uploads/' }); +app.post('/api/v2/import', upload.single('file'), importController); app.use(express.json()); +app.use(express.urlencoded({ extended: true })); const port = process.env.OVERIGE_OBJECTEN_API_PORT; // Centralized error handler middleware diff --git a/apps/overige-objecten-api/src/strapi-product-type.ts b/apps/overige-objecten-api/src/strapi-product-type.ts index d32cce5e..35b3fd3e 100644 --- a/apps/overige-objecten-api/src/strapi-product-type.ts +++ b/apps/overige-objecten-api/src/strapi-product-type.ts @@ -177,3 +177,11 @@ export interface Section { component?: string; internal_field: InternalField; } + +export type CreateVacResponse = { + data: { + createVac: { + data: DataVacItem; + }; + }; +}; diff --git a/apps/overige-objecten-api/src/utils/index.ts b/apps/overige-objecten-api/src/utils/index.ts index 9c4a140c..5299685f 100644 --- a/apps/overige-objecten-api/src/utils/index.ts +++ b/apps/overige-objecten-api/src/utils/index.ts @@ -18,3 +18,5 @@ export { processData } from './processData'; export { convertSpotlightToHTML } from './convertSpotlightToHTML'; export { convertMultiColumnsButtonToHTML } from './convertMultiColumnsButtonToHTML'; export { createHTMLFiles } from './createHTMLFiles'; +export { processCsvFile } from './processCsvFile'; +export { sanitizeHTML } from './sanitizeHTML'; diff --git a/apps/overige-objecten-api/src/utils/processCsvFile.ts b/apps/overige-objecten-api/src/utils/processCsvFile.ts new file mode 100644 index 00000000..7a1b582f --- /dev/null +++ b/apps/overige-objecten-api/src/utils/processCsvFile.ts @@ -0,0 +1,54 @@ +import csvParser from 'csv-parser'; +import fs from 'node:fs'; +import { v4 } from 'uuid'; +import { sanitizeHTML } from './sanitizeHTML'; + +export const processCsvFile = (filePath: string, requiredColumns: string[]) => { + const results: Record[] = []; + let hasRequiredColumns = true; + + // Create read stream for CSV file + return new Promise((resolve, reject) => { + fs.createReadStream(filePath) + .pipe(csvParser()) + .on('headers', (headers) => { + // Check if required columns exist in the CSV file headers + const missingColumns = requiredColumns.filter((col) => !headers.includes(col)); + + if (missingColumns.length > 0) { + // If any required columns are missing, reject with error + hasRequiredColumns = false; + reject({ error: 'Missing required columns', missingColumns }); + } + }) + .on('data', (data) => { + // If the columns are valid, push the data into results + if (hasRequiredColumns) { + results.push({ vraag: data.vraag, antwoord: data.antwoord }); + } + }) + .on('end', () => { + if (hasRequiredColumns) { + const sanitizedResults = results.map((result) => { + const DOMPurifyHTML = sanitizeHTML(result?.antwoord); + return { + vac: { + vraag: result?.vraag, + antwoord: { + content: DOMPurifyHTML, + }, + doelgroep: null, + uuid: v4(), + }, + }; + }); + resolve(sanitizedResults); + } else { + reject({ error: 'CSV failed validation' }); + } + }) + .on('error', (err) => { + reject({ error: 'Failed to parse CSV', details: err.message }); + }); + }); +}; diff --git a/apps/overige-objecten-api/src/utils/sanitizeHTML.ts b/apps/overige-objecten-api/src/utils/sanitizeHTML.ts new file mode 100644 index 00000000..e0c4bd0a --- /dev/null +++ b/apps/overige-objecten-api/src/utils/sanitizeHTML.ts @@ -0,0 +1,9 @@ +import DOMPurify from 'dompurify'; +import { JSDOM } from 'jsdom'; +import { memoize } from 'lodash'; // For memoization of sanitize function +const createDOMPurify = memoize(() => { + const { window } = new JSDOM(); + return DOMPurify(window); +}); +const domPurify = createDOMPurify(); +export const sanitizeHTML = memoize((html: string) => domPurify.sanitize(html, { FORBID_ATTR: ['style'] })); diff --git a/yarn.lock b/yarn.lock index fe48e7df..e3df114d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -11759,6 +11759,13 @@ csstype@^3.0.2, csstype@^3.1.2: resolved "https://registry.yarnpkg.com/csstype/-/csstype-3.1.3.tgz#d80ff294d114fb0e6ac500fbf85b60137d7eff81" integrity sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw== +csv-parser@3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/csv-parser/-/csv-parser-3.0.0.tgz#b88a6256d79e090a97a1b56451f9327b01d710e7" + integrity sha512-s6OYSXAK3IdKqYO33y09jhypG/bSDHPuyCme/IdEHfWpLf/jKcpitVFyOC6UemgGk8v7Q5u2XE0vvwmanxhGlQ== + dependencies: + minimist "^1.2.0" + csvtojson@2.0.10: version "2.0.10" resolved "https://registry.yarnpkg.com/csvtojson/-/csvtojson-2.0.10.tgz#11e7242cc630da54efce7958a45f443210357574" @@ -21395,6 +21402,13 @@ p-is-promise@^3.0.0: resolved "https://registry.yarnpkg.com/p-is-promise/-/p-is-promise-3.0.0.tgz#58e78c7dfe2e163cf2a04ff869e7c1dba64a5971" integrity sha512-Wo8VsW4IRQSKVXsJCn7TomUaVtyfjVDn3nUP7kE967BQk0CwFpdbZs0X0uk5sW9mkBa9eNM7hCMaG93WUAwxYQ== +p-limit@3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/p-limit/-/p-limit-3.0.0.tgz#8a9da09ee359017af6a3aa6b8ede13f5894224ec" + integrity sha512-2FnzNu8nBx8Se231yrvScYw34Is5J5MtvKOQt7Lii+DGpM89xnCT7kIH/HJwniNkQpjB7zy/O3LckEfMVqYvFg== + dependencies: + p-try "^2.0.0" + p-limit@3.1.0, p-limit@^3.0.2, p-limit@^3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/p-limit/-/p-limit-3.1.0.tgz#e1daccbe78d0d1388ca18c64fea38e3e57e3706b"