diff --git a/.editorconfig b/.editorconfig index 0eda16d..443c1ab 100644 --- a/.editorconfig +++ b/.editorconfig @@ -17,3 +17,11 @@ tab_width = 2 indent_size = 2 tab_width = 2 +[package-lock.json] +indent_size = 2 +tab_width = 2 + +[package.json] +indent_size = 2 +tab_width = 2 + diff --git a/.gitignore b/.gitignore index c2e2554..992416e 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,8 @@ tdm-fe/.next/ .gitlab-ci.yml helm/* tdm-be/environment-recette.ts -.idea \ No newline at end of file +.idea +tdm-be/uploads/* +!tdm-be/uploads/.keep +tdm-be/dynamic-config.json +tdm-be/logs/* diff --git a/Dockerfile b/Dockerfile index 263fade..2563321 100644 --- a/Dockerfile +++ b/Dockerfile @@ -58,7 +58,7 @@ FROM node:18.19-bullseye-slim AS application # Add ezmaster config file RUN echo '{ \ "httpPort": 3000, \ - "configPath": "/app/config/default.json", \ + "configPath": "/app/config/production.json", \ "dataPath": "/app" \ }' > /etc/ezmaster.json diff --git a/Makefile b/Makefile index e41a010..c0886bc 100644 --- a/Makefile +++ b/Makefile @@ -4,3 +4,14 @@ install: cd ../tdm-fe && \ npm install && \ cd .. + + +run-dev: + docker compose -f docker-compose.dev.yml up --force-recreate + +update-front-api: + cd tdm-be && \ + npm run swagger-autogen && \ + cd ../tdm-fe && \ + npm run generate-api && \ + cd .. diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml new file mode 100644 index 0000000..78ff1f7 --- /dev/null +++ b/docker-compose.dev.yml @@ -0,0 +1,44 @@ +version: "3.5" + +services: + tdm-factory-backend: + container_name: tdm-factory-dev-backend + image: node:18.19-bullseye-slim + environment: + NODE_ENV: development + ports: + - "3000:3000" + volumes: + - ./tdm-be:/app + working_dir: /app + command: npm run dev + networks: + - tdm-factory-network + + tdm-factory-frontend: + container_name: tdm-factory-dev-frontend + image: node:18.19-bullseye-slim + environment: + NODE_ENV: development + ports: + - "8080:8080" + volumes: + - ./tdm-fe:/app + working_dir: /app + command: npm run dev + networks: + - tdm-factory-network + + tdm-factory-maildev: + container_name: tdm-factory-dev-maildev + image: maildev/maildev + ports: + - "1025:1025" + - "1080:1080" + networks: + - tdm-factory-network + +networks: + tdm-factory-network: + driver: bridge + name: tdm-factory-network diff --git a/tdm-be/config/default.json b/tdm-be/config/default.json index 51c48fa..77c0552 100644 --- a/tdm-be/config/default.json +++ b/tdm-be/config/default.json @@ -1,34 +1,9 @@ { - "port": 3000, - "password": "", - "hosts": { - "internal": { - "host": "", - "isHttps": false - }, - "external": { - "host": "", - "isHttps": false + "port": 3000, + "password": "", + "fileFolder": "uploads/", + "cron": { + "schedule": "0 0 * * *", + "deleteFileOlderThan": 7 } - }, - "smtp": { - "host": "maildev", - "port": 1025, - "auth": { - "user": "", - "pass": "" - }, - "secure": false, - "tls": { - "ignore": true, - "rejectUnauthorized": false - } - }, - "fileFolder": "uploads/", - "dumpFile": "dump.tar.gz", - "finalFile": "final.tar.gz", - "cron": { - "schedule": "0 0 * * *", - "deleteFileOlderThan": 7 - } -} \ No newline at end of file +} diff --git a/tdm-be/config/development.json b/tdm-be/config/development.json new file mode 100644 index 0000000..44d26cb --- /dev/null +++ b/tdm-be/config/development.json @@ -0,0 +1,21 @@ +{ + "hosts": { + "internal": { + "host": "{ngrok host}", + "isHttps": true + }, + "external": { + "host": "http://localhost:8080", + "isHttps": false + } + }, + "smtp": { + "host": "tdm-factory-dev-maildev", + "port": 1025, + "secure": false, + "tls": { + "ignore": true, + "rejectUnauthorized": false + } + } +} diff --git a/tdm-be/config/production.json b/tdm-be/config/production.json new file mode 100644 index 0000000..1078240 --- /dev/null +++ b/tdm-be/config/production.json @@ -0,0 +1,26 @@ +{ + "password": "{very secure password}", + "hosts": { + "internal": { + "host": "{machine ip or internal host}", + "isHttps": true + }, + "external": { + "host": "{external host / revers proxy host}", + "isHttps": false + } + }, + "smtp": { + "host": "{mail services host}", + "port": 1025, + "secure": false, + "tls": { + "ignore": true, + "rejectUnauthorized": false + } + }, + "cron": { + "schedule": "0 0 * * *", + "deleteFileOlderThan": 7 + } +} diff --git a/tdm-be/package-lock.json b/tdm-be/package-lock.json index 5839e02..585f582 100644 --- a/tdm-be/package-lock.json +++ b/tdm-be/package-lock.json @@ -14,6 +14,7 @@ "cors": "^2.8.5", "express": "^4.18.2", "express-basic-auth": "^1.2.1", + "md5": "^2.3.0", "multer": "^1.4.5-lts.1", "node-cron": "^3.0.3", "nodemailer": "^6.9.9", @@ -26,6 +27,7 @@ "@types/config": "^3.3.3", "@types/cors": "^2.8.17", "@types/express": "^4.17.21", + "@types/md5": "^2.3.5", "@types/multer": "^1.4.11", "@types/node": "^18.19.15", "@types/node-cron": "^3.0.11", @@ -469,6 +471,12 @@ "integrity": "sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ==", "dev": true }, + "node_modules/@types/md5": { + "version": "2.3.5", + "resolved": "https://registry.npmjs.org/@types/md5/-/md5-2.3.5.tgz", + "integrity": "sha512-/i42wjYNgE6wf0j2bcTX6kuowmdL/6PE4IVitMpm2eYKBUuYCprdcWVK+xEF0gcV6ufMCRhtxmReGfc6hIK7Jw==", + "dev": true + }, "node_modules/@types/mime": { "version": "1.3.5", "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.5.tgz", @@ -1294,6 +1302,14 @@ "url": "https://github.com/chalk/chalk?sponsor=1" } }, + "node_modules/charenc": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/charenc/-/charenc-0.0.2.tgz", + "integrity": "sha512-yrLQ/yVUFXkzg7EDQsPieE/53+0RlaWTs+wBrvW36cyilJ2SaDWfl4Yj7MtLTXleV9uEKefbAGUPv2/iWSooRA==", + "engines": { + "node": "*" + } + }, "node_modules/color": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/color/-/color-3.2.1.tgz", @@ -1480,6 +1496,14 @@ "node": ">= 8" } }, + "node_modules/crypt": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/crypt/-/crypt-0.0.2.tgz", + "integrity": "sha512-mCxBlsHFYh9C+HVpiEacem8FEBnMXgU9gy4zmNC+SXAZNB/1idgp/aulFJ4FgCi7GPEVbfyng092GqL2k2rmow==", + "engines": { + "node": "*" + } + }, "node_modules/debug": { "version": "2.6.9", "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", @@ -2743,6 +2767,11 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/is-buffer": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz", + "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==" + }, "node_modules/is-callable": { "version": "1.2.7", "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.7.tgz", @@ -3099,6 +3128,16 @@ "integrity": "sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==", "dev": true }, + "node_modules/md5": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/md5/-/md5-2.3.0.tgz", + "integrity": "sha512-T1GITYmFaKuO91vxyoQMFETst+O71VUPEU3ze5GNzDm0OWdP8v1ziTaAEPUr/3kLsY3Sftgz242A1SetQiDL7g==", + "dependencies": { + "charenc": "0.0.2", + "crypt": "0.0.2", + "is-buffer": "~1.1.6" + } + }, "node_modules/media-typer": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", diff --git a/tdm-be/package.json b/tdm-be/package.json index 6070f18..b76e142 100644 --- a/tdm-be/package.json +++ b/tdm-be/package.json @@ -23,6 +23,7 @@ "cors": "^2.8.5", "express": "^4.18.2", "express-basic-auth": "^1.2.1", + "md5": "^2.3.0", "multer": "^1.4.5-lts.1", "node-cron": "^3.0.3", "nodemailer": "^6.9.9", @@ -35,6 +36,7 @@ "@types/config": "^3.3.3", "@types/cors": "^2.8.17", "@types/express": "^4.17.21", + "@types/md5": "^2.3.5", "@types/multer": "^1.4.11", "@types/node": "^18.19.15", "@types/node-cron": "^3.0.11", diff --git a/tdm-be/src/controller/traitment.ts b/tdm-be/src/controller/traitment.ts index 86284f8..480e2b2 100644 --- a/tdm-be/src/controller/traitment.ts +++ b/tdm-be/src/controller/traitment.ts @@ -1,7 +1,7 @@ import environment from '../lib/config'; import logger from '../lib/logger'; import { StatusEnum } from '../model/StatusEnum'; -import currentTraitments from '../model/Traitment'; +import { addTraitement, getTraitement } from '../model/Traitment'; import { sendEmail } from '../service/email-sender'; import axios from 'axios'; import express from 'express'; @@ -65,9 +65,9 @@ const router = express.Router(); //Route to start traitment wrapping and then enrichment router.post( '/start', - (req: Request<{}, {}, Traitment>, res) => { + (req: Request, res) => { const traitment: Traitment = req.body; - currentTraitments.push(traitment); + addTraitement(traitment); const url = traitment.wrapper?.url ? traitment.wrapper?.url : ''; const urlEnrichment = traitment.enrichment?.url ? traitment.enrichment?.url : ''; const fileData = fs.readFileSync(`${environment.fileFolder}${traitment.file}`); @@ -209,7 +209,7 @@ router.post('/upload', upload.single('file'), (req: any, res: Response) => { //Route to retrieve traitment status router.get('/status', (req, res) => { const { id } = req.query; - const traitment: Traitment = currentTraitments.filter((t) => t.timestamp + '' === id)[0]; + const traitment: Traitment = getTraitement().filter((t) => t.timestamp + '' === id)[0]; let status: StatusEnum = StatusEnum.UNKNOWN; if (traitment) { status = traitment.status; diff --git a/tdm-be/src/controller/webhook.ts b/tdm-be/src/controller/webhook.ts index 5f4e1ea..8f8c19f 100644 --- a/tdm-be/src/controller/webhook.ts +++ b/tdm-be/src/controller/webhook.ts @@ -1,7 +1,7 @@ import logger from '../lib/logger'; import configModel from '../model/Config'; import { StatusEnum } from '../model/StatusEnum'; -import currentTraitments from '../model/Traitment'; +import { getTraitement, setTraitement } from '../model/Traitment'; import { sendEmail } from '../service/email-sender'; import axios from 'axios'; import express from 'express'; @@ -19,7 +19,7 @@ router.post( '/success', (req, res) => { const { id } = req.query; - const traitment: Traitment = currentTraitments.filter((t) => t.timestamp + '' === id)[0]; + const traitment: Traitment = getTraitement().filter((t) => t.timestamp + '' === id)[0]; const enrichment = config.enrichments.filter( (enrichment: SwaggerApi) => traitment.enrichment.url.indexOf(enrichment.url) > -1, )[0]; @@ -65,10 +65,10 @@ router.post( '/failure', (req, res) => { const { id } = req.query; - const traitment: Traitment = currentTraitments.filter((traitment) => traitment.timestamp === id)[0]; + const traitment: Traitment = getTraitement().filter((traitment) => traitment.timestamp === id)[0]; traitment.status = StatusEnum.FINISHED_ERROR; if (traitment) { - currentTraitments = currentTraitments.filter((t) => t.timestamp !== traitment.timestamp); + setTraitement(getTraitement().filter((t) => t.timestamp !== traitment.timestamp)); // Process the payload as needed logger.info('Received webhook error:', id); const mailOptions: EmailOptions = { diff --git a/tdm-be/src/model/Traitment.ts b/tdm-be/src/model/Traitment.ts index 56da6e4..01318e0 100644 --- a/tdm-be/src/model/Traitment.ts +++ b/tdm-be/src/model/Traitment.ts @@ -11,6 +11,16 @@ export class Traitment { status: StatusEnum = StatusEnum.UNKNOWN; } -const currentTraitments: Traitment[] = []; +let currentTraitments: Traitment[] = []; -export default currentTraitments; +export const addTraitement = (traitement: Traitment) => { + currentTraitments.push(traitement); +}; + +export const getTraitement = () => { + return currentTraitments; +}; + +export const setTraitement = (traitements: Traitment[]) => { + currentTraitments = traitements; +}; diff --git a/tdm-be/src/swagger/swagger.js b/tdm-be/src/swagger/swagger.js index f12bebd..17f8d04 100644 --- a/tdm-be/src/swagger/swagger.js +++ b/tdm-be/src/swagger/swagger.js @@ -15,7 +15,7 @@ const swaggerOptions = { }, ], }, - apis: ['./routers/data-enrichment.ts', './routers/data-wrapper.ts', './routers/traitment.ts'], // Specify the file containing your routes + apis: ['./src/controller/data-enrichment.ts', './src/controller/data-wrapper.ts', './src/controller/traitment.ts'], // Specify the file containing your routes }; const swaggerOptionsConfig = { @@ -32,19 +32,19 @@ const swaggerOptionsConfig = { }, ], }, - apis: ['./routers/config.ts'], // Specify the file containing your routes + apis: ['./src/controller/config.ts'], // Specify the file containing your routes }; const swaggerSpec = swaggerJsdoc(swaggerOptions); // Write Swagger JSON to a file const swaggerJson = JSON.stringify(swaggerSpec, null, 2); -fs.writeFileSync('swagger.json', swaggerJson); +fs.writeFileSync(__dirname + '/swagger.json', swaggerJson); const swaggerSpecConfig = swaggerJsdoc(swaggerOptionsConfig); // Write Swagger JSON to a file const swaggerJsonConfig = JSON.stringify(swaggerSpecConfig, null, 2); -fs.writeFileSync('swagger-config.json', swaggerJsonConfig); +fs.writeFileSync(__dirname + '/swagger-config.json', swaggerJsonConfig); console.log('Swagger JSON file generated successfully'); diff --git a/tdm-be/uploads/.keep b/tdm-be/uploads/.keep new file mode 100644 index 0000000..e69de29 diff --git a/tdm-fe/package.json b/tdm-fe/package.json index 25d7523..bee7301 100644 --- a/tdm-fe/package.json +++ b/tdm-fe/package.json @@ -1,41 +1,41 @@ { - "name": "tdm-fe", - "version": "1.0.0", - "private": true, - "scripts": { - "dev": "next dev", - "build": "next build", - "start": "next start -p 8080", - "generate-api": "openapi-generator-cli generate -i ../tdm-be/src/swagger/swagger.json -g typescript-axios -o src/generated", - "lint": "next lint" - }, - "dependencies": { - "@emotion/react": "^11.11.1", - "@emotion/styled": "^11.11.0", - "@google-cloud/recaptcha-enterprise": "^5.1.0", - "@mui/icons-material": "^5.14.19", - "@mui/material": "^5.14.18", - "mui-file-input": "^4.0.2", - "next": "14.0.2", - "react": "^18.2.0", - "react-dom": "^18.2.0", - "react-google-recaptcha": "^3.1.0", - "react-markdown": "^9.0.1", - "react-router-dom": "^6.21.1" - }, - "devDependencies": { - "@draconides/eslint-config-react": "^1.2.0", - "@draconides/eslint-config-ts": "^1.2.0", - "@openapitools/openapi-generator-cli": "^2.7.0", - "@types/node": "^18.19.15", - "@types/react": "^18.2.55", - "@types/react-dom": "^18.2.19", - "@types/react-google-recaptcha": "^2.1.8", - "autoprefixer": "^10.0.1", - "eslint": "^8.56.0", - "eslint-config-next": "14.0.2", - "postcss": "^8.4.35", - "tailwindcss": "^3.3.0", - "typescript": "^5.2.2" - } + "name": "tdm-fe", + "version": "1.0.0", + "private": true, + "scripts": { + "dev": "next dev -p 8080", + "build": "next build", + "start": "next start -p 8080", + "generate-api": "openapi-generator-cli generate -i ../tdm-be/src/swagger/swagger.json -g typescript-axios -o src/generated", + "lint": "next lint" + }, + "dependencies": { + "@emotion/react": "^11.11.1", + "@emotion/styled": "^11.11.0", + "@google-cloud/recaptcha-enterprise": "^5.1.0", + "@mui/icons-material": "^5.14.19", + "@mui/material": "^5.14.18", + "mui-file-input": "^4.0.2", + "next": "14.0.2", + "react": "^18.2.0", + "react-dom": "^18.2.0", + "react-google-recaptcha": "^3.1.0", + "react-markdown": "^9.0.1", + "react-router-dom": "^6.21.1" + }, + "devDependencies": { + "@draconides/eslint-config-react": "^1.2.0", + "@draconides/eslint-config-ts": "^1.2.0", + "@openapitools/openapi-generator-cli": "^2.7.0", + "@types/node": "^18.19.15", + "@types/react": "^18.2.55", + "@types/react-dom": "^18.2.19", + "@types/react-google-recaptcha": "^2.1.8", + "autoprefixer": "^10.0.1", + "eslint": "^8.56.0", + "eslint-config-next": "14.0.2", + "postcss": "^8.4.35", + "tailwindcss": "^3.3.0", + "typescript": "^5.2.2" + } }