From cbf2f6267ee2f549c2319b74970976eff20bf2db Mon Sep 17 00:00:00 2001 From: Thuan Le Date: Thu, 14 Mar 2024 17:28:34 +0700 Subject: [PATCH] feat: web summarizer plugin --- package-lock.json | 616 ++++++++++++++++-- package.json | 3 + src/api-docs/openAPIDocumentGenerator.ts | 8 +- src/routes/webContent/webContentModel.ts | 10 + .../webContent/webContentModelRouter.ts | 94 +++ src/server.ts | 2 + 6 files changed, 687 insertions(+), 46 deletions(-) create mode 100644 src/routes/webContent/webContentModel.ts create mode 100644 src/routes/webContent/webContentModelRouter.ts diff --git a/package-lock.json b/package-lock.json index 47cb67e..bc003ab 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,13 +10,17 @@ "license": "MIT", "dependencies": { "@asteasolutions/zod-to-openapi": "^6.4.0", + "@types/got": "^9.6.12", "body-parser": "^1.20.2", + "cheerio": "^1.0.0-rc.12", "cors": "^2.8.5", "dotenv": "^16.4.5", "envalid": "^8.0.0", "express": "^4.18.2", "express-rate-limit": "^7.2.0", "fs": "^0.0.1-security", + "got": "^14.2.1", + "got-scraping": "^4.0.4", "helmet": "^7.1.0", "http-status-codes": "^2.3.0", "path": "^0.12.7", @@ -1670,7 +1674,6 @@ "version": "5.6.0", "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-5.6.0.tgz", "integrity": "sha512-TV7t8GKYaJWsn00tFDqBw8+Uqmr8A0fRU1tvTQhyZzGv0sJCGRQL3JGMI3ucuKo3XIZdUP+Lx7/gh2t3lewy7g==", - "dev": true, "engines": { "node": ">=14.16" }, @@ -1694,7 +1697,6 @@ "version": "5.0.1", "resolved": "https://registry.npmjs.org/@szmarczak/http-timer/-/http-timer-5.0.1.tgz", "integrity": "sha512-+PmQX0PiAYPMeVYe237LJAYvOMYW1j2rH5YROyS3b4CTVJum34HfRvKvAzozHAQG0TnHNdUfY9nCeUyRAs//cw==", - "dev": true, "dependencies": { "defer-to-connect": "^2.0.1" }, @@ -1781,11 +1783,33 @@ "@types/send": "*" } }, + "node_modules/@types/got": { + "version": "9.6.12", + "resolved": "https://registry.npmjs.org/@types/got/-/got-9.6.12.tgz", + "integrity": "sha512-X4pj/HGHbXVLqTpKjA2ahI4rV/nNBc9mGO2I/0CgAra+F2dKgMXnENv2SRpemScBzBAI4vMelIVYViQxlSE6xA==", + "dependencies": { + "@types/node": "*", + "@types/tough-cookie": "*", + "form-data": "^2.5.0" + } + }, + "node_modules/@types/got/node_modules/form-data": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.5.1.tgz", + "integrity": "sha512-m21N3WOmEEURgk6B9GLOE4RuWOFf28Lhh9qGYeNlGq4VDXUlJy2th2slBNU8Gp8EzloYZOibZJ7t5ecIrFSjVA==", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.6", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 0.12" + } + }, "node_modules/@types/http-cache-semantics": { "version": "4.0.4", "resolved": "https://registry.npmjs.org/@types/http-cache-semantics/-/http-cache-semantics-4.0.4.tgz", - "integrity": "sha512-1m0bIFVc7eJWyve9S0RnuRgcQqF/Xd5QsUZAZeQFr1Q3/p9JWoQQEqmVy+DPTNpGXwhgIetAoYF8JSc33q29QA==", - "dev": true + "integrity": "sha512-1m0bIFVc7eJWyve9S0RnuRgcQqF/Xd5QsUZAZeQFr1Q3/p9JWoQQEqmVy+DPTNpGXwhgIetAoYF8JSc33q29QA==" }, "node_modules/@types/http-errors": { "version": "2.0.4", @@ -1821,7 +1845,6 @@ "version": "20.11.25", "resolved": "https://registry.npmjs.org/@types/node/-/node-20.11.25.tgz", "integrity": "sha512-TBHyJxk2b7HceLVGFcpAUjsa5zIdsPWlR6XHfyGzd0SFu+/NFgQgMAl96MSDZgQDvJAvV6BKsFOrt6zIL09JDw==", - "dev": true, "dependencies": { "undici-types": "~5.26.4" } @@ -1902,6 +1925,11 @@ "@types/serve-static": "*" } }, + "node_modules/@types/tough-cookie": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.5.tgz", + "integrity": "sha512-/Ad8+nIOV7Rl++6f1BdKxFSMgmoqEoYbHRpPcx3JEfv8VRsQe9Z4mCXeJBzxs7mbHY/XOZZuXlRNfhpVPbs6ZA==" + }, "node_modules/@types/uuid": { "version": "8.3.4", "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-8.3.4.tgz", @@ -2273,6 +2301,14 @@ "integrity": "sha512-qQLMr+8o0WC4FZGQTcJiKBVC59JylcPSrTtk6usvmIDFUOCKegapy1VHQwRbFMOFyb/inzUVqHs+eMYKDM1YeQ==", "dev": true }, + "node_modules/adm-zip": { + "version": "0.5.12", + "resolved": "https://registry.npmjs.org/adm-zip/-/adm-zip-0.5.12.tgz", + "integrity": "sha512-6TVU49mK6KZb4qG6xWaaM4C7sA/sgUMLy/JYMOzkcp3BvVLpW0fXDFQiIzAuxFCt/2+xD7fNIiPFAoLZPhVNLQ==", + "engines": { + "node": ">=6.0" + } + }, "node_modules/agent-base": { "version": "7.1.0", "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.0.tgz", @@ -2528,8 +2564,7 @@ "node_modules/asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", - "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", - "dev": true + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" }, "node_modules/atomic-sleep": { "version": "1.0.0", @@ -2650,6 +2685,11 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" }, + "node_modules/boolbase": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", + "integrity": "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==" + }, "node_modules/boxen": { "version": "7.1.1", "resolved": "https://registry.npmjs.org/boxen/-/boxen-7.1.1.tgz", @@ -2772,6 +2812,37 @@ "node": ">=8" } }, + "node_modules/browserslist": { + "version": "4.23.0", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.23.0.tgz", + "integrity": "sha512-QW8HiM1shhT2GuzkvklfjcKDiWFXHOeFCIA/huJPwHsslwcydgk7X+z2zXpEijP98UCY7HbubZt5J2Zgvf0CaQ==", + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "dependencies": { + "caniuse-lite": "^1.0.30001587", + "electron-to-chromium": "^1.4.668", + "node-releases": "^2.0.14", + "update-browserslist-db": "^1.0.13" + }, + "bin": { + "browserslist": "cli.js" + }, + "engines": { + "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" + } + }, "node_modules/buffer": { "version": "5.7.1", "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", @@ -2853,7 +2924,6 @@ "version": "7.0.0", "resolved": "https://registry.npmjs.org/cacheable-lookup/-/cacheable-lookup-7.0.0.tgz", "integrity": "sha512-+qJyx4xiKra8mZrcwhjMRMUhD5NR1R8esPkzIYxX96JiecFoxAXFuz/GpR3+ev4PE1WamHip78wV0vcmPQtp8w==", - "dev": true, "engines": { "node": ">=14.16" } @@ -2862,7 +2932,6 @@ "version": "10.2.14", "resolved": "https://registry.npmjs.org/cacheable-request/-/cacheable-request-10.2.14.tgz", "integrity": "sha512-zkDT5WAF4hSSoUgyfg5tFIxz8XQK+25W/TLVojJTMKBaxevLBBtLxgqguAuVQB8PVW79FVjHcU+GJ9tVbDZ9mQ==", - "dev": true, "dependencies": { "@types/http-cache-semantics": "^4.0.2", "get-stream": "^6.0.1", @@ -2880,7 +2949,6 @@ "version": "6.0.1", "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz", "integrity": "sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==", - "dev": true, "engines": { "node": ">=10" }, @@ -2910,7 +2978,6 @@ "version": "3.1.0", "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", - "dev": true, "engines": { "node": ">=6" } @@ -2927,6 +2994,25 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/caniuse-lite": { + "version": "1.0.30001597", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001597.tgz", + "integrity": "sha512-7LjJvmQU6Sj7bL0j5b5WY/3n7utXUJvAe1lxhsHDbLmwX9mdL86Yjtr+5SRCyf8qME4M7pU2hswj0FpyBVCv9w==", + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/caniuse-lite" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ] + }, "node_modules/centra": { "version": "2.6.0", "resolved": "https://registry.npmjs.org/centra/-/centra-2.6.0.tgz", @@ -2981,6 +3067,42 @@ "node": "*" } }, + "node_modules/cheerio": { + "version": "1.0.0-rc.12", + "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.0.0-rc.12.tgz", + "integrity": "sha512-VqR8m68vM46BNnuZ5NtnGBKIE/DfN0cRIzg9n40EIq9NOv90ayxLBXA8fXC5gquFRGJSTRqBq25Jt2ECLR431Q==", + "dependencies": { + "cheerio-select": "^2.1.0", + "dom-serializer": "^2.0.0", + "domhandler": "^5.0.3", + "domutils": "^3.0.1", + "htmlparser2": "^8.0.1", + "parse5": "^7.0.0", + "parse5-htmlparser2-tree-adapter": "^7.0.0" + }, + "engines": { + "node": ">= 6" + }, + "funding": { + "url": "https://github.com/cheeriojs/cheerio?sponsor=1" + } + }, + "node_modules/cheerio-select": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/cheerio-select/-/cheerio-select-2.1.0.tgz", + "integrity": "sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g==", + "dependencies": { + "boolbase": "^1.0.0", + "css-select": "^5.1.0", + "css-what": "^6.1.0", + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3", + "domutils": "^3.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, "node_modules/chokidar": { "version": "3.6.0", "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.6.0.tgz", @@ -3223,7 +3345,6 @@ "version": "1.0.8", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", - "dev": true, "dependencies": { "delayed-stream": "~1.0.0" }, @@ -3679,6 +3800,32 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/css-select": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/css-select/-/css-select-5.1.0.tgz", + "integrity": "sha512-nwoRF1rvRRnnCqqY7updORDsuqKzqYJ28+oSMaJMMgOauh3fvwHqMS7EZpIPqK8GL+g9mKxF1vP/ZjSeNjEVHg==", + "dependencies": { + "boolbase": "^1.0.0", + "css-what": "^6.1.0", + "domhandler": "^5.0.2", + "domutils": "^3.0.1", + "nth-check": "^2.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, + "node_modules/css-what": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/css-what/-/css-what-6.1.0.tgz", + "integrity": "sha512-HTUrgRJ7r4dsZKU6GjmpfRK1O76h97Z8MfS1G0FozR+oF2kG6Vfe8JE6zwrkbxigziPHinCJ+gCPjA9EaBDtRw==", + "engines": { + "node": ">= 6" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, "node_modules/dargs": { "version": "8.1.0", "resolved": "https://registry.npmjs.org/dargs/-/dargs-8.1.0.tgz", @@ -3730,7 +3877,6 @@ "version": "6.0.0", "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz", "integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==", - "dev": true, "dependencies": { "mimic-response": "^3.1.0" }, @@ -3745,7 +3891,6 @@ "version": "3.1.0", "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz", "integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==", - "dev": true, "engines": { "node": ">=10" }, @@ -3824,7 +3969,6 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/defer-to-connect/-/defer-to-connect-2.0.1.tgz", "integrity": "sha512-4tvttepXG1VaYGrRibk5EwJd1t4udunSOVMdLSAL6mId1ix438oPwPZMALY41FCijukO1L0twNcGsdzS7dHgDg==", - "dev": true, "engines": { "node": ">=10" } @@ -3892,7 +4036,6 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", - "dev": true, "engines": { "node": ">=0.4.0" } @@ -3963,6 +4106,57 @@ "node": ">=6.0.0" } }, + "node_modules/dom-serializer": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz", + "integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==", + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.2", + "entities": "^4.2.0" + }, + "funding": { + "url": "https://github.com/cheeriojs/dom-serializer?sponsor=1" + } + }, + "node_modules/domelementtype": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz", + "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ] + }, + "node_modules/domhandler": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz", + "integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==", + "dependencies": { + "domelementtype": "^2.3.0" + }, + "engines": { + "node": ">= 4" + }, + "funding": { + "url": "https://github.com/fb55/domhandler?sponsor=1" + } + }, + "node_modules/domutils": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-3.1.0.tgz", + "integrity": "sha512-H78uMmQtI2AhgDJjWeQmHwJJ2bLPD3GMmO7Zja/ZZh84wkm+4ut+IUnUdRa8uCGX88DiVx1j6FRe1XfxEgjEZA==", + "dependencies": { + "dom-serializer": "^2.0.0", + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3" + }, + "funding": { + "url": "https://github.com/fb55/domutils?sponsor=1" + } + }, "node_modules/dot-prop": { "version": "5.3.0", "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-5.3.0.tgz", @@ -3997,6 +4191,11 @@ "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==" }, + "node_modules/electron-to-chromium": { + "version": "1.4.704", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.704.tgz", + "integrity": "sha512-OK01+86Qvby1V6cTiowVbhp25aX4DLZnwar+NocAOXdzKAByd+jq5156bmo4kHwevWMknznW18Y/Svfk2dU91A==" + }, "node_modules/emoji-regex": { "version": "10.3.0", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-10.3.0.tgz", @@ -4020,6 +4219,17 @@ "once": "^1.4.0" } }, + "node_modules/entities": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", + "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, "node_modules/env-paths": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/env-paths/-/env-paths-2.2.1.tgz", @@ -4222,7 +4432,6 @@ "version": "3.1.2", "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.2.tgz", "integrity": "sha512-ErCHMCae19vR8vQGe50xIsVomy19rg6gFu3+r3jkEO46suLMWBksvVyoGgQV+jOfl84ZSOSlmv6Gxa89PmTGmA==", - "dev": true, "engines": { "node": ">=6" } @@ -5121,7 +5330,6 @@ "version": "2.1.4", "resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-2.1.4.tgz", "integrity": "sha512-yDYSgNMraqvnxiEXO4hi88+YZxaHC6QKzb5N84iRCTDeRO7ZALpir/lVmf/uXUhnwUr2O4HU8s/n6x+yNjQkHw==", - "dev": true, "engines": { "node": ">= 14.17" } @@ -5243,6 +5451,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/generative-bayesian-network": { + "version": "2.1.49", + "resolved": "https://registry.npmjs.org/generative-bayesian-network/-/generative-bayesian-network-2.1.49.tgz", + "integrity": "sha512-UYkPB7g82ij21Ag4C6Cujr03GO2mgR5vh9EWLoaMp8jE+sV4OwdjVjraQESP6RKyyxFGPFxSPK7eJtk6afyFQQ==", + "dependencies": { + "adm-zip": "^0.5.9", + "tslib": "^2.4.0" + } + }, "node_modules/get-caller-file": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", @@ -5294,7 +5511,6 @@ "version": "8.0.1", "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-8.0.1.tgz", "integrity": "sha512-VaUJspBffn/LMCJVoMvSAdmscJyS1auj5Zulnn5UoYcY531UWmdwhRWkcGKnGU93m5HSXP9LP2usOryrBtQowA==", - "dev": true, "engines": { "node": ">=16" }, @@ -5548,10 +5764,61 @@ } }, "node_modules/got": { + "version": "14.2.1", + "resolved": "https://registry.npmjs.org/got/-/got-14.2.1.tgz", + "integrity": "sha512-KOaPMremmsvx6l9BLC04LYE6ZFW4x7e4HkTe3LwBmtuYYQwpeS4XKqzhubTIkaQ1Nr+eXxeori0zuwupXMovBQ==", + "dependencies": { + "@sindresorhus/is": "^6.1.0", + "@szmarczak/http-timer": "^5.0.1", + "cacheable-lookup": "^7.0.0", + "cacheable-request": "^10.2.14", + "decompress-response": "^6.0.0", + "form-data-encoder": "^4.0.2", + "get-stream": "^8.0.1", + "http2-wrapper": "^2.2.1", + "lowercase-keys": "^3.0.0", + "p-cancelable": "^4.0.1", + "responselike": "^3.0.0" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sindresorhus/got?sponsor=1" + } + }, + "node_modules/got-scraping": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/got-scraping/-/got-scraping-4.0.4.tgz", + "integrity": "sha512-KJ//aqYWqj2Wyz08gmEgJr1lp0WWoUrI3FeADbhLLfSqimJkkytoToTtHhnEa0LFlxR94E3ud+MsOqpCAOt9LA==", + "dependencies": { + "got": "^13.0.0", + "header-generator": "^2.1.41", + "http2-wrapper": "^2.2.0", + "mimic-response": "^4.0.0", + "ow": "^1.1.1", + "quick-lru": "^7.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16" + } + }, + "node_modules/got-scraping/node_modules/get-stream": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz", + "integrity": "sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/got-scraping/node_modules/got": { "version": "13.0.0", "resolved": "https://registry.npmjs.org/got/-/got-13.0.0.tgz", "integrity": "sha512-XfBk1CxOOScDcMr9O1yKkNaQyy865NbYs+F7dr4H0LZMVgCj2Le59k6PqbNHoL5ToeaEQUYh6c6yMfVcc6SJxA==", - "dev": true, "dependencies": { "@sindresorhus/is": "^5.2.0", "@szmarczak/http-timer": "^5.0.1", @@ -5572,18 +5839,44 @@ "url": "https://github.com/sindresorhus/got?sponsor=1" } }, - "node_modules/got/node_modules/get-stream": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz", - "integrity": "sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==", - "dev": true, + "node_modules/got-scraping/node_modules/quick-lru": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-7.0.0.tgz", + "integrity": "sha512-MX8gB7cVYTrYcFfAnfLlhRd0+Toyl8yX8uBx1MrX7K0jegiz9TumwOK27ldXrgDlHRdVi+MqU9Ssw6dr4BNreg==", "engines": { - "node": ">=10" + "node": ">=18" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/got/node_modules/@sindresorhus/is": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-6.2.0.tgz", + "integrity": "sha512-yM/IGPkVnYGblhDosFBwq0ZGdnVSBkNV4onUtipGMOjZd4kB6GAu3ys91aftSbyMHh6A2GPdt+KDI5NoWP63MQ==", + "engines": { + "node": ">=16" + }, + "funding": { + "url": "https://github.com/sindresorhus/is?sponsor=1" + } + }, + "node_modules/got/node_modules/form-data-encoder": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-4.0.2.tgz", + "integrity": "sha512-KQVhvhK8ZkWzxKxOr56CPulAhH3dobtuQ4+hNQ+HekH/Wp5gSOafqRAeTphQUJAIk0GBvHZgJ2ZGRWd5kphMuw==", + "engines": { + "node": ">= 18" + } + }, + "node_modules/got/node_modules/p-cancelable": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-4.0.1.tgz", + "integrity": "sha512-wBowNApzd45EIKdO1LaU+LrMBwAcjfPaYtVzV3lmfM3gf8Z4CHZsiIqlM8TZZ8okYvh5A1cP6gTfCRQtwUpaUg==", + "engines": { + "node": ">=14.16" + } + }, "node_modules/graceful-fs": { "version": "4.2.11", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", @@ -5694,6 +5987,63 @@ "node": ">= 0.4" } }, + "node_modules/header-generator": { + "version": "2.1.49", + "resolved": "https://registry.npmjs.org/header-generator/-/header-generator-2.1.49.tgz", + "integrity": "sha512-hGrZjRzwvLiVeggy92nOAmBvxXBnJaaQ0DEfcIziZiFawzBkjZ07uf8NFjuyg/6eegKHv7YcP0NQeFRWPwy3bg==", + "dependencies": { + "browserslist": "^4.21.1", + "generative-bayesian-network": "^2.1.49", + "ow": "^0.28.1", + "tslib": "^2.4.0" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/header-generator/node_modules/@sindresorhus/is": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-4.6.0.tgz", + "integrity": "sha512-t09vSN3MdfsyCHoFcTRCH/iUtG7OJ0CsjzB8cjAmKc/va/kIgeDI/TxsigdncE/4be734m0cvIYwNaV4i2XqAw==", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sindresorhus/is?sponsor=1" + } + }, + "node_modules/header-generator/node_modules/dot-prop": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-6.0.1.tgz", + "integrity": "sha512-tE7ztYzXHIeyvc7N+hR3oi7FIbf/NIjVP9hmAt3yMXzrQ072/fpjGLx2GxNxGxUl5V73MEqYzioOMoVhGMJ5cA==", + "dependencies": { + "is-obj": "^2.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/header-generator/node_modules/ow": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/ow/-/ow-0.28.2.tgz", + "integrity": "sha512-dD4UpyBh/9m4X2NVjA+73/ZPBRF+uF4zIMFvvQsabMiEK8x41L3rQ8EENOi35kyyoaJwNxEeJcP6Fj1H4U409Q==", + "dependencies": { + "@sindresorhus/is": "^4.2.0", + "callsites": "^3.1.0", + "dot-prop": "^6.0.1", + "lodash.isequal": "^4.5.0", + "vali-date": "^1.0.0" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/helmet": { "version": "7.1.0", "resolved": "https://registry.npmjs.org/helmet/-/helmet-7.1.0.tgz", @@ -5735,11 +6085,28 @@ "integrity": "sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==", "dev": true }, + "node_modules/htmlparser2": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-8.0.2.tgz", + "integrity": "sha512-GYdjWKDkbRLkZ5geuHs5NY1puJ+PXwP7+fHPRz06Eirsb9ugf6d8kkXav6ADhcODhFFPMIXyxkxSuMf3D6NCFA==", + "funding": [ + "https://github.com/fb55/htmlparser2?sponsor=1", + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3", + "domutils": "^3.0.1", + "entities": "^4.4.0" + } + }, "node_modules/http-cache-semantics": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-4.1.1.tgz", - "integrity": "sha512-er295DKPVsV82j5kw1Gjt+ADA/XYHsajl82cGNQG2eyoPkvgUhX+nDIyelzhIWbbsXP39EHcI6l5tYs2FYqYXQ==", - "dev": true + "integrity": "sha512-er295DKPVsV82j5kw1Gjt+ADA/XYHsajl82cGNQG2eyoPkvgUhX+nDIyelzhIWbbsXP39EHcI6l5tYs2FYqYXQ==" }, "node_modules/http-errors": { "version": "2.0.0", @@ -5778,7 +6145,6 @@ "version": "2.2.1", "resolved": "https://registry.npmjs.org/http2-wrapper/-/http2-wrapper-2.2.1.tgz", "integrity": "sha512-V5nVw1PAOgfI3Lmeaj2Exmeg7fenjhRUgz1lPSezy1CuhPYbgQtbQj4jZfEAEMlaL+vupsvhjqCyjzob0yxsmQ==", - "dev": true, "dependencies": { "quick-lru": "^5.1.1", "resolve-alpn": "^1.2.0" @@ -6559,7 +6925,6 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/is-obj/-/is-obj-2.0.0.tgz", "integrity": "sha512-drqDG3cbczxxEJRoOXcOjtdp1J/lyp1mNn0xaznRs8+muBhgQcrnbspox5X5fOw0HnMnbfDzvnEMEtqDEJEo8w==", - "dev": true, "engines": { "node": ">=8" } @@ -6899,8 +7264,7 @@ "node_modules/json-buffer": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", - "integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==", - "dev": true + "integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==" }, "node_modules/json-parse-even-better-errors": { "version": "2.3.1", @@ -6973,7 +7337,6 @@ "version": "4.5.4", "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz", "integrity": "sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==", - "dev": true, "dependencies": { "json-buffer": "3.0.1" } @@ -7129,6 +7492,11 @@ "integrity": "sha512-TM9YBvyC84ZxE3rgfefxUWiQKLilstD6k7PTGt6wfbtXF8ixIJLOL3VYyV/z+ZiPLsVxAsKAFVwWlWeb2Y8Yyw==", "dev": true }, + "node_modules/lodash.isequal": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.isequal/-/lodash.isequal-4.5.0.tgz", + "integrity": "sha512-pDo3lu8Jhfjqls6GkMgpahsF9kCyayhgykjyLMNFTKWrpVdAQtYyB4muAMWozBB4ig/dtWAmsMxLEI8wuz+DYQ==" + }, "node_modules/lodash.isplainobject": { "version": "4.0.6", "resolved": "https://registry.npmjs.org/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz", @@ -7313,7 +7681,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-3.0.0.tgz", "integrity": "sha512-ozCC6gdQ+glXOQsveKD0YsDy8DSQFjDTz4zyzEHNV5+JP5D62LmfDZ6o1cycFx9ouG940M5dE8C8CTewdj2YWQ==", - "dev": true, "engines": { "node": "^12.20.0 || ^14.13.1 || >=16.0.0" }, @@ -7487,7 +7854,6 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-4.0.0.tgz", "integrity": "sha512-e5ISH9xMYU0DzrT+jl8q2ze9D6eWBto+I8CNpe+VI+K2J/F/k3PdkdTdz4wvGVH4NTpo+NRYTVIuMQEMMcsLqg==", - "dev": true, "engines": { "node": "^12.20.0 || ^14.13.1 || >=16.0.0" }, @@ -7677,6 +8043,11 @@ "url": "https://opencollective.com/node-fetch" } }, + "node_modules/node-releases": { + "version": "2.0.14", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.14.tgz", + "integrity": "sha512-y10wOWt8yZpqXmOgRo77WaHEmhYQYGNA6y421PKsKYWEK8aW+cqAphborZDhqfyKrbZEN92CN1X2KbafY2s7Yw==" + }, "node_modules/normalize-package-data": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/normalize-package-data/-/normalize-package-data-6.0.0.tgz", @@ -7705,7 +8076,6 @@ "version": "8.0.0", "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-8.0.0.tgz", "integrity": "sha512-uVFpKhj5MheNBJRTiMZ9pE/7hD1QTeEvugSJW/OmLzAp78PB5O6adfMNTvmfKhXBkvCzC+rqifWcVYpGFwTjnw==", - "dev": true, "engines": { "node": ">=14.16" }, @@ -7740,6 +8110,17 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/nth-check": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-2.1.1.tgz", + "integrity": "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==", + "dependencies": { + "boolbase": "^1.0.0" + }, + "funding": { + "url": "https://github.com/fb55/nth-check?sponsor=1" + } + }, "node_modules/object-assign": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", @@ -7944,11 +8325,64 @@ "node": ">=0.10.0" } }, + "node_modules/ow": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ow/-/ow-1.1.1.tgz", + "integrity": "sha512-sJBRCbS5vh1Jp9EOgwp1Ws3c16lJrUkJYlvWTYC03oyiYVwS/ns7lKRWow4w4XjDyTrA2pplQv4B2naWSR6yDA==", + "dependencies": { + "@sindresorhus/is": "^5.3.0", + "callsites": "^4.0.0", + "dot-prop": "^7.2.0", + "lodash.isequal": "^4.5.0", + "vali-date": "^1.0.0" + }, + "engines": { + "node": ">=14.16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/ow/node_modules/callsites": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/callsites/-/callsites-4.1.0.tgz", + "integrity": "sha512-aBMbD1Xxay75ViYezwT40aQONfr+pSXTHwNKvIXhXD6+LY3F1dLIcceoC5OZKBVHbXcysz1hL9D2w0JJIMXpUw==", + "engines": { + "node": ">=12.20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/ow/node_modules/dot-prop": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-7.2.0.tgz", + "integrity": "sha512-Ol/IPXUARn9CSbkrdV4VJo7uCy1I3VuSiWCaFSg+8BdUOzF9n3jefIpcgAydvUZbTdEBZs2vEiTiS9m61ssiDA==", + "dependencies": { + "type-fest": "^2.11.2" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/ow/node_modules/type-fest": { + "version": "2.19.0", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-2.19.0.tgz", + "integrity": "sha512-RAH822pAdBgcNMAfWnCBU3CFZcfZ/i1eZjwFU/dsLKumyuuP3niueg2UAukXYF0E2AAoc82ZSSf9J0WQBinzHA==", + "engines": { + "node": ">=12.20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/p-cancelable": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-3.0.0.tgz", "integrity": "sha512-mlVgR3PGuzlo0MmTdk4cXqXWlwQDLnONTAg6sm62XkMJEiRxN3GL3SffkYvqwonbkJBcrI7Uvv5Zh9yjvn2iUw==", - "dev": true, "engines": { "node": ">=12.20" } @@ -8118,6 +8552,29 @@ "parse-path": "^7.0.0" } }, + "node_modules/parse5": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-7.1.2.tgz", + "integrity": "sha512-Czj1WaSVpaoj0wbhMzLmWD69anp2WH7FXMB9n1Sy8/ZFF9jolSQVMu1Ij5WIyGmcBmhk7EOndpO4mIpihVqAXw==", + "dependencies": { + "entities": "^4.4.0" + }, + "funding": { + "url": "https://github.com/inikulin/parse5?sponsor=1" + } + }, + "node_modules/parse5-htmlparser2-tree-adapter": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/parse5-htmlparser2-tree-adapter/-/parse5-htmlparser2-tree-adapter-7.0.0.tgz", + "integrity": "sha512-B77tOZrqqfUfnVcOrUvfdLbz4pu4RopLD/4vmu3HUPswwTA8OH0EMW9BlWR2B0RCoiZRAHEUu7IxeP1Pd1UU+g==", + "dependencies": { + "domhandler": "^5.0.2", + "parse5": "^7.0.0" + }, + "funding": { + "url": "https://github.com/inikulin/parse5?sponsor=1" + } + }, "node_modules/parseurl": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", @@ -8228,8 +8685,7 @@ "node_modules/picocolors": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.0.0.tgz", - "integrity": "sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ==", - "dev": true + "integrity": "sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ==" }, "node_modules/picomatch": { "version": "2.3.1", @@ -8726,7 +9182,6 @@ "version": "5.1.1", "resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-5.1.1.tgz", "integrity": "sha512-WuyALRjWPDGtt/wzJiadO5AXY+8hZ80hVpe6MyivgraREW751X3SbhRvG3eLKOYN+8VEvqLcf3wdnt44Z4S4SA==", - "dev": true, "engines": { "node": ">=10" }, @@ -9084,6 +9539,18 @@ } } }, + "node_modules/release-it/node_modules/get-stream": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz", + "integrity": "sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==", + "dev": true, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/release-it/node_modules/globby": { "version": "14.0.1", "resolved": "https://registry.npmjs.org/globby/-/globby-14.0.1.tgz", @@ -9104,6 +9571,31 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/release-it/node_modules/got": { + "version": "13.0.0", + "resolved": "https://registry.npmjs.org/got/-/got-13.0.0.tgz", + "integrity": "sha512-XfBk1CxOOScDcMr9O1yKkNaQyy865NbYs+F7dr4H0LZMVgCj2Le59k6PqbNHoL5ToeaEQUYh6c6yMfVcc6SJxA==", + "dev": true, + "dependencies": { + "@sindresorhus/is": "^5.2.0", + "@szmarczak/http-timer": "^5.0.1", + "cacheable-lookup": "^7.0.0", + "cacheable-request": "^10.2.8", + "decompress-response": "^6.0.0", + "form-data-encoder": "^2.1.2", + "get-stream": "^6.0.1", + "http2-wrapper": "^2.1.10", + "lowercase-keys": "^3.0.0", + "p-cancelable": "^3.0.0", + "responselike": "^3.0.0" + }, + "engines": { + "node": ">=16" + }, + "funding": { + "url": "https://github.com/sindresorhus/got?sponsor=1" + } + }, "node_modules/release-it/node_modules/path-type": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/path-type/-/path-type-5.0.0.tgz", @@ -9166,8 +9658,7 @@ "node_modules/resolve-alpn": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/resolve-alpn/-/resolve-alpn-1.2.1.tgz", - "integrity": "sha512-0a1F4l73/ZFZOakJnQ3FvkJ2+gSTQWz/r2KE5OdDY0TxPm5h4GkqkWWfM47T7HsbnOtcJVEF4epCVy6u7Q3K+g==", - "dev": true + "integrity": "sha512-0a1F4l73/ZFZOakJnQ3FvkJ2+gSTQWz/r2KE5OdDY0TxPm5h4GkqkWWfM47T7HsbnOtcJVEF4epCVy6u7Q3K+g==" }, "node_modules/resolve-from": { "version": "5.0.0", @@ -9191,7 +9682,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/responselike/-/responselike-3.0.0.tgz", "integrity": "sha512-40yHxbNcl2+rzXvZuVkrYohathsSJlMTXKryG5y8uciHv1+xDLHQpgjG64JUO9nrEq2jGLH6IZ8BcZyw3wrweg==", - "dev": true, "dependencies": { "lowercase-keys": "^3.0.0" }, @@ -10820,8 +11310,7 @@ "node_modules/undici-types": { "version": "5.26.5", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", - "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", - "dev": true + "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==" }, "node_modules/unicorn-magic": { "version": "0.1.0", @@ -10873,6 +11362,35 @@ "node": ">= 0.8" } }, + "node_modules/update-browserslist-db": { + "version": "1.0.13", + "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.0.13.tgz", + "integrity": "sha512-xebP81SNcPuNpPP3uzeW1NYXxI3rxyJzF3pD6sH4jE7o/IX+WtSpwnVU+qIsDPyk0d3hmFQ7mjqc6AtV604hbg==", + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "dependencies": { + "escalade": "^3.1.1", + "picocolors": "^1.0.0" + }, + "bin": { + "update-browserslist-db": "cli.js" + }, + "peerDependencies": { + "browserslist": ">= 4.21.0" + } + }, "node_modules/update-notifier": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/update-notifier/-/update-notifier-7.0.0.tgz", @@ -10975,6 +11493,14 @@ "node": ">=10.12.0" } }, + "node_modules/vali-date": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/vali-date/-/vali-date-1.0.0.tgz", + "integrity": "sha512-sgECfZthyaCKW10N0fm27cg8HYTFK5qMWgypqkXMQ4Wbl/zZKx7xZICgcoxIIE+WFAP/MBL2EFwC/YvLxw3Zeg==", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/validate-npm-package-license": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/validate-npm-package-license/-/validate-npm-package-license-3.0.4.tgz", diff --git a/package.json b/package.json index 4ad89d6..c0087c6 100644 --- a/package.json +++ b/package.json @@ -19,13 +19,16 @@ }, "dependencies": { "@asteasolutions/zod-to-openapi": "^6.4.0", + "@types/got": "^9.6.12", "body-parser": "^1.20.2", + "cheerio": "^1.0.0-rc.12", "cors": "^2.8.5", "dotenv": "^16.4.5", "envalid": "^8.0.0", "express": "^4.18.2", "express-rate-limit": "^7.2.0", "fs": "^0.0.1-security", + "got": "^14.2.1", "helmet": "^7.1.0", "http-status-codes": "^2.3.0", "path": "^0.12.7", diff --git a/src/api-docs/openAPIDocumentGenerator.ts b/src/api-docs/openAPIDocumentGenerator.ts index 86d4ba2..8794819 100644 --- a/src/api-docs/openAPIDocumentGenerator.ts +++ b/src/api-docs/openAPIDocumentGenerator.ts @@ -2,10 +2,16 @@ import { OpenApiGeneratorV3, OpenAPIRegistry } from '@asteasolutions/zod-to-open import { healthCheckRegistry } from '@/routes/healthCheck/healthCheckRouter'; import { stabilityRegistry } from '@/routes/stability/stabilityRouter'; +import { webContentRegistry } from '@/routes/webContent/webContentModelRouter'; import { transcriptRegistry } from '@/routes/youtubeTranscript/transcriptRouter'; export function generateOpenAPIDocument() { - const registry = new OpenAPIRegistry([healthCheckRegistry, stabilityRegistry, transcriptRegistry]); + const registry = new OpenAPIRegistry([ + healthCheckRegistry, + stabilityRegistry, + transcriptRegistry, + webContentRegistry, + ]); const generator = new OpenApiGeneratorV3(registry.definitions); return generator.generateDocument({ diff --git a/src/routes/webContent/webContentModel.ts b/src/routes/webContent/webContentModel.ts new file mode 100644 index 0000000..9c8a415 --- /dev/null +++ b/src/routes/webContent/webContentModel.ts @@ -0,0 +1,10 @@ +import { extendZodWithOpenApi } from '@asteasolutions/zod-to-openapi'; +import { z } from 'zod'; + +extendZodWithOpenApi(z); + +export type Transcript = z.infer; +export const WebContentSchema = z.object({ + title: z.string(), + content: z.string(), +}); diff --git a/src/routes/webContent/webContentModelRouter.ts b/src/routes/webContent/webContentModelRouter.ts new file mode 100644 index 0000000..5066c28 --- /dev/null +++ b/src/routes/webContent/webContentModelRouter.ts @@ -0,0 +1,94 @@ +import { OpenAPIRegistry } from '@asteasolutions/zod-to-openapi'; +import * as cheerio from 'cheerio'; +import express, { Request, Response, Router } from 'express'; +import got from 'got'; +import { StatusCodes } from 'http-status-codes'; + +import { createApiResponse } from '@/api-docs/openAPIResponseBuilders'; +import { ResponseStatus, ServiceResponse } from '@/common/models/serviceResponse'; +import { handleServiceResponse } from '@/common/utils/httpHandlers'; + +import { WebContentSchema } from './webContentModel'; + +export const webContentRegistry = new OpenAPIRegistry(); +webContentRegistry.register('WebContent', WebContentSchema); + +const removeUnwantedElements = (_cheerio: any) => { + const elementsToRemove = [ + 'footer', + 'header', + 'nav', + 'script', + 'style', + 'link', + 'meta', + 'noscript', + 'img', + 'picture', + 'video', + 'audio', + 'iframe', + 'object', + 'embed', + 'param', + 'track', + 'source', + 'canvas', + 'map', + 'area', + 'svg', + 'math', + 'table', + 'caption', + 'colgroup', + 'col', + ]; + + elementsToRemove.forEach((element) => _cheerio(element).remove()); +}; + +const fetchAndCleanContent = async (url: string) => { + const { body } = await got(url); + const $ = cheerio.load(body); + const title = $('title').text(); + removeUnwantedElements($); + const bodyContent = $.text().replace(/\s\s+/g, ' ').trim(); + + return { title, content: bodyContent }; +}; + +export const webContentRouter: Router = (() => { + const router = express.Router(); + + webContentRegistry.registerPath({ + method: 'get', + path: '/content', + tags: ['WebContent'], + responses: createApiResponse(WebContentSchema, 'Success'), + }); + + router.get('/', async (_req: Request, res: Response) => { + const { url } = _req.query; + + if (typeof url !== 'string') { + return new ServiceResponse(ResponseStatus.Failed, 'URL must be a string', null, StatusCodes.BAD_REQUEST); + } + + try { + const content = await fetchAndCleanContent(url); + const serviceResponse = new ServiceResponse( + ResponseStatus.Success, + 'Service is healthy', + content, + StatusCodes.OK + ); + handleServiceResponse(serviceResponse, res); + } catch (error) { + console.error(`Error fetching content ${(error as Error).message}`); + const errorMessage = `Error fetching content $${(error as Error).message}`; + return new ServiceResponse(ResponseStatus.Failed, errorMessage, null, StatusCodes.INTERNAL_SERVER_ERROR); + } + }); + + return router; +})(); diff --git a/src/server.ts b/src/server.ts index ce0289e..c163d02 100644 --- a/src/server.ts +++ b/src/server.ts @@ -11,6 +11,7 @@ import requestLogger from '@/common/middleware/requestLogger'; import { healthCheckRouter } from '@/routes/healthCheck/healthCheckRouter'; import { stabilityRouter } from './routes/stability/stabilityRouter'; +import { webContentRouter } from './routes/webContent/webContentModelRouter'; import { transcriptRouter } from './routes/youtubeTranscript/transcriptRouter'; const logger = pino({ name: 'server start' }); const app: Express = express(); @@ -36,6 +37,7 @@ app.use('/health-check', healthCheckRouter); app.use('/stability', stabilityRouter); app.use('/images', express.static('public/images')); app.use('/transcript', transcriptRouter); +app.use('/get-content', webContentRouter); // Swagger UI app.use(openAPIRouter);