diff --git a/package-lock.json b/package-lock.json index d5e8746991..8c648b2e69 100644 --- a/package-lock.json +++ b/package-lock.json @@ -26,7 +26,7 @@ "@typescript-eslint/eslint-plugin": "^7.0.0", "@typescript-eslint/parser": "^7.0.0", "commitlint": "^19.3.0", - "crawlee": "^3.11.1", + "crawlee": "^3.11.5", "eslint": "^8.57.0", "fs-extra": "^11.2.0", "gen-esm-wrapper": "^1.1.3", @@ -579,17 +579,17 @@ } }, "node_modules/@crawlee/basic": { - "version": "3.11.1", - "resolved": "https://registry.npmjs.org/@crawlee/basic/-/basic-3.11.1.tgz", - "integrity": "sha512-bWQIAkV4DhCGCsZ1Tb3GHH2Ouz1QTvxAiYvEifu6y/mxAxRcIdH25iyLlvI/5ii7hyQ5sTdhiw8k487ohMvRVA==", + "version": "3.11.5", + "resolved": "https://registry.npmjs.org/@crawlee/basic/-/basic-3.11.5.tgz", + "integrity": "sha512-P8fvvQLAk1rxIRdez+996d+SSGfXbcwkaOOlXSN267/POWdU8pFo0q7zOjstHdrLEf1UAWAek6vI35udukgW6g==", "license": "Apache-2.0", "dependencies": { "@apify/log": "^2.4.0", "@apify/timeout": "^0.3.0", "@apify/utilities": "^2.7.10", - "@crawlee/core": "3.11.1", - "@crawlee/types": "3.11.1", - "@crawlee/utils": "3.11.1", + "@crawlee/core": "3.11.5", + "@crawlee/types": "3.11.5", + "@crawlee/utils": "3.11.5", "csv-stringify": "^6.2.0", "fs-extra": "^11.0.0", "got-scraping": "^4.0.0", @@ -603,16 +603,16 @@ } }, "node_modules/@crawlee/browser": { - "version": "3.11.1", - "resolved": "https://registry.npmjs.org/@crawlee/browser/-/browser-3.11.1.tgz", - "integrity": "sha512-B7M9NcACdKz5vyRCUam227bcXr//szpQ3xo47rykTDEXmAD2KMoXiugc7PZXXfoBOLj9BoSdKXkeX3O2+DQYiQ==", + "version": "3.11.5", + "resolved": "https://registry.npmjs.org/@crawlee/browser/-/browser-3.11.5.tgz", + "integrity": "sha512-j7YGNTRYY8UVkhcsoixzA/je5cH42YZYZLvYaTglIKJbHZ6VJlh+Ua5f6rOaDWFwql6IIgivYghs12/p4d24Ww==", "license": "Apache-2.0", "dependencies": { "@apify/timeout": "^0.3.0", - "@crawlee/basic": "3.11.1", - "@crawlee/browser-pool": "3.11.1", - "@crawlee/types": "3.11.1", - "@crawlee/utils": "3.11.1", + "@crawlee/basic": "3.11.5", + "@crawlee/browser-pool": "3.11.5", + "@crawlee/types": "3.11.5", + "@crawlee/utils": "3.11.5", "ow": "^0.28.1", "tslib": "^2.4.0", "type-fest": "^4.0.0" @@ -634,15 +634,15 @@ } }, "node_modules/@crawlee/browser-pool": { - "version": "3.11.1", - "resolved": "https://registry.npmjs.org/@crawlee/browser-pool/-/browser-pool-3.11.1.tgz", - "integrity": "sha512-Svf/5Sn8pFEZSUO7Ro0dxyW+YJC19H+jm+gQn3ekf/MluLD+wSK+NJJIb4qjpgqleq0N+1Akm3jDInGV8gu3qg==", + "version": "3.11.5", + "resolved": "https://registry.npmjs.org/@crawlee/browser-pool/-/browser-pool-3.11.5.tgz", + "integrity": "sha512-zAc7+RWRKjfgD7VEtdKBUvp7/wrzQdOI5OxVc0xk3X6HVEJ/9z25OAGU381Km8Lpw9PUrZU6KpwWes6IQPBt4Q==", "license": "Apache-2.0", "dependencies": { "@apify/log": "^2.4.0", "@apify/timeout": "^0.3.0", - "@crawlee/core": "3.11.1", - "@crawlee/types": "3.11.1", + "@crawlee/core": "3.11.5", + "@crawlee/types": "3.11.5", "fingerprint-generator": "^2.0.6", "fingerprint-injector": "^2.0.5", "lodash.merge": "^4.6.2", @@ -671,15 +671,15 @@ } }, "node_modules/@crawlee/cheerio": { - "version": "3.11.1", - "resolved": "https://registry.npmjs.org/@crawlee/cheerio/-/cheerio-3.11.1.tgz", - "integrity": "sha512-PFQNp9LniBVYVuJvPn1WIUrEpioTxYOSsd3CcbMF+pGN1tCyeeqt6O8bJAfAs+P99AEoZNruCkblv0uPg8q3wQ==", + "version": "3.11.5", + "resolved": "https://registry.npmjs.org/@crawlee/cheerio/-/cheerio-3.11.5.tgz", + "integrity": "sha512-ZTbEHDCTfav+vm4eO5j7K3WKuW6ZvacgqjN3U1h3QP8PXCiYOSaiM4tnMKavus1mUivpTmgtIW8SJZze5QIsMw==", "license": "Apache-2.0", "dependencies": { - "@crawlee/http": "3.11.1", - "@crawlee/types": "3.11.1", - "@crawlee/utils": "3.11.1", - "cheerio": "^1.0.0-rc.12", + "@crawlee/http": "3.11.5", + "@crawlee/types": "3.11.5", + "@crawlee/utils": "3.11.5", + "cheerio": "1.0.0-rc.12", "htmlparser2": "^9.0.0", "tslib": "^2.4.0" }, @@ -688,12 +688,12 @@ } }, "node_modules/@crawlee/cli": { - "version": "3.11.1", - "resolved": "https://registry.npmjs.org/@crawlee/cli/-/cli-3.11.1.tgz", - "integrity": "sha512-XX4naPjGIRbyr/+ceCFUPbGEXKfUA/xILUa6DWrqp1NOt7+LftIEoQO16WGg6vDo8efYx+ZWpsreei9qPSMPzw==", + "version": "3.11.5", + "resolved": "https://registry.npmjs.org/@crawlee/cli/-/cli-3.11.5.tgz", + "integrity": "sha512-/4NJmqtTC775mUDY9Gc2bf//GGRRSOR7weJ6KpFpEE6MB4ydrQB1OaKR+cLLxEt6QDvB6B31DywRD9auV4Zt0A==", "license": "Apache-2.0", "dependencies": { - "@crawlee/templates": "3.11.1", + "@crawlee/templates": "3.11.5", "ansi-colors": "^4.1.3", "fs-extra": "^11.0.0", "inquirer": "^8.2.4", @@ -709,9 +709,9 @@ } }, "node_modules/@crawlee/core": { - "version": "3.11.1", - "resolved": "https://registry.npmjs.org/@crawlee/core/-/core-3.11.1.tgz", - "integrity": "sha512-Sy7qZJ3pJiolJVQ6YqNp+q8VbGuJfwWaYMFmaWqT+oY74V7Kz3BGw+o677shJ7cIQ+MlXGb8DD+G5fQqBCKswg==", + "version": "3.11.5", + "resolved": "https://registry.npmjs.org/@crawlee/core/-/core-3.11.5.tgz", + "integrity": "sha512-gxB5ZC++WhU+B6Fr2mTc6cbrl9KHJ31ASJ3W32R/qmItpi7l8MkdnBxqxYfu9iXroe2DconZbceVsFm9Gt6kkQ==", "license": "Apache-2.0", "dependencies": { "@apify/consts": "^2.20.0", @@ -720,11 +720,10 @@ "@apify/pseudo_url": "^2.0.30", "@apify/timeout": "^0.3.0", "@apify/utilities": "^2.7.10", - "@crawlee/memory-storage": "3.11.1", - "@crawlee/types": "3.11.1", - "@crawlee/utils": "3.11.1", + "@crawlee/memory-storage": "3.11.5", + "@crawlee/types": "3.11.5", + "@crawlee/utils": "3.11.5", "@sapphire/async-queue": "^1.5.1", - "@types/tough-cookie": "^4.0.2", "@vladfrangu/async_event_emitter": "^2.2.2", "csv-stringify": "^6.2.0", "fs-extra": "^11.0.0", @@ -732,10 +731,9 @@ "json5": "^2.2.3", "minimatch": "^9.0.0", "ow": "^0.28.1", - "stream-chain": "^2.2.5", - "stream-json": "^1.7.4", + "stream-json": "^1.8.0", "tldts": "^6.0.0", - "tough-cookie": "^4.0.0", + "tough-cookie": "^5.0.0", "tslib": "^2.4.0", "type-fest": "^4.0.0" }, @@ -743,19 +741,31 @@ "node": ">=16.0.0" } }, + "node_modules/@crawlee/core/node_modules/tough-cookie": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-5.0.0.tgz", + "integrity": "sha512-FRKsF7cz96xIIeMZ82ehjC3xW2E+O2+v11udrDYewUbszngYhsGa8z6YUMMzO9QJZzzyd0nGGXnML/TReX6W8Q==", + "license": "BSD-3-Clause", + "dependencies": { + "tldts": "^6.1.32" + }, + "engines": { + "node": ">=16" + } + }, "node_modules/@crawlee/http": { - "version": "3.11.1", - "resolved": "https://registry.npmjs.org/@crawlee/http/-/http-3.11.1.tgz", - "integrity": "sha512-Jn9U6JQBfmN+ALMpv3JX472qATrTrU6bakoSXC4pY9c5Sr9I+SQqoZq8yEFmE2XZjQgPAhjv8WrysY/yenFMwA==", + "version": "3.11.5", + "resolved": "https://registry.npmjs.org/@crawlee/http/-/http-3.11.5.tgz", + "integrity": "sha512-9eZ4V0WqVzhuiGACyZeozYmf+VXlKtUeplZLTVHXl10II4VAm9ix8GrXuSXby6hJO4aru/sNUHoScYfMTz4PFg==", "license": "Apache-2.0", "dependencies": { "@apify/timeout": "^0.3.0", "@apify/utilities": "^2.7.10", - "@crawlee/basic": "3.11.1", - "@crawlee/types": "3.11.1", - "@crawlee/utils": "3.11.1", + "@crawlee/basic": "3.11.5", + "@crawlee/types": "3.11.5", + "@crawlee/utils": "3.11.5", "@types/content-type": "^1.1.5", - "cheerio": "^1.0.0-rc.12", + "cheerio": "1.0.0-rc.12", "content-type": "^1.0.4", "got-scraping": "^4.0.0", "iconv-lite": "^0.6.3", @@ -769,19 +779,19 @@ } }, "node_modules/@crawlee/jsdom": { - "version": "3.11.1", - "resolved": "https://registry.npmjs.org/@crawlee/jsdom/-/jsdom-3.11.1.tgz", - "integrity": "sha512-NhUQUDz4l9qJHSpmHDGUBMvHUwQJTYTBF5u4tTMF1ehMJEw8ofJA1uXUrV3VtW/nBOvBqCM0XAvWOlzH95LUCA==", + "version": "3.11.5", + "resolved": "https://registry.npmjs.org/@crawlee/jsdom/-/jsdom-3.11.5.tgz", + "integrity": "sha512-Swer2KP0m8Xua4YH/FIHeXiyh/bjkoPRCT7M+IoeeV6aD0AOb7EQDLevnMHxhn4/wjAz1nRVpb+Ps5/Jz3Zx+w==", "license": "Apache-2.0", "dependencies": { "@apify/timeout": "^0.3.0", "@apify/utilities": "^2.7.10", - "@crawlee/http": "3.11.1", - "@crawlee/types": "3.11.1", - "@crawlee/utils": "3.11.1", + "@crawlee/http": "3.11.5", + "@crawlee/types": "3.11.5", + "@crawlee/utils": "3.11.5", "@types/jsdom": "^21.0.0", - "cheerio": "^1.0.0-rc.12", - "jsdom": "^24.0.0", + "cheerio": "1.0.0-rc.12", + "jsdom": "^25.0.0", "ow": "^0.28.2", "tslib": "^2.4.0" }, @@ -789,56 +799,16 @@ "node": ">=16.0.0" } }, - "node_modules/@crawlee/jsdom/node_modules/jsdom": { - "version": "24.1.3", - "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-24.1.3.tgz", - "integrity": "sha512-MyL55p3Ut3cXbeBEG7Hcv0mVM8pp8PBNWxRqchZnSfAiES1v1mRnMeFfaHWIPULpwsYfvO+ZmMZz5tGCnjzDUQ==", - "license": "MIT", - "dependencies": { - "cssstyle": "^4.0.1", - "data-urls": "^5.0.0", - "decimal.js": "^10.4.3", - "form-data": "^4.0.0", - "html-encoding-sniffer": "^4.0.0", - "http-proxy-agent": "^7.0.2", - "https-proxy-agent": "^7.0.5", - "is-potential-custom-element-name": "^1.0.1", - "nwsapi": "^2.2.12", - "parse5": "^7.1.2", - "rrweb-cssom": "^0.7.1", - "saxes": "^6.0.0", - "symbol-tree": "^3.2.4", - "tough-cookie": "^4.1.4", - "w3c-xmlserializer": "^5.0.0", - "webidl-conversions": "^7.0.0", - "whatwg-encoding": "^3.1.1", - "whatwg-mimetype": "^4.0.0", - "whatwg-url": "^14.0.0", - "ws": "^8.18.0", - "xml-name-validator": "^5.0.0" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "canvas": "^2.11.2" - }, - "peerDependenciesMeta": { - "canvas": { - "optional": true - } - } - }, "node_modules/@crawlee/linkedom": { - "version": "3.11.1", - "resolved": "https://registry.npmjs.org/@crawlee/linkedom/-/linkedom-3.11.1.tgz", - "integrity": "sha512-Ete9ShFBhKsYOQRcUEs8GB+Tkx7lDVnr+QQs1hw+gdemuXChX9Nppo/Iz0OH8JYWh9nZUOyTvvqrrBJLFKWtow==", + "version": "3.11.5", + "resolved": "https://registry.npmjs.org/@crawlee/linkedom/-/linkedom-3.11.5.tgz", + "integrity": "sha512-P22M7IMFSAWMcXTZnTarWXKPOeQcgnXOUVk+bTUNof1LbuLmy3sSCrpAI0OrACA0yUNmta/wegWSAicHWU/svw==", "license": "Apache-2.0", "dependencies": { "@apify/timeout": "^0.3.0", "@apify/utilities": "^2.7.10", - "@crawlee/http": "3.11.1", - "@crawlee/types": "3.11.1", + "@crawlee/http": "3.11.5", + "@crawlee/types": "3.11.5", "linkedom": "^0.18.0", "ow": "^0.28.2", "tslib": "^2.4.0" @@ -848,13 +818,13 @@ } }, "node_modules/@crawlee/memory-storage": { - "version": "3.11.1", - "resolved": "https://registry.npmjs.org/@crawlee/memory-storage/-/memory-storage-3.11.1.tgz", - "integrity": "sha512-k/rgBmoHE1CC68Jj5flaXJC9wWbScedvJFnqJy52bc2UiYooZvLVykkRXIBfY+aBuuILWvJdrZoT1OIEpm2EgQ==", + "version": "3.11.5", + "resolved": "https://registry.npmjs.org/@crawlee/memory-storage/-/memory-storage-3.11.5.tgz", + "integrity": "sha512-mm6dilvz64HK8m6kdUAOrPdMPzTufgy8PjsF5PFL1zsIJF2v9GTs8JcXUKZsT7cXqsfj40lvt65s5Lf9ZBz+MQ==", "license": "Apache-2.0", "dependencies": { "@apify/log": "^2.4.0", - "@crawlee/types": "3.11.1", + "@crawlee/types": "3.11.5", "@sapphire/async-queue": "^1.5.0", "@sapphire/shapeshift": "^3.0.0", "content-type": "^1.0.4", @@ -869,20 +839,20 @@ } }, "node_modules/@crawlee/playwright": { - "version": "3.11.1", - "resolved": "https://registry.npmjs.org/@crawlee/playwright/-/playwright-3.11.1.tgz", - "integrity": "sha512-baPuOHZW81pn/tOmJ54ySa+lSq3jpQEVXZbr52JWt5a/GEyMneJlyDNnk8fEVea0GMLZL3d3D5Bgm3NNsMHOzQ==", + "version": "3.11.5", + "resolved": "https://registry.npmjs.org/@crawlee/playwright/-/playwright-3.11.5.tgz", + "integrity": "sha512-j02e6jQb2g+bxVhWLkEttRXdaHpp5jRKdU+RkBAC+doKgQFrbN+jfQBO+4BsiPAMxldF5S3AEgBCXiwMxKC7BQ==", "license": "Apache-2.0", "dependencies": { "@apify/datastructures": "^2.0.0", "@apify/log": "^2.4.0", "@apify/timeout": "^0.3.1", - "@crawlee/browser": "3.11.1", - "@crawlee/browser-pool": "3.11.1", - "@crawlee/core": "3.11.1", - "@crawlee/types": "3.11.1", - "@crawlee/utils": "3.11.1", - "cheerio": "^1.0.0-rc.12", + "@crawlee/browser": "3.11.5", + "@crawlee/browser-pool": "3.11.5", + "@crawlee/core": "3.11.5", + "@crawlee/types": "3.11.5", + "@crawlee/utils": "3.11.5", + "cheerio": "1.0.0-rc.12", "idcac-playwright": "^0.1.2", "jquery": "^3.6.0", "lodash.isequal": "^4.5.0", @@ -905,18 +875,18 @@ } }, "node_modules/@crawlee/puppeteer": { - "version": "3.11.1", - "resolved": "https://registry.npmjs.org/@crawlee/puppeteer/-/puppeteer-3.11.1.tgz", - "integrity": "sha512-1BMIHbmMyKooGR6n6inOKH2uM+cFQclg0mdyn9IQRl/O7tULSNL1jUE0UhdSCNYbKmQV+oxocw6cT/BtqEaCmw==", + "version": "3.11.5", + "resolved": "https://registry.npmjs.org/@crawlee/puppeteer/-/puppeteer-3.11.5.tgz", + "integrity": "sha512-y3mbG7LIXZLgrRSbcXRmlXZYopAHCXUe0hTqvkSCYG3hxio783LJCSYuT5254dNqyWYT7yujI13mmvY4kYdM+g==", "license": "Apache-2.0", "dependencies": { "@apify/datastructures": "^2.0.0", "@apify/log": "^2.4.0", - "@crawlee/browser": "3.11.1", - "@crawlee/browser-pool": "3.11.1", - "@crawlee/types": "3.11.1", - "@crawlee/utils": "3.11.1", - "cheerio": "^1.0.0-rc.12", + "@crawlee/browser": "3.11.5", + "@crawlee/browser-pool": "3.11.5", + "@crawlee/types": "3.11.5", + "@crawlee/utils": "3.11.5", + "cheerio": "1.0.0-rc.12", "devtools-protocol": "*", "idcac-playwright": "^0.1.2", "jquery": "^3.6.0", @@ -936,9 +906,9 @@ } }, "node_modules/@crawlee/templates": { - "version": "3.11.1", - "resolved": "https://registry.npmjs.org/@crawlee/templates/-/templates-3.11.1.tgz", - "integrity": "sha512-dkd+oSH0gysSZWOpACMtiTWGkIKkCz0OlEqHRBmx10Q6rA4fzoR3gzYOnnmu6MR6YY3n+HFdOuegMttFGy1EHQ==", + "version": "3.11.5", + "resolved": "https://registry.npmjs.org/@crawlee/templates/-/templates-3.11.5.tgz", + "integrity": "sha512-RUDCxSPYWtVWKLUyKqpIfVBQan4EBl4Rka3+OLOpEijd5hqbIWREVoTYt+qxqb8uhOiMy66e3wYUrHd5pINlzg==", "license": "Apache-2.0", "dependencies": { "ansi-colors": "^4.1.3", @@ -961,9 +931,9 @@ } }, "node_modules/@crawlee/templates/node_modules/inquirer": { - "version": "9.3.6", - "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-9.3.6.tgz", - "integrity": "sha512-riK/iQB2ctwkpWYgjjWIRv3MBLt2gzb2Sj0JNQNbyTXgyXsLWcDPJ5WS5ZDTCx7BRFnJsARtYh+58fjP5M2Y0Q==", + "version": "9.3.7", + "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-9.3.7.tgz", + "integrity": "sha512-LJKFHCSeIRq9hanN14IlOtPSTe3lNES7TYDTE2xxdAy1LS5rYphajK1qtwvj3YmQXvvk0U2Vbmcni8P9EIQW9w==", "license": "MIT", "dependencies": { "@inquirer/figures": "^1.0.3", @@ -1002,9 +972,9 @@ } }, "node_modules/@crawlee/types": { - "version": "3.11.1", - "resolved": "https://registry.npmjs.org/@crawlee/types/-/types-3.11.1.tgz", - "integrity": "sha512-rbI8HIQonnjMFz8PBWss7AZ1Vmbiwx3C5BbJYYz7z9rd6hoDle/3K61GJJWLjClWpafZr1ywLpeJ2IKTGsonRw==", + "version": "3.11.5", + "resolved": "https://registry.npmjs.org/@crawlee/types/-/types-3.11.5.tgz", + "integrity": "sha512-3BHP+WgFRqVKK5hVQKhBb1+AMKvcgFzfkg0H9JafzrhLvwdMkiGsEQ/7TVajD76kCD2Ujs3hhhJXlheb0D1lWQ==", "license": "Apache-2.0", "dependencies": { "tslib": "^2.4.0" @@ -1014,16 +984,16 @@ } }, "node_modules/@crawlee/utils": { - "version": "3.11.1", - "resolved": "https://registry.npmjs.org/@crawlee/utils/-/utils-3.11.1.tgz", - "integrity": "sha512-PeNQDte8V7rr6t4N5L1Rdvt+KmeMrnMb19H6QqXtBTFBawJfgfyYxTHRgDcDJ/a1ZZ77gZE7UgZ2PsX9DqwAsA==", + "version": "3.11.5", + "resolved": "https://registry.npmjs.org/@crawlee/utils/-/utils-3.11.5.tgz", + "integrity": "sha512-Orz5xHp9KrO2FwIwRDbNQpRSGovbpQ2Xgt3Ex/4S5zzCz3/YzHMihAg3/+JamzbJkUcTIjWoERG4PUwOC+OGMg==", "license": "Apache-2.0", "dependencies": { "@apify/log": "^2.4.0", "@apify/ps-tree": "^1.2.0", - "@crawlee/types": "3.11.1", + "@crawlee/types": "3.11.5", "@types/sax": "^1.2.7", - "cheerio": "^1.0.0-rc.12", + "cheerio": "1.0.0-rc.12", "file-type": "^19.0.0", "got-scraping": "^4.0.3", "ow": "^0.28.1", @@ -1639,9 +1609,9 @@ } }, "node_modules/@inquirer/figures": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/@inquirer/figures/-/figures-1.0.5.tgz", - "integrity": "sha512-79hP/VWdZ2UVc9bFGJnoQ/lQMpL74mGgzSYX1xUqCVk7/v73vJCMw1VuyWN1jGkZ9B3z7THAbySqGbCNefcjfA==", + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/@inquirer/figures/-/figures-1.0.6.tgz", + "integrity": "sha512-yfZzps3Cso2UbM7WlxKwZQh2Hs6plrbjs1QnzQDZhK2DgyCo6D8AaHps9olkNcUFlcYERMqU3uJSp1gmy3s/qQ==", "license": "MIT", "engines": { "node": ">=18" @@ -6538,23 +6508,23 @@ } }, "node_modules/crawlee": { - "version": "3.11.1", - "resolved": "https://registry.npmjs.org/crawlee/-/crawlee-3.11.1.tgz", - "integrity": "sha512-8YS4qM7k0O/dr5aLJvCUK/V/9wrVtauIc2DQZ12TDy+Lr9Sal6jlHUNRiMSwfo0L3kFopZCppuagtFvMwu6lXw==", + "version": "3.11.5", + "resolved": "https://registry.npmjs.org/crawlee/-/crawlee-3.11.5.tgz", + "integrity": "sha512-jXE7NkN+k9oSYPMozNYDfYgjA2QV7tEr0jba/foCfGqLog+B7DAtE/RYqCMXoxzkdlq/xsjBDz72lTxmlMyIbw==", "license": "Apache-2.0", "dependencies": { - "@crawlee/basic": "3.11.1", - "@crawlee/browser": "3.11.1", - "@crawlee/browser-pool": "3.11.1", - "@crawlee/cheerio": "3.11.1", - "@crawlee/cli": "3.11.1", - "@crawlee/core": "3.11.1", - "@crawlee/http": "3.11.1", - "@crawlee/jsdom": "3.11.1", - "@crawlee/linkedom": "3.11.1", - "@crawlee/playwright": "3.11.1", - "@crawlee/puppeteer": "3.11.1", - "@crawlee/utils": "3.11.1", + "@crawlee/basic": "3.11.5", + "@crawlee/browser": "3.11.5", + "@crawlee/browser-pool": "3.11.5", + "@crawlee/cheerio": "3.11.5", + "@crawlee/cli": "3.11.5", + "@crawlee/core": "3.11.5", + "@crawlee/http": "3.11.5", + "@crawlee/jsdom": "3.11.5", + "@crawlee/linkedom": "3.11.5", + "@crawlee/playwright": "3.11.5", + "@crawlee/puppeteer": "3.11.5", + "@crawlee/utils": "3.11.5", "import-local": "^3.1.0", "tslib": "^2.4.0" }, @@ -11445,7 +11415,6 @@ "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-25.0.0.tgz", "integrity": "sha512-OhoFVT59T7aEq75TVw9xxEfkXgacpqAhQaYgP9y/fDqWQCMB/b1H66RfmPm/MaeaAIU9nDwMOVTlPN51+ao6CQ==", "license": "MIT", - "peer": true, "dependencies": { "cssstyle": "^4.0.1", "data-urls": "^5.0.0", @@ -12185,9 +12154,9 @@ } }, "node_modules/linkedom": { - "version": "0.18.4", - "resolved": "https://registry.npmjs.org/linkedom/-/linkedom-0.18.4.tgz", - "integrity": "sha512-JhLErxMIEOKByMi3fURXgI1fYOzR87L1Cn0+MI9GlMckFrqFZpV1SUGox1jcKtsKN3y6JgclcQf0FzZT//BuGw==", + "version": "0.18.5", + "resolved": "https://registry.npmjs.org/linkedom/-/linkedom-0.18.5.tgz", + "integrity": "sha512-JGLaGGtqtu+eOhYrC1wkWYTBcpVWL4AsnwAtMtgO1Q0gI0PuPJKI0zBBE+a/1BrhOE3Uw8JI/ycByAv5cLrAuQ==", "license": "ISC", "dependencies": { "css-select": "^5.1.0", diff --git a/package.json b/package.json index 457c1637ce..1779d52cac 100644 --- a/package.json +++ b/package.json @@ -64,7 +64,7 @@ "@typescript-eslint/eslint-plugin": "^7.0.0", "@typescript-eslint/parser": "^7.0.0", "commitlint": "^19.3.0", - "crawlee": "^3.11.1", + "crawlee": "^3.11.5", "eslint": "^8.57.0", "fs-extra": "^11.2.0", "gen-esm-wrapper": "^1.1.3", diff --git a/packages/apify/src/proxy_configuration.ts b/packages/apify/src/proxy_configuration.ts index 79153a623b..3292a53421 100644 --- a/packages/apify/src/proxy_configuration.ts +++ b/packages/apify/src/proxy_configuration.ts @@ -271,7 +271,8 @@ export class ProxyConfiguration extends CoreProxyConfiguration { sessionId, groups, countryCode, - password: password ?? '', + // this.password is not encoded, but the password from the URL will be, we need to normalize + password: this.usesApifyProxy ? (password ?? '') : decodeURIComponent(password!), hostname, port: port!, }; @@ -353,9 +354,12 @@ export class ProxyConfiguration extends CoreProxyConfiguration { protected composeDefaultUrl(sessionId?: string): string { const username = this._getUsername(sessionId); - const { password, hostname, port } = this; + const url = new URL(`http://${this.hostname}:${this.port}`); + url.username = `${username}`; + url.password = `${this.password}`; + const urlString = url.toString(); - return `http://${username}:${password}@${hostname}:${port}`; + return urlString.substring(0, urlString.length - 1); } /** @@ -464,7 +468,7 @@ export class ProxyConfiguration extends CoreProxyConfiguration { * @internal */ protected _throwCannotCombineCustomWithApify() { - throw new Error('Cannot combine custom proxies with Apify Proxy!' + throw new Error('Cannot combine custom proxies with Apify Proxy! ' + 'It is not allowed to set "options.proxyUrls" or "options.newUrlFunction" combined with ' + '"options.groups" or "options.apifyProxyGroups" and "options.countryCode" or "options.apifyProxyCountry".'); } diff --git a/test/apify/proxy_configuration.test.ts b/test/apify/proxy_configuration.test.ts index a2fd55ccb6..04a0d02257 100644 --- a/test/apify/proxy_configuration.test.ts +++ b/test/apify/proxy_configuration.test.ts @@ -78,6 +78,21 @@ describe('ProxyConfiguration', () => { expect(await proxyConfiguration.newProxyInfo(sessionId)).toEqual(proxyInfo); }); + test('newProxyInfo() works with special characters', async () => { + const url = 'http://user%40name:pass%40word@proxy.com:1111'; + const proxyConfiguration = new ProxyConfiguration({ proxyUrls: [url] }); + + const proxyInfo = { + sessionId: `${sessionId}`, + url, + username: 'user@name', + password: 'pass@word', + hostname: 'proxy.com', + port: '1111', + }; + expect(await proxyConfiguration.newProxyInfo(sessionId)).toEqual(proxyInfo); + }); + test('actor UI input schema should work', () => { const apifyProxyGroups = ['GROUP1', 'GROUP2']; const apifyProxyCountry = 'CZ';