From b62e5353557ffcbe998fbc6ef6f554d787f06af9 Mon Sep 17 00:00:00 2001 From: Paulo Victor Gomes Date: Sun, 29 Sep 2024 10:50:43 +0200 Subject: [PATCH 1/2] start scam adviser --- .../modules/provider/providers/scamadviser.ts | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 src/app/modules/provider/providers/scamadviser.ts diff --git a/src/app/modules/provider/providers/scamadviser.ts b/src/app/modules/provider/providers/scamadviser.ts new file mode 100644 index 0000000..f8485ff --- /dev/null +++ b/src/app/modules/provider/providers/scamadviser.ts @@ -0,0 +1,44 @@ +import axios from 'axios'; +import * as cheerio from 'cheerio'; + +export class ScamAdviser { + private url: string; + + async factory(url: string) { + try { + this.url = this.enrichUrl(url); + const response = await this.fetchPageData(this.url); + const parsedData = this.parseData(response.data); + return parsedData; + } catch (error) { + return { + crawledUrl: this.url, + error: error.message || 'Error during request', + }; + } + } + + private async fetchPageData(url: string) { + const response = await axios.get(url); + return response; + } + + private enrichUrl(url: string): string { + if (!url.startsWith('https://www.scamadviser.com/check-website/') && !url.startsWith('https://www.scamadviser.com/')) { + console.log(`https://www.scamadviser.com/check-website/${url}`); + return `https://www.scamadviser.com/check-website/${url}`; + } + return url; + } + + private parseData(html: string) { + const $ = cheerio.load(html); + const reviewSummary = $('[class^="quick-review-text"]').text().trim(); + const [description, totalReviews] = reviewSummary.split('based on').map(part => part.trim()); + const [rating] = description.split(' / ').map(part => part.trim()); + return { + rating, + totalReviews + }; + } +} From fb2aff61ea7618e46a6243766ac49c6ce8cd0d05 Mon Sep 17 00:00:00 2001 From: Paulo Victor Gomes Date: Sun, 29 Sep 2024 10:57:32 +0200 Subject: [PATCH 2/2] voce bosta --- package-lock.json | 43 +++++++++++- package.json | 7 +- .../provider/providers/scamadviser.ts.spec.ts | 65 +++++++++++++++++++ 3 files changed, 109 insertions(+), 6 deletions(-) create mode 100644 src/app/modules/provider/providers/scamadviser.ts.spec.ts diff --git a/package-lock.json b/package-lock.json index 69783b4..33812bb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -29,20 +29,21 @@ "@nestjs/schematics": "^10.0.0", "@nestjs/testing": "^10.0.0", "@types/express": "^4.17.17", - "@types/jest": "^29.5.2", + "@types/jest": "^29.5.13", "@types/mongoose": "^5.11.97", "@types/node": "^20.3.1", "@types/supertest": "^6.0.0", "@typescript-eslint/eslint-plugin": "^6.0.0", "@typescript-eslint/parser": "^6.0.0", + "axios-mock-adapter": "^2.0.0", "eslint": "^8.42.0", "eslint-config-prettier": "^9.0.0", "eslint-plugin-prettier": "^5.0.0", - "jest": "^29.5.0", + "jest": "^29.7.0", "prettier": "^3.0.0", "source-map-support": "^0.5.21", "supertest": "^6.3.3", - "ts-jest": "^29.1.0", + "ts-jest": "^29.2.5", "ts-loader": "^9.4.3", "ts-node": "^10.9.1", "tsconfig-paths": "^4.2.0", @@ -2909,6 +2910,19 @@ "proxy-from-env": "^1.1.0" } }, + "node_modules/axios-mock-adapter": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/axios-mock-adapter/-/axios-mock-adapter-2.0.0.tgz", + "integrity": "sha512-D/K0J5Zm6KvaMTnsWrBQZWLzKN9GxUFZEa0mx2qeEHXDeTugCoplWehy8y36dj5vuSjhe1u/Dol8cZ8lzzmDew==", + "dev": true, + "dependencies": { + "fast-deep-equal": "^3.1.3", + "is-buffer": "^2.0.5" + }, + "peerDependencies": { + "axios": ">= 0.17.0" + } + }, "node_modules/babel-jest": { "version": "29.7.0", "resolved": "https://registry.npmjs.org/babel-jest/-/babel-jest-29.7.0.tgz", @@ -5409,6 +5423,29 @@ "node": ">=8" } }, + "node_modules/is-buffer": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-2.0.5.tgz", + "integrity": "sha512-i2R6zNFDwgEHJyQUtJEk0XFi1i0dPFn/oqjK3/vPCcDeJvW5NQ83V8QbicfF1SupOaB0h8ntgBC2YiE7dfyctQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "engines": { + "node": ">=4" + } + }, "node_modules/is-core-module": { "version": "2.15.1", "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.15.1.tgz", diff --git a/package.json b/package.json index ded55da..a1a45ec 100644 --- a/package.json +++ b/package.json @@ -40,20 +40,21 @@ "@nestjs/schematics": "^10.0.0", "@nestjs/testing": "^10.0.0", "@types/express": "^4.17.17", - "@types/jest": "^29.5.2", + "@types/jest": "^29.5.13", "@types/mongoose": "^5.11.97", "@types/node": "^20.3.1", "@types/supertest": "^6.0.0", "@typescript-eslint/eslint-plugin": "^6.0.0", "@typescript-eslint/parser": "^6.0.0", + "axios-mock-adapter": "^2.0.0", "eslint": "^8.42.0", "eslint-config-prettier": "^9.0.0", "eslint-plugin-prettier": "^5.0.0", - "jest": "^29.5.0", + "jest": "^29.7.0", "prettier": "^3.0.0", "source-map-support": "^0.5.21", "supertest": "^6.3.3", - "ts-jest": "^29.1.0", + "ts-jest": "^29.2.5", "ts-loader": "^9.4.3", "ts-node": "^10.9.1", "tsconfig-paths": "^4.2.0", diff --git a/src/app/modules/provider/providers/scamadviser.ts.spec.ts b/src/app/modules/provider/providers/scamadviser.ts.spec.ts new file mode 100644 index 0000000..7530c4d --- /dev/null +++ b/src/app/modules/provider/providers/scamadviser.ts.spec.ts @@ -0,0 +1,65 @@ +import axios from 'axios'; +import MockAdapter from 'axios-mock-adapter'; +import { ScamAdviser } from './ScamAdviser'; +import * as cheerio from 'cheerio'; + +describe('ScamAdviser', () => { + let scamAdviser: ScamAdviser; + let mockAxios: MockAdapter; + + beforeEach(() => { + scamAdviser = new ScamAdviser(); + mockAxios = new MockAdapter(axios); + + // Mock console.log to avoid seeing logs during the test + jest.spyOn(console, 'log').mockImplementation(() => {}); + }); + + afterEach(() => { + mockAxios.reset(); + jest.restoreAllMocks(); // Restore console.log and other mocks after each test + }); + + describe('factory method', () => { + + it('should return an error message when the request fails', async () => { + const fakeUrl = 'example.com'; + const enrichedUrl = 'https://www.scamadviser.com/check-website/example.com'; + const errorMessage = 'Network Error'; + + mockAxios.onGet(enrichedUrl).networkError(); + + const result = await scamAdviser.factory(fakeUrl); + + expect(result).toEqual({ + crawledUrl: enrichedUrl, + error: errorMessage, + }); + }); + }); + + describe('enrichUrl method', () => { + it('should add ScamAdviser URL if not present', () => { + const result = scamAdviser['enrichUrl']('example.com'); + expect(result).toBe('https://www.scamadviser.com/check-website/example.com'); + }); + + it('should not modify URL if already in ScamAdviser format', () => { + const existingUrl = 'https://www.scamadviser.com/check-website/example.com'; + const result = scamAdviser['enrichUrl'](existingUrl); + expect(result).toBe(existingUrl); + }); + }); + + describe('parseData method', () => { + it('should extract rating and total reviews from the HTML', () => { + const fakeHtml = '
85 / 100 based on 200 reviews
'; + const $ = cheerio.load(fakeHtml); + const result = scamAdviser['parseData'](fakeHtml); + expect(result).toEqual({ + rating: '85', + totalReviews: '200 reviews', + }); + }); + }); +});