From cc03c9644fae0dc8db35b88f895372778ad02add Mon Sep 17 00:00:00 2001 From: Alex Yarmosh Date: Thu, 23 Nov 2023 16:51:13 +0100 Subject: [PATCH] feat: setting data for adopted probes (#442) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: ignore syncing of custom fields * refactor: move fetch sockets from ws server * test: fix tests * refactor: rename fetchsockets * feat: update probe info with custom data * feat: add index values * feat: get username from dashboard * test: fix tests * feat: return probe info after send code to the probe * fix: fix filtering by user tags * fix: fix test ts build * feat: log and send to the probe updated location * fix: tests * feat: do not apply adoption city data if country doesn`t match * fix: test db * fix: update init sql script * test: add GET int test * fix: nock geo ip * test: add adoption POST test * test: add adopted-probes tests * test: add more adopted sockets tests * feat: update state of the adopted probes in US cities * feat: update gihtub ci * feat: use github field instead of last_name field * test: update userId value * fix: treat null and undefined values as equal * refactor: update sql query * fix: remove user tags from index * feat: rename normalize function * feat: update country field of the adopted probe * feat: send notification to the user * test: add tests for edit of country field * feat: get rid of duplicate notification messages * feat: update dev mariadb password * test: fix tests * feat: add new cities altnames --------- Co-authored-by: Martin Kolárik --- .github/workflows/ci.yml | 28 +- config/init.sql | 75 +++- docker-compose.yml | 2 +- public/demo/measurements.vue.js | 2 +- src/adoption-code/route/adoption-code.ts | 13 +- src/adoption-code/sender.ts | 11 +- src/lib/adopted-probes.ts | 221 +++++++++-- src/lib/geoip/altnames.ts | 2 + src/lib/geoip/utils.ts | 2 + src/lib/server.ts | 4 + src/lib/ws/fetch-sockets.ts | 60 +++ src/lib/ws/gateway.ts | 7 +- src/lib/ws/helper/probe-ip-limit.ts | 2 +- src/lib/ws/helper/reconnect-probes.ts | 14 +- src/lib/ws/server.ts | 24 +- src/probe/builder.ts | 49 ++- src/probe/route/get-probes.ts | 3 +- src/probe/router.ts | 2 +- src/probe/sockets-location-filter.ts | 2 +- src/probe/types.ts | 2 +- test/mocks/nock-geoip.json | 2 +- test/setup.ts | 3 + .../adoption-code/adoption-code.test.ts | 11 +- .../measurement/create-measurement.test.ts | 161 ++++---- .../integration/middleware/compress.test.ts | 12 +- .../integration/probes/get-probes.test.ts | 142 ++++--- test/tests/unit/adopted-probes.test.ts | 345 ++++++++++++++---- test/tests/unit/geoip/client.test.ts | 10 +- test/tests/unit/probe/router.test.ts | 22 +- test/tests/unit/ws/fetch-sockets.test.ts | 182 +++++++++ test/tests/unit/ws/reconnect-probes.test.ts | 43 +++ test/tests/unit/ws/server.test.ts | 45 +-- test/utils/nock-geo-ip.ts | 32 +- 33 files changed, 1116 insertions(+), 419 deletions(-) create mode 100644 src/lib/ws/fetch-sockets.ts create mode 100644 test/tests/unit/ws/fetch-sockets.test.ts create mode 100644 test/tests/unit/ws/reconnect-probes.test.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 639095e1..85d10434 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,23 +24,25 @@ jobs: --health-timeout 5s --health-retries 5 + mariadb: + image: mariadb:10.11.5 + ports: + - 3306:3306 + env: + MARIADB_DATABASE: directus + MARIADB_USER: directus + MARIADB_PASSWORD: password + MARIADB_RANDOM_ROOT_PASSWORD: 1 + options: >- + --health-cmd "mysqladmin ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + steps: - uses: actions/checkout@v3 - - - name: Shutdown Ubuntu MySQL - run: sudo service mysql stop - - - name: Setup MariaDB - uses: getong/mariadb-action@v1.1 - with: - mariadb version: '10.11.5' - mysql user: 'directus' - mysql password: 'password' - mysql database: 'directus' - - name: Init MariaDB run: | - sleep 5 mysql -u directus -ppassword --database=directus --protocol=tcp < config/init.sql - uses: actions/setup-node@v3 diff --git a/config/init.sql b/config/init.sql index d2e96b8e..9f1c9289 100644 --- a/config/init.sql +++ b/config/init.sql @@ -10,15 +10,72 @@ CREATE TABLE IF NOT EXISTS adopted_probes ( userId VARCHAR(255) NOT NULL, ip VARCHAR(255) NOT NULL, uuid VARCHAR(255), - lastSyncDate DATE, - status VARCHAR(255), - version VARCHAR(255), - country VARCHAR(255), + lastSyncDate DATE NOT NULL, + isCustomCity TINYINT DEFAULT 0, + tags LONGTEXT, + status VARCHAR(255) NOT NULL, + version VARCHAR(255) NOT NULL, + country VARCHAR(255) NOT NULL, city VARCHAR(255), - latitude FLOAT, - longitude FLOAT, - asn INTEGER, - network VARCHAR(255) + state VARCHAR(255), + latitude FLOAT(10, 5), + longitude FLOAT(10, 5), + asn INTEGER NOT NULL, + network VARCHAR(255) NOT NULL, + countryOfCustomCity VARCHAR(255) ); -INSERT IGNORE INTO adopted_probes (id, userId, ip) VALUES ('1', '6191378', '79.205.97.254'); +CREATE TABLE IF NOT EXISTS directus_users ( + id CHAR(36), + github VARCHAR(255) +); + +CREATE TABLE IF NOT EXISTS directus_notifications ( + id CHAR(10), + recipient CHAR(36), + timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + subject VARCHAR(255), + message TEXT +); + +INSERT IGNORE INTO adopted_probes ( + userId, + lastSyncDate, + ip, + uuid, + isCustomCity, + tags, + status, + version, + country, + city, + latitude, + longitude, + network, + asn, + countryOfCustomCity +) VALUES ( + '89da69bd-a236-4ab7-9c5d-b5f52ce09959', + CURRENT_DATE, + '51.158.22.211', + 'c77f021d-23ff-440a-aa96-35e82c73e731', + 1, + '["mytag1"]', + 'ready', + '0.26.0', + 'FR', + 'Marseille', + '43.29695', + '5.38107', + 'SCALEWAY S.A.S.', + 12876, + 'FR' +); + +INSERT IGNORE INTO directus_users ( + id, + github +) VALUES ( + '89da69bd-a236-4ab7-9c5d-b5f52ce09959', + 'jimaek' +); diff --git a/docker-compose.yml b/docker-compose.yml index de99b53f..690bd6f2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,7 +15,7 @@ services: - MARIADB_DATABASE=directus - MARIADB_USER=directus - MARIADB_PASSWORD=password - - MARIADB_RANDOM_ROOT_PASSWORD=1 + - MARIADB_ROOT_PASSWORD=root ports: - "3306:3306" volumes: diff --git a/public/demo/measurements.vue.js b/public/demo/measurements.vue.js index a9dec0ec..6bdb3a07 100644 --- a/public/demo/measurements.vue.js +++ b/public/demo/measurements.vue.js @@ -176,7 +176,7 @@ const app = () => ({ const locations = this.query.locations.map(({ limit, fields }) => ({ fields: fields.map(field => ({ ...field, - value: field.type === 'tags' ? field.value.split(',') : field.value, + value: field.type === 'tags' ? field.value.split(',').map(tag => tag.trim()) : field.value, })), ...(limit ? { limit } : {}), })); diff --git a/src/adoption-code/route/adoption-code.ts b/src/adoption-code/route/adoption-code.ts index cb53a517..67c65968 100644 --- a/src/adoption-code/route/adoption-code.ts +++ b/src/adoption-code/route/adoption-code.ts @@ -9,10 +9,19 @@ import { codeSender } from '../sender.js'; const handle = async (ctx: Context): Promise => { const request = ctx.request.body as AdoptionCodeRequest; - const result = await codeSender.sendCode(request); + const socket = await codeSender.sendCode(request); ctx.body = { - result, + uuid: socket.data.probe.uuid, + version: socket.data.probe.version, + status: socket.data.probe.status, + city: socket.data.probe.location.city, + state: socket.data.probe.location.state, + country: socket.data.probe.location.country, + latitude: socket.data.probe.location.latitude, + longitude: socket.data.probe.location.longitude, + asn: socket.data.probe.location.asn, + network: socket.data.probe.location.network, }; }; diff --git a/src/adoption-code/sender.ts b/src/adoption-code/sender.ts index 4855bb94..b19c485e 100644 --- a/src/adoption-code/sender.ts +++ b/src/adoption-code/sender.ts @@ -1,12 +1,13 @@ import createHttpError from 'http-errors'; -import { fetchSockets, RemoteProbeSocket } from '../lib/ws/server.js'; +import type { RemoteProbeSocket } from '../lib/ws/server.js'; +import { fetchSockets } from '../lib/ws/fetch-sockets.js'; import type { AdoptionCodeRequest } from './types.js'; export class CodeSender { constructor (private readonly fetchWsSockets: typeof fetchSockets) {} - async sendCode (request: AdoptionCodeRequest): Promise { + async sendCode (request: AdoptionCodeRequest): Promise { const socket = await this.findSocketByIp(request.ip); if (!socket) { @@ -15,7 +16,7 @@ export class CodeSender { this.sendToSocket(socket, request.code); - return 'Code was sent to the probe.'; + return socket; } private async findSocketByIp (ip: string) { @@ -23,8 +24,8 @@ export class CodeSender { return sockets.find(socket => socket.data.probe.ipAddress === ip); } - private sendToSocket (sockets: RemoteProbeSocket, code: string) { - sockets.emit('probe:adoption:code', { + private sendToSocket (socket: RemoteProbeSocket, code: string) { + socket.emit('probe:adoption:code', { code, }); } diff --git a/src/lib/adopted-probes.ts b/src/lib/adopted-probes.ts index 087bcf79..0b0b9e26 100644 --- a/src/lib/adopted-probes.ts +++ b/src/lib/adopted-probes.ts @@ -1,49 +1,125 @@ import type { Knex } from 'knex'; import Bluebird from 'bluebird'; import _ from 'lodash'; - import { scopedLogger } from './logger.js'; import { client } from './sql/client.js'; -import { fetchSockets } from './ws/server.js'; +import { fetchRawSockets } from './ws/server.js'; import type { Probe } from '../probe/types.js'; +import { normalizeFromPublicName } from './geoip/utils.js'; const logger = scopedLogger('adopted-probes'); -const TABLE_NAME = 'adopted_probes'; +export const ADOPTED_PROBES_TABLE = 'adopted_probes'; +export const USERS_TABLE = 'directus_users'; +export const NOTIFICATIONS_TABLE = 'directus_notifications'; -type AdoptedProbe = { +export type AdoptedProbe = { + userId: string; + username: string; ip: string; uuid: string; - lastSyncDate: string; + lastSyncDate: Date; + tags: string[]; + isCustomCity: boolean; status: string; version: string; country: string; - city: string; - latitude: number; - longitude: number; + countryOfCustomCity?: string; + city?: string; + state?: string; + latitude?: number; + longitude?: number; asn: number; network: string; } +type Row = Omit & { + tags: string; + isCustomCity: number; +} + export class AdoptedProbes { private connectedIpToProbe: Map = new Map(); private connectedUuidToIp: Map = new Map(); + private adoptedIpToProbe: Map = new Map(); private readonly adoptedFieldToConnectedField = { - status: 'status', - version: 'version', - country: 'location.country', - city: 'location.city', - latitude: 'location.latitude', - longitude: 'location.longitude', - asn: 'location.asn', - network: 'location.network', + status: { + connectedField: 'status', + shouldUpdateIfCustomCity: true, + }, + version: { + connectedField: 'version', + shouldUpdateIfCustomCity: true, + }, + asn: { + connectedField: 'location.asn', + shouldUpdateIfCustomCity: true, + }, + network: { + connectedField: 'location.network', + shouldUpdateIfCustomCity: true, + }, + country: { + connectedField: 'location.country', + shouldUpdateIfCustomCity: true, + }, + city: { + connectedField: 'location.city', + shouldUpdateIfCustomCity: false, + }, + state: { + connectedField: 'location.state', + shouldUpdateIfCustomCity: false, + }, + latitude: { + connectedField: 'location.latitude', + shouldUpdateIfCustomCity: false, + }, + longitude: { + connectedField: 'location.longitude', + shouldUpdateIfCustomCity: false, + }, }; constructor ( private readonly sql: Knex, - private readonly fetchWsSockets: typeof fetchSockets, + private readonly fetchSocketsRaw: typeof fetchRawSockets, ) {} + getByIp (ip: string) { + return this.adoptedIpToProbe.get(ip); + } + + getUpdatedLocation (probe: Probe) { + const adoptedProbe = this.getByIp(probe.ipAddress); + + if (!adoptedProbe || !adoptedProbe.isCustomCity || adoptedProbe.countryOfCustomCity !== probe.location.country) { + return probe.location; + } + + return { + ...probe.location, + city: adoptedProbe.city!, + normalizedCity: normalizeFromPublicName(adoptedProbe.city!), + latitude: adoptedProbe.latitude!, + longitude: adoptedProbe.longitude!, + ...(adoptedProbe.state && { state: adoptedProbe.state }), + }; + } + + getUpdatedTags (probe: Probe) { + const adoptedProbe = this.getByIp(probe.ipAddress); + + if (!adoptedProbe || !adoptedProbe.tags.length) { + return probe.tags; + } + + return [ + ...probe.tags, + ...adoptedProbe.tags.map(tag => ({ type: 'user' as const, value: `u-${adoptedProbe.username}-${tag}` })), + ]; + } + scheduleSync () { setTimeout(() => { this.syncDashboardData() @@ -53,14 +129,38 @@ export class AdoptedProbes { } async syncDashboardData () { - const allSockets = await this.fetchWsSockets(); + const allSockets = await this.fetchSocketsRaw(); this.connectedIpToProbe = new Map(allSockets.map(socket => [ socket.data.probe.ipAddress, socket.data.probe ])); this.connectedUuidToIp = new Map(allSockets.map(socket => [ socket.data.probe.uuid, socket.data.probe.ipAddress ])); - const adoptedProbes = await this.sql(TABLE_NAME).select('ip', 'uuid', 'lastSyncDate', ...Object.keys(this.adoptedFieldToConnectedField)); - await Bluebird.map(adoptedProbes, ({ ip, uuid }) => this.syncProbeIds(ip, uuid), { concurrency: 8 }); - await Bluebird.map(adoptedProbes, adoptedProbe => this.syncProbeData(adoptedProbe), { concurrency: 8 }); - await Bluebird.map(adoptedProbes, ({ ip, lastSyncDate }) => this.updateSyncDate(ip, lastSyncDate), { concurrency: 8 }); + await this.fetchAdoptedProbes(); + await Bluebird.map(this.adoptedIpToProbe.values(), ({ ip, uuid }) => this.syncProbeIds(ip, uuid), { concurrency: 8 }); + await Bluebird.map(this.adoptedIpToProbe.values(), adoptedProbe => this.syncProbeData(adoptedProbe), { concurrency: 8 }); + await Bluebird.map(this.adoptedIpToProbe.values(), ({ ip, lastSyncDate }) => this.updateSyncDate(ip, lastSyncDate), { concurrency: 8 }); + } + + private async fetchAdoptedProbes () { + const rows = await this.sql({ probes: ADOPTED_PROBES_TABLE }) + .join({ users: USERS_TABLE }, 'probes.userId', '=', 'users.id') + .select({ + username: 'users.github', + userId: 'probes.userId', + ip: 'probes.ip', + uuid: 'probes.uuid', + lastSyncDate: 'probes.lastSyncDate', + isCustomCity: 'probes.isCustomCity', + countryOfCustomCity: 'probes.countryOfCustomCity', + tags: 'probes.tags', + ...Object.fromEntries(Object.keys(this.adoptedFieldToConnectedField).map(field => [ field, `probes.${field}` ])), + }); + + const adoptedProbes: AdoptedProbe[] = rows.map(row => ({ + ...row, + tags: row.tags ? JSON.parse(row.tags) as string[] : [], + isCustomCity: Boolean(row.isCustomCity), + })); + + this.adoptedIpToProbe = new Map(adoptedProbes.map(probe => [ probe.ip, probe ])); } private async syncProbeIds (ip: string, uuid: string) { @@ -84,6 +184,7 @@ export class AdoptedProbes { private async syncProbeData (adoptedProbe: AdoptedProbe) { const connectedProbe = this.connectedIpToProbe.get(adoptedProbe.ip); + const isCustomCity = adoptedProbe.isCustomCity; if (!connectedProbe) { return; @@ -91,21 +192,34 @@ export class AdoptedProbes { const updateObject: Record = {}; - Object.entries(this.adoptedFieldToConnectedField).forEach(([ adoptedField, connectedField ]) => { + Object.entries(this.adoptedFieldToConnectedField).forEach(([ adoptedField, { connectedField, shouldUpdateIfCustomCity }]) => { + if (isCustomCity && !shouldUpdateIfCustomCity) { + return; + } + const adoptedValue = _.get(adoptedProbe, adoptedField) as string | number; const connectedValue = _.get(connectedProbe, connectedField) as string | number; + if (!adoptedValue && !connectedValue) { // undefined and null values are treated equal and don't require sync + return; + } + if (adoptedValue !== connectedValue) { updateObject[adoptedField] = connectedValue; } }); + // if country of probe changes, but there is a custom city in prev country, send notification to user + if (updateObject['country'] && adoptedProbe.country === adoptedProbe.countryOfCustomCity) { + await this.sendNotification(adoptedProbe, connectedProbe); + } + if (!_.isEmpty(updateObject)) { - await this.updateProbeData(adoptedProbe.ip, updateObject); + await this.updateProbeData(adoptedProbe, updateObject); } } - private async updateSyncDate (ip: string, lastSyncDate: string) { + private async updateSyncDate (ip: string, lastSyncDate: Date) { if (this.isToday(lastSyncDate)) { // date is already synced return; } @@ -123,40 +237,71 @@ export class AdoptedProbes { } private async updateUuid (ip: string, uuid: string) { - await this.sql(TABLE_NAME).where({ ip }).update({ uuid }); + await this.sql(ADOPTED_PROBES_TABLE).where({ ip }).update({ uuid }); + const adoptedProbe = this.adoptedIpToProbe.get(ip); + + if (adoptedProbe) { adoptedProbe.uuid = uuid; } } private async updateIp (ip: string, uuid: string) { - await this.sql(TABLE_NAME).where({ uuid }).update({ ip }); + await this.sql(ADOPTED_PROBES_TABLE).where({ uuid }).update({ ip }); + const entry = [ ...this.adoptedIpToProbe.entries() ].find(([ , adoptedProbe ]) => adoptedProbe.uuid === uuid); + + if (entry) { + const [ prevIp, adoptedProbe ] = entry; + adoptedProbe.ip = ip; + this.adoptedIpToProbe.delete(prevIp); + this.adoptedIpToProbe.set(ip, adoptedProbe); + } } - private async updateProbeData (ip: string, updateObject: Record) { - await this.sql(TABLE_NAME).where({ ip }).update(updateObject); + private async updateProbeData (adoptedProbe: AdoptedProbe, updateObject: Record) { + await this.sql(ADOPTED_PROBES_TABLE).where({ ip: adoptedProbe.ip }).update(updateObject); + + for (const [ field, value ] of Object.entries(updateObject)) { + (adoptedProbe as unknown as Record)[field] = value; + } } private async updateLastSyncDate (ip: string) { - await this.sql(TABLE_NAME).where({ ip }).update({ lastSyncDate: new Date() }); + const date = new Date(); + await this.sql(ADOPTED_PROBES_TABLE).where({ ip }).update({ lastSyncDate: date }); + const adoptedProbe = this.adoptedIpToProbe.get(ip); + + if (adoptedProbe) { + adoptedProbe.lastSyncDate = date; + } } private async deleteAdoptedProbe (ip: string) { - await this.sql(TABLE_NAME).where({ ip }).delete(); + await this.sql(ADOPTED_PROBES_TABLE).where({ ip }).delete(); + this.adoptedIpToProbe.delete(ip); + } + + private async sendNotification (adoptedProbe: AdoptedProbe, connectedProbe: Probe) { + await this.sql.raw(` + INSERT INTO directus_notifications (recipient, subject, message) SELECT :recipient, :subject, :message + WHERE NOT EXISTS (SELECT 1 FROM directus_notifications WHERE recipient = :recipient AND message = :message AND DATE(timestamp) = CURRENT_DATE) + `, { + recipient: adoptedProbe.userId, + subject: 'Adopted probe country change', + message: `Globalping API detected that your adopted probe with ip: ${adoptedProbe.ip} is located at "${connectedProbe.location.country}". So its country value changed from "${adoptedProbe.country}" to "${connectedProbe.location.country}", and custom city value "${adoptedProbe.city}" is not applied right now.\n\nIf this change is not right please report in [that issue](https://github.com/jsdelivr/globalping/issues/268).`, + }); } - private isToday (dateString: string) { + private isToday (date: Date) { const currentDate = new Date(); - const currentDateString = currentDate.toISOString().split('T')[0]; - return dateString === currentDateString; + return date.toDateString() === currentDate.toDateString(); } - private isMoreThan30DaysAgo (dateString: string) { - const inputDate = new Date(dateString); + private isMoreThan30DaysAgo (date: Date) { const currentDate = new Date(); - const timeDifference = currentDate.getTime() - inputDate.getTime(); + const timeDifference = currentDate.getTime() - date.getTime(); const daysDifference = timeDifference / (24 * 3600 * 1000); return daysDifference > 30; } } -export const adoptedProbes = new AdoptedProbes(client, fetchSockets); +export const adoptedProbes = new AdoptedProbes(client, fetchRawSockets); diff --git a/src/lib/geoip/altnames.ts b/src/lib/geoip/altnames.ts index 3196cba6..1bf28dc9 100644 --- a/src/lib/geoip/altnames.ts +++ b/src/lib/geoip/altnames.ts @@ -2,4 +2,6 @@ export const cities: Record = { 'Geneve': 'Geneva', 'Frankfurt am Main': 'Frankfurt', 'New York City': 'New York', + 'Santiago de Queretaro': 'Queretaro', + 'Nurnberg': 'Nuremberg', }; diff --git a/src/lib/geoip/utils.ts b/src/lib/geoip/utils.ts index 8f68ff2c..000f3770 100644 --- a/src/lib/geoip/utils.ts +++ b/src/lib/geoip/utils.ts @@ -9,4 +9,6 @@ export const normalizeCityNamePublic = (name: string): string => { export const normalizeCityName = (name: string): string => normalizeCityNamePublic(name).toLowerCase(); +export const normalizeFromPublicName = (name: string): string => name.toLowerCase(); + export const normalizeNetworkName = (name: string): string => name.toLowerCase(); diff --git a/src/lib/server.ts b/src/lib/server.ts index 59b1ea6c..4bc65a51 100644 --- a/src/lib/server.ts +++ b/src/lib/server.ts @@ -7,6 +7,7 @@ import { populateMemList as populateMemIpRangesList } from './ip-ranges.js'; import { populateMemList as populateIpWhiteList } from './geoip/whitelist.js'; import { populateCitiesList } from './geoip/city-approximation.js'; import { adoptedProbes } from './adopted-probes.js'; +import { reconnectProbes } from './ws/helper/reconnect-probes.js'; export const createServer = async (): Promise => { await initRedis(); @@ -22,8 +23,11 @@ export const createServer = async (): Promise => { await initWsServer(); + await adoptedProbes.syncDashboardData(); adoptedProbes.scheduleSync(); + reconnectProbes(); + const { getWsServer } = await import('./ws/server.js'); const { getHttpServer } = await import('./http/server.js'); diff --git a/src/lib/ws/fetch-sockets.ts b/src/lib/ws/fetch-sockets.ts new file mode 100644 index 00000000..70062859 --- /dev/null +++ b/src/lib/ws/fetch-sockets.ts @@ -0,0 +1,60 @@ +import config from 'config'; +import { throttle, LRUOptions } from './helper/throttle.js'; +import { fetchRawSockets, RemoteProbeSocket } from './server.js'; +import { adoptedProbes } from '../adopted-probes.js'; +import { getIndex } from '../../probe/builder.js'; + +const throttledFetchSockets = throttle( + async () => { + const sockets = await fetchRawSockets(); + const withAdoptedData = addAdoptedProbesData(sockets); + return withAdoptedData; + }, + config.get('ws.fetchSocketsCacheTTL'), +); + +const addAdoptedProbesData = (sockets: RemoteProbeSocket[]) => { + return sockets.map((socket) => { + const adopted = adoptedProbes.getByIp(socket.data.probe.ipAddress); + + if (!adopted) { + return socket; + } + + const isCustomCity = adopted.isCustomCity; + const hasUserTags = adopted.tags && adopted.tags.length; + + if (!isCustomCity && !hasUserTags) { + return socket; + } + + const newLocation = adoptedProbes.getUpdatedLocation(socket.data.probe); + + const newTags = adoptedProbes.getUpdatedTags(socket.data.probe); + + const result = { + ...socket, + data: { + ...socket.data, + probe: { + ...socket.data.probe, + location: newLocation, + tags: newTags, + index: getIndex(newLocation, newTags), + }, + }, + } as RemoteProbeSocket; + + // We need to copy prototype to the 'result' object, so socket methods like 'disconnect' are available + Object.setPrototypeOf(result, Object.getPrototypeOf(socket) as object); + + return result; + }); +}; + +export const fetchSockets = async (options?: LRUOptions) => { + const sockets = await throttledFetchSockets(options); + return sockets; +}; + +export type ThrottledFetchSockets = typeof fetchSockets; diff --git a/src/lib/ws/gateway.ts b/src/lib/ws/gateway.ts index ba680e30..6adbe2a2 100644 --- a/src/lib/ws/gateway.ts +++ b/src/lib/ws/gateway.ts @@ -10,6 +10,7 @@ import { getWsServer, PROBES_NAMESPACE, ServerSocket } from './server.js'; import { probeMetadata } from './middleware/probe-metadata.js'; import { errorHandler } from './helper/error-handler.js'; import { subscribeWithHandler } from './helper/subscribe-handler.js'; +import { adoptedProbes } from '../adopted-probes.js'; const io = getWsServer(); const logger = scopedLogger('gateway'); @@ -20,8 +21,10 @@ io .use(probeMetadata) .on('connect', errorHandler(async (socket: ServerSocket) => { const probe = socket.data.probe; - socket.emit('api:connect:location', probe.location); - logger.info(`ws client ${socket.id} connected from ${probe.location.city}, ${probe.location.country} [${probe.ipAddress} - ${probe.location.network}]`); + const location = adoptedProbes.getUpdatedLocation(probe); + + socket.emit('api:connect:location', location); + logger.info(`ws client ${socket.id} connected from ${location.city}, ${location.country} [${probe.ipAddress} - ${location.network}]`); // Handlers socket.on('probe:status:update', handleStatusUpdate(probe)); diff --git a/src/lib/ws/helper/probe-ip-limit.ts b/src/lib/ws/helper/probe-ip-limit.ts index a9cc27b7..6cde057d 100644 --- a/src/lib/ws/helper/probe-ip-limit.ts +++ b/src/lib/ws/helper/probe-ip-limit.ts @@ -1,4 +1,4 @@ -import { fetchSockets } from '../server.js'; +import { fetchSockets } from '../fetch-sockets.js'; import { scopedLogger } from '../../logger.js'; import { ProbeError } from '../../probe-error.js'; import type { LRUOptions } from './throttle.js'; diff --git a/src/lib/ws/helper/reconnect-probes.ts b/src/lib/ws/helper/reconnect-probes.ts index 98d5eb60..b43478e0 100644 --- a/src/lib/ws/helper/reconnect-probes.ts +++ b/src/lib/ws/helper/reconnect-probes.ts @@ -1,11 +1,21 @@ -import type { ThrottledFetchSockets } from '../server.js'; +import { scopedLogger } from '../../logger.js'; +import { fetchSockets } from '../fetch-sockets.js'; +const logger = scopedLogger('reconnect-probes'); + +const TIME_UNTIL_VM_BECOMES_HEALTHY = 8000; const TIME_TO_RECONNECT_PROBES = 2 * 60 * 1000; -export const reconnectProbes = async (fetchSockets: ThrottledFetchSockets) => { // passing fetchSockets in arguments to avoid cycle dependency +const disconnectProbes = async () => { const sockets = await fetchSockets(); for (const socket of sockets) { setTimeout(() => socket.disconnect(), Math.random() * TIME_TO_RECONNECT_PROBES); } }; + +export const reconnectProbes = () => { + setTimeout(() => { + disconnectProbes().catch(error => logger.error(error)); + }, TIME_UNTIL_VM_BECOMES_HEALTHY); +}; diff --git a/src/lib/ws/server.ts b/src/lib/ws/server.ts index 1c3dd6d1..be6acfd7 100644 --- a/src/lib/ws/server.ts +++ b/src/lib/ws/server.ts @@ -1,13 +1,9 @@ -import config from 'config'; import { type RemoteSocket, Server, Socket } from 'socket.io'; import { createAdapter } from '@socket.io/redis-adapter'; // eslint-disable-next-line n/no-missing-import import type { DefaultEventsMap } from 'socket.io/dist/typed-events.js'; import type { Probe } from '../../probe/types.js'; import { getRedisClient } from '../redis/client.js'; -import { reconnectProbes } from './helper/reconnect-probes.js'; -import { throttle, LRUOptions } from './helper/throttle.js'; -import { scopedLogger } from '../logger.js'; export type SocketData = { probe: Probe; @@ -20,11 +16,8 @@ export type ServerSocket = Socket; export const PROBES_NAMESPACE = '/probes'; -const TIME_UNTIL_VM_BECOMES_HEALTHY = 8000; -const logger = scopedLogger('ws-server'); let io: WsServer; -let throttledFetchSockets: (options?: LRUOptions) => Promise; export const initWsServer = async () => { const pubClient = getRedisClient().duplicate(); @@ -40,15 +33,6 @@ export const initWsServer = async () => { }); io.adapter(createAdapter(pubClient, subClient)); - - throttledFetchSockets = throttle( - io.of(PROBES_NAMESPACE).fetchSockets.bind(io.of(PROBES_NAMESPACE)), - config.get('ws.fetchSocketsCacheTTL'), - ); - - setTimeout(() => { - reconnectProbes(fetchSockets).catch(error => logger.error(error)); - }, TIME_UNTIL_VM_BECOMES_HEALTHY); }; export const getWsServer = (): WsServer => { @@ -59,14 +43,12 @@ export const getWsServer = (): WsServer => { return io; }; -export const fetchSockets = async (options?: LRUOptions) => { - if (!io || !throttledFetchSockets) { +export const fetchRawSockets = async () => { + if (!io) { throw new Error('WS server not initialized yet'); } - const sockets = await throttledFetchSockets(options); + const sockets = await io.of(PROBES_NAMESPACE).fetchSockets(); return sockets; }; - -export type ThrottledFetchSockets = typeof fetchSockets; diff --git a/src/probe/builder.ts b/src/probe/builder.ts index 5783071d..5a34f54c 100644 --- a/src/probe/builder.ts +++ b/src/probe/builder.ts @@ -13,15 +13,20 @@ import { } from '../lib/location/location.js'; import { ProbeError } from '../lib/probe-error.js'; import { createGeoipClient } from '../lib/geoip/client.js'; +import type GeoipClient from '../lib/geoip/client.js'; import getProbeIp from '../lib/get-probe-ip.js'; import { getRegion } from '../lib/ip-ranges.js'; import type { Probe, ProbeLocation, Tag } from './types.js'; import { verifyIpLimit } from '../lib/ws/helper/probe-ip-limit.js'; import { fakeLookup } from '../lib/geoip/fake-client.js'; -const geoipClient = createGeoipClient(); +let geoipClient: GeoipClient; export const buildProbe = async (socket: Socket): Promise => { + if (!geoipClient) { + geoipClient = createGeoipClient(); + } + const version = String(socket.handshake.query['version']); const nodeVersion = String(socket.handshake.query['nodeVersion']); @@ -58,24 +63,7 @@ export const buildProbe = async (socket: Socket): Promise => { const tags = getTags(ip, ipInfo); - // Storing index as string[][] so every category will have it's exact position in the index array across all probes - const index = [ - [ location.country ], - [ getCountryIso3ByIso2(location.country) ], - [ getCountryByIso(location.country) ], - getCountryAliases(location.country), - [ location.normalizedCity ], - location.state ? [ location.state ] : [], - location.state ? [ getStateNameByIso(location.state) ] : [], - [ location.continent ], - getContinentAliases(location.continent), - [ location.region ], - getRegionAliases(location.region), - [ `as${location.asn}` ], - tags.filter(tag => tag.type === 'system').map(tag => tag.value), - [ location.normalizedNetwork ], - getNetworkAliases(location.normalizedNetwork), - ].map(category => category.map(s => s.toLowerCase().replaceAll('-', ' '))); + const index = getIndex(location, tags); // Todo: add validation and handle missing or partial data return { @@ -100,6 +88,29 @@ export const buildProbe = async (socket: Socket): Promise => { }; }; +export const getIndex = (location: ProbeLocation, tags: Tag[]) => { + // Storing index as string[][] so every category will have it's exact position in the index array across all probes + const index = [ + [ location.country ], + [ getCountryIso3ByIso2(location.country) ], + [ getCountryByIso(location.country) ], + getCountryAliases(location.country), + [ location.normalizedCity ], + location.state ? [ location.state ] : [], + location.state ? [ getStateNameByIso(location.state) ] : [], + [ location.continent ], + getContinentAliases(location.continent), + [ location.region ], + getRegionAliases(location.region), + [ `as${location.asn}` ], + tags.filter(tag => tag.type === 'system').map(tag => tag.value), + [ location.normalizedNetwork ], + getNetworkAliases(location.normalizedNetwork), + ].map(category => category.map(s => s.toLowerCase().replaceAll('-', ' '))); + + return index; +}; + const getLocation = (ipInfo: ProbeLocation): ProbeLocation => ({ continent: ipInfo.continent, region: ipInfo.region, diff --git a/src/probe/route/get-probes.ts b/src/probe/route/get-probes.ts index 364385e0..3e006bc5 100644 --- a/src/probe/route/get-probes.ts +++ b/src/probe/route/get-probes.ts @@ -1,6 +1,7 @@ import type { DefaultContext, DefaultState, ParameterizedContext } from 'koa'; import type Router from '@koa/router'; -import { fetchSockets, type RemoteProbeSocket } from '../../lib/ws/server.js'; +import type { RemoteProbeSocket } from '../../lib/ws/server.js'; +import { fetchSockets } from '../../lib/ws/fetch-sockets.js'; const handle = async (ctx: ParameterizedContext): Promise => { const { isAdmin } = ctx; diff --git a/src/probe/router.ts b/src/probe/router.ts index 997af9b1..f4c0b7c2 100644 --- a/src/probe/router.ts +++ b/src/probe/router.ts @@ -1,6 +1,6 @@ import _ from 'lodash'; import type { RemoteProbeSocket } from '../lib/ws/server.js'; -import { fetchSockets } from '../lib/ws/server.js'; +import { fetchSockets } from '../lib/ws/fetch-sockets.js'; import type { LocationWithLimit } from '../measurement/types.js'; import type { Location } from '../lib/location/types.js'; import type { Probe } from './types.js'; diff --git a/src/probe/sockets-location-filter.ts b/src/probe/sockets-location-filter.ts index df9bb90f..9035bb83 100644 --- a/src/probe/sockets-location-filter.ts +++ b/src/probe/sockets-location-filter.ts @@ -51,7 +51,7 @@ export class SocketsLocationFilter { } static hasTag (socket: RemoteProbeSocket, tag: string) { - return socket.data.probe.tags.some(({ type, value }) => type === 'system' && value === tag); + return socket.data.probe.tags.some(({ value }) => value === tag); } public filterGloballyDistibuted (sockets: RemoteProbeSocket[], limit: number): RemoteProbeSocket[] { diff --git a/src/probe/types.ts b/src/probe/types.ts index a8579f34..22205a40 100644 --- a/src/probe/types.ts +++ b/src/probe/types.ts @@ -7,7 +7,7 @@ export type ProbeLocation = { asn: number; latitude: number; longitude: number; - state: string | undefined; + state?: string | undefined; network: string; normalizedNetwork: string; isHosting?: boolean | undefined; diff --git a/test/mocks/nock-geoip.json b/test/mocks/nock-geoip.json index 23b7ef38..7e6600c3 100644 --- a/test/mocks/nock-geoip.json +++ b/test/mocks/nock-geoip.json @@ -23,7 +23,7 @@ "newYork": { "country_code": "US", "region_name": "New York", - "city_name": "New York", + "city_name": "New York City", "latitude": 40.7143, "longitude": -74.0060, "asn": "80001", diff --git a/test/setup.ts b/test/setup.ts index 365bb0f3..c196d38a 100644 --- a/test/setup.ts +++ b/test/setup.ts @@ -13,12 +13,15 @@ import { import chaiOas from './plugins/oas/index.js'; import { getRedisClient, initRedis } from '../src/lib/redis/client.js'; +import { client } from '../src/lib/sql/client.js'; +import { USERS_TABLE } from '../src/lib/adopted-probes.js'; before(async () => { chai.use(await chaiOas({ specPath: path.join(fileURLToPath(new URL('.', import.meta.url)), '../public/v1/spec.yaml') })); await initRedis(); const redis = getRedisClient(); await redis.flushDb(); + await client(USERS_TABLE).insert({ id: '89da69bd-a236-4ab7-9c5d-b5f52ce09959', github: 'jimaek' }).onConflict().ignore(); nock.disableNetConnect(); nock.enableNetConnect('127.0.0.1'); diff --git a/test/tests/integration/adoption-code/adoption-code.test.ts b/test/tests/integration/adoption-code/adoption-code.test.ts index 822981c7..902b655c 100644 --- a/test/tests/integration/adoption-code/adoption-code.test.ts +++ b/test/tests/integration/adoption-code/adoption-code.test.ts @@ -37,7 +37,16 @@ describe('Adoption code', () => { }) .expect(200).expect((response) => { expect(response.body).to.deep.equal({ - result: 'Code was sent to the probe.', + uuid: '1-1-1-1-1', + version: '0.14.0', + status: 'initializing', + city: 'Dallas', + state: 'TX', + country: 'US', + latitude: 32.7831, + longitude: -96.8067, + asn: 20004, + network: 'The Constant Company LLC', }); }); diff --git a/test/tests/integration/measurement/create-measurement.test.ts b/test/tests/integration/measurement/create-measurement.test.ts index 7bcb1ff2..c1db2fab 100644 --- a/test/tests/integration/measurement/create-measurement.test.ts +++ b/test/tests/integration/measurement/create-measurement.test.ts @@ -5,16 +5,21 @@ import * as td from 'testdouble'; import nock from 'nock'; import type { Socket } from 'socket.io-client'; import nockGeoIpProviders from '../../../utils/nock-geo-ip.js'; +import { client } from '../../../../src/lib/sql/client.js'; +import type { AdoptedProbes } from '../../../../src/lib/adopted-probes.js'; describe('Create measurement', () => { let addFakeProbe: () => Promise; let deleteFakeProbe: (socket: Socket) => Promise; let getTestServer; let requestAgent: SuperTest; + let adoptedProbes: AdoptedProbes; + let ADOPTED_PROBES_TABLE: string; before(async () => { await td.replaceEsm('../../../../src/lib/ip-ranges.ts', { getRegion: () => 'gcp-us-west4', populateMemList: () => Promise.resolve() }); ({ getTestServer, addFakeProbe, deleteFakeProbe } = await import('../../../utils/server.js')); + ({ adoptedProbes, ADOPTED_PROBES_TABLE } = await import('../../../../src/lib/adopted-probes.js')); const app = await getTestServer(); requestAgent = request(app); }); @@ -30,9 +35,6 @@ describe('Create measurement', () => { type: 'ping', target: 'example.com', locations: [{ country: 'US' }], - measurementOptions: { - packets: 4, - }, limit: 2, }) .expect(422) @@ -58,6 +60,11 @@ describe('Create measurement', () => { before(async () => { nockGeoIpProviders(); probe = await addFakeProbe(); + probe.emit('probe:status:update', 'ready'); + }); + + afterEach(() => { + probe.emit('probe:status:update', 'ready'); }); after(async () => { @@ -66,43 +73,13 @@ describe('Create measurement', () => { }); it('should respond with error if there are no ready probes', async () => { - await requestAgent.post('/v1/measurements') - .send({ - type: 'ping', - target: 'example.com', - locations: [{ country: 'US' }], - measurementOptions: { - packets: 4, - }, - limit: 2, - }) - .expect(422) - .expect((response) => { - expect(response.body).to.deep.equal({ - error: { - message: 'No suitable probes found.', - type: 'no_probes_found', - }, - links: { - documentation: 'https://www.jsdelivr.com/docs/api.globalping.io#post-/v1/measurements', - }, - }); - - expect(response).to.matchApiSchema(); - }); - }); - - it('should respond with error if probe emitted non-"ready" "probe:status:update"', async () => { - probe.emit('probe:status:update', 'sigterm'); + probe.emit('probe:status:update', 'initializing'); await requestAgent.post('/v1/measurements') .send({ type: 'ping', target: 'example.com', locations: [{ country: 'US' }], - measurementOptions: { - packets: 4, - }, limit: 2, }) .expect(422) @@ -121,8 +98,7 @@ describe('Create measurement', () => { }); }); - it('should respond with error if probe emitted non-"ready" "probe:status:update" after being "ready"', async () => { - probe.emit('probe:status:update', 'ready'); + it('should respond with error if probe emitted non-"ready" "probe:status:update"', async () => { probe.emit('probe:status:update', 'sigterm'); await requestAgent.post('/v1/measurements') @@ -130,9 +106,6 @@ describe('Create measurement', () => { type: 'ping', target: 'example.com', locations: [{ country: 'US' }], - measurementOptions: { - packets: 4, - }, limit: 2, }) .expect(422) @@ -152,16 +125,11 @@ describe('Create measurement', () => { }); it('should create measurement with global limit', async () => { - probe.emit('probe:status:update', 'ready'); - await requestAgent.post('/v1/measurements') .send({ type: 'ping', target: 'example.com', locations: [{ country: 'US' }], - measurementOptions: { - packets: 4, - }, limit: 2, }) .expect(202) @@ -217,9 +185,6 @@ describe('Create measurement', () => { type: 'ping', target: 'example.com', locations: [{ country: 'US', limit: 2 }], - measurementOptions: { - packets: 4, - }, }) .expect(202) .expect((response) => { @@ -303,15 +268,10 @@ describe('Create measurement', () => { }); it('should create measurement for globally distributed probes', async () => { - probe.emit('probe:status:update', 'ready'); - await requestAgent.post('/v1/measurements') .send({ type: 'ping', target: 'example.com', - measurementOptions: { - packets: 4, - }, limit: 2, }) .expect(202) @@ -324,16 +284,11 @@ describe('Create measurement', () => { }); it('should create measurement with "magic: world" location', async () => { - probe.emit('probe:status:update', 'ready'); - await requestAgent.post('/v1/measurements') .send({ type: 'ping', target: 'example.com', locations: [{ magic: 'world', limit: 2 }], - measurementOptions: { - packets: 4, - }, }) .expect(202) .expect((response) => { @@ -345,16 +300,11 @@ describe('Create measurement', () => { }); it('should create measurement with "magic" value in any case', async () => { - probe.emit('probe:status:update', 'ready'); - await requestAgent.post('/v1/measurements') .send({ type: 'ping', target: 'example.com', locations: [{ magic: 'Na' }], - measurementOptions: { - packets: 4, - }, }) .expect(202) .expect((response) => { @@ -366,16 +316,11 @@ describe('Create measurement', () => { }); it('should create measurement with partial tag value "magic: GCP-us-West4" location', async () => { - probe.emit('probe:status:update', 'ready'); - await requestAgent.post('/v1/measurements') .send({ type: 'ping', target: 'example.com', locations: [{ magic: 'GCP-us-West4', limit: 2 }], - measurementOptions: { - packets: 4, - }, }) .expect(202) .expect((response) => { @@ -387,16 +332,11 @@ describe('Create measurement', () => { }); it('should not create measurement with "magic: non-existing-tag" location', async () => { - probe.emit('probe:status:update', 'ready'); - await requestAgent.post('/v1/measurements') .send({ type: 'ping', target: 'example.com', locations: [{ magic: 'non-existing-tag', limit: 2 }], - measurementOptions: { - packets: 4, - }, }) .expect(422) .expect((response) => { @@ -415,16 +355,11 @@ describe('Create measurement', () => { }); it('should create measurement with "tags: ["tag-value"]" location', async () => { - probe.emit('probe:status:update', 'ready'); - await requestAgent.post('/v1/measurements') .send({ type: 'ping', target: 'example.com', locations: [{ tags: [ 'gcp-us-west4' ], limit: 2 }], - measurementOptions: { - packets: 4, - }, }) .expect(202) .expect((response) => { @@ -434,5 +369,77 @@ describe('Create measurement', () => { expect(response).to.matchApiSchema(); }); }); + + describe('adopted probes', () => { + before(async () => { + await client(ADOPTED_PROBES_TABLE).insert({ + userId: '89da69bd-a236-4ab7-9c5d-b5f52ce09959', + lastSyncDate: new Date(), + ip: '1.2.3.4', + uuid: '1-1-1-1-1', + isCustomCity: 1, + tags: '["dashboard-tag"]', + status: 'ready', + version: '0.26.0', + country: 'US', + countryOfCustomCity: 'US', + city: 'Oklahoma City', + latitude: '35.46756', + longitude: '-97.51643', + network: 'InterBS S.R.L. (BAEHOST)', + asn: 61004, + }); + + await adoptedProbes.syncDashboardData(); + }); + + after(async () => { + await client(ADOPTED_PROBES_TABLE).where({ city: 'Oklahoma City' }).delete(); + }); + + it('should create measurement with adopted "city: Oklahoma City" location', async () => { + await requestAgent.post('/v1/measurements') + .send({ + type: 'ping', + target: 'example.com', + locations: [{ city: 'Oklahoma City', limit: 2 }], + }) + .expect(202) + .expect((response) => { + expect(response.body.id).to.exist; + expect(response.header.location).to.exist; + expect(response.body.probesCount).to.equal(1); + expect(response).to.matchApiSchema(); + }); + }); + + it('should create measurement with adopted "tags: ["u-jimaek-dashboard-tag"]" location', async () => { + await requestAgent.post('/v1/measurements') + .send({ + type: 'ping', + target: 'example.com', + locations: [{ tags: [ 'u-jimaek-dashboard-tag' ], limit: 2 }], + }) + .expect(202) + .expect((response) => { + expect(response.body.id).to.exist; + expect(response.header.location).to.exist; + expect(response.body.probesCount).to.equal(1); + expect(response).to.matchApiSchema(); + }); + }); + + it('should not use create measurement with adopted tag in magic field "magic: ["u-jimaek-dashboard-tag"]" location', async () => { + await requestAgent.post('/v1/measurements') + .send({ + type: 'ping', + target: 'example.com', + locations: [{ magic: 'u-jimaek-dashboard-tag', limit: 2 }], + }) + .expect(422).expect((response) => { + expect(response.body.error.message).to.equal('No suitable probes found.'); + }); + }); + }); }); }); diff --git a/test/tests/integration/middleware/compress.test.ts b/test/tests/integration/middleware/compress.test.ts index 3f8d1d45..47a8a833 100644 --- a/test/tests/integration/middleware/compress.test.ts +++ b/test/tests/integration/middleware/compress.test.ts @@ -3,7 +3,7 @@ import { expect } from 'chai'; import nock from 'nock'; import type { Socket } from 'socket.io-client'; import { getTestServer, addFakeProbe, deleteFakeProbe } from '../../../utils/server.js'; -import { geoIpMocks } from '../../../utils/nock-geo-ip.js'; +import geoIpMocks from '../../../mocks/nock-geoip.json' assert { type: 'json' }; describe('compression', () => { let requestAgent: any; @@ -21,11 +21,11 @@ describe('compression', () => { }); it('should include compression headers', async () => { - nock('https://ipmap-api.ripe.net/v1/locate/').get(/.*/).times(10).reply(200, geoIpMocks['ipmap'].default); - nock('https://api.ip2location.io').get(/.*/).times(10).reply(200, geoIpMocks['ip2location'].default); - nock('https://globalping-geoip.global.ssl.fastly.net').get(/.*/).times(10).reply(200, geoIpMocks['fastly'].default); - nock('https://ipinfo.io').get(/.*/).times(10).reply(200, geoIpMocks['ipinfo'].default); - nock('https://geoip.maxmind.com/geoip/v2.1/city/').get(/.*/).times(10).reply(200, geoIpMocks['maxmind'].default); + nock('https://ipmap-api.ripe.net/v1/locate/').get(/.*/).times(10).reply(200, geoIpMocks.ipmap.default); + nock('https://api.ip2location.io').get(/.*/).times(10).reply(200, geoIpMocks.ip2location.default); + nock('https://globalping-geoip.global.ssl.fastly.net').get(/.*/).times(10).reply(200, geoIpMocks.fastly.default); + nock('https://ipinfo.io').get(/.*/).times(10).reply(200, geoIpMocks.ipinfo.default); + nock('https://geoip.maxmind.com/geoip/v2.1/city/').get(/.*/).times(10).reply(200, geoIpMocks.maxmind.default); probes = await Promise.all(Array.from({ length: 10 }).map(() => addFakeProbe())); for (const probe of probes) { diff --git a/test/tests/integration/probes/get-probes.test.ts b/test/tests/integration/probes/get-probes.test.ts index 8512b50d..1fafaa25 100644 --- a/test/tests/integration/probes/get-probes.test.ts +++ b/test/tests/integration/probes/get-probes.test.ts @@ -5,6 +5,8 @@ import request, { type SuperTest, type Test } from 'supertest'; import type { Socket } from 'socket.io-client'; import { getTestServer, addFakeProbe, deleteFakeProbe } from '../../../utils/server.js'; import nockGeoIpProviders from '../../../utils/nock-geo-ip.js'; +import { adoptedProbes, ADOPTED_PROBES_TABLE } from '../../../../src/lib/adopted-probes.js'; +import { client } from '../../../../src/lib/sql/client.js'; describe('Get Probes', () => { let requestAgent: SuperTest; @@ -54,87 +56,6 @@ describe('Get Probes', () => { }); }); - it('should detect 1 probe in "ready: true" status', async () => { - nockGeoIpProviders({ maxmind: 'argentina', ipinfo: 'argentina', fastly: 'argentina' }); - - const probe = await addProbe(); - probe.emit('probe:status:update', 'ready'); - - await requestAgent.get('/v1/probes') - .send() - .expect(200) - .expect((response) => { - expect(response.body).to.deep.equal([{ - version: '0.14.0', - location: { - continent: 'SA', - region: 'South America', - country: 'AR', - city: 'Buenos Aires', - asn: 61004, - latitude: -34.6131, - longitude: -58.3772, - network: 'InterBS S.R.L. (BAEHOST)', - }, - tags: [], - resolvers: [], - }]); - - expect(response).to.matchApiSchema(); - }); - }); - - it('should detect 2 probes in "ready: true" status', async () => { - nockGeoIpProviders({ ip2location: 'argentina', ipmap: 'argentina', maxmind: 'argentina', ipinfo: 'argentina', fastly: 'argentina' }); - nockGeoIpProviders(); - - const probe1 = await addProbe(); - const probe2 = await addProbe(); - probe1.emit('probe:status:update', 'ready'); - probe2.emit('probe:status:update', 'ready'); - - await requestAgent.get('/v1/probes') - .send() - .expect(200) - .expect((response) => { - expect(response.body).to.deep.equal([ - { - version: '0.14.0', - location: { - continent: 'SA', - region: 'South America', - country: 'AR', - city: 'Buenos Aires', - asn: 61004, - latitude: -34.6131, - longitude: -58.3772, - network: 'InterBS S.R.L. (BAEHOST)', - }, - tags: [], - resolvers: [], - }, - { - version: '0.14.0', - location: { - continent: 'NA', - region: 'Northern America', - country: 'US', - state: 'TX', - city: 'Dallas', - asn: 20004, - latitude: 32.7831, - longitude: -96.8067, - network: 'The Constant Company LLC', - }, - tags: [ 'datacenter-network' ], - resolvers: [], - }, - ]); - - expect(response).to.matchApiSchema(); - }); - }); - it('should detect 4 probes in "ready: true" status', async () => { nockGeoIpProviders({ ip2location: 'argentina', ipmap: 'argentina', maxmind: 'argentina', ipinfo: 'argentina', fastly: 'argentina' }); nockGeoIpProviders(); @@ -269,6 +190,8 @@ describe('Get Probes', () => { expect(response.body[0]).to.deep.include({ version: '0.14.0', host: '', + ipAddress: '1.2.3.4', + uuid: '1-1-1-1-1', location: { continent: 'SA', region: 'South America', @@ -289,5 +212,62 @@ describe('Get Probes', () => { expect(response).to.matchApiSchema(); }); }); + + describe('adopted probes', () => { + before(async () => { + await client(ADOPTED_PROBES_TABLE).insert({ + userId: '89da69bd-a236-4ab7-9c5d-b5f52ce09959', + lastSyncDate: new Date(), + ip: '1.2.3.4', + uuid: '1-1-1-1-1', + isCustomCity: 1, + tags: '["dashboardtag1"]', + status: 'ready', + version: '0.26.0', + country: 'AR', + countryOfCustomCity: 'AR', + city: 'Cordoba', + latitude: '-31.4135', + longitude: '-64.18105', + network: 'InterBS S.R.L. (BAEHOST)', + asn: 61004, + }); + + await adoptedProbes.syncDashboardData(); + }); + + after(async () => { + await client(ADOPTED_PROBES_TABLE).where({ city: 'Cordoba' }).delete(); + }); + + it('should update probes data', async () => { + nockGeoIpProviders({ ip2location: 'argentina', ipmap: 'argentina', maxmind: 'argentina', ipinfo: 'argentina', fastly: 'argentina' }); + const probe = await addProbe(); + probe.emit('probe:status:update', 'ready'); + + await requestAgent.get('/v1/probes') + .send() + .expect(200) + .expect((response) => { + expect(response.body[0]).to.deep.equal({ + version: '0.14.0', + location: { + continent: 'SA', + region: 'South America', + country: 'AR', + city: 'Cordoba', + latitude: -31.4135, + longitude: -64.18105, + asn: 61004, + network: 'InterBS S.R.L. (BAEHOST)', + }, + tags: [ 'u-jimaek-dashboardtag1' ], + resolvers: [], + }); + + expect(response).to.matchApiSchema(); + }); + }); + }); }); }); diff --git a/test/tests/unit/adopted-probes.test.ts b/test/tests/unit/adopted-probes.test.ts index babdefb6..146284e8 100644 --- a/test/tests/unit/adopted-probes.test.ts +++ b/test/tests/unit/adopted-probes.test.ts @@ -2,18 +2,75 @@ import { expect } from 'chai'; import type { Knex } from 'knex'; import * as sinon from 'sinon'; import { AdoptedProbes } from '../../../src/lib/adopted-probes.js'; +import type { Probe } from '../../../src/probe/types.js'; + +const defaultAdoptedProbe = { + userId: '3cff97ae-4a0a-4f34-9f1a-155e6def0a45', + username: 'jimaek', + ip: '1.1.1.1', + uuid: '1-1-1-1-1', + lastSyncDate: new Date('1970-01-01'), + tags: '["dashboardtag"]', + isCustomCity: 0, + status: 'ready', + version: '0.26.0', + country: 'IE', + countryOfCustomCity: '', + city: 'Dublin', + latitude: 53.3331, + longitude: -6.2489, + asn: 16509, + network: 'Amazon.com, Inc.', +}; + +const defaultConnectedProbe: Probe = { + ipAddress: '1.1.1.1', + uuid: '1-1-1-1-1', + status: 'ready', + version: '0.26.0', + nodeVersion: 'v18.17.0', + location: { + continent: 'EU', + region: 'Northern Europe', + country: 'IE', + city: 'Dublin', + normalizedCity: 'dublin', + asn: 16509, + latitude: 53.3331, + longitude: -6.2489, + network: 'Amazon.com, Inc.', + normalizedNetwork: 'amazon.com, inc.', + }, + tags: [], + index: [], + client: '', + host: '', + resolvers: [], + stats: { + cpu: { + count: 0, + load: [], + }, + jobs: { count: 0 }, + }, +}; const selectStub = sinon.stub(); const updateStub = sinon.stub(); const deleteStub = sinon.stub(); +const rawStub = sinon.stub(); const whereStub = sinon.stub().returns({ update: updateStub, delete: deleteStub, }); -const sqlStub = sinon.stub().returns({ +const joinStub = sinon.stub().returns({ select: selectStub, - where: whereStub, }); +const sqlStub = sinon.stub().returns({ + join: joinStub, + where: whereStub, +}) as sinon.SinonStub & {raw: any}; +sqlStub.raw = rawStub; const fetchSocketsStub = sinon.stub().resolves([]); let sandbox: sinon.SinonSandbox; @@ -21,6 +78,8 @@ describe('AdoptedProbes', () => { beforeEach(() => { sandbox = sinon.createSandbox({ useFakeTimers: true }); sinon.resetHistory(); + selectStub.resolves([ defaultAdoptedProbe ]); + fetchSocketsStub.resolves([{ data: { probe: defaultConnectedProbe } }]); }); afterEach(() => { @@ -29,7 +88,6 @@ describe('AdoptedProbes', () => { it('syncDashboardData method should sync the data', async () => { const adoptedProbes = new AdoptedProbes(sqlStub as unknown as Knex, fetchSocketsStub); - selectStub.resolves([{ ip: '1.1.1.1', uuid: '1-1-1-1-1', lastSyncDate: '1970-01-01' }]); expect(sqlStub.callCount).to.equal(0); expect(selectStub.callCount).to.equal(0); @@ -37,16 +95,37 @@ describe('AdoptedProbes', () => { await adoptedProbes.syncDashboardData(); expect(sqlStub.callCount).to.equal(1); - expect(sqlStub.args[0]).deep.equal([ 'adopted_probes' ]); + expect(sqlStub.args[0]).deep.equal([{ probes: 'adopted_probes' }]); + expect(joinStub.callCount).to.equal(1); + expect(joinStub.args[0]).deep.equal([{ users: 'directus_users' }, 'probes.userId', '=', 'users.id' ]); expect(selectStub.callCount).to.equal(1); - expect(selectStub.args[0]).deep.equal([ 'ip', 'uuid', 'lastSyncDate', 'status', 'version', 'country', 'city', 'latitude', 'longitude', 'asn', 'network' ]); + expect(selectStub.args[0]).deep.equal([ + { + userId: 'probes.userId', + username: 'users.github', + ip: 'probes.ip', + uuid: 'probes.uuid', + lastSyncDate: 'probes.lastSyncDate', + isCustomCity: 'probes.isCustomCity', + tags: 'probes.tags', + status: 'probes.status', + version: 'probes.version', + asn: 'probes.asn', + network: 'probes.network', + country: 'probes.country', + countryOfCustomCity: 'probes.countryOfCustomCity', + city: 'probes.city', + state: 'probes.state', + latitude: 'probes.latitude', + longitude: 'probes.longitude', + }, + ]); }); it('class should update uuid if it is wrong', async () => { const adoptedProbes = new AdoptedProbes(sqlStub as unknown as Knex, fetchSocketsStub); - selectStub.resolves([{ ip: '1.1.1.1', uuid: '1-1-1-1-1', lastSyncDate: '1970-01-01' }]); - fetchSocketsStub.resolves([{ data: { probe: { ipAddress: '1.1.1.1', uuid: '2-2-2-2-2' } } }]); + fetchSocketsStub.resolves([{ data: { probe: { ...defaultConnectedProbe, uuid: '2-2-2-2-2' } } }]); await adoptedProbes.syncDashboardData(); @@ -58,8 +137,7 @@ describe('AdoptedProbes', () => { it('class should update ip if it is wrong', async () => { const adoptedProbes = new AdoptedProbes(sqlStub as unknown as Knex, fetchSocketsStub); - selectStub.resolves([{ ip: '1.1.1.1', uuid: '1-1-1-1-1', lastSyncDate: '1970-01-01' }]); - fetchSocketsStub.resolves([{ data: { probe: { ipAddress: '2.2.2.2', uuid: '1-1-1-1-1' } } }]); + fetchSocketsStub.resolves([{ data: { probe: { ...defaultConnectedProbe, ipAddress: '2.2.2.2' } } }]); await adoptedProbes.syncDashboardData(); @@ -71,7 +149,7 @@ describe('AdoptedProbes', () => { it('class should do nothing if adopted probe was not found and lastSyncDate < 30 days away', async () => { const adoptedProbes = new AdoptedProbes(sqlStub as unknown as Knex, fetchSocketsStub); - selectStub.resolves([{ ip: '1.1.1.1', uuid: '1-1-1-1-1', lastSyncDate: '1969-12-15' }]); + selectStub.resolves([{ ...defaultAdoptedProbe, lastSyncDate: new Date('1969-12-15') }]); fetchSocketsStub.resolves([]); await adoptedProbes.syncDashboardData(); @@ -83,7 +161,7 @@ describe('AdoptedProbes', () => { it('class should delete adoption if adopted probe was not found and lastSyncDate > 30 days away', async () => { const adoptedProbes = new AdoptedProbes(sqlStub as unknown as Knex, fetchSocketsStub); - selectStub.resolves([{ ip: '1.1.1.1', uuid: '1-1-1-1-1', lastSyncDate: '1969-11-15' }]); + selectStub.resolves([{ ...defaultAdoptedProbe, lastSyncDate: new Date('1969-11-15') }]); fetchSocketsStub.resolves([]); await adoptedProbes.syncDashboardData(); @@ -94,21 +172,9 @@ describe('AdoptedProbes', () => { expect(deleteStub.callCount).to.equal(1); }); - it('class should do nothing if probe data is actual', async () => { + it('class should update lastSyncDate if probe is connected and lastSyncDate < 30 days away', async () => { const adoptedProbes = new AdoptedProbes(sqlStub as unknown as Knex, fetchSocketsStub); - selectStub.resolves([{ ip: '1.1.1.1', uuid: '1-1-1-1-1', lastSyncDate: '1970-01-01' }]); - fetchSocketsStub.resolves([{ data: { probe: { ipAddress: '1.1.1.1', uuid: '1-1-1-1-1' } } }]); - - await adoptedProbes.syncDashboardData(); - - expect(whereStub.callCount).to.equal(0); - expect(updateStub.callCount).to.equal(0); - }); - - it('class update lastSyncDate if probe is connected and lastSyncDate < 30 days away', async () => { - const adoptedProbes = new AdoptedProbes(sqlStub as unknown as Knex, fetchSocketsStub); - selectStub.resolves([{ ip: '1.1.1.1', uuid: '1-1-1-1-1', lastSyncDate: '1969-12-31' }]); - fetchSocketsStub.resolves([{ data: { probe: { ipAddress: '1.1.1.1', uuid: '1-1-1-1-1' } } }]); + selectStub.resolves([{ ...defaultAdoptedProbe, lastSyncDate: new Date('1969-12-31') }]); await adoptedProbes.syncDashboardData(); @@ -119,10 +185,9 @@ describe('AdoptedProbes', () => { expect(deleteStub.callCount).to.equal(0); }); - it('class update lastSyncDate if probe is connected and lastSyncDate > 30 days away', async () => { + it('class should update lastSyncDate if probe is connected and lastSyncDate > 30 days away', async () => { const adoptedProbes = new AdoptedProbes(sqlStub as unknown as Knex, fetchSocketsStub); - selectStub.resolves([{ ip: '1.1.1.1', uuid: '1-1-1-1-1', lastSyncDate: '1969-11-15' }]); - fetchSocketsStub.resolves([{ data: { probe: { ipAddress: '1.1.1.1', uuid: '1-1-1-1-1' } } }]); + selectStub.resolves([{ ...defaultAdoptedProbe, lastSyncDate: new Date('1969-11-15') }]); await adoptedProbes.syncDashboardData(); @@ -133,10 +198,8 @@ describe('AdoptedProbes', () => { expect(deleteStub.callCount).to.equal(0); }); - it('class update lastSyncDate should not update anything if lastSyncDate is today', async () => { + it('class should update lastSyncDate should not update anything if lastSyncDate is today', async () => { const adoptedProbes = new AdoptedProbes(sqlStub as unknown as Knex, fetchSocketsStub); - selectStub.resolves([{ ip: '1.1.1.1', uuid: '1-1-1-1-1', lastSyncDate: '1970-01-01' }]); - fetchSocketsStub.resolves([{ data: { probe: { ipAddress: '1.1.1.1', uuid: '1-1-1-1-1' } } }]); await adoptedProbes.syncDashboardData(); @@ -145,21 +208,8 @@ describe('AdoptedProbes', () => { expect(deleteStub.callCount).to.equal(0); }); - it('class should update probe meta info if it is outdated', async () => { + it('class should update probe meta info if it is outdated and "isCustomCity: false"', async () => { const adoptedProbes = new AdoptedProbes(sqlStub as unknown as Knex, fetchSocketsStub); - selectStub.resolves([{ - ip: '1.1.1.1', - uuid: '1-1-1-1-1', - lastSyncDate: '1970-01-01', - status: 'ready', - version: '0.26.0', - country: 'IE', - city: 'Dublin', - latitude: 53.3331, - longitude: -6.2489, - asn: 16509, - network: 'Amazon.com, Inc.', - }]); fetchSocketsStub.resolves([{ data: { @@ -175,8 +225,8 @@ describe('AdoptedProbes', () => { country: 'GB', city: 'London', asn: 20473, - latitude: 53.3331, - longitude: -6.2489, + latitude: 51.50853, + longitude: -0.12574, network: 'The Constant Company, LLC', }, }, @@ -192,46 +242,87 @@ describe('AdoptedProbes', () => { expect(updateStub.args[0]).to.deep.equal([{ status: 'initializing', version: '0.27.0', - country: 'GB', - city: 'London', asn: 20473, network: 'The Constant Company, LLC', + country: 'GB', + city: 'London', + latitude: 51.50853, + longitude: -0.12574, }]); }); - it('class should update probe meta info if it is outdated', async () => { + it('class should update country and send notification if country of the probe changes', async () => { const adoptedProbes = new AdoptedProbes(sqlStub as unknown as Knex, fetchSocketsStub); - selectStub.resolves([{ - ip: '1.1.1.1', - uuid: '1-1-1-1-1', - lastSyncDate: '1970-01-01', - status: 'ready', - version: '0.26.0', - country: 'IE', - city: 'Dublin', - latitude: 53.3331, - longitude: -6.2489, - asn: 16509, - network: 'Amazon.com, Inc.', + selectStub.resolves([{ ...defaultAdoptedProbe, countryOfCustomCity: 'IE', isCustomCity: true }]); + + fetchSocketsStub.resolves([{ + data: { + probe: { + ipAddress: '1.1.1.1', + uuid: '1-1-1-1-1', + status: 'initializing', + version: '0.27.0', + nodeVersion: 'v18.17.0', + location: { + continent: 'EU', + region: 'Northern Europe', + country: 'GB', + city: 'London', + asn: 20473, + latitude: 51.50853, + longitude: -0.12574, + network: 'The Constant Company, LLC', + }, + }, + }, }]); + await adoptedProbes.syncDashboardData(); + + expect(rawStub.callCount).to.equal(1); + + expect(rawStub.args[0]![1]).to.deep.equal({ + recipient: '3cff97ae-4a0a-4f34-9f1a-155e6def0a45', + subject: 'Adopted probe country change', + message: 'Globalping API detected that your adopted probe with ip: 1.1.1.1 is located at "GB". So its country value changed from "IE" to "GB", and custom city value "Dublin" is not applied right now.\n\nIf this change is not right please report in [that issue](https://github.com/jsdelivr/globalping/issues/268).', + }); + + expect(whereStub.callCount).to.equal(1); + expect(whereStub.args[0]).to.deep.equal([{ ip: '1.1.1.1' }]); + expect(updateStub.callCount).to.equal(1); + + expect(updateStub.args[0]).to.deep.equal([ + { + status: 'initializing', + version: '0.27.0', + asn: 20473, + network: 'The Constant Company, LLC', + country: 'GB', + }, + ]); + }); + + it('class should partially update probe meta info if it is outdated and "isCustomCity: true"', async () => { + const adoptedProbes = new AdoptedProbes(sqlStub as unknown as Knex, fetchSocketsStub); + selectStub.resolves([{ ...defaultAdoptedProbe, countryOfCustomCity: 'IE', isCustomCity: true }]); + fetchSocketsStub.resolves([{ data: { probe: { ipAddress: '1.1.1.1', uuid: '1-1-1-1-1', - status: 'ready', - version: '0.26.0', + status: 'initializing', + version: '0.27.0', nodeVersion: 'v18.17.0', location: { continent: 'EU', region: 'Northern Europe', - country: 'IE', - city: 'Dublin', - asn: 16509, - latitude: 53.3331, - longitude: -6.2489, - network: 'Amazon.com, Inc.', + country: 'GB', + city: 'London', + asn: 20473, + latitude: 51.50853, + longitude: -0.12574, + network: 'The Constant Company, LLC', }, }, }, @@ -239,8 +330,122 @@ describe('AdoptedProbes', () => { await adoptedProbes.syncDashboardData(); + expect(whereStub.callCount).to.equal(1); + expect(whereStub.args[0]).to.deep.equal([{ ip: '1.1.1.1' }]); + expect(updateStub.callCount).to.equal(1); + + expect(updateStub.args[0]).to.deep.equal([{ + status: 'initializing', + version: '0.27.0', + country: 'GB', + asn: 20473, + network: 'The Constant Company, LLC', + }]); + }); + + it('class should not update probe meta info if it is actual', async () => { + const adoptedProbes = new AdoptedProbes(sqlStub as unknown as Knex, fetchSocketsStub); + + await adoptedProbes.syncDashboardData(); + expect(whereStub.callCount).to.equal(0); expect(updateStub.callCount).to.equal(0); expect(deleteStub.callCount).to.equal(0); }); + + it('class should treat null and undefined values as equal', async () => { + const adoptedProbes = new AdoptedProbes(sqlStub as unknown as Knex, fetchSocketsStub); + selectStub.resolves([{ ...defaultAdoptedProbe, state: null }]); + + await adoptedProbes.syncDashboardData(); + + expect(whereStub.callCount).to.equal(0); + expect(updateStub.callCount).to.equal(0); + expect(deleteStub.callCount).to.equal(0); + }); + + it('getByIp method should return adopted probe data', async () => { + const adoptedProbes = new AdoptedProbes(sqlStub as unknown as Knex, fetchSocketsStub); + + await adoptedProbes.syncDashboardData(); + + const adoptedProbe = adoptedProbes.getByIp('1.1.1.1'); + expect(adoptedProbe).to.deep.equal({ ...defaultAdoptedProbe, tags: [ 'dashboardtag' ], isCustomCity: false }); + }); + + it('getUpdatedLocation method should return updated location', async () => { + const adoptedProbes = new AdoptedProbes(sqlStub as unknown as Knex, fetchSocketsStub); + selectStub.resolves([{ + ...defaultAdoptedProbe, + city: 'Dundalk', + countryOfCustomCity: 'IE', + isCustomCity: true, + latitude: 54, + longitude: -6.41667, + }]); + + await adoptedProbes.syncDashboardData(); + const updatedLocation = adoptedProbes.getUpdatedLocation(defaultConnectedProbe); + expect(updatedLocation).to.deep.equal({ + continent: 'EU', + region: 'Northern Europe', + country: 'IE', + city: 'Dundalk', + normalizedCity: 'dundalk', + asn: 16509, + latitude: 54, + longitude: -6.41667, + network: 'Amazon.com, Inc.', + normalizedNetwork: 'amazon.com, inc.', + }); + }); + + it('getUpdatedLocation method should return same location object if connected.country !== adopted.countryOfCustomCity', async () => { + const adoptedProbes = new AdoptedProbes(sqlStub as unknown as Knex, fetchSocketsStub); + selectStub.resolves([{ + ...defaultAdoptedProbe, + country: 'IE', + countryOfCustomCity: 'GB', + city: 'London', + isCustomCity: true, + latitude: 51.50853, + longitude: -0.12574, + }]); + + await adoptedProbes.syncDashboardData(); + const updatedLocation = adoptedProbes.getUpdatedLocation(defaultConnectedProbe); + expect(updatedLocation).to.equal(defaultConnectedProbe.location); + }); + + it('getUpdatedLocation method should return same location object if "isCustomCity: false"', async () => { + const adoptedProbes = new AdoptedProbes(sqlStub as unknown as Knex, fetchSocketsStub); + selectStub.resolves([{ + ...defaultAdoptedProbe, + city: 'Dundalk', + isCustomCity: false, + latitude: 54, + longitude: -6.41667, + }]); + + await adoptedProbes.syncDashboardData(); + const updatedLocation = adoptedProbes.getUpdatedLocation(defaultConnectedProbe); + expect(updatedLocation).to.equal(defaultConnectedProbe.location); + }); + + it('getUpdatedTags method should return updated tags', async () => { + const adoptedProbes = new AdoptedProbes(sqlStub as unknown as Knex, fetchSocketsStub); + + await adoptedProbes.syncDashboardData(); + const updatedTags = adoptedProbes.getUpdatedTags(defaultConnectedProbe); + expect(updatedTags).to.deep.equal([{ type: 'user', value: 'u-jimaek-dashboardtag' }]); + }); + + it('getUpdatedTags method should return same tags array if user tags are empty', async () => { + const adoptedProbes = new AdoptedProbes(sqlStub as unknown as Knex, fetchSocketsStub); + selectStub.resolves([{ ...defaultAdoptedProbe, tags: undefined }]); + + await adoptedProbes.syncDashboardData(); + const updatedTags = adoptedProbes.getUpdatedTags(defaultConnectedProbe); + expect(updatedTags).to.equal(defaultConnectedProbe.tags); + }); }); diff --git a/test/tests/unit/geoip/client.test.ts b/test/tests/unit/geoip/client.test.ts index 2c254487..7f9623e9 100644 --- a/test/tests/unit/geoip/client.test.ts +++ b/test/tests/unit/geoip/client.test.ts @@ -3,8 +3,10 @@ import mockFs from 'mock-fs'; import { expect } from 'chai'; import GeoipClient, { type LocationInfo } from '../../../../src/lib/geoip/client.js'; import NullCache from '../../../../src/lib/cache/null-cache.js'; -import nockGeoIpProviders, { geoIpMocks } from '../../../utils/nock-geo-ip.js'; +import nockGeoIpProviders from '../../../utils/nock-geo-ip.js'; import type { CacheInterface } from '../../../../src/lib/cache/cache-interface.js'; +import geoIpMocks from '../../../mocks/nock-geoip.json' assert { type: 'json' }; + const MOCK_IP = '131.255.7.26'; @@ -78,8 +80,8 @@ describe('geoip service', () => { it('should choose top prioritized provider if some providers are down', async () => { nock('https://ipmap-api.ripe.net/v1/locate/').get(/.*/).reply(400); nock('https://api.ip2location.io').get(/.*/).reply(400); - nock('https://globalping-geoip.global.ssl.fastly.net').get(/.*/).reply(200, geoIpMocks['fastly'].argentina); - nock('https://ipinfo.io').get(/.*/).reply(200, geoIpMocks['ipinfo'].argentina); + nock('https://globalping-geoip.global.ssl.fastly.net').get(/.*/).reply(200, geoIpMocks.fastly.argentina); + nock('https://ipinfo.io').get(/.*/).reply(200, geoIpMocks.ipinfo.argentina); nock('https://geoip.maxmind.com/geoip/v2.1/city/').get(/.*/).reply(400); const info = await client.lookup(MOCK_IP); @@ -187,7 +189,7 @@ describe('geoip service', () => { it('should fail when only fastly reports the data', async () => { nock('https://ipmap-api.ripe.net/v1/locate/').get(/.*/).reply(400); nock('https://api.ip2location.io').get(/.*/).reply(400); - nock('https://globalping-geoip.global.ssl.fastly.net').get(/.*/).reply(200, geoIpMocks['fastly'].default); + nock('https://globalping-geoip.global.ssl.fastly.net').get(/.*/).reply(200, geoIpMocks.fastly.default); nock('https://ipinfo.io').get(/.*/).reply(400); nock('https://geoip.maxmind.com/geoip/v2.1/city/').get(/.*/).reply(400); diff --git a/test/tests/unit/probe/router.test.ts b/test/tests/unit/probe/router.test.ts index 5fd4d9f8..0632f390 100644 --- a/test/tests/unit/probe/router.test.ts +++ b/test/tests/unit/probe/router.test.ts @@ -380,7 +380,6 @@ describe('probe router', () => { const location = { continent: 'NA', region: getRegionByCountry('US'), - normalizedRegion: 'northern america', country: 'US', state: 'NY', city: 'The New York City', @@ -818,5 +817,26 @@ describe('probe router', () => { expect(probes.length).to.equal(0); }); + + it('should return match for user tag', async () => { + const socket = await buildSocket(String(Date.now()), location); + socket.data.probe.tags = [ + ...socket.data.probe.tags, + { type: 'user', value: 'u-jimaek-dashboardtag' }, + ]; + + const sockets: DeepPartial = [ socket ]; + + const locations: Location[] = [ + { tags: [ 'u-jimaek-dashboardtag' ] }, + ]; + + fetchSocketsMock.resolves(sockets as never); + + const probes = await router.findMatchingProbes(locations, 100); + + expect(probes.length).to.equal(1); + expect(probes[0]!.location.country).to.equal('GB'); + }); }); }); diff --git a/test/tests/unit/ws/fetch-sockets.test.ts b/test/tests/unit/ws/fetch-sockets.test.ts new file mode 100644 index 00000000..43ae1c5d --- /dev/null +++ b/test/tests/unit/ws/fetch-sockets.test.ts @@ -0,0 +1,182 @@ +import * as sinon from 'sinon'; +import * as td from 'testdouble'; +import { expect } from 'chai'; + +import type { LRUOptions } from '../../../../src/lib/ws/helper/throttle.js'; +import type { RemoteProbeSocket } from '../../../../src/lib/ws/server.js'; + +const fetchRawSockets = sinon.stub().resolves([]); +const getByIp = sinon.stub(); +const getUpdatedLocation = sinon.stub(); +const getUpdatedTags = sinon.stub(); + +describe('fetchSockets', () => { + let fetchSockets: (options?: LRUOptions) => Promise; + + beforeEach(async () => { + sinon.resetHistory(); + td.reset(); + await td.replaceEsm('../../../../src/lib/ws/server.ts', { fetchRawSockets }); + await td.replaceEsm('../../../../src/lib/adopted-probes.ts', { adoptedProbes: { getByIp, getUpdatedLocation, getUpdatedTags } }); + ({ fetchSockets } = await import('../../../../src/lib/ws/fetch-sockets.js')); + }); + + after(() => { + td.reset(); + }); + + it('should apply adopted probes data to the result sockets', async () => { + fetchRawSockets.resolves([{ + data: { + probe: { + client: 'pf2pER5jLnhxTgBqAAAB', + version: '0.26.0', + nodeVersion: 'v18.17.0', + uuid: 'c873cd81-5ede-4fff-9314-5905ad2bdb58', + ipAddress: '1.1.1.1', + host: '', + location: { + continent: 'EU', + region: 'Western Europe', + country: 'FR', + state: undefined, + city: 'Paris', + normalizedCity: 'paris', + asn: 12876, + latitude: 48.8534, + longitude: 2.3488, + network: 'SCALEWAY S.A.S.', + normalizedNetwork: 'scaleway s.a.s.', + }, + index: [ + [ 'fr' ], + [ 'fra' ], + [ 'france' ], + [], + [ 'paris' ], + [], + [], + [ 'eu' ], + [ 'eu', 'europe' ], + [ 'western europe' ], + [ 'western europe', 'west europe' ], + [ 'as12876' ], + [ 'datacenter network' ], + [ 'scaleway s.a.s.' ], + [], + ], + resolvers: [ 'private' ], + tags: [{ type: 'system', value: 'datacenter-network' }], + stats: { cpu: { count: 8, load: [] }, jobs: { count: 0 } }, + status: 'ready', + }, + }, + }]); + + getByIp.returns({ + username: 'jimaek', + ip: '1.1.1.1', + uuid: 'c873cd81-5ede-4fff-9314-5905ad2bdb58', + lastSyncDate: '2023-10-29T21:00:00.000Z', + isCustomCity: 1, + tags: [ 'my-tag-1' ], + status: 'ready', + version: '0.26.0', + asn: 12876, + network: 'SCALEWAY S.A.S.', + country: 'FR', + city: 'Marseille', + latitude: 43.29695, + longitude: 5.38107, + }); + + getUpdatedLocation.returns({ + continent: 'EU', + region: 'Western Europe', + country: 'FR', + state: undefined, + city: 'Marseille', + normalizedCity: 'marseille', + asn: 12876, + latitude: 43.29695, + longitude: 5.38107, + network: 'SCALEWAY S.A.S.', + normalizedNetwork: 'scaleway s.a.s.', + }); + + getUpdatedTags.returns([{ type: 'system', value: 'datacenter-network' }, { type: 'user', value: 'u-jimaek-my-tag-1' }]); + + const result = await fetchSockets(); + expect(result).to.deep.equal([ + { + data: { + probe: { + client: 'pf2pER5jLnhxTgBqAAAB', + version: '0.26.0', + nodeVersion: 'v18.17.0', + uuid: 'c873cd81-5ede-4fff-9314-5905ad2bdb58', + ipAddress: '1.1.1.1', + host: '', + location: { + continent: 'EU', + region: 'Western Europe', + country: 'FR', + state: undefined, + city: 'Marseille', + normalizedCity: 'marseille', + asn: 12876, + latitude: 43.29695, + longitude: 5.38107, + network: 'SCALEWAY S.A.S.', + normalizedNetwork: 'scaleway s.a.s.', + }, + index: [ + [ 'fr' ], + [ 'fra' ], + [ 'france' ], + [], + [ 'marseille' ], + [], + [], + [ 'eu' ], + [ 'eu', 'europe' ], + [ 'western europe' ], + [ 'western europe', 'west europe' ], + [ 'as12876' ], + [ 'datacenter network' ], + [ 'scaleway s.a.s.' ], + [], + ], + resolvers: [ 'private' ], + tags: [{ type: 'system', value: 'datacenter-network' }, { type: 'user', value: 'u-jimaek-my-tag-1' }], + stats: { cpu: { count: 8, load: [] }, jobs: { count: 0 } }, + status: 'ready', + }, + }, + }, + ]); + }); + + it('should return same socket object if there is no adoption data or it is not edited', async () => { + const socket1 = { id: '1', data: { probe: { ipAddress: '' } } }; + const socket2 = { id: '2', data: { probe: { ipAddress: '' } } }; + fetchRawSockets.resolves([ socket1, socket2 ]); + getByIp.onCall(0).returns(undefined); + getByIp.onCall(1).returns({ isCustomCity: false, tags: [] }); + const result = await fetchSockets(); + expect(result[0]).to.equal(socket1); + expect(result[1]).to.equal(socket2); + }); + + it('multiple calls to fetchSockets should result in one socket.io fetchSockets call', async () => { + expect(fetchRawSockets.callCount).to.equal(0); + + await Promise.all([ + fetchSockets(), + fetchSockets(), + fetchSockets(), + ]); + + expect(fetchRawSockets.callCount).to.equal(1); + }); +}); diff --git a/test/tests/unit/ws/reconnect-probes.test.ts b/test/tests/unit/ws/reconnect-probes.test.ts new file mode 100644 index 00000000..485110e6 --- /dev/null +++ b/test/tests/unit/ws/reconnect-probes.test.ts @@ -0,0 +1,43 @@ +import * as sinon from 'sinon'; +import * as td from 'testdouble'; +import { expect } from 'chai'; + +const disconnect = sinon.stub(); +const fetchSockets = sinon.stub().resolves([{ disconnect }, { disconnect }]); + +describe('reconnectProbes', () => { + let sandbox: sinon.SinonSandbox; + let reconnectProbes: () => void; + + before(async () => { + await td.replaceEsm('../../../../src/lib/ws/fetch-sockets.ts', { + fetchSockets, + }); + + ({ reconnectProbes } = await import('../../../../src/lib/ws/helper/reconnect-probes.js')); + }); + + beforeEach(() => { + sandbox = sinon.createSandbox({ useFakeTimers: true }); + }); + + afterEach(() => { + sandbox.restore(); + }); + + after(() => { + td.reset(); + }); + + it('should disconnect every probe in configured time', async () => { + reconnectProbes(); + + expect(fetchSockets.callCount).to.equal(0); + expect(disconnect.callCount).to.equal(0); + + await sandbox.clock.tickAsync(8000 + 2 * 60_000 + 1000); + + expect(fetchSockets.callCount).to.equal(1); + expect(disconnect.callCount).to.equal(2); + }); +}); diff --git a/test/tests/unit/ws/server.test.ts b/test/tests/unit/ws/server.test.ts index 998e00a1..caf895e9 100644 --- a/test/tests/unit/ws/server.test.ts +++ b/test/tests/unit/ws/server.test.ts @@ -3,8 +3,7 @@ import * as td from 'testdouble'; import { expect } from 'chai'; describe('ws server', () => { - let sandbox: sinon.SinonSandbox; - let initWsServer: () => any, getWsServer: () => any, fetchSockets: () => any; + let initWsServer: () => any, getWsServer: () => any; const redisClient = { duplicate: () => redisClient, @@ -13,12 +12,11 @@ describe('ws server', () => { const disconnect = sinon.stub(); const fetchSocketsSocketIo = sinon.stub(); const getRedisClient = sinon.stub().returns(redisClient); - const of = sinon.stub().returns({ - fetchSockets: fetchSocketsSocketIo, - }); const io = { adapter: sinon.stub(), - of, + of: sinon.stub().returns({ + fetchSockets: fetchSocketsSocketIo, + }), }; before(async () => { @@ -27,29 +25,11 @@ describe('ws server', () => { }); beforeEach(async () => { - ({ initWsServer, getWsServer, fetchSockets } = await import('../../../../src/lib/ws/server.js')); - sandbox = sinon.createSandbox({ useFakeTimers: true }); + ({ initWsServer, getWsServer } = await import('../../../../src/lib/ws/server.js')); fetchSocketsSocketIo.reset(); fetchSocketsSocketIo.resolves([{ disconnect }, { disconnect }]); }); - afterEach(() => { - sandbox.restore(); - }); - - after(() => { - td.reset(); - }); - - it('initWsServer should reconnect the probes on start', async () => { - await initWsServer(); - await sandbox.clock.tickAsync(8000 + 2 * 60_000 + 1000); - - expect(io.adapter.callCount).to.equal(1); - expect(redisClient.connect.callCount).to.equal(2); - expect(disconnect.callCount).to.equal(2); - }); - it('getWsServer should return the same instance every time', async () => { await initWsServer(); const wsServer1 = getWsServer(); @@ -58,19 +38,4 @@ describe('ws server', () => { expect(wsServer1).to.equal(wsServer2); expect(wsServer1).to.equal(wsServer3); }); - - it('multiple calls to fetchSockets should result in one socket.io fetchSockets call', async () => { - await initWsServer(); - await sandbox.clock.tickAsync(8000 + 60_000 + 1000); - fetchSocketsSocketIo.reset(); - fetchSocketsSocketIo.resolves([]); - - await Promise.all([ - fetchSockets(), - fetchSockets(), - fetchSockets(), - ]); - - expect(fetchSocketsSocketIo.callCount).to.equal(1); - }); }); diff --git a/test/utils/nock-geo-ip.ts b/test/utils/nock-geo-ip.ts index 9c1d96e0..109cf948 100644 --- a/test/utils/nock-geo-ip.ts +++ b/test/utils/nock-geo-ip.ts @@ -1,24 +1,16 @@ -import * as fs from 'node:fs'; import nock from 'nock'; - -export const geoIpMocks = JSON.parse(fs.readFileSync('./test/mocks/nock-geoip.json').toString()) as Record; +import geoIpMocks from '../mocks/nock-geoip.json' assert { type: 'json' }; type ProviderToMockname = { - ipmap?: string; - ip2location?: string; - maxmind?: string; - ipinfo?: string; - fastly?: string; + ipmap?: keyof(typeof geoIpMocks.ipmap); + ip2location?: keyof(typeof geoIpMocks.ip2location); + maxmind?: keyof(typeof geoIpMocks.maxmind); + ipinfo?: keyof(typeof geoIpMocks.ipinfo); + fastly?: keyof(typeof geoIpMocks.fastly); }; const nockGeoIpProviders = (providersToMockname: ProviderToMockname = {}) => { - Object.entries(providersToMockname).forEach(([ provider, mockname ]) => { - if (mockname && !geoIpMocks[provider][mockname]) { - throw new Error(`No ${mockname} mock for ${provider} provider`); - } - }); - - const mockNames = { + const mockNames: Required = { ipmap: 'default', ip2location: 'default', maxmind: 'default', @@ -27,11 +19,11 @@ const nockGeoIpProviders = (providersToMockname: ProviderToMockname = {}) => { ...providersToMockname, }; - nock('https://ipmap-api.ripe.net/v1/locate/').get(/.*/).reply(200, geoIpMocks['ipmap'][mockNames.ipmap]); - nock('https://api.ip2location.io').get(/.*/).reply(200, geoIpMocks['ip2location'][mockNames.ip2location]); - nock('https://geoip.maxmind.com/geoip/v2.1/city/').get(/.*/).reply(200, geoIpMocks['maxmind'][mockNames.maxmind]); - nock('https://ipinfo.io').get(/.*/).reply(200, geoIpMocks['ipinfo'][mockNames.ipinfo]); - nock('https://globalping-geoip.global.ssl.fastly.net').get(/.*/).reply(200, geoIpMocks['fastly'][mockNames.fastly]); + nock('https://ipmap-api.ripe.net/v1/locate/').get(/.*/).reply(200, geoIpMocks.ipmap[mockNames.ipmap]); + nock('https://api.ip2location.io').get(/.*/).reply(200, geoIpMocks.ip2location[mockNames.ip2location]); + nock('https://geoip.maxmind.com/geoip/v2.1/city/').get(/.*/).reply(200, geoIpMocks.maxmind[mockNames.maxmind]); + nock('https://ipinfo.io').get(/.*/).reply(200, geoIpMocks.ipinfo[mockNames.ipinfo]); + nock('https://globalping-geoip.global.ssl.fastly.net').get(/.*/).reply(200, geoIpMocks.fastly[mockNames.fastly]); }; export default nockGeoIpProviders;