diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0603745a..3c05db1c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -10,16 +10,16 @@ Hi! We're really excited that you're interested in contributing! Before submitti ## Project setup -In order to run the Globalping API locally you will need Node.js 18 and Redis with [RedisJSON](https://oss.redis.com/redisjson/) module (included in docker-compose.yml file). You will also need to run a development instance of the [Globalping Probe](https://github.com/jsdelivr/globalping-probe) at the same time when testing. +In order to run the Globalping API locally you will need Node.js 18 and Redis with [RedisJSON](https://oss.redis.com/redisjson/) module and MariaDB. All of them are included in docker-compose.yml file. You will also need to run a development instance of the [Globalping Probe](https://github.com/jsdelivr/globalping-probe) at the same time when testing. The API uses 3000 port by default. This can be overridden by `PORT` environment variable. You can run the project by following these steps: 1. Clone this repository. -2. `docker-compose up -d` - Run Redis +2. `docker-compose up -d` - Run Redis and MariaDB 3. `npm install` -4. Run `npm run dev` +4. Run `npm run start:dev` Once the API is live, you can spin up a probe instance by running as described at https://github.com/jsdelivr/globalping-probe/blob/master/CONTRIBUTING.md. @@ -40,5 +40,7 @@ Most IDEs have plugins integrating the used linter (eslint), including support f ### Environment variables -- `NEW_RELIC_LICENSE_KEY={value}` environment variable should be used in production to send APM metrics to new relic -- `NEW_RELIC_APP_NAME={value}` environment variable should be used in production to send APM mentrics to new relic +- `NEW_RELIC_LICENSE_KEY={value}` used in production to send APM metrics to new relic +- `NEW_RELIC_APP_NAME={value}` used in production to send APM mentrics to new relic +- `NEW_RELIC_ENABLED=false` useid in development to disable newrelic monitoring +- `FAKE_PROBE_IP={api|probe}` used in development to either use a random fake ip from the API or a fake ip from Probe diff --git a/config/default.cjs b/config/default.cjs index 9b0749e8..d98a2271 100644 --- a/config/default.cjs +++ b/config/default.cjs @@ -11,6 +11,9 @@ module.exports = { tls: false, }, }, + sql: { + url: 'mysql://root:@localhost:3306/directus', + }, admin: { key: '', }, diff --git a/config/init.sql b/config/init.sql new file mode 100644 index 00000000..655382b0 --- /dev/null +++ b/config/init.sql @@ -0,0 +1,16 @@ +CREATE DATABASE IF NOT EXISTS directus; + +USE directus; + +CREATE TABLE IF NOT EXISTS adopted_probes ( + id INT(10) UNSIGNED AUTO_INCREMENT PRIMARY KEY, + user_created CHAR(36), + date_created TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + user_updated CHAR(36), + date_updated TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + userId VARCHAR(255) NOT NULL, + ip VARCHAR(255) NOT NULL, + uuid VARCHAR(255) +); + +INSERT IGNORE INTO adopted_probes (id, userId, ip) VALUES ('1', '6191378', '65.19.141.130'); diff --git a/docker-compose.yml b/docker-compose.yml index abc8ed8b..2bf04396 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,3 +8,12 @@ services: volumes: - ./config/redis.conf:/usr/local/etc/redis/redis.conf command: /usr/local/etc/redis/redis.conf + + mariadb: + image: mariadb:latest + environment: + - MARIADB_ALLOW_EMPTY_ROOT_PASSWORD=1 + ports: + - "3306:3306" + volumes: + - ./config/init.sql:/docker-entrypoint-initdb.d/init.sql diff --git a/src/adoption/route/adoption-code.ts b/src/adoption-code/route/adoption-code.ts similarity index 90% rename from src/adoption/route/adoption-code.ts rename to src/adoption-code/route/adoption-code.ts index 61ec348b..96aed1c6 100644 --- a/src/adoption/route/adoption-code.ts +++ b/src/adoption-code/route/adoption-code.ts @@ -5,9 +5,7 @@ import { bodyParser } from '../../lib/http/middleware/body-parser.js'; import { validate } from '../../lib/http/middleware/validate.js'; import { schema } from '../schema.js'; import { onlyAdminMiddleware } from '../../lib/http/middleware/only-admin.js'; -import { getCodeSender } from '../sender.js'; - -const codeSender = getCodeSender(); +import { codeSender } from '../sender.js'; const handle = async (ctx: Context): Promise => { const request = ctx.request.body as AdoptionCodeRequest; diff --git a/src/adoption/schema.ts b/src/adoption-code/schema.ts similarity index 100% rename from src/adoption/schema.ts rename to src/adoption-code/schema.ts diff --git a/src/adoption/sender.ts b/src/adoption-code/sender.ts similarity index 85% rename from src/adoption/sender.ts rename to src/adoption-code/sender.ts index 44ce5b21..65bb017a 100644 --- a/src/adoption/sender.ts +++ b/src/adoption-code/sender.ts @@ -30,12 +30,4 @@ export class CodeSender { } } -let sender: CodeSender; - -export const getCodeSender = () => { - if (!sender) { - sender = new CodeSender(fetchSockets); - } - - return sender; -}; +export const codeSender = new CodeSender(fetchSockets); diff --git a/src/adoption/types.ts b/src/adoption-code/types.ts similarity index 100% rename from src/adoption/types.ts rename to src/adoption-code/types.ts diff --git a/src/lib/adopted-probes/adopted-probes.ts b/src/lib/adopted-probes.ts similarity index 76% rename from src/lib/adopted-probes/adopted-probes.ts rename to src/lib/adopted-probes.ts index e4c7f75d..92a85eaa 100644 --- a/src/lib/adopted-probes/adopted-probes.ts +++ b/src/lib/adopted-probes.ts @@ -1,9 +1,11 @@ import type { Knex } from 'knex'; -import { scopedLogger } from '../logger.js'; -import { client } from '../sql/client.js'; +import { scopedLogger } from './logger.js'; +import { client } from './sql/client.js'; const logger = scopedLogger('adopted-probes'); +const TABLE_NAME = 'adopted_probes'; + type AdoptedProbe = { ip: string; uuid: string; @@ -21,7 +23,7 @@ export class AdoptedProbes { this.scheduleSync(); } - async syncProbeData (probeIp: string, probeUuid: string) { + async syncProbeIds (probeIp: string, probeUuid: string) { const adoptedProbeByIp = this.adoptedProbesByIp.get(probeIp); if (adoptedProbeByIp && adoptedProbeByIp.uuid === probeUuid) { // Probe is synced @@ -39,18 +41,18 @@ export class AdoptedProbes { } private async updateUuid (ip: string, uuid: string) { - await this.sql('adopted_probes').where({ ip }).update({ uuid }); + await this.sql(TABLE_NAME).where({ ip }).update({ uuid }); } private async updateIp (ip: string, uuid: string) { - await this.sql('adopted_probes').where({ uuid }).update({ ip }); + await this.sql(TABLE_NAME).where({ uuid }).update({ ip }); } private async syncDashboardData () { - const probes = await this.sql.select('ip', 'uuid').from('adopted_probes'); + const probes = await this.sql(TABLE_NAME).select('ip', 'uuid'); + // Storing city as emtpy string until https://github.com/jsdelivr/globalping/issues/427 is implemented this.adoptedProbesByIp = new Map(probes.map(probe => [ probe.ip, { uuid: probe.uuid, city: '' }])); this.adoptedProbesByUuid = new Map(probes.map(probe => [ probe.uuid, { ip: probe.ip, city: '' }])); - console.log('this.adoptedProbes', this.adoptedProbesByIp); } scheduleSync () { diff --git a/src/lib/get-probe-ip.ts b/src/lib/get-probe-ip.ts index 2585efbc..a9883d3a 100644 --- a/src/lib/get-probe-ip.ts +++ b/src/lib/get-probe-ip.ts @@ -6,16 +6,16 @@ const getProbeIp = (socket: Socket) => { // Use random ip assigned by the API if (process.env['FAKE_PROBE_IP'] === 'api') { return _.sample([ - // '18.200.0.1', - // '34.140.0.10', - // '95.155.94.127', - // '65.49.22.66', - // '185.229.226.83', - // '51.158.22.211', - // '131.255.7.26', - // '213.136.174.80', - // '94.214.253.78', - // '79.205.97.254', + '18.200.0.1', + '34.140.0.10', + '95.155.94.127', + '65.49.22.66', + '185.229.226.83', + '51.158.22.211', + '131.255.7.26', + '213.136.174.80', + '94.214.253.78', + '79.205.97.254', '65.19.141.130', ]); } diff --git a/src/lib/http/server.ts b/src/lib/http/server.ts index 8afdeb9b..468219b9 100644 --- a/src/lib/http/server.ts +++ b/src/lib/http/server.ts @@ -14,7 +14,7 @@ import cjsDependencies from '../../cjs-dependencies.cjs'; import { registerGetProbesRoute } from '../../probe/route/get-probes.js'; import { registerGetMeasurementRoute } from '../../measurement/route/get-measurement.js'; import { registerCreateMeasurementRoute } from '../../measurement/route/create-measurement.js'; -import { registerSendCodeRoute } from '../../adoption/route/adoption-code.js'; +import { registerSendCodeRoute } from '../../adoption-code/route/adoption-code.js'; import { registerHealthRoute } from '../../health/route/get.js'; import { registerSpecRoute } from './spec.js'; import { errorHandler } from './error-handler.js'; diff --git a/src/lib/server.ts b/src/lib/server.ts index 2b98db57..cbfd5c06 100644 --- a/src/lib/server.ts +++ b/src/lib/server.ts @@ -6,7 +6,7 @@ import { populateMemList as populateMemMalwareList } from './malware/client.js'; import { populateMemList as populateMemIpRangesList } from './ip-ranges.js'; import { populateMemList as populateIpWhiteList } from './geoip/whitelist.js'; import { populateCitiesList } from './geoip/city-approximation.js'; -import { adoptedProbes } from './adopted-probes/adopted-probes.js'; +import { adoptedProbes } from './adopted-probes.js'; export const createServer = async (): Promise => { await initRedis(); diff --git a/src/lib/sql/client.ts b/src/lib/sql/client.ts new file mode 100644 index 00000000..acc985c1 --- /dev/null +++ b/src/lib/sql/client.ts @@ -0,0 +1,7 @@ +import config from 'config'; +import knex, { Knex } from 'knex'; + +export const client: Knex = knex({ + client: 'mysql', + connection: config.get('sql.url'), +}); diff --git a/src/probe/builder.ts b/src/probe/builder.ts index 6c5c86b2..bf9c7b6d 100644 --- a/src/probe/builder.ts +++ b/src/probe/builder.ts @@ -18,7 +18,7 @@ import { getRegion } from '../lib/ip-ranges.js'; import type { Probe, ProbeLocation, Tag } from './types.js'; import { verifyIpLimit } from '../lib/ws/helper/probe-ip-limit.js'; import { fakeLookup } from '../lib/geoip/fake-client.js'; -import { adoptedProbes } from '../lib/adopted-probes/adopted-probes.js'; +import { adoptedProbes } from '../lib/adopted-probes.js'; const geoipClient = createGeoipClient(); @@ -27,7 +27,7 @@ export const buildProbe = async (socket: Socket): Promise => { const nodeVersion = String(socket.handshake.query['nodeVersion']); - const id = String(socket.handshake.query['id']); + const uuid = String(socket.handshake.query['uuid']); const host = process.env['HOSTNAME'] ?? ''; @@ -55,7 +55,7 @@ export const buildProbe = async (socket: Socket): Promise => { await verifyIpLimit(ip, socket.id); - await adoptedProbes.syncProbeData(ip, id); + await adoptedProbes.syncProbeIds(ip, uuid); const location = getLocation(ipInfo); @@ -85,7 +85,7 @@ export const buildProbe = async (socket: Socket): Promise => { client: socket.id, version, nodeVersion, - id, + uuid, ipAddress: ip, host, location, diff --git a/src/probe/types.ts b/src/probe/types.ts index b4f566d4..69a0da4d 100644 --- a/src/probe/types.ts +++ b/src/probe/types.ts @@ -36,7 +36,7 @@ export type Probe = { client: string; version: string; nodeVersion: string; - id: string; + uuid: string; ipAddress: string; host: string; location: ProbeLocation; diff --git a/test/tests/integration/measurement/probe-communication.test.ts b/test/tests/integration/measurement/probe-communication.test.ts index 641691a0..9646a93f 100644 --- a/test/tests/integration/measurement/probe-communication.test.ts +++ b/test/tests/integration/measurement/probe-communication.test.ts @@ -313,7 +313,7 @@ describe('Create measurement request', () => { status: 'initializing', version: '0.14.0', nodeVersion: 'v18.17.0', - id: '1-1-1-1-1', + uuid: '1-1-1-1-1', location: { continent: 'NA', region: 'Northern America', diff --git a/test/utils/server.ts b/test/utils/server.ts index d0a3703a..165762a6 100644 --- a/test/utils/server.ts +++ b/test/utils/server.ts @@ -26,7 +26,7 @@ export const addFakeProbe = async (events: Record = {}): Promise