feat: add mariadb to docker compose

CONTRIBUTING.md

@@ -10,16 +10,16 @@ Hi! We're really excited that you're interested in contributing! Before submitti
 
 ## Project setup
 
-In order to run the Globalping API locally you will need Node.js 18 and Redis with [RedisJSON](https://oss.redis.com/redisjson/) module (included in docker-compose.yml file). You will also need to run a development instance of the [Globalping Probe](https://github.com/jsdelivr/globalping-probe) at the same time when testing.
+In order to run the Globalping API locally you will need Node.js 18 and Redis with [RedisJSON](https://oss.redis.com/redisjson/) module and MariaDB. All of them are included in docker-compose.yml file. You will also need to run a development instance of the [Globalping Probe](https://github.com/jsdelivr/globalping-probe) at the same time when testing.
 
 The API uses 3000 port by default. This can be overridden by `PORT` environment variable.
 
 You can run the project by following these steps:
 
 1. Clone this repository.
-2. `docker-compose up -d` - Run Redis
+2. `docker-compose up -d` - Run Redis and MariaDB
 3. `npm install`
-4. Run `npm run dev`
+4. Run `npm run start:dev`
 
 Once the API is live, you can spin up a probe instance by running as described at https://github.com/jsdelivr/globalping-probe/blob/master/CONTRIBUTING.md.
 
@@ -40,5 +40,7 @@ Most IDEs have plugins integrating the used linter (eslint), including support f
 
 ### Environment variables
 
-- `NEW_RELIC_LICENSE_KEY={value}` environment variable should be used in production to send APM metrics to new relic
-- `NEW_RELIC_APP_NAME={value}` environment variable should be used in production to send APM mentrics to new relic
+- `NEW_RELIC_LICENSE_KEY={value}` used in production to send APM metrics to new relic
+- `NEW_RELIC_APP_NAME={value}` used in production to send APM mentrics to new relic
+- `NEW_RELIC_ENABLED=false` useid in development to disable newrelic monitoring
+- `FAKE_PROBE_IP={api|probe}` used in development to either use a random fake ip from the API or a fake ip from Probe

config/default.cjs

@@ -11,6 +11,9 @@ module.exports = {
 			tls: false,
 		},
 	},
+	sql: {
+		url: 'mysql://root:@localhost:3306/directus',
+	},
 	admin: {
 		key: '',
 	},

config/init.sql

@@ -0,0 +1,16 @@
+CREATE DATABASE IF NOT EXISTS directus;
+
+USE directus;
+
+CREATE TABLE IF NOT EXISTS adopted_probes (
+  id INT(10) UNSIGNED AUTO_INCREMENT PRIMARY KEY,
+  user_created CHAR(36),
+  date_created TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+  user_updated CHAR(36),
+  date_updated TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+  userId VARCHAR(255) NOT NULL,
+  ip VARCHAR(255) NOT NULL,
+  uuid VARCHAR(255)
+);
+
+INSERT IGNORE INTO adopted_probes (id, userId, ip) VALUES ('1', '6191378', '65.19.141.130');

docker-compose.yml

@@ -8,3 +8,12 @@ services:
     volumes:
       - ./config/redis.conf:/usr/local/etc/redis/redis.conf
     command: /usr/local/etc/redis/redis.conf
+
+  mariadb:
+    image: mariadb:latest
+    environment:
+    - MARIADB_ALLOW_EMPTY_ROOT_PASSWORD=1
+    ports:
+      - "3306:3306"
+    volumes:
+      - ./config/init.sql:/docker-entrypoint-initdb.d/init.sql

src/adoption/route/adoption-code.ts → src/adoption-code/route/adoption-code.ts

@@ -5,9 +5,7 @@ import { bodyParser } from '../../lib/http/middleware/body-parser.js';
 import { validate } from '../../lib/http/middleware/validate.js';
 import { schema } from '../schema.js';
 import { onlyAdminMiddleware } from '../../lib/http/middleware/only-admin.js';
-import { getCodeSender } from '../sender.js';
-
-const codeSender = getCodeSender();
+import { codeSender } from '../sender.js';
 
 const handle = async (ctx: Context): Promise<void> => {
 	const request = ctx.request.body as AdoptionCodeRequest;

src/adoption/sender.ts → src/adoption-code/sender.ts

@@ -30,12 +30,4 @@ export class CodeSender {
 	}
 }
 
-let sender: CodeSender;
-
-export const getCodeSender = () => {
-	if (!sender) {
-		sender = new CodeSender(fetchSockets);
-	}
-
-	return sender;
-};
+export const codeSender = new CodeSender(fetchSockets);

src/lib/adopted-probes/adopted-probes.ts → src/lib/adopted-probes.ts

@@ -1,9 +1,11 @@
 import type { Knex } from 'knex';
-import { scopedLogger } from '../logger.js';
-import { client } from '../sql/client.js';
+import { scopedLogger } from './logger.js';
+import { client } from './sql/client.js';
 
 const logger = scopedLogger('adopted-probes');
 
+const TABLE_NAME = 'adopted_probes';
+
 type AdoptedProbe = {
   ip: string;
   uuid: string;
@@ -21,7 +23,7 @@ export class AdoptedProbes {
 		this.scheduleSync();
 	}
 
-	async syncProbeData (probeIp: string, probeUuid: string) {
+	async syncProbeIds (probeIp: string, probeUuid: string) {
 		const adoptedProbeByIp = this.adoptedProbesByIp.get(probeIp);
 
 		if (adoptedProbeByIp && adoptedProbeByIp.uuid === probeUuid) { // Probe is synced
@@ -39,18 +41,18 @@ export class AdoptedProbes {
 	}
 
 	private async updateUuid (ip: string, uuid: string) {
-		await this.sql('adopted_probes').where({ ip }).update({ uuid });
+		await this.sql(TABLE_NAME).where({ ip }).update({ uuid });
 	}
 
 	private async updateIp (ip: string, uuid: string) {
-		await this.sql('adopted_probes').where({ uuid }).update({ ip });
+		await this.sql(TABLE_NAME).where({ uuid }).update({ ip });
 	}
 
 	private async syncDashboardData () {
-		const probes = await this.sql.select<AdoptedProbe[]>('ip', 'uuid').from('adopted_probes');
+		const probes = await this.sql(TABLE_NAME).select<AdoptedProbe[]>('ip', 'uuid');
+		// Storing city as emtpy string until https://github.com/jsdelivr/globalping/issues/427 is implemented
 		this.adoptedProbesByIp = new Map(probes.map(probe => [ probe.ip, { uuid: probe.uuid, city: '' }]));
 		this.adoptedProbesByUuid = new Map(probes.map(probe => [ probe.uuid, { ip: probe.ip, city: '' }]));
-		console.log('this.adoptedProbes', this.adoptedProbesByIp);
 	}
 
 	scheduleSync () {

src/lib/get-probe-ip.ts

@@ -6,16 +6,16 @@ const getProbeIp = (socket: Socket) => {
 	// Use random ip assigned by the API
 	if (process.env['FAKE_PROBE_IP'] === 'api') {
 		return _.sample([
-			// '18.200.0.1',
-			// '34.140.0.10',
-			// '95.155.94.127',
-			// '65.49.22.66',
-			// '185.229.226.83',
-			// '51.158.22.211',
-			// '131.255.7.26',
-			// '213.136.174.80',
-			// '94.214.253.78',
-			// '79.205.97.254',
+			'18.200.0.1',
+			'34.140.0.10',
+			'95.155.94.127',
+			'65.49.22.66',
+			'185.229.226.83',
+			'51.158.22.211',
+			'131.255.7.26',
+			'213.136.174.80',
+			'94.214.253.78',
+			'79.205.97.254',
 			'65.19.141.130',
 		]);
 	}

src/lib/http/server.ts

@@ -14,7 +14,7 @@ import cjsDependencies from '../../cjs-dependencies.cjs';
 import { registerGetProbesRoute } from '../../probe/route/get-probes.js';
 import { registerGetMeasurementRoute } from '../../measurement/route/get-measurement.js';
 import { registerCreateMeasurementRoute } from '../../measurement/route/create-measurement.js';
-import { registerSendCodeRoute } from '../../adoption/route/adoption-code.js';
+import { registerSendCodeRoute } from '../../adoption-code/route/adoption-code.js';
 import { registerHealthRoute } from '../../health/route/get.js';
 import { registerSpecRoute } from './spec.js';
 import { errorHandler } from './error-handler.js';

src/lib/server.ts

@@ -6,7 +6,7 @@ import { populateMemList as populateMemMalwareList } from './malware/client.js';
 import { populateMemList as populateMemIpRangesList } from './ip-ranges.js';
 import { populateMemList as populateIpWhiteList } from './geoip/whitelist.js';
 import { populateCitiesList } from './geoip/city-approximation.js';
-import { adoptedProbes } from './adopted-probes/adopted-probes.js';
+import { adoptedProbes } from './adopted-probes.js';
 
 export const createServer = async (): Promise<Server> => {
 	await initRedis();

src/lib/sql/client.ts

@@ -0,0 +1,7 @@
+import config from 'config';
+import knex, { Knex } from 'knex';
+
+export const client: Knex = knex({
+	client: 'mysql',
+	connection: config.get<string>('sql.url'),
+});

src/probe/builder.ts

@@ -18,7 +18,7 @@ import { getRegion } from '../lib/ip-ranges.js';
 import type { Probe, ProbeLocation, Tag } from './types.js';
 import { verifyIpLimit } from '../lib/ws/helper/probe-ip-limit.js';
 import { fakeLookup } from '../lib/geoip/fake-client.js';
-import { adoptedProbes } from '../lib/adopted-probes/adopted-probes.js';
+import { adoptedProbes } from '../lib/adopted-probes.js';
 
 const geoipClient = createGeoipClient();
 
@@ -27,7 +27,7 @@ export const buildProbe = async (socket: Socket): Promise<Probe> => {
 
 	const nodeVersion = String(socket.handshake.query['nodeVersion']);
 
-	const id = String(socket.handshake.query['id']);
+	const uuid = String(socket.handshake.query['uuid']);
 
 	const host = process.env['HOSTNAME'] ?? '';
 
@@ -55,7 +55,7 @@ export const buildProbe = async (socket: Socket): Promise<Probe> => {
 
 	await verifyIpLimit(ip, socket.id);
 
-	await adoptedProbes.syncProbeData(ip, id);
+	await adoptedProbes.syncProbeIds(ip, uuid);
 
 	const location = getLocation(ipInfo);
 
@@ -85,7 +85,7 @@ export const buildProbe = async (socket: Socket): Promise<Probe> => {
 		client: socket.id,
 		version,
 		nodeVersion,
-		id,
+		uuid,
 		ipAddress: ip,
 		host,
 		location,

src/probe/types.ts

@@ -36,7 +36,7 @@ export type Probe = {
 	client: string;
 	version: string;
 	nodeVersion: string;
-	id: string;
+	uuid: string;
 	ipAddress: string;
 	host: string;
 	location: ProbeLocation;

test/tests/integration/measurement/probe-communication.test.ts

@@ -313,7 +313,7 @@ describe('Create measurement request', () => {
 					status: 'initializing',
 					version: '0.14.0',
 					nodeVersion: 'v18.17.0',
-					id: '1-1-1-1-1',
+					uuid: '1-1-1-1-1',
 					location: {
 						continent: 'NA',
 						region: 'Northern America',

test/utils/server.ts

@@ -26,7 +26,7 @@ export const addFakeProbe = async (events: Record<string, any> = {}): Promise<So
 			query: {
 				version: '0.14.0',
 				nodeVersion: 'v18.17.0',
-				id: '1-1-1-1-1',
+				uuid: '1-1-1-1-1',
 			},
 		});