feat: staging env set up (#423)

* feat: add ability to handle dev env probes

* test: fix tests and local dev probes

* feat: udpate redis configuration

* feat: add redis setup script

* feat: add api and probes setup scripts

* fix: fix docker compose stop after cli logout

* feat: auto start the probes process

* feat: update remote-dev-env script

* feat: use fake lookup

* feat: update remote dev env script

* feat: update doc

* refactor: move redis config to /config

CONTRIBUTING.md

@@ -25,7 +25,7 @@ Once the API is live, you can spin up a probe instance by running as described a
 
 ### Environment variables
 - `PORT=3000` environment variable can start the API on another port (default is 3000)
-- `FAKE_PROBE_IP=1` environment variable can be used to make debug easier. When defined, every Probe
+- `FAKE_PROBE_IP=api` environment variable can be used to make debug easier. When defined, every Probe
   that connects to the API will get an IP address from the list of predefined "real" addresses.
 
 ### Testing

config/redis.conf

@@ -0,0 +1,2 @@
+maxmemory-policy volatile-ttl
+loadmodule /usr/lib/redis/modules/rejson.so

docker-compose.yml

@@ -5,3 +5,6 @@ services:
     image: redislabs/redismod:latest
     ports:
       - "6379:6379"
+    volumes:
+      - ./config/redis.conf:/usr/local/etc/redis/redis.conf
+    command: /usr/local/etc/redis/redis.conf

docs/staging-env.md

@@ -0,0 +1,129 @@
+# Running a remote staging environment
+
+This guide outlines the steps to set up and run a staging environment on a remote host. It is aimed to be used mostly for performance benchmarking. The environment consists of three parts: Redis, API, and Probes. For every part there is a bash script which prepares and runs everything. Every script is executed on a separate Ubuntu server.
+
+## Redis
+
+Starts a redis db inside docker. Make sure port 6379 is open for connections.
+
+```bash
+# Update that variables before start
+REDIS_PASSWORD=<your_value>
+REDIS_MAX_MEMORY=500mb
+
+# Copy and enter the repository
+git clone https://github.com/jsdelivr/globalping.git
+cd globalping/
+
+# Add redis config lines
+echo "requirepass $REDIS_PASSWORD" >> redis.conf
+echo "maxmemory $REDIS_MAX_MEMORY" >> redis.conf
+
+# Install docker
+curl -fsSL https://get.docker.com -o get-docker.sh
+sudo sh get-docker.sh
+
+# Allow to run docker without sudo
+sudo su -c "sudo usermod -aG docker ubuntu && exit" -
+
+# Relogin and start
+echo "Need to relogin, please get back and run:
+cd globalping/ && docker compose up -d
+"
+exit
+```
+
+## API
+
+Runs 2 API instances on ports 3001 and 3002 behind the haproxy on port 80. Geoip client is mocked so all probes get same location. `FAKE_PROBE_IP=probe` makes API to use fake ip provided by the probe.
+
+```bash
+# Update that variables before start
+REDIS_PASSWORD=<your_value>
+REDIS_HOST=<your_value>
+
+# Install haproxy
+sudo apt-get update
+sudo apt -y install haproxy
+
+# Configure and start haproxy
+sudo chmod a+w /etc/haproxy/haproxy.cfg
+cat <<EOF | sudo tee -a /etc/haproxy/haproxy.cfg > /dev/null
+frontend gp_fe
+    bind *:80
+    default_backend gp_be
+
+backend gp_be
+    balance roundrobin
+    option httpchk GET /health
+    server server1 127.0.0.1:3001 check
+    server server2 127.0.0.1:3002 check
+EOF
+sudo systemctl stop haproxy
+sudo systemctl start haproxy
+
+# Install node
+sudo apt-get install -y ca-certificates curl gnupg
+sudo mkdir -p /etc/apt/keyrings
+curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | sudo gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg
+NODE_MAJOR=18
+echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main" | sudo tee /etc/apt/sources.list.d/nodesource.list
+sudo apt-get update
+sudo apt-get install nodejs -y
+
+# Copy and build the repository
+git clone https://github.com/jsdelivr/globalping.git
+cd globalping/
+npm i
+npm run build
+
+# Run the app
+echo 'Run 2 app instances using:
+PORT=3001 HOSTNAME=3001 REDIS_URL=redis://default:$REDIS_PASSWORD@$REDIS_HOST:6379 NODE_ENV=production ADMIN_KEY=admin FAKE_PROBE_IP=probe NEW_RELIC_ENABLED=false NEW_RELIC_LOG_ENABLED=false node dist/index.js
+and
+PORT=3002 HOSTNAME=3002 REDIS_URL=redis://default:$REDIS_PASSWORD@$REDIS_HOST:6379 NODE_ENV=production ADMIN_KEY=admin FAKE_PROBE_IP=probe NEW_RELIC_ENABLED=false NEW_RELIC_LOG_ENABLED=false node dist/index.js
+'
+```
+
+## Probe
+
+Runs `PROBES_COUNT` number of probe processes. They all get a random fake ip which is passed to the API. Value of the `FAKE_PROBE_IP` is the first octet of the fake ip. Each probe process requires ~40mb of RAM.
+
+```bash
+# Update that variables before start
+API_HOST=<your_value>
+FAKE_PROBE_IP=1
+PROBES_COUNT=300
+
+# Install node
+sudo apt-get install -y ca-certificates curl gnupg
+sudo mkdir -p /etc/apt/keyrings
+curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | sudo gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg
+NODE_MAJOR=18
+echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main" | sudo tee /etc/apt/sources.list.d/nodesource.list
+sudo apt-get update
+sudo apt-get install nodejs -y
+
+# Copy and build the repository
+git clone https://github.com/jsdelivr/globalping-probe.git
+cd globalping-probe
+npm i
+npm run build
+
+# Install unbuffer
+ARCHLOCAL=$(dpkg --print-architecture)
+curl "http://ftp.nl.debian.org/debian/pool/main/e/expect/tcl-expect_5.45.4-2+b1_${ARCHLOCAL}.deb" -o "/tmp/tcl-expect.deb"
+sudo dpkg --extract "/tmp/tcl-expect.deb" /
+curl "http://ftp.nl.debian.org/debian/pool/main/t/tcl8.6/libtcl8.6_8.6.11+dfsg-1_${ARCHLOCAL}.deb" -o "/tmp/libtcl.deb"
+sudo dpkg --extract "/tmp/libtcl.deb" /
+curl "http://ftp.nl.debian.org/debian/pool/main/t/tcl8.6/tcl8.6_8.6.11+dfsg-1_${ARCHLOCAL}.deb" -o "/tmp/tcl.deb"
+sudo dpkg --extract "/tmp/tcl.deb" /
+curl "http://ftp.nl.debian.org/debian/pool/main/e/expect/expect_5.45.4-2+b1_${ARCHLOCAL}.deb" -o "/tmp/expect.deb"
+sudo dpkg --extract "/tmp/expect.deb" /
+
+# Auto start the probes
+sudo npm i -g pm2
+sudo env PATH=$PATH:/usr/bin /usr/lib/node_modules/pm2/bin/pm2 startup systemd -u ubuntu --hp /home/ubuntu
+FAKE_PROBE_IP=$FAKE_PROBE_IP NODE_ENV=development PROBES_COUNT=$PROBES_COUNT API_HOST=ws://$API_HOST pm2 start dist/index.js
+pm2 save
+```

package.json

@@ -106,14 +106,14 @@
 	},
 	"scripts": {
 		"build": "tsc && npm run download:files",
-		"coverage": "npm run clean && NODE_ENV=test FAKE_PROBE_IP=1 NEW_RELIC_ENABLED=false NEW_RELIC_LOG_ENABLED=false c8 mocha",
+		"coverage": "npm run clean && NODE_ENV=test FAKE_PROBE_IP=api NEW_RELIC_ENABLED=false NEW_RELIC_LOG_ENABLED=false c8 mocha",
 		"clean": "rimraf coverage",
 		"download:files": "tsx src/lib/download-files.ts",
 		"prepare": "npm run download:files; husky install || echo 'Failed to install husky'",
 		"stats": "NEW_RELIC_ENABLED=false NEW_RELIC_LOG_ENABLED=false tsx probes-stats/known.ts",
 		"start": "NODE_ENV=production node --max_old_space_size=3584 --max-semi-space-size=128 --experimental-loader newrelic/esm-loader.mjs dist/index.js",
-		"start:dev": "NODE_ENV=development FAKE_PROBE_IP=1 NEW_RELIC_ENABLED=false tsx src/index.ts",
-		"start:test": "NODE_ENV=test FAKE_PROBE_IP=1 NEW_RELIC_ENABLED=false tsx src/index.ts",
+		"start:dev": "NODE_ENV=development FAKE_PROBE_IP=api NEW_RELIC_ENABLED=false tsx src/index.ts",
+		"start:test": "NODE_ENV=test FAKE_PROBE_IP=api NEW_RELIC_ENABLED=false tsx src/index.ts",
 		"lint": "npm run lint:js && npm run lint:types && npm run lint:docs",
 		"lint:fix": "npm run lint:js:fix && npm run lint:types && npm run lint:docs",
 		"lint:docs": "spectral lint public/**/spec.yaml",
@@ -122,8 +122,8 @@
 		"lint:types": "tsc --noEmit",
 		"test": "npm run lint && npm run test:mocha && npm run test:portman",
 		"test:dist": "node --test-reporter=spec test/dist.js",
-		"test:mocha": "NODE_ENV=test FAKE_PROBE_IP=1 NEW_RELIC_ENABLED=false NEW_RELIC_LOG_ENABLED=false mocha",
-		"test:mocha:dev": "TS_NODE_TRANSPILE_ONLY=true NODE_ENV=test FAKE_PROBE_IP=1 NEW_RELIC_ENABLED=false NEW_RELIC_LOG_ENABLED=false mocha",
+		"test:mocha": "NODE_ENV=test FAKE_PROBE_IP=api NEW_RELIC_ENABLED=false NEW_RELIC_LOG_ENABLED=false mocha",
+		"test:mocha:dev": "TS_NODE_TRANSPILE_ONLY=true NODE_ENV=test FAKE_PROBE_IP=api NEW_RELIC_ENABLED=false NEW_RELIC_LOG_ENABLED=false mocha",
 		"test:perf": "tsx test-perf/index.ts",
 		"test:portman": "TEST_DONT_RESTART_WORKERS=1 start-test 'npm run start:test' http://localhost:3000/health 'npm run test:portman:create && npm run test:portman:run' 2> /dev/null || E=$?; rm -rf tmp; exit $E;",
 		"test:portman:create": "portman --cliOptionsFile test/tests/contract/portman-cli.json",

src/lib/geoip/fake-client.ts

@@ -0,0 +1,18 @@
+import type { ProbeLocation } from '../../probe/types';
+
+export const fakeLookup = (): ProbeLocation => {
+	return {
+		continent: 'SA',
+		country: 'AR',
+		state: undefined,
+		city: 'Buenos Aires',
+		region: 'South America',
+		normalizedRegion: 'south america',
+		normalizedCity: 'buenos aires',
+		asn: 61003,
+		latitude: -34.6131,
+		longitude: -58.3772,
+		network: 'InterBS S.R.L. (BAEHOST)',
+		normalizedNetwork: 'interbs s.r.l. (baehost)',
+	};
+};

src/lib/get-probe-ip.ts

@@ -1,8 +1,30 @@
-import type { IncomingMessage } from 'node:http';
+import _ from 'lodash';
 import requestIp from 'request-ip';
+import type { Socket } from 'socket.io';
 
-const getProbeIp = (request: IncomingMessage) => {
-	const clientIp = requestIp.getClientIp(request);
+const getProbeIp = (socket: Socket) => {
+	// Use random ip assigned by the API
+	if (process.env['FAKE_PROBE_IP'] === 'api') {
+		return _.sample([
+			'18.200.0.1',
+			'34.140.0.10',
+			'95.155.94.127',
+			'65.49.22.66',
+			'185.229.226.83',
+			'51.158.22.211',
+			'131.255.7.26',
+			'213.136.174.80',
+			'94.214.253.78',
+			'79.205.97.254',
+		]);
+	}
+
+	// Use fake ip provided by the probe
+	if (process.env['FAKE_PROBE_IP'] === 'probe') {
+		return socket.handshake.query['fakeIp'] as string;
+	}
+
+	const clientIp = requestIp.getClientIp(socket.request);
 
 	if (!clientIp) {
 		return null;

src/lib/ws/helper/error-handler.ts

@@ -20,7 +20,7 @@ const isError = (error: unknown): error is Error => Boolean(error as Error['mess
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 export const errorHandler = (next: NextArgument) => (socket: Socket, mwNext?: (error?: any) => void) => {
 	next(socket, mwNext!).catch((error) => { // eslint-disable-line @typescript-eslint/no-non-null-assertion
-		const clientIp = getProbeIp(socket.request) ?? '';
+		const clientIp = getProbeIp(socket) ?? '';
 		const reason = isError(error) ? error.message : 'unknown';
 
 		logger.info(`disconnecting client ${socket.id} for (${reason}) [${clientIp}]`);

src/lib/ws/helper/probe-ip-limit.ts

@@ -1,4 +1,3 @@
-import * as process from 'node:process';
 import { fetchSockets } from '../server.js';
 import { scopedLogger } from '../../logger.js';
 import { InternalError } from '../../internal-error.js';
@@ -7,7 +6,7 @@ import type { LRUOptions } from './throttle.js';
 const logger = scopedLogger('ws:limit');
 
 export const verifyIpLimit = async (ip: string, socketId: string): Promise<void> => {
-	if (process.env['FAKE_PROBE_IP']) {
+	if (process.env['FAKE_PROBE_IP'] === 'api') {
 		return;
 	}
 

src/lib/ws/helper/throttle.ts

@@ -1,5 +1,6 @@
 import { LRUCache } from 'lru-cache';
 
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
 export type LRUOptions = LRUCache.FetchOptions<object, any, unknown>;
 
 export const throttle = <Value>(func: () => Promise<Value>, time: number) => {

src/lib/ws/middleware/probe-metadata.ts

@@ -10,7 +10,7 @@ import getProbeIp from '../../get-probe-ip.js';
 const logger = scopedLogger('probe-metadata');
 
 export const probeMetadata = errorHandler(async (socket: Socket, next: (error?: ExtendedError) => void) => {
-	const clientIp = getProbeIp(socket.request);
+	const clientIp = getProbeIp(socket);
 
 	try {
 		socket.data['probe'] = await buildProbe(socket);

src/probe/builder.ts

@@ -1,5 +1,4 @@
 import * as process from 'node:process';
-import _ from 'lodash';
 import type { Socket } from 'socket.io';
 import { isIpPrivate } from '../lib/private-ip.js';
 import semver from 'semver';
@@ -18,21 +17,7 @@ import getProbeIp from '../lib/get-probe-ip.js';
 import { getRegion } from '../lib/ip-ranges.js';
 import type { Probe, ProbeLocation, Tag } from './types.js';
 import { verifyIpLimit } from '../lib/ws/helper/probe-ip-limit.js';
-
-const fakeIpForDebug = () => {
-	return _.sample([
-		'18.200.0.1', // aws-eu-west-1
-		'34.140.0.10', // gcp-europe-west1
-		'95.155.94.127',
-		'65.49.22.66',
-		'185.229.226.83',
-		'51.158.22.211',
-		'131.255.7.26',
-		'213.136.174.80',
-		'94.214.253.78',
-		'79.205.97.254',
-	]);
-};
+import { fakeLookup } from '../lib/geoip/fake-client.js';
 
 const geoipClient = createGeoipClient();
 
@@ -43,7 +28,7 @@ export const buildProbe = async (socket: Socket): Promise<Probe> => {
 
 	const host = process.env['HOSTNAME'] ?? '';
 
-	const clientIp = process.env['FAKE_PROBE_IP'] ? fakeIpForDebug() : getProbeIp(socket.request);
+	const clientIp = getProbeIp(socket);
 
 	if (!clientIp) {
 		throw new Error('failed to detect ip address of connected probe');
@@ -55,7 +40,9 @@ export const buildProbe = async (socket: Socket): Promise<Probe> => {
 
 	let ipInfo;
 
-	if (!isIpPrivate(clientIp)) {
+	if (process.env['FAKE_PROBE_IP'] === 'probe') {
+		ipInfo = fakeLookup();
+	} else if (!isIpPrivate(clientIp)) {
 		ipInfo = await geoipClient.lookup(clientIp);
 	}
 

test-perf/artillery.yml

@@ -12,11 +12,9 @@ scenarios:
   - name: "Ping 100 probes"
     flow:
       - post:
-          url: "/measurements"
+          url: "/v1/measurements?adminkey=admin"
           json:
             target: "google.com"
-            type: "ping"
-            measurementOptions:
-              packets: 16
+            type: "mtr"
             limit: "{{ $processEnvironment.LIMIT }}"
             locations: []

test-perf/index.ts

@@ -6,12 +6,13 @@ import fs from 'node:fs';
 // Config
 
 const config = {
-	host: 'https://api.globalping.io/v1',
-	delay: 300, // Time to wait between measurements
+	host: 'http://localhost:3000',
+	delay: 5 * 60, // Time to wait between measurements
 	measurements: [
-		{ probes: 100, rps: 1, duration: 240 },
-		{ probes: 100, rps: 2, duration: 240 },
-		{ probes: 100, rps: 5, duration: 240 },
+		{ probes: 200, rps: 1, duration: 2 * 60 },
+		{ probes: 300, rps: 1, duration: 2 * 60 },
+		{ probes: 500, rps: 1, duration: 2 * 60 },
+		{ probes: 500, rps: 2, duration: 2 * 60 },
 	],
 };