From 0361f2374cdfad9cb55c1177e70c012a0c20c0d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Kol=C3=A1rik?= Date: Fri, 6 Oct 2023 18:22:17 +0200 Subject: [PATCH] feat: add network tags --- public/v1/components/examples.yaml | 23 +++++++++++----- public/v1/components/schemas.yaml | 1 + src/lib/geoip/client.ts | 3 +++ src/lib/geoip/providers/ipinfo.ts | 4 +++ src/probe/builder.ts | 16 +++++++++-- src/probe/types.ts | 1 + test/mocks/nock-geoip.json | 15 ++++++++--- .../measurement/probe-communication.test.ts | 10 +++---- .../measurement/timeout-result.test.ts | 4 +-- .../integration/probes/get-probes.test.ts | 27 ++++++++++++++++--- test/tests/unit/geoip/client.test.ts | 22 +++++++++++++++ 11 files changed, 104 insertions(+), 22 deletions(-) diff --git a/public/v1/components/examples.yaml b/public/v1/components/examples.yaml index 09e928c3..5f3b987b 100644 --- a/public/v1/components/examples.yaml +++ b/public/v1/components/examples.yaml @@ -17,7 +17,8 @@ components: "longitude": -77.4875 }, "tags": [ - "aws-us-east-1" + "aws-us-east-1", + "datacenter-network" ], "resolvers": [ "private" @@ -106,7 +107,9 @@ components: "longitude": 174.76667, "latitude": -36.86667, "network": "Zappie Host LLC", - "tags": [ ], + "tags": [ + "datacenter-network" + ], "resolvers": [ "1.1.1.1", "8.8.8.8" @@ -146,7 +149,9 @@ components: "longitude": 151.207052, "latitude": -33.86778, "network": "OVH SAS", - "tags": [ ], + "tags": [ + "datacenter-network" + ], "resolvers": [ "213.186.33.99" ] @@ -195,7 +200,9 @@ components: "longitude": 151.207052, "latitude": -33.86778, "network": "G-Core Labs S.A.", - "tags": [ ], + "tags": [ + "datacenter-network" + ], "resolvers": [ "8.8.8.8" ] @@ -352,7 +359,9 @@ components: "longitude": 28.04355, "latitude": -26.20225, "network": "G-Core Labs S.A.", - "tags": [ ], + "tags": [ + "datacenter-network" + ], "resolvers": [ "private" ] @@ -417,7 +426,9 @@ components: "longitude": 126.977207, "latitude": 37.566309, "network": "Psychz Networks", - "tags": [ ], + "tags": [ + "datacenter-network" + ], "resolvers": [ "8.8.8.8", "8.8.4.4" diff --git a/public/v1/components/schemas.yaml b/public/v1/components/schemas.yaml index 530da269..30eba39b 100644 --- a/public/v1/components/schemas.yaml +++ b/public/v1/components/schemas.yaml @@ -709,6 +709,7 @@ components: An array of additional values that can be used to target the probe. Probes hosted in [AWS](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions) and [Google Cloud](https://cloud.google.com/compute/docs/regions-zones#available) are automatically assigned the service region code. + Probes are also automatically assigned `datacenter-network` and `eyeball-network` tags to distinguish between datacenter and end-user locations. items: type: string TestResult: diff --git a/src/lib/geoip/client.ts b/src/lib/geoip/client.ts index 95c6120a..8432a546 100644 --- a/src/lib/geoip/client.ts +++ b/src/lib/geoip/client.ts @@ -37,6 +37,7 @@ export default class GeoipClient { constructor (private readonly cache: CacheInterface) {} async lookup (addr: string): Promise { + let isHosting = undefined; const results = await Promise .allSettled([ this.lookupWithCache(`geoip:ipinfo:${addr}`, async () => ipinfoLookup(addr)), @@ -46,6 +47,7 @@ export default class GeoipClient { this.lookupWithCache(`geoip:fastly:${addr}`, async () => fastlyLookup(addr)), ]) .then(([ ipinfo, ip2location, maxmind, ipmap, fastly ]) => { + isHosting = ipinfo.status === 'fulfilled' ? ipinfo.value.isHosting : undefined; const fulfilled: (LocationInfoWithProvider | null)[] = []; // Providers here are pushed in a desc prioritized order @@ -92,6 +94,7 @@ export default class GeoipClient { longitude: Number(match.longitude), network: networkMatch.network, normalizedNetwork: networkMatch.normalizedNetwork, + isHosting, }; } diff --git a/src/lib/geoip/providers/ipinfo.ts b/src/lib/geoip/providers/ipinfo.ts index e49b275f..ce6664c0 100644 --- a/src/lib/geoip/providers/ipinfo.ts +++ b/src/lib/geoip/providers/ipinfo.ts @@ -15,6 +15,9 @@ type IpinfoResponse = { region: string | undefined; org: string | undefined; loc: string | undefined; + privacy?: { + hosting: boolean; + } }; export const ipinfoLookup = async (addr: string): Promise => { @@ -40,5 +43,6 @@ export const ipinfoLookup = async (addr: string): Promise => { longitude: Number(lon), network, normalizedNetwork: normalizeNetworkName(network), + isHosting: result.privacy?.hosting, }; }; diff --git a/src/probe/builder.ts b/src/probe/builder.ts index 7b2f891b..de509bc3 100644 --- a/src/probe/builder.ts +++ b/src/probe/builder.ts @@ -54,7 +54,7 @@ export const buildProbe = async (socket: Socket): Promise => { const location = getLocation(ipInfo); - const tags = getTags(clientIp); + const tags = getTags(clientIp, ipInfo); // Storing index as string[][] so every category will have it's exact position in the index array across all probes const index = [ @@ -112,7 +112,7 @@ const getLocation = (ipInfo: ProbeLocation): ProbeLocation => ({ normalizedNetwork: ipInfo.normalizedNetwork, }); -const getTags = (clientIp: string) => { +const getTags = (clientIp: string, ipInfo: ProbeLocation) => { const tags: Tag[] = []; const cloudRegion = getRegion(clientIp); @@ -123,5 +123,17 @@ const getTags = (clientIp: string) => { }); } + if (ipInfo.isHosting === true) { + tags.push({ + type: 'system', + value: 'datacenter-network', + }); + } else if (ipInfo.isHosting === false) { + tags.push({ + type: 'system', + value: 'eyeball-network', + }); + } + return tags; }; diff --git a/src/probe/types.ts b/src/probe/types.ts index 047992c6..cf9665c9 100644 --- a/src/probe/types.ts +++ b/src/probe/types.ts @@ -11,6 +11,7 @@ export type ProbeLocation = { state: string | undefined; network: string; normalizedNetwork: string; + isHosting?: boolean | undefined; }; export type ProbeStats = { diff --git a/test/mocks/nock-geoip.json b/test/mocks/nock-geoip.json index 237612f0..23b7ef38 100644 --- a/test/mocks/nock-geoip.json +++ b/test/mocks/nock-geoip.json @@ -328,7 +328,10 @@ "region": "Texas", "country": "US", "loc": "32.7831,-96.8067", - "org": "AS20004 The Constant Company LLC" + "org": "AS20004 The Constant Company LLC", + "privacy": { + "hosting": true + } }, "argentina": { "city": "Buenos Aires", @@ -363,14 +366,20 @@ "region": "New York", "country": "US", "loc": "40.7143,-74.0060", - "org": "AS80004 The Constant Company LLC" + "org": "AS80004 The Constant Company LLC", + "privacy": { + "hosting": true + } }, "washington": { "city": "Washington", "region": "Washington, D.C.", "country": "US", "loc": "38.89539,-77.039476", - "org": "AS701 Verizon Business" + "org": "AS701 Verizon Business", + "privacy": { + "hosting": false + } }, "emptyNetwork": { "city": "Dallas", diff --git a/test/tests/integration/measurement/probe-communication.test.ts b/test/tests/integration/measurement/probe-communication.test.ts index 23bd41dc..74aec936 100644 --- a/test/tests/integration/measurement/probe-communication.test.ts +++ b/test/tests/integration/measurement/probe-communication.test.ts @@ -113,7 +113,7 @@ describe('Create measurement request', () => { longitude: -96.8067, latitude: 32.7831, network: 'The Constant Company LLC', - tags: [ 'gcp-us-west4' ], + tags: [ 'gcp-us-west4', 'datacenter-network' ], resolvers: [], }, result: { @@ -159,7 +159,7 @@ describe('Create measurement request', () => { longitude: -96.8067, latitude: 32.7831, network: 'The Constant Company LLC', - tags: [ 'gcp-us-west4' ], + tags: [ 'gcp-us-west4', 'datacenter-network' ], resolvers: [], }, result: { status: 'in-progress', rawOutput: 'abc' }, @@ -200,7 +200,7 @@ describe('Create measurement request', () => { longitude: -96.8067, latitude: 32.7831, network: 'The Constant Company LLC', - tags: [ 'gcp-us-west4' ], + tags: [ 'gcp-us-west4', 'datacenter-network' ], resolvers: [], }, result: { status: 'in-progress', rawOutput: 'abcdef' }, @@ -255,7 +255,7 @@ describe('Create measurement request', () => { longitude: -96.8067, latitude: 32.7831, network: 'The Constant Company LLC', - tags: [ 'gcp-us-west4' ], + tags: [ 'gcp-us-west4', 'datacenter-network' ], resolvers: [], }, result: { @@ -324,7 +324,7 @@ describe('Create measurement request', () => { longitude: -96.8067, network: 'The Constant Company LLC', }, - tags: [ 'gcp-us-west4' ], + tags: [ 'gcp-us-west4', 'datacenter-network' ], resolvers: [], host: '', stats: { diff --git a/test/tests/integration/measurement/timeout-result.test.ts b/test/tests/integration/measurement/timeout-result.test.ts index dbfd5bf6..52a19c27 100644 --- a/test/tests/integration/measurement/timeout-result.test.ts +++ b/test/tests/integration/measurement/timeout-result.test.ts @@ -73,7 +73,7 @@ describe('Timeout results', () => { longitude: -96.8067, latitude: 32.7831, network: 'The Constant Company LLC', - tags: [ 'gcp-us-west4' ], + tags: [ 'gcp-us-west4', 'datacenter-network' ], resolvers: [], }, result: { status: 'in-progress', rawOutput: '' }, @@ -112,7 +112,7 @@ describe('Timeout results', () => { longitude: -96.8067, latitude: 32.7831, network: 'The Constant Company LLC', - tags: [ 'gcp-us-west4' ], + tags: [ 'gcp-us-west4', 'datacenter-network' ], resolvers: [], }, result: { status: 'failed', rawOutput: '\n\nThe measurement timed out' }, diff --git a/test/tests/integration/probes/get-probes.test.ts b/test/tests/integration/probes/get-probes.test.ts index 01c21f08..8512b50d 100644 --- a/test/tests/integration/probes/get-probes.test.ts +++ b/test/tests/integration/probes/get-probes.test.ts @@ -126,7 +126,7 @@ describe('Get Probes', () => { longitude: -96.8067, network: 'The Constant Company LLC', }, - tags: [], + tags: [ 'datacenter-network' ], resolvers: [], }, ]); @@ -135,17 +135,20 @@ describe('Get Probes', () => { }); }); - it('should detect 3 probes in "ready: true" status', async () => { + it('should detect 4 probes in "ready: true" status', async () => { nockGeoIpProviders({ ip2location: 'argentina', ipmap: 'argentina', maxmind: 'argentina', ipinfo: 'argentina', fastly: 'argentina' }); nockGeoIpProviders(); nockGeoIpProviders({ ip2location: 'newYork', ipmap: 'argentina', maxmind: 'newYork', ipinfo: 'newYork', fastly: 'newYork' }); + nockGeoIpProviders({ ip2location: 'washington', ipmap: 'argentina', maxmind: 'default', ipinfo: 'washington', fastly: 'newYork' }); const probe1 = await addProbe(); const probe2 = await addProbe(); const probe3 = await addProbe(); + const probe4 = await addProbe(); probe1.emit('probe:status:update', 'ready'); probe2.emit('probe:status:update', 'ready'); probe3.emit('probe:status:update', 'ready'); + probe4.emit('probe:status:update', 'ready'); await requestAgent.get('/v1/probes') .send() @@ -180,7 +183,7 @@ describe('Get Probes', () => { longitude: -96.8067, network: 'The Constant Company LLC', }, - tags: [], + tags: [ 'datacenter-network' ], resolvers: [], }, { @@ -196,7 +199,23 @@ describe('Get Probes', () => { longitude: -74.0060, network: 'The Constant Company LLC', }, - tags: [], + tags: [ 'datacenter-network' ], + resolvers: [], + }, + { + version: '0.14.0', + location: { + asn: 701, + city: 'Washington', + continent: 'NA', + country: 'US', + latitude: 38.89539, + longitude: -77.039476, + network: 'Verizon Business', + region: 'Northern America', + state: 'DC', + }, + tags: [ 'eyeball-network' ], resolvers: [], }, ]); diff --git a/test/tests/unit/geoip/client.test.ts b/test/tests/unit/geoip/client.test.ts index 119b78e0..bfa6185e 100644 --- a/test/tests/unit/geoip/client.test.ts +++ b/test/tests/unit/geoip/client.test.ts @@ -50,6 +50,7 @@ describe('geoip service', () => { longitude: -96.8067, network: 'The Constant Company LLC', normalizedNetwork: 'the constant company llc', + isHosting: true, }); }); }); @@ -72,6 +73,7 @@ describe('geoip service', () => { longitude: -58.3772, network: 'InterBS S.R.L. (BAEHOST)', normalizedNetwork: 'interbs s.r.l. (baehost)', + isHosting: true, }); }); @@ -97,6 +99,7 @@ describe('geoip service', () => { longitude: -58.3772, network: 'InterBS S.R.L. (BAEHOST)', normalizedNetwork: 'interbs s.r.l. (baehost)', + isHosting: undefined, }); }); @@ -118,6 +121,7 @@ describe('geoip service', () => { longitude: -96.8067, network: 'The Constant Company LLC', normalizedNetwork: 'the constant company llc', + isHosting: undefined, }); }); @@ -139,6 +143,7 @@ describe('geoip service', () => { longitude: -96.8067, network: 'The Constant Company LLC', normalizedNetwork: 'the constant company llc', + isHosting: undefined, }); }); @@ -160,6 +165,7 @@ describe('geoip service', () => { longitude: -58.3772, network: 'InterBS S.R.L. (BAEHOST)', normalizedNetwork: 'interbs s.r.l. (baehost)', + isHosting: undefined, }); }); @@ -181,6 +187,7 @@ describe('geoip service', () => { longitude: -58.3772, network: 'InterBS S.R.L. (BAEHOST)', normalizedNetwork: 'interbs s.r.l. (baehost)', + isHosting: undefined, }); }); @@ -215,6 +222,7 @@ describe('geoip service', () => { longitude: 26.487, network: 'The Constant Company LLC', normalizedNetwork: 'the constant company llc', + isHosting: undefined, }); }); @@ -236,6 +244,7 @@ describe('geoip service', () => { longitude: 12.371563, network: 'Hetzner Online GmbH', normalizedNetwork: 'hetzner online gmbh', + isHosting: undefined, }); }); @@ -257,6 +266,7 @@ describe('geoip service', () => { longitude: 12.371563, network: 'Hetzner Online GmbH', normalizedNetwork: 'hetzner online gmbh', + isHosting: undefined, }); }); @@ -278,6 +288,7 @@ describe('geoip service', () => { longitude: -96.8067, network: 'The Constant Company LLC', normalizedNetwork: 'the constant company llc', + isHosting: true, }); }); @@ -299,6 +310,7 @@ describe('geoip service', () => { longitude: -58.3772, network: 'InterBS S.R.L. (BAEHOST)', normalizedNetwork: 'interbs s.r.l. (baehost)', + isHosting: undefined, }); }); @@ -320,6 +332,7 @@ describe('geoip service', () => { longitude: -74.0060, network: 'The Constant Company LLC', normalizedNetwork: 'the constant company llc', + isHosting: true, }); }); @@ -341,6 +354,7 @@ describe('geoip service', () => { longitude: -96.8067, network: 'The Constant Company LLC', normalizedNetwork: 'the constant company llc', + isHosting: undefined, }); }); @@ -363,6 +377,7 @@ describe('geoip service', () => { longitude: -96.8067, network: 'The Constant Company LLC', normalizedNetwork: 'the constant company llc', + isHosting: undefined, }); }); @@ -384,6 +399,7 @@ describe('geoip service', () => { longitude: -96.8067, network: 'The Constant Company LLC', normalizedNetwork: 'the constant company llc', + isHosting: undefined, }); }); @@ -405,6 +421,7 @@ describe('geoip service', () => { longitude: -96.8067, network: 'The Constant Company LLC', normalizedNetwork: 'the constant company llc', + isHosting: undefined, }); }); @@ -426,6 +443,7 @@ describe('geoip service', () => { longitude: -77.039476, network: 'Psychz Networks', normalizedNetwork: 'psychz networks', + isHosting: undefined, }); nockGeoIpProviders({ ip2location: 'empty', ipmap: 'empty', maxmind: 'empty', ipinfo: 'washington', fastly: 'empty' }); @@ -445,6 +463,7 @@ describe('geoip service', () => { longitude: -77.039476, network: 'Verizon Business', normalizedNetwork: 'verizon business', + isHosting: false, }); }); @@ -480,6 +499,7 @@ describe('geoip service', () => { longitude: -96.8067, network: 'The Constant Company LLC', normalizedNetwork: 'the constant company llc', + isHosting: true, }); }); @@ -501,6 +521,7 @@ describe('geoip service', () => { longitude: -96.8067, network: 'The Constant Company LLC', normalizedNetwork: 'the constant company llc', + isHosting: true, }); }); @@ -530,6 +551,7 @@ describe('geoip service', () => { longitude: -96.8067, network: 'The Constant Company LLC', normalizedNetwork: 'the constant company llc', + isHosting: true, }); });