Skip to content

Commit daf893f

Browse files
PatrickNercessianPatrickNercessian
committed
Fix: Migrate to dedicated station count endpoints (CheckerNetwork#93)
* Migrate to dedicated station count endpoints * Improved local helper function SQL security * Renamed request helper methods
1 parent d9846dd commit daf893f

File tree

5 files changed

+209
-53
lines changed

5 files changed

+209
-53
lines changed

‎lib/platform-routes.js

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,25 @@
11
import { getStatsWithFilterAndCaching } from './request-helpers.js'
2-
import { fetchDailyStationMetrics } from './platform-stats-fetchers.js'
2+
import { fetchDailyStationCount, fetchMonthlyStationCount } from './platform-stats-fetchers.js'
33

44
export const handlePlatformRoutes = async (req, res, pgPool) => {
55
// Caveat! `new URL('//foo', 'http://127.0.0.1')` would produce "http://foo/" - not what we want!
66
const { pathname, searchParams } = new URL(`http://127.0.0.1${req.url}`)
77
const segs = pathname.split('/').filter(Boolean)
8-
if (req.method === 'GET' && segs[0] === 'stations' && segs[1] === 'raw' && segs.length === 2) {
8+
if (req.method === 'GET' && segs[0] === 'stations' && segs[1] === 'daily' && segs.length === 2) {
99
await getStatsWithFilterAndCaching(
1010
pathname,
1111
searchParams,
1212
res,
1313
pgPool,
14-
fetchDailyStationMetrics)
14+
fetchDailyStationCount)
15+
return true
16+
} else if (req.method === 'GET' && segs[0] === 'stations' && segs[1] === 'monthly' && segs.length === 2) {
17+
await getStatsWithFilterAndCaching(
18+
pathname,
19+
searchParams,
20+
res,
21+
pgPool,
22+
fetchMonthlyStationCount)
1523
return true
1624
} else if (req.method === 'GET' && segs.length === 0) {
1725
// health check - required by Grafana datasources

‎lib/platform-stats-fetchers.js

Lines changed: 33 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,42 @@
1+
import { getDailyDistinctCount, getMonthlyDistinctCount } from './request-helpers.js'
2+
13
/**
24
* @param {import('pg').Pool} pgPool
35
* @param {import('./typings').Filter} filter
46
*/
5-
export const fetchDailyStationMetrics = async (pgPool, filter) => {
7+
export const fetchDailyStationCount = async (pgPool, filter) => {
8+
return await getDailyDistinctCount({
9+
pgPool,
10+
table: 'daily_stations',
11+
column: 'station_id',
12+
filter
13+
})
14+
}
15+
16+
/**
17+
* @param {import('pg').Pool} pgPool
18+
* @param {import('./typings').Filter} filter
19+
*/
20+
export const fetchMonthlyStationCount = async (pgPool, filter) => {
21+
return await getMonthlyDistinctCount({
22+
pgPool,
23+
table: 'daily_stations',
24+
column: 'station_id',
25+
filter
26+
})
27+
}
28+
29+
/**
30+
* @param {import('pg').Pool} pgPool
31+
* @param {import('./typings').Filter} filter
32+
*/
33+
export const fetchDailyStationAcceptedMeasurementCount = async (pgPool, filter) => {
634
const { rows } = await pgPool.query(`
7-
SELECT day::TEXT, station_id, accepted_measurement_count
35+
SELECT day::TEXT, SUM(accepted_measurement_count) as accepted_measurement_count
836
FROM daily_stations
937
WHERE day >= $1 AND day <= $2
10-
GROUP BY day, station_id
11-
`, [
12-
filter.from,
13-
filter.to
14-
])
38+
GROUP BY day
39+
ORDER BY day
40+
`, [filter.from, filter.to])
1541
return rows
1642
}

‎lib/request-helpers.js

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import assert from 'http-assert'
22
import { json } from 'http-responders'
33
import { URLSearchParams } from 'node:url'
4+
import pg from 'pg'
45

56
const getDayAsISOString = (d) => d.toISOString().split('T')[0]
67

@@ -84,3 +85,72 @@ const setCacheControlForStatsResponse = (res, filter) => {
8485
res.setHeader('cache-control', `public, max-age=${365 * 24 * 3600}, immutable`)
8586
}
8687
}
88+
89+
/**
90+
* @param {object} args
91+
* @param {pg.Pool} args.pgPool
92+
* @param {string} args.table
93+
* @param {string} args.column
94+
* @param {import('./typings').Filter} args.filter
95+
* @param {string} [args.asColumn]
96+
*/
97+
export const getDailyDistinctCount = async ({
98+
pgPool,
99+
table,
100+
column,
101+
filter,
102+
asColumn = null
103+
}) => {
104+
if (!asColumn) asColumn = column + '_count'
105+
const safeTable = pg.escapeIdentifier(table)
106+
const safeColumn = pg.escapeIdentifier(column)
107+
const safeAsColumn = pg.escapeIdentifier(asColumn)
108+
109+
// Fetch the "day" (DATE) as a string (TEXT) to prevent node-postgres from converting it into
110+
// a JavaScript Date with a timezone, as that could change the date one day forward or back.
111+
const { rows } = await pgPool.query(`
112+
SELECT day::TEXT, COUNT(DISTINCT ${safeColumn})::INT as ${safeAsColumn}
113+
FROM ${safeTable}
114+
WHERE day >= $1 AND day <= $2
115+
GROUP BY day
116+
ORDER BY day
117+
`, [filter.from, filter.to])
118+
return rows
119+
}
120+
121+
/**
122+
* @param {object} args
123+
* @param {pg.Pool} args.pgPool
124+
* @param {string} args.table
125+
* @param {string} args.column
126+
* @param {import('./typings').Filter} args.filter
127+
* @param {string} [args.asColumn]
128+
*/
129+
export const getMonthlyDistinctCount = async ({
130+
pgPool,
131+
table,
132+
column,
133+
filter,
134+
asColumn = null
135+
}) => {
136+
if (!asColumn) asColumn = column + '_count'
137+
const safeTable = pg.escapeIdentifier(table)
138+
const safeColumn = pg.escapeIdentifier(column)
139+
const safeAsColumn = pg.escapeIdentifier(asColumn)
140+
141+
// Fetch the "day" (DATE) as a string (TEXT) to prevent node-postgres from converting it into
142+
// a JavaScript Date with a timezone, as that could change the date one day forward or back.
143+
const { rows } = await pgPool.query(`
144+
SELECT
145+
date_trunc('month', day)::DATE::TEXT as month,
146+
COUNT(DISTINCT ${safeColumn})::INT as ${safeAsColumn}
147+
FROM ${safeTable}
148+
WHERE
149+
day >= date_trunc('month', $1::DATE)
150+
AND day < date_trunc('month', $2::DATE) + INTERVAL '1 month'
151+
GROUP BY month
152+
ORDER BY month
153+
`, [filter.from, filter.to]
154+
)
155+
return rows
156+
}

‎lib/stats-fetchers.js

Lines changed: 16 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
import { getDailyDistinctCount, getMonthlyDistinctCount } from './request-helpers.js'
2+
13
/**
24
* @param {import('pg').Pool} pgPool
35
* @param {import('./typings').Filter} filter
@@ -22,48 +24,24 @@ export const fetchRetrievalSuccessRate = async (pgPool, filter) => {
2224
return stats
2325
}
2426

25-
/**
26-
* @param {import('pg').Pool} pgPool
27-
* @param {import('./typings').Filter} filter
28-
*/
2927
export const fetchDailyParticipants = async (pgPool, filter) => {
30-
// Fetch the "day" (DATE) as a string (TEXT) to prevent node-postgres from converting it into
31-
// a JavaScript Date with a timezone, as that could change the date one day forward or back.
32-
const { rows } = await pgPool.query(`
33-
SELECT day::TEXT, COUNT(DISTINCT participant_id)::INT as participants
34-
FROM daily_participants
35-
WHERE day >= $1 AND day <= $2
36-
GROUP BY day
37-
ORDER BY day
38-
`, [
39-
filter.from,
40-
filter.to
41-
])
42-
return rows
28+
return await getDailyDistinctCount({
29+
pgPool,
30+
table: 'daily_participants',
31+
column: 'participant_id',
32+
filter,
33+
asColumn: 'participants'
34+
})
4335
}
4436

45-
/**
46-
* @param {import('pg').Pool} pgPool
47-
* @param {import('./typings').Filter} filter
48-
*/
4937
export const fetchMonthlyParticipants = async (pgPool, filter) => {
50-
// Fetch the "day" (DATE) as a string (TEXT) to prevent node-postgres from converting it into
51-
// a JavaScript Date with a timezone, as that could change the date one day forward or back.
52-
const { rows } = await pgPool.query(`
53-
SELECT
54-
date_trunc('month', day)::DATE::TEXT as month,
55-
COUNT(DISTINCT participant_id)::INT as participants
56-
FROM daily_participants
57-
WHERE
58-
day >= date_trunc('month', $1::DATE)
59-
AND day < date_trunc('month', $2::DATE) + INTERVAL '1 month'
60-
GROUP BY month
61-
ORDER BY month
62-
`, [
63-
filter.from,
64-
filter.to
65-
])
66-
return rows
38+
return await getMonthlyDistinctCount({
39+
pgPool,
40+
table: 'daily_participants',
41+
column: 'participant_id',
42+
filter,
43+
asColumn: 'participants'
44+
})
6745
}
6846

6947
/**

‎test/platform-routes.test.js

Lines changed: 79 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ describe('Platform Routes HTTP request handler', () => {
4646
await pgPool.query('DELETE FROM daily_stations')
4747
})
4848

49-
describe('GET /stations/raw', () => {
49+
describe('GET /stations/daily', () => {
5050
it('returns daily station metrics for the given date range', async () => {
5151
await givenDailyStationMetrics(pgPool, '2024-01-10', [
5252
{ station_id: 'station1', accepted_measurement_count: 1 }
@@ -64,7 +64,7 @@ describe('Platform Routes HTTP request handler', () => {
6464

6565
const res = await fetch(
6666
new URL(
67-
'/stations/raw?from=2024-01-11&to=2024-01-12',
67+
'/stations/daily?from=2024-01-11&to=2024-01-12',
6868
baseUrl
6969
), {
7070
redirect: 'manual'
@@ -73,9 +73,83 @@ describe('Platform Routes HTTP request handler', () => {
7373
await assertResponseStatus(res, 200)
7474
const metrics = await res.json()
7575
assert.deepStrictEqual(metrics, [
76-
{ day: '2024-01-11', station_id: 'station2', accepted_measurement_count: 1 },
77-
{ day: '2024-01-12', station_id: 'station2', accepted_measurement_count: 2 },
78-
{ day: '2024-01-12', station_id: 'station3', accepted_measurement_count: 1 }
76+
{ day: '2024-01-11', station_id_count: 1 },
77+
{ day: '2024-01-12', station_id_count: 2 }
78+
])
79+
})
80+
})
81+
82+
describe('GET /stations/monthly', () => {
83+
it('returns monthly station metrics for the given date range ignoring the day number', async () => {
84+
// before the date range
85+
await givenDailyStationMetrics(pgPool, '2023-12-31', [
86+
{ station_id: 'station1', accepted_measurement_count: 1 }
87+
])
88+
// in the date range
89+
await givenDailyStationMetrics(pgPool, '2024-01-10', [
90+
{ station_id: 'station1', accepted_measurement_count: 1 }
91+
])
92+
await givenDailyStationMetrics(pgPool, '2024-01-11', [
93+
{ station_id: 'station2', accepted_measurement_count: 1 }
94+
])
95+
await givenDailyStationMetrics(pgPool, '2024-01-12', [
96+
{ station_id: 'station2', accepted_measurement_count: 2 },
97+
{ station_id: 'station3', accepted_measurement_count: 1 }
98+
])
99+
await givenDailyStationMetrics(pgPool, '2024-02-13', [
100+
{ station_id: 'station1', accepted_measurement_count: 1 }
101+
])
102+
// after the date range
103+
await givenDailyStationMetrics(pgPool, '2024-03-01', [
104+
{ station_id: 'station1', accepted_measurement_count: 1 }
105+
])
106+
107+
const res = await fetch(
108+
new URL(
109+
'/stations/monthly?from=2024-01-11&to=2024-02-11',
110+
baseUrl
111+
), {
112+
redirect: 'manual'
113+
}
114+
)
115+
await assertResponseStatus(res, 200)
116+
const metrics = await res.json()
117+
assert.deepStrictEqual(metrics, [
118+
{ month: '2024-01-01', station_id_count: 3 },
119+
{ month: '2024-02-01', station_id_count: 1 }
120+
])
121+
})
122+
})
123+
124+
describe('GET /measurements/daily', () => {
125+
it('returns daily total accepted measurement count for the given date range', async () => {
126+
await givenDailyStationMetrics(pgPool, '2024-01-10', [
127+
{ station_id: 'station1', accepted_measurement_count: 1 }
128+
])
129+
await givenDailyStationMetrics(pgPool, '2024-01-11', [
130+
{ station_id: 'station2', accepted_measurement_count: 1 }
131+
])
132+
await givenDailyStationMetrics(pgPool, '2024-01-12', [
133+
{ station_id: 'station2', accepted_measurement_count: 2 },
134+
{ station_id: 'station3', accepted_measurement_count: 1 }
135+
])
136+
await givenDailyStationMetrics(pgPool, '2024-01-13', [
137+
{ station_id: 'station1', accepted_measurement_count: 1 }
138+
])
139+
140+
const res = await fetch(
141+
new URL(
142+
'/measurements/daily?from=2024-01-11&to=2024-01-12',
143+
baseUrl
144+
), {
145+
redirect: 'manual'
146+
}
147+
)
148+
await assertResponseStatus(res, 200)
149+
const metrics = await res.json()
150+
assert.deepStrictEqual(metrics, [
151+
{ day: '2024-01-11', accepted_measurement_count: '1' },
152+
{ day: '2024-01-12', accepted_measurement_count: '3' }
79153
])
80154
})
81155
})

0 commit comments

Comments
 (0)