-
Notifications
You must be signed in to change notification settings - Fork 2
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Feat: Created daily_node_metrics table and received/stored station_id (…
…#188) * Created daily_node_metrics table and received/stored station_id * Enriching tests and small syntax/structure changes * Converted 'metric_date' column to 'day' * Use bulk insert for updating daily_node_metrics * Accept 88-char station IDs instead, naming changes --------- Co-authored-by: Miroslav Bajtoš <oss@bajtos.net> Co-authored-by: Julian Gruber <julian@juliangruber.com>
- Loading branch information
1 parent
aafadc7
commit 05da282
Showing
8 changed files
with
132 additions
and
1 deletion.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,29 @@ | ||
import createDebug from 'debug' | ||
|
||
const debug = createDebug('spark:platform-stats') | ||
|
||
/** | ||
* @param {import('pg').Client} pgClient | ||
* @param {import('./preprocess').Measurement[]} honestMeasurements | ||
*/ | ||
export const updatePlatformStats = async (pgClient, honestMeasurements) => { | ||
await updateDailyStationStats(pgClient, honestMeasurements) | ||
} | ||
|
||
/** | ||
* @param {import('pg').Client} pgClient | ||
* @param {import('./preprocess').Measurement[]} honestMeasurements | ||
*/ | ||
export const updateDailyStationStats = async (pgClient, honestMeasurements) => { | ||
// TODO: when we add more fields, we will update the ON CONFLICT clause | ||
// to update those fields, and we won't just use a Set for the stationIds | ||
// which currently removes all granular measurement details | ||
const stationIds = [...new Set(honestMeasurements.map(m => m.stationId))] | ||
debug('Updating daily station stats, unique_count=%s', stationIds.length) | ||
|
||
await pgClient.query(` | ||
INSERT INTO daily_stations (station_id, day) | ||
SELECT unnest($1::text[]), now() | ||
ON CONFLICT (station_id, day) DO NOTHING | ||
`, [stationIds]) | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,5 @@ | ||
CREATE TABLE daily_stations ( | ||
day DATE NOT NULL, | ||
station_id TEXT NOT NULL, | ||
PRIMARY KEY (day, station_id) | ||
) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,81 @@ | ||
import assert from 'node:assert' | ||
import pg from 'pg' | ||
import { beforeEach, describe, it } from 'mocha' | ||
|
||
import { DATABASE_URL } from '../lib/config.js' | ||
import { migrateWithPgClient } from '../lib/migrate.js' | ||
import { VALID_MEASUREMENT, VALID_STATION_ID } from './helpers/test-data.js' | ||
import { updateDailyStationStats } from '../lib/platform-stats.js' | ||
|
||
const createPgClient = async () => { | ||
const pgClient = new pg.Client({ connectionString: DATABASE_URL }) | ||
await pgClient.connect() | ||
return pgClient | ||
} | ||
|
||
describe('platform-stats', () => { | ||
let pgClient | ||
before(async () => { | ||
pgClient = await createPgClient() | ||
await migrateWithPgClient(pgClient) | ||
}) | ||
|
||
let today | ||
beforeEach(async () => { | ||
await pgClient.query('DELETE FROM daily_stations') | ||
|
||
// Run all tests inside a transaction to ensure `now()` always returns the same value | ||
// See https://dba.stackexchange.com/a/63549/125312 | ||
// This avoids subtle race conditions when the tests are executed around midnight. | ||
await pgClient.query('BEGIN TRANSACTION') | ||
today = await getCurrentDate() | ||
}) | ||
|
||
afterEach(async () => { | ||
await pgClient.query('END TRANSACTION') | ||
}) | ||
|
||
after(async () => { | ||
await pgClient.end() | ||
}) | ||
|
||
describe('updateDailyStationStats', () => { | ||
it('updates daily station stats for today with multiple measurements', async () => { | ||
const validStationId2 = VALID_STATION_ID.slice(0, -1) + '1' | ||
const honestMeasurements = [ | ||
{ ...VALID_MEASUREMENT, stationId: VALID_STATION_ID }, | ||
{ ...VALID_MEASUREMENT, stationId: validStationId2 } | ||
] | ||
|
||
await updateDailyStationStats(pgClient, honestMeasurements) | ||
|
||
const { rows } = await pgClient.query(` | ||
SELECT station_id, day::TEXT FROM daily_stations | ||
ORDER BY station_id` | ||
) | ||
assert.strictEqual(rows.length, 2) | ||
assert.deepStrictEqual(rows, [ | ||
{ station_id: VALID_STATION_ID, day: today }, | ||
{ station_id: validStationId2, day: today } | ||
]) | ||
}) | ||
|
||
it('ignores duplicate measurements for the same station on the same day', async () => { | ||
const honestMeasurements = [ | ||
{ ...VALID_MEASUREMENT, stationId: VALID_STATION_ID }, | ||
{ ...VALID_MEASUREMENT, stationId: VALID_STATION_ID } | ||
] | ||
|
||
await updateDailyStationStats(pgClient, honestMeasurements) | ||
|
||
const { rows } = await pgClient.query('SELECT station_id, day::TEXT FROM daily_stations') | ||
assert.strictEqual(rows.length, 1) | ||
assert.deepStrictEqual(rows, [{ station_id: VALID_STATION_ID, day: today }]) | ||
}) | ||
}) | ||
|
||
const getCurrentDate = async () => { | ||
const { rows: [{ today }] } = await pgClient.query('SELECT now()::DATE::TEXT as today') | ||
return today | ||
} | ||
}) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters