Skip to content
This repository was archived by the owner on Jun 18, 2021. It is now read-only.

Commit b1235d4

Browse files
teogebteogeb
authored
NET-261: Improve logging (#263)
Logs using Logger from streamr-network New run-all.js script which runs all brokers and trackers and colorizes the output. We can set log level either globally or separately for each application: export LOG_LEVEL='debug' export LOG_LEVEL_T1='info' Uses short names T1, T2, T3 , S1, B1 and B2 for Trackers/Brokers in development environment. Also disabled reporting from development nodes. Changed log level for some classes from debug to trace.
1 parent b6990d5 commit b1235d4

30 files changed

+288
-161
lines changed

‎bin/run-all.js‎

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
#!/usr/bin/env node
2+
const { fork } = require('child_process')
3+
const chalk = require('chalk')
4+
const { Transform } = require('stream')
5+
const { Logger } = require('streamr-network')
6+
7+
const processes = new Map()
8+
9+
const killProcesses = () => {
10+
processes.forEach((p) => {
11+
if (!p.killed) {
12+
p.kill()
13+
}
14+
})
15+
}
16+
17+
const createCollectLineTransform = () => {
18+
let prevMsg = ''
19+
return new Transform({
20+
transform: (buf, _encoding, done) => {
21+
const msg = buf.toString()
22+
if (msg.endsWith('\n')) {
23+
const fullMsg = prevMsg + msg
24+
done(null, fullMsg)
25+
prevMsg = ''
26+
} else {
27+
prevMsg = msg
28+
done()
29+
}
30+
}
31+
})
32+
}
33+
34+
const createColorizeTransform = (defaultColor) => {
35+
const getLevelColor = (id) => {
36+
if (id === 'ERROR') {
37+
return chalk.hex('#FF0000')
38+
} else if (id === 'WARN') {
39+
return chalk.hex('#FFFF00')
40+
} else {
41+
return undefined
42+
}
43+
}
44+
const META_PATTERN = new RegExp(`^(ERROR|WARN|INFO|DEBUG|TRACE) \\[(.{23})\\] \\((.{${Logger.NAME_LENGTH}})\\):`)
45+
return new Transform({
46+
transform: (buf, _encoding, done) => {
47+
const msg = buf.toString()
48+
const lines = msg.split('\n').map((line) => {
49+
const groups = line.match(META_PATTERN)
50+
if (groups !== null) {
51+
const match = groups[0]
52+
const level = groups[1]
53+
const time = groups[2]
54+
const name = groups[3]
55+
const levelColor = getLevelColor(level)
56+
const metaPrefix = `${level.substring(0, 1)} ${time} ${name} `
57+
const msg = line.substring(match.length)
58+
if (levelColor !== undefined) {
59+
return levelColor(`${metaPrefix}${msg}`)
60+
} else {
61+
return `${defaultColor(metaPrefix)}${msg}`
62+
}
63+
} else {
64+
return line
65+
}
66+
})
67+
done(null, lines.join('\n'))
68+
}
69+
})
70+
}
71+
72+
const pipeOutputStreams = (p, defaultColor) => {
73+
p.stdout
74+
.pipe(createCollectLineTransform())
75+
.pipe(createColorizeTransform(defaultColor))
76+
.pipe(process.stdout)
77+
p.stderr
78+
.pipe(process.stderr)
79+
}
80+
81+
const getProcessLogLevel = (processName) => {
82+
const key = `LOG_LEVEL_${processName}`
83+
return process.env[key] || process.env.LOG_LEVEL
84+
}
85+
86+
const forkProcess = (processName, filePath, args, color) => {
87+
const p = fork(filePath, args, {
88+
silent: true,
89+
env: {
90+
STREAMR_APPLICATION_ID: processName,
91+
LOG_LEVEL: getProcessLogLevel(processName),
92+
LOG_COLORS: 'false'
93+
}
94+
})
95+
processes.set(processName, p)
96+
pipeOutputStreams(p, color)
97+
p.on('close', () => {
98+
killProcesses()
99+
})
100+
}
101+
102+
forkProcess('T1', './tracker.js', ['0xa4fc7b912ed4588080ac9f2044cb3d6cab961bcf0d96de6fa9e773860bf59e1c', 'T1', '--port=30301'], chalk.hex('#66CC66')) // 0xDE11165537ef6C01260ee89A850a281525A5b63F
103+
forkProcess('T2', './tracker.js', ['0x3b328103c0d043cd1b34f1203c0313eb70c98443444dfdf7125f2d4479b89eb9', 'T2', '--port=30302'], chalk.hex('#00FF66')) // 0xDE22222da3F861c2Ec63b03e16a1dce153Cf069c
104+
forkProcess('T3', './tracker.js', ['0xf456bb2a958574416f4b9e1b1b642cb3092953aa6131aba6e36f7529d1137795', 'T3', '--port=30303'], chalk.hex('#66FFAA')) // 0xDE33390cC85aBf61d9c27715Fa61d8E5efC61e75
105+
106+
forkProcess('S1', './broker.js', ['../configs/development-1.env.json'], chalk.hex('#8888FF')) // 0xde1112f631486CfC759A50196853011528bC5FA0
107+
forkProcess('B1', './broker.js', ['../configs/development-2.env.json'], chalk.hex('#0088FF')) // 0xde222E8603FCf641F928E5F66a0CBf4de70d5352
108+
forkProcess('B2', './broker.js', ['../configs/development-3.env.json'], chalk.hex('#88CCFF')) // 0xde3331cA6B8B636E0b82Bf08E941F727B8927442

‎bin/tracker.js‎

Lines changed: 9 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,12 @@
11
#!/usr/bin/env node
22
const program = require('commander')
3-
const { startTracker } = require('streamr-network')
3+
const { startTracker, Logger } = require('streamr-network')
44
const Sentry = require('@sentry/node')
5-
const pino = require('pino')
65
const ethers = require('ethers')
76

87
const CURRENT_VERSION = require('../package.json').version
9-
const { getLogger } = require('../dist/src/helpers/logger')
108

11-
const logger = getLogger('streamr:broker:tracker')
9+
const logger = new Logger(module)
1210

1311
program
1412
.version(CURRENT_VERSION)
@@ -79,20 +77,20 @@ async function main() {
7977
...trackerObj
8078
})
8179
} catch (err) {
82-
pino.final(logger).error(err, 'tracker bin catch')
80+
logger.getFinalLogger().error(err, 'tracker bin catch')
8381
process.exit(1)
8482
}
8583
}
8684

8785
main()
8886

8987
// pino.finalLogger
90-
process.on('uncaughtException', pino.final(logger, (err, finalLogger) => {
91-
finalLogger.error(err, 'uncaughtException')
88+
process.on('uncaughtException', (err) => {
89+
logger.getFinalLogger().error(err, 'uncaughtException')
9290
process.exit(1)
93-
}))
91+
})
9492

95-
process.on('unhandledRejection', pino.final(logger, (err, finalLogger) => {
96-
finalLogger.error(err, 'unhandledRejection')
93+
process.on('unhandledRejection', (err) => {
94+
logger.getFinalLogger().error(err, 'unhandledRejection')
9795
process.exit(1)
98-
}))
96+
})

‎configs/development-1.env.json‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
{
22
"ethereumPrivateKey": "0xaa7a3b3bb9b4a662e756e978ad8c6464412e7eef1b871f19e5120d4747bce966",
33
"network": {
4-
"name": "development-node-1",
4+
"name": "S1",
55
"hostname": "127.0.0.1",
66
"port": 30315,
77
"advertisedWsUrl": null,
@@ -31,7 +31,7 @@
3131
"refreshInterval": 10000
3232
},
3333
"reporting": {
34-
"intervalInSeconds": 10,
34+
"intervalInSeconds": 0,
3535
"sentry": null,
3636
"streamr": null,
3737
"perNodeMetrics": null

‎configs/development-2.env.json‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
{
22
"ethereumPrivateKey": "0xa51ef2b7a1f160e53cc3a17c5a74ea0563a50030644e9f6e1b1f4a7d0afd088e",
33
"network": {
4-
"name": "development-node-2",
4+
"name": "B1",
55
"hostname": "127.0.0.1",
66
"port": 30316,
77
"advertisedWsUrl": null,
@@ -21,7 +21,7 @@
2121
"cassandra": null,
2222
"storageConfig": null,
2323
"reporting": {
24-
"intervalInSeconds": 10,
24+
"intervalInSeconds": 0,
2525
"sentry": null,
2626
"streamr": null,
2727
"perNodeMetrics": null

‎configs/development-3.env.json‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
{
22
"ethereumPrivateKey": "0x2ba0e218041e81b8e5c88ee4134995c1e358c2c01664676d975792057c57e333",
33
"network": {
4-
"name": "development-node-3",
4+
"name": "B2",
55
"hostname": "127.0.0.1",
66
"port": 30317,
77
"advertisedWsUrl": null,
@@ -21,7 +21,7 @@
2121
"cassandra": null,
2222
"storageConfig": null,
2323
"reporting": {
24-
"intervalInSeconds": 10,
24+
"intervalInSeconds": 0,
2525
"sentry": null,
2626
"streamr": null,
2727
"perNodeMetrics": null

‎package-lock.json‎

Lines changed: 69 additions & 31 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@
6666
"qs": "^6.10.1",
6767
"split2": "^3.2.2",
6868
"streamr-client": "^5.2.1",
69-
"streamr-network": "^24.2.2",
69+
"streamr-network": "^24.3.0",
7070
"ts-jest": "^26.5.4",
7171
"typescript": "^4.2.4",
7272
"uWebSockets.js": "github:uNetworking/uWebSockets.js#v18.14.0",

‎src/StreamFetcher.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,12 @@
11
import fetch, { Headers, Response } from 'node-fetch'
22
import memoize from 'memoizee'
3-
import { getLogger } from './helpers/logger'
3+
import { Logger } from 'streamr-network'
44
import { HttpError } from './errors/HttpError'
55
// TODO do all REST operations to E&E via StreamrClient
66
import StreamrClient from 'streamr-client'
77
import { Todo } from './types'
88

9-
const logger = getLogger('streamr:StreamFetcher')
9+
const logger = new Logger(module)
1010

1111
const MAX_AGE = 15 * 60 * 1000 // 15 minutes
1212
const MAX_AGE_MINUTE = 1000 // 1 minutes

‎src/StreamMetrics.ts‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,8 @@
11
import { StreamOperation, StreamrClient } from "streamr-client"
22
import { MetricsContext } from 'streamr-network'
3+
import { Logger } from 'streamr-network'
34

4-
import { getLogger } from './helpers/logger'
5-
6-
const logger = getLogger('streamr:StreamMetrics')
5+
const logger = new Logger(module)
76

87
const throttledAvg = (avg: number, avgInterval: number) => {
98
return (0.8 * avg) + (0.2 * avgInterval)

0 commit comments

Comments
 (0)