Skip to content

Commit

Permalink
feat: pino logs
Browse files Browse the repository at this point in the history
  • Loading branch information
@sanoel
sanoel committed Oct 10, 2024
1 parent eb0af15 commit 8533c02
Show file tree
Hide file tree
Showing 7 changed files with 261 additions and 194 deletions.
12 changes: 6 additions & 6 deletions package.json
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "@qlever-llc/target-helper",
"version": "2.1.0",
"version": "2.2.0",
"description": "A Trellis microservice to fill in around Target integration",
"author": "Andrew Balmos <andrew@qlever.io>",
"license": "Apache-2.0",
Expand Down Expand Up @@ -47,11 +47,11 @@
]
},
"dependencies": {
"@oada/client": "^5.1.0",
"@oada/jobs": "^4.6.0",
"@oada/lib-prom": "^3.8.0",
"@oada/list-lib": "^6.0.2",
"@oada/pino-debug": "^3.10.0",
"@oada/client": "^5.2.2",
"@oada/jobs": "^4.7.1",
"@oada/lib-prom": "^4.0.0",
"@oada/list-lib": "^6.1.0",
"@oada/pino-debug": "^4.0.1",
"@oada/types": "^3.5.3",
"@overleaf/o-error": "^3.4.0",
"@trellisfw/signatures": "^2.1.3",
Expand Down
30 changes: 14 additions & 16 deletions src/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,15 +15,13 @@
* limitations under the License.
*/

// Load config first so it can set up env
import config from './config.js';

// Needs to be imported _before_ debug
import '@oada/pino-debug';
// Load config first so it can set up env
import config from './config.js';

import debug from 'debug';
import { pino } from '@oada/pino-debug';

import '@oada/lib-prom';
import { Service } from '@oada/jobs';

/* Import {
Expand All @@ -37,10 +35,7 @@ import {
} from './pdfJob.js';
import { jobHandler as transcriptionOnlyJobHandler } from './transcriptionOnly.js';

const error = debug('target-helper:error');
const info = debug('target-helper:info');
const trace = debug('target-helper:trace');
const warn = debug('target-helper:warn');
const log = pino({base: {service: 'target-helper'}});

const tokens = config.get('oada.token');
const domain = config.get('oada.domain');
Expand All @@ -49,11 +44,11 @@ if (domain.startsWith('http')) {
// Domain = domain.replace(/^https?:\/\//, '');
}

trace('Using token(s) = %s', tokens);
info('Using domain = %s', domain);
log.trace('Using token(s) = %s', tokens);
log.info('Using domain = %s', domain);

process.on('unhandledRejection', (reason, promise) => {
warn({ promise, reason }, 'Unhandled Rejection');
log.warn({ promise, reason }, 'Unhandled Rejection');
// Application specific logging, throwing an error, or other logic here
});

Expand All @@ -66,15 +61,18 @@ await Promise.all(
name: 'target',
oada: { domain, token },
opts: {
/*
finishReporters: [
{
type: 'slack',
status: 'failure',
posturl: config.get('slack.posturl'),
},
],
*/
},
concurrency: jobsConcurrency,
log
});

// --------------------------------------------------
Expand All @@ -94,25 +92,25 @@ await Promise.all(

// --------------------------------------------------
// Start the jobs watching service
info(
log.info(
`Initializing target-helper service. Version: ${process.env.npm_package_version}`,
);
const serviceP = service.start();

// Start the things watching to create jobs
info('Started pdf job creator processes');
log.info('Started pdf job creator processes');
const pdfP = pdfStartJobCreator({ domain, token });
// Const asnP = asnStartJobCreator({ domain, token });

// Catch errors?
try {
await Promise.all([serviceP, pdfP]);
} catch (cError: unknown) {
error(cError);
log.error(cError);
// eslint-disable-next-line no-process-exit, unicorn/no-process-exit
process.exit(1);
}

info('Ready');
log.info('Ready');
}),
);
3 changes: 2 additions & 1 deletion src/pdfJob.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,8 @@ import oError from '@overleaf/o-error';

import { AssumeState, ChangeType, ListWatch } from '@oada/list-lib';
import type { Change, JsonObject, OADAClient } from '@oada/client';
import type { Job, Json, Logger, WorkerFunction } from '@oada/jobs';
import type { Job, Json, WorkerFunction } from '@oada/jobs';
import type { Logger } from '@oada/pino-debug';

import type { Link } from '@oada/types/oada/link/v1.js';
import type Resource from '@oada/types/oada/resource.js';
Expand Down
55 changes: 28 additions & 27 deletions src/pdfJobPostProc.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,22 +19,17 @@ import config from './config.js';

import { readFile } from 'node:fs/promises';

import debug from 'debug';

import type { Json, Logger } from '@oada/jobs';
import type { Json } from '@oada/jobs';
import type { JWK } from '@trellisfw/signatures';
import type { Link } from '@oada/types/oada/link/v1.js';
import type { Logger } from '@oada/pino-debug';
import type { OADAClient } from '@oada/client';
import tSignatures from '@trellisfw/signatures';

import type { ExpandIndex, TargetJob } from './pdfJob.js';
import { has, treeForDocumentType } from './utils.js';
import { tree } from './tree.js';

const error = debug('target-helper:error');
const info = debug('target-helper:info');
const trace = debug('target-helper:trace');

// You can generate a signing key pair by running `oada-certs --create-keys`
const keyFile = await readFile(config.get('signing.privateJWK'));
const prvKey = JSON.parse(keyFile.toString()) as JWK;
Expand Down Expand Up @@ -67,7 +62,7 @@ export async function handleShares(
const shares: Shares = [];

for await (const [dockey, document] of Object.entries(doclist)) {
trace(
log.trace(
'Fetching lookups for doctype = %s, doc = %O, getting /%s/_meta/lookups',
doctype,
document,
Expand All @@ -78,12 +73,13 @@ export async function handleShares(
})) as unknown as {
data: Record<string, Record<string, { _ref: string }>>;
};
trace(lookups, 'lookups');
log.trace(lookups, 'lookups');
let facilityID;
switch (doctype) {
case 'fsqa-audits': {
facilityID = lookups['fsqa-audit']!.organization!._ref;
await pushSharesForFacility({
log,
facilityid: facilityID,
doc: document,
dockey,
Expand All @@ -96,6 +92,7 @@ export async function handleShares(
case 'fsqa-certificates': {
facilityID = lookups['fsqa-certificate']!.organization!._ref;
await pushSharesForFacility({
log,
facilityid: facilityID,
doc: document,
dockey,
Expand All @@ -111,7 +108,7 @@ export async function handleShares(
})) as unknown as {
data: { 'trading-partners': Record<string, { _id: string }> };
};
trace({ holder }, 'Retrieved holder');
log.trace({ holder }, 'Retrieved holder');
for await (const tpLink of Object.values(
holder['trading-partners'],
)) {
Expand All @@ -138,7 +135,7 @@ export async function handleShares(
})) as unknown as {
data: { 'trading-partners': Record<string, { _id: string }> };
};
trace('Retrieved buyer %O', buyer);
log.trace('Retrieved buyer %O', buyer);
for await (const tpLink of Object.values(buyer['trading-partners'])) {
const { data: tp } = (await oada.get({
path: `/${tpLink._id}`,
Expand All @@ -157,7 +154,7 @@ export async function handleShares(
}
}

void log.info(
log.info(
'sharing',
`Posting ${shares.length} shares jobs for doctype ${doctype} resulting from this transcription`,
);
Expand All @@ -173,8 +170,8 @@ export async function handleShares(
// HACK FOR DEMO UNTIL WE GET MASKING SETTINGS:
let mask: Mask | boolean = false;
if (tpKey.includes('REDDYRAW')) {
info('COPY WILL MASK LOCATIONS FOR REDDYRAW');
trace(
log.info('COPY WILL MASK LOCATIONS FOR REDDYRAW');
log.trace(
'pdf is only generated for fsqa-audits or cois, doctype is %s',
doctype,
);
Expand All @@ -185,12 +182,13 @@ export async function handleShares(
}
// END HACK FOR DEMO

await postSharesJob({ doc, doctype, dockey, tpKey, user, mask, oada });
await postSharesJob({ log, doc, doctype, dockey, tpKey, user, mask, oada });
}
}
}

async function postSharesJob({
log,
doc,
doctype,
dockey,
Expand All @@ -199,6 +197,7 @@ async function postSharesJob({
mask,
oada,
}: {
log: Logger;
doc: Link;
doctype: string;
dockey: string;
Expand Down Expand Up @@ -236,7 +235,7 @@ async function postSharesJob({
});
const resourceID = location?.replace(/^\//, '');
const reskey = resourceID?.replace(/^resources\//, '');
trace('Shares job posted as resId = %s', resourceID);
log.trace('Shares job posted as resId = %s', resourceID);
const {
headers: { 'content-location': jobpath },
} = await oada.put({
Expand All @@ -245,7 +244,7 @@ async function postSharesJob({
tree,
});
const jobkey = jobpath?.replace(/^\/resources\/[^/]+\//, '');
trace('Posted jobkey %s for shares', jobkey);
log.trace('Posted jobkey %s for shares', jobkey);
}

export async function recursiveSignLinks(
Expand Down Expand Up @@ -273,12 +272,14 @@ export async function recursiveSignLinks(
}

async function pushSharesForFacility({
log,
facilityid,
doc,
dockey,
shares,
expandIndex,
}: {
log: Logger;
facilityid: string;
doc: { _id: string };
dockey: string;
Expand All @@ -287,7 +288,7 @@ async function pushSharesForFacility({
}) {
const ei = expandIndex['trading-partners'];
const values = Object.values(ei);
trace(
log.trace(
'Looking for facility %s in %d trading partners',
facilityid,
values.length,
Expand All @@ -307,7 +308,7 @@ async function pushSharesForFacility({
const { id, ...tp } = structuredClone(tpv);
const s = { tp: { _id: id, ...tp }, doc, dockey };
shares.push(s);
trace('Added share to running list: %O', s);
log.trace('Added share to running list: %O', s);
}
}

Expand All @@ -328,7 +329,7 @@ async function signResourceForTarget({

try {
// Test first if this thing already has a transcription signature. If so, skip it.
trace('#signResourceForTarget: Checking for existing signature...');
log.trace('#signResourceForTarget: Checking for existing signature...');
// eslint-disable-next-line no-inner-declarations
async function hasTranscriptionSignature(resource: {
signatures?: string[];
Expand All @@ -339,7 +340,7 @@ async function signResourceForTarget({

const { trusted, valid, unchanged, payload, original } =
await tSignatures.verify(resource);
trace(
log.trace(
'#signResourceForTarget: Checked for signature, got back trusted %s valid %s unchanged %s payload %O',
trusted,
valid,
Expand All @@ -358,14 +359,14 @@ async function signResourceForTarget({
}

if (await hasTranscriptionSignature(data)) {
void log.error(
log.error(
`#signResourceForTarget: Item ${_id} already has a transcription signature on it, choose to skip it and not apply a new one`,
{},
);
return true;
}

trace(
log.trace(
'#signResourceForTarget: Did not find existing %s signature, signing...',
signatureType,
);
Expand All @@ -377,19 +378,19 @@ async function signResourceForTarget({
type: signatureType,
});

info('PUTing signed signatures key only to /%s/signatures', _id);
log.info('PUTing signed signatures key only to /%s/signatures', _id);
try {
await oada.put({ path: `/${_id}/signatures`, data: signatures });
} catch (cError: unknown) {
error(cError, `Failed to apply signature to /${_id}/signatures`);
log.error(cError, `Failed to apply signature to /${_id}/signatures`);
throw new Error(`Failed to apply signature to /${_id}/signatures`);
}
} catch (cError: unknown) {
error(cError, `Could not apply signature to resource ${_id}`);
log.error(cError, `Could not apply signature to resource ${_id}`);
throw new Error(`Could not apply signature to resource ${_id}`);
}

void log.info('signed', `Signed resource ${_id} successfully`);
log.info('signed', `Signed resource ${_id} successfully`);
return true; // Success!
}

Expand Down
Loading

0 comments on commit 8533c02

Please sign in to comment.