From abd91e867b8a60f60c96b7478599eca32637b056 Mon Sep 17 00:00:00 2001 From: Andris Reinman <andris@reinman.eu> Date: Fri, 9 Feb 2024 14:31:42 +0200 Subject: [PATCH] Use Gmail API client instead of IMAP client when account is created with API scopes --- lib/api-client/gmail-client.js | 145 ++++++++++++++++++++++----------- package.json | 2 +- workers/imap.js | 47 +++++++---- 3 files changed, 129 insertions(+), 65 deletions(-) diff --git a/lib/api-client/gmail-client.js b/lib/api-client/gmail-client.js index 473304d12..d785d07a6 100644 --- a/lib/api-client/gmail-client.js +++ b/lib/api-client/gmail-client.js @@ -5,11 +5,12 @@ const { oauth2Apps } = require('../oauth2-apps'); const getSecret = require('../get-secret'); const msgpack = require('msgpack5')(); const logger = require('../logger'); -const util = require('util'); const addressparser = require('nodemailer/lib/addressparser'); const libmime = require('libmime'); const he = require('he'); +const { REDIS_PREFIX } = require('../consts'); + const fs = require('fs'); const GMAIL_API_BASE = 'https://gmail.googleapis.com'; @@ -68,8 +69,45 @@ class GmailClient { constructor(account, options) { this.account = account; this.options = options || {}; + + this.accountLogger = options.accountLogger; this.redis = options.redis; this.logger = options.logger || logger; + + this.subconnections = []; + } + + async init() { + // No-op + } + + async delete() { + // No-op + } + + getAccountKey() { + return `${REDIS_PREFIX}iad:${this.account}`; + } + + getMailboxListKey() { + return `${REDIS_PREFIX}ial:${this.account}`; + } + + getMailboxHashKey() { + return `${REDIS_PREFIX}iah:${this.account}`; + } + + getLogKey() { + // this format ensures that the key is deleted when user is removed + return `${REDIS_PREFIX}iam:${this.account}:g`; + } + + getLoggedAccountsKey() { + return `${REDIS_PREFIX}iaz:logged`; + } + + currentState() { + return 'connected'; } async getAccount() { @@ -108,12 +146,13 @@ class GmailClient { } async listMailboxes(options) { + console.log('LIST MAILBOXES', options); await this.prepare(); - + console.log(1); const accessToken = await this.getToken(); - + console.log(2, accessToken); let labelsResult = await this.oAuth2Client.request(accessToken, `${GMAIL_API_BASE}/gmail/v1/users/me/labels`); - + console.log(3, labelsResult); let labels = labelsResult.labels.filter(label => !SKIP_LABELS.includes(label.id)); let resultLabels; @@ -127,7 +166,9 @@ class GmailClient { resultLabels = labels; } - resultLabels = resultLabels + console.log(3, resultLabels); + + let mailboxes = resultLabels .map(label => { let pathParts = label.name.split('/'); let name = pathParts.pop(); @@ -191,8 +232,9 @@ class GmailClient { return a.path.toLowerCase().localeCompare(b.path.toLowerCase()); }); + console.log(555, mailboxes); - return { mailboxes: resultLabels }; + return mailboxes; } getEnvelope(messageData) { @@ -433,6 +475,7 @@ class GmailClient { } async listMessages(query) { + console.log('LIST MESSAGES', query); await this.prepare(); const accessToken = await this.getToken(); @@ -711,62 +754,66 @@ class GmailClient { module.exports = { GmailClient }; -const { redis } = require('../db'); +if (/gmail-client\.js$/.test(process.argv[1])) { + console.log('RUN AS STANDALONE'); -let main = async () => { - let gmailClient = new GmailClient('andris', { redis }); + let main = async () => { + const { redis } = require('../db'); - let mailboxes = await gmailClient.listMailboxes(); - console.log(mailboxes); + let gmailClient = new GmailClient('andris', { redis }); - let messages = await gmailClient.listMessages({ path: 'INBOX' }); - console.log(JSON.stringify(messages, false, 2)); + let mailboxes = await gmailClient.listMailboxes(); + console.log(mailboxes); - let deleted = false; + let messages = await gmailClient.listMessages({ path: 'INBOX' }); + console.log(JSON.stringify(messages, false, 2)); - for (let msg of messages) { - if (/testkiri/i.test(msg.subject) && !deleted) { - deleted = true; + let deleted = false; - console.log('DELETING', msg.id); - let y = await gmailClient.deleteMessage(msg.id, true); - console.log('DELETE RESULT', y); - } + for (let msg of messages) { + if (/testkiri/i.test(msg.subject) && !deleted) { + deleted = true; + + console.log('DELETING', msg.id); + let y = await gmailClient.deleteMessage(msg.id, true); + console.log('DELETE RESULT', y); + } - if (msg.attachments && msg.attachments.length) { - await gmailClient.getMessage(msg.id, { textType: '*' }); + if (msg.attachments && msg.attachments.length) { + await gmailClient.getMessage(msg.id, { textType: '*' }); - const textContent = await gmailClient.getText(msg.text.id, { textType: '*' }); - console.log('TEXT CONTENT', textContent); + const textContent = await gmailClient.getText(msg.text.id, { textType: '*' }); + console.log('TEXT CONTENT', textContent); - console.log('MOVE MESSAGE'); - let moveRes = await gmailClient.moveMessage(msg.id, { path: 'Inbox' }); - console.log('MOVE RES', moveRes); + console.log('MOVE MESSAGE'); + let moveRes = await gmailClient.moveMessage(msg.id, { path: 'Inbox' }); + console.log('MOVE RES', moveRes); - let raw = await gmailClient.getRawMessage(msg.id); - await fs.promises.writeFile(`/Users/andris/Desktop/${msg.id}.eml`, raw); - for (let a of msg.attachments) { - let attachment = await gmailClient.getAttachment(a.id); - console.log(attachment); - let s = fs.createWriteStream(`/Users/andris/Desktop/${a.filename}`); + let raw = await gmailClient.getRawMessage(msg.id); + await fs.promises.writeFile(`/Users/andris/Desktop/${msg.id}.eml`, raw); + for (let a of msg.attachments) { + let attachment = await gmailClient.getAttachment(a.id); + console.log(attachment); + let s = fs.createWriteStream(`/Users/andris/Desktop/${a.filename}`); - console.log('PIPING TO STREAM'); - await new Promise((r, e) => { - s.once('finish', r); - s.once('error', e); + console.log('PIPING TO STREAM'); + await new Promise((r, e) => { + s.once('finish', r); + s.once('error', e); - s.write(attachment.data); - s.end(); - }); - console.log('DONE'); + s.write(attachment.data); + s.end(); + }); + console.log('DONE'); - //await fs.promises.writeFile(`/Users/andris/Desktop/${a.filename}`, attachment); - process.exit(); + //await fs.promises.writeFile(`/Users/andris/Desktop/${a.filename}`, attachment); + process.exit(); + } } } - } -}; + }; -main() - .catch(err => console.error(util.inspect(err, false, 22))) - .finally(() => process.exit()); + main() + .catch(err => console.error(require('util').inspect(err, false, 22))) + .finally(() => process.exit()); +} diff --git a/package.json b/package.json index 324490c70..9a5789cf0 100644 --- a/package.json +++ b/package.json @@ -8,7 +8,7 @@ "scripts": { "start": "node server.js", "dev": "EE_OPENAPI_VERBOSE=true EENGINE_LOG_RAW=true node --tls-keylog=keylog.txt server --dbs.redis='redis://127.0.0.1:6379/9' --api.port=7003 --api.host=0.0.0.0 | tee $HOME/ee.log.dev.txt | pino-pretty", - "single": "EE_OPENAPI_VERBOSE=true EENGINE_LOG_RAW=true EENGINE_WORKERS=1 node --inspect server --dbs.redis='redis://127.0.0.1:6379/10' --api.port=7009 --api.host=0.0.0.0 | tee $HOME/ee.log.single.txt | pino-pretty", + "single": "EE_OPENAPI_VERBOSE=true EENGINE_LOG_RAW=true EENGINE_WORKERS=1 node --inspect server --dbs.redis='redis://127.0.0.1:6379/10' --api.port=7003 --api.host=0.0.0.0 | tee $HOME/ee.log.single.txt | pino-pretty", "gmail": "EE_OPENAPI_VERBOSE=true EENGINE_LOG_RAW=true EENGINE_FEATURE_GMAIL_API=true EENGINE_WORKERS=1 node --inspect server --dbs.redis='redis://127.0.0.1:6379/11' --api.port=7003 --api.host=0.0.0.0 | tee $HOME/ee.log.gmail.txt | pino-pretty", "test": "grunt && node --test test/", "swagger": "./getswagger.sh", diff --git a/workers/imap.js b/workers/imap.js index 0954d32ed..20cc9db29 100644 --- a/workers/imap.js +++ b/workers/imap.js @@ -33,7 +33,9 @@ if (readEnvValue('BUGSNAG_API_KEY')) { } const { Connection } = require('../lib/connection'); +const { GmailClient } = require('../lib/api-client/gmail-client'); const { Account } = require('../lib/account'); +const { oauth2Apps } = require('../lib/oauth2-apps'); const { redis, notifyQueue, submitQueue, documentsQueue, getFlowProducer } = require('../lib/db'); const { MessagePortWritable } = require('../lib/message-port-stream'); const { getESClient } = require('../lib/document-store'); @@ -138,22 +140,37 @@ class ConnectionHandler { }); this.accounts.set(account, accountObject); - accountObject.connection = new Connection({ - account, - accountObject, - redis, - secret, - notifyQueue, - submitQueue, - documentsQueue, - flowProducer, - accountLogger, - call: msg => this.call(msg), - logRaw: EENGINE_LOG_RAW - }); - accountObject.logger = accountObject.connection.logger; - let accountData = await accountObject.loadAccountData(); + const accountData = await accountObject.loadAccountData(); + + if (accountData.oauth2 && accountData.oauth2.auth) { + const oauth2App = await oauth2Apps.get(accountData.oauth2.provider); + if (oauth2App.baseScopes === 'api') { + // Use API instead of IMAP + accountObject.connection = new GmailClient(account, { + redis, + accountLogger + }); + accountData.state = 'connected'; + } + } + + if (!accountObject.connection) { + accountObject.connection = new Connection({ + account, + accountObject, + redis, + secret, + notifyQueue, + submitQueue, + documentsQueue, + flowProducer, + accountLogger, + call: msg => this.call(msg), + logRaw: EENGINE_LOG_RAW + }); + accountObject.logger = accountObject.connection.logger; + } if (accountData.state) { await redis.hSetExists(accountObject.connection.getAccountKey(), 'state', accountData.state);