From d771bad89b8816804d6859ac30b297f31357ab4c Mon Sep 17 00:00:00 2001 From: Andris Reinman Date: Mon, 8 Jan 2024 17:52:43 +0200 Subject: [PATCH] Additional Gmail API requests --- lib/api-client/gmail-client.js | 155 ++++++++++++++++++++++++++++++--- package.json | 3 +- 2 files changed, 144 insertions(+), 14 deletions(-) diff --git a/lib/api-client/gmail-client.js b/lib/api-client/gmail-client.js index 2215fb8c4..cf9214d18 100644 --- a/lib/api-client/gmail-client.js +++ b/lib/api-client/gmail-client.js @@ -9,6 +9,9 @@ const util = require('util'); const addressparser = require('nodemailer/lib/addressparser'); const libmime = require('libmime'); +const GMAIL_API_BASE = 'https://gmail.googleapis.com'; +const LIST_BATCH_SIZE = 10; // how many listing requests to run at the same time + const SKIP_LABELS = ['UNREAD', 'STARRED', 'IMPORTANT', 'CHAT', 'CATEGORY_PERSONAL']; const SYSTEM_LABELS = { @@ -75,7 +78,7 @@ class GmailClient { const accessToken = await this.getToken(); - let labelsResult = await this.oAuth2Client.request(accessToken, 'https://gmail.googleapis.com/gmail/v1/users/me/labels'); + let labelsResult = await this.oAuth2Client.request(accessToken, `${GMAIL_API_BASE}/gmail/v1/users/me/labels`); let labels = labelsResult.labels.filter(label => !SKIP_LABELS.includes(label.id)); @@ -83,7 +86,7 @@ class GmailClient { if (query && query.counters) { resultLabels = []; for (let label of labels) { - let labelResult = await this.oAuth2Client.request(accessToken, `https://gmail.googleapis.com/gmail/v1/users/me/labels/${label.id}`); + let labelResult = await this.oAuth2Client.request(accessToken, `${GMAIL_API_BASE}/gmail/v1/users/me/labels/${label.id}`); resultLabels.push(labelResult); } } else { @@ -176,10 +179,13 @@ class GmailClient { return envelope; } - getAttachmentList(messageData) { + getAttachmentList(messageData, options) { + options = options || {}; + let encodedTextSize = {}; const attachments = []; const textParts = [[], [], []]; + const textContents = [[], [], []]; let walk = (node, isRelated) => { if (node.mimeType === 'multipart/related') { @@ -236,12 +242,21 @@ class GmailClient { switch (type) { case 'plain': textParts[0].push(node.partId); + if ([type, '*'].includes(options.textType)) { + textContents[0].push(Buffer.from(node.body.data, 'base64')); + } break; case 'html': textParts[1].push(node.partId); + if ([type, '*'].includes(options.textType)) { + textContents[1].push(Buffer.from(node.body.data, 'base64')); + } break; default: textParts[2].push(node.partId); + if (['*'].includes(options.textType)) { + textContents[0].push(Buffer.from(node.body.data, 'base64')); + } break; } } @@ -254,15 +269,20 @@ class GmailClient { walk(messageData.payload, false); + for (let i = 0; i < textContents.length; i++) { + textContents[i] = textContents[i].length ? Buffer.concat(textContents[i]) : null; + } + return { attachments, textId: msgpack.encode([messageData.id, textParts]).toString('base64url'), - encodedTextSize + encodedTextSize, + textContents }; } formatMessage(messageData, options) { - let { extended, path } = options || {}; + let { extended, path, textType } = options || {}; let date = messageData.internalDate && !isNaN(messageData.internalDate) ? new Date(Number(messageData.internalDate)) : undefined; if (date.toString() === 'Invalid Date') { @@ -314,7 +334,7 @@ class GmailClient { } } - const { attachments, textId, encodedTextSize } = this.getAttachmentList(messageData); + const { attachments, textId, encodedTextSize, textContents } = this.getAttachmentList(messageData, { textType }); const result = { id: messageData.id, @@ -356,13 +376,23 @@ class GmailClient { text: textId ? { id: textId, - encodedSize: encodedTextSize + encodedSize: encodedTextSize, + plain: textContents?.[0]?.toString(), + html: textContents?.[1]?.toString(), + hasMore: textContents?.[0] || textContents?.[1] ? false : undefined } : undefined, preview: messageData.snippet }; + for (let specialUseTag of ['\\Junk', '\\Sent', '\\Trash', '\\Inbox', '\\Drafts']) { + if (result.labels && result.labels.includes(specialUseTag)) { + result.messageSpecialUse = specialUseTag; + break; + } + } + return result; } @@ -383,7 +413,7 @@ class GmailClient { .join('/') .replace(/^INBOX(\/|$)/gi, 'INBOX'); - let labelsResult = await this.oAuth2Client.request(accessToken, 'https://gmail.googleapis.com/gmail/v1/users/me/labels'); + let labelsResult = await this.oAuth2Client.request(accessToken, `${GMAIL_API_BASE}/gmail/v1/users/me/labels`); let label = labelsResult.labels.find(entry => entry.name === path || entry.id === path); if (!label) { return false; @@ -392,17 +422,103 @@ class GmailClient { } let messageList = []; - let listingResult = await this.oAuth2Client.request(accessToken, 'https://gmail.googleapis.com/gmail/v1/users/me/messages', 'get', requestQuery); + let listingResult = await this.oAuth2Client.request(accessToken, `${GMAIL_API_BASE}/gmail/v1/users/me/messages`, 'get', requestQuery); + + let promises = []; + + let resolvePromises = async () => { + if (!promises.length) { + return; + } + let resultList = await Promise.allSettled(promises); + for (let entry of resultList) { + if (entry.status === 'rejected') { + throw entry.reason; + } + if (entry.value) { + messageList.push(this.formatMessage(entry.value, { path })); + } + } + promises = []; + }; + for (let { id: message } of listingResult.messages) { - let messageData = await this.oAuth2Client.request(accessToken, `https://gmail.googleapis.com/gmail/v1/users/me/messages/${message}`); - //console.log(util.inspect(messageData, false, 22, true)); - if (messageData) { - messageList.push(this.formatMessage(messageData, { path })); + promises.push(this.oAuth2Client.request(accessToken, `${GMAIL_API_BASE}/gmail/v1/users/me/messages/${message}`)); + if (promises.length > LIST_BATCH_SIZE) { + await resolvePromises(); } } + await resolvePromises(); return messageList; } + + async getRawMessage(messageId) { + await this.prepare(); + + const accessToken = await this.getToken(); + + const requestQuery = {}; + const result = await this.oAuth2Client.request( + accessToken, + `${GMAIL_API_BASE}/gmail/v1/users/me/messages/${messageId}?format=raw`, + 'get', + requestQuery + ); + + console.log(result); + + return result?.raw ? Buffer.from(result?.raw, 'base64url') : null; + } + + async getAttachmentContent(attachmentId) { + let sepPos = attachmentId.indexOf('_'); + if (sepPos < 0) { + return null; + } + const messageId = attachmentId.substring(0, sepPos); + const id = attachmentId.substring(sepPos + 1); + + await this.prepare(); + + const accessToken = await this.getToken(); + + const requestQuery = {}; + const result = await this.oAuth2Client.request( + accessToken, + `${GMAIL_API_BASE}/gmail/v1/users/me/messages/${messageId}/attachments/${id}`, + 'get', + requestQuery + ); + + console.log(result); + + return result?.data ? Buffer.from(result?.data, 'base64url') : null; + } + + async getMessage(messageId, options) { + options = options || {}; + await this.prepare(); + + const accessToken = await this.getToken(); + + const requestQuery = {}; + const messageData = await this.oAuth2Client.request( + accessToken, + `${GMAIL_API_BASE}/gmail/v1/users/me/messages/${messageId}?format=full`, + 'get', + requestQuery + ); + + let result = this.formatMessage(messageData, { extended: true, textType: options.textType }); + + console.log('---MESSAGE----'); + console.log(JSON.stringify(messageData)); + + console.log(result); + + return result; + } } module.exports = { GmailClient }; @@ -417,6 +533,19 @@ let main = async () => { let messages = await gmailClient.listMessages({ path: 'INBOX' }); console.log(JSON.stringify(messages, false, 2)); + for (let msg of messages) { + if (msg.attachments && msg.attachments.length) { + let s = await gmailClient.getMessage(msg.id, { textType: '*' }); + + let raw = await gmailClient.getRawMessage(msg.id); + await require('fs').promises.writeFile(`/Users/andris/Desktop/${msg.id}.eml`, raw); + for (let a of msg.attachments) { + let attachment = await gmailClient.getAttachmentContent(a.id); + await require('fs').promises.writeFile(`/Users/andris/Desktop/${a.filename}`, attachment); + process.exit(); + } + } + } }; main() diff --git a/package.json b/package.json index 13e56066c..b5267a25b 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,8 @@ "licenses": "license-checker --excludePackages emailengine-app --json | node license-table.js > static/licenses.html", "gettext": "find ./views -name \"*.hbs\" -print0 | xargs -0 xgettext-template -L Handlebars -o translations/messages.pot --force-po && jsxgettext lib/routes-ui.js workers/api.js lib/tools.js -j -o translations/messages.pot", "prepare-docker": "echo \"EE_DOCKER_LEGACY=$EE_DOCKER_LEGACY\" >> system.env && cat system.env", - "update": "rm -rf node_modules package-lock.json && ncu -u && npm install && ./copy-static-files.sh && npm run licenses && npm run gettext" + "update": "rm -rf node_modules package-lock.json && ncu -u && npm install && ./copy-static-files.sh && npm run licenses && npm run gettext", + "test-gmail-api": "node lib/api-client/gmail-client.js --dbs.redis=redis://127.0.0.1/11" }, "keywords": [ "IMAP",