Skip to content

Commit

Permalink
Additional Gmail API requests
Browse files Browse the repository at this point in the history
  • Loading branch information
@andris9
andris9 committed Jan 8, 2024
1 parent e7fd102 commit d771bad
Show file tree
Hide file tree
Showing 2 changed files with 144 additions and 14 deletions.
155 changes: 142 additions & 13 deletions lib/api-client/gmail-client.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,9 @@ const util = require('util');
const addressparser = require('nodemailer/lib/addressparser');
const libmime = require('libmime');

const GMAIL_API_BASE = 'https://gmail.googleapis.com';
const LIST_BATCH_SIZE = 10; // how many listing requests to run at the same time

const SKIP_LABELS = ['UNREAD', 'STARRED', 'IMPORTANT', 'CHAT', 'CATEGORY_PERSONAL'];

const SYSTEM_LABELS = {
Expand Down Expand Up @@ -75,15 +78,15 @@ class GmailClient {

const accessToken = await this.getToken();

let labelsResult = await this.oAuth2Client.request(accessToken, 'https://gmail.googleapis.com/gmail/v1/users/me/labels');
let labelsResult = await this.oAuth2Client.request(accessToken, `${GMAIL_API_BASE}/gmail/v1/users/me/labels`);

let labels = labelsResult.labels.filter(label => !SKIP_LABELS.includes(label.id));

let resultLabels;
if (query && query.counters) {
resultLabels = [];
for (let label of labels) {
let labelResult = await this.oAuth2Client.request(accessToken, `https://gmail.googleapis.com/gmail/v1/users/me/labels/${label.id}`);
let labelResult = await this.oAuth2Client.request(accessToken, `${GMAIL_API_BASE}/gmail/v1/users/me/labels/${label.id}`);
resultLabels.push(labelResult);
}
} else {
Expand Down Expand Up @@ -176,10 +179,13 @@ class GmailClient {
return envelope;
}

getAttachmentList(messageData) {
getAttachmentList(messageData, options) {
options = options || {};

let encodedTextSize = {};
const attachments = [];
const textParts = [[], [], []];
const textContents = [[], [], []];

let walk = (node, isRelated) => {
if (node.mimeType === 'multipart/related') {
Expand Down Expand Up @@ -236,12 +242,21 @@ class GmailClient {
switch (type) {
case 'plain':
textParts[0].push(node.partId);
if ([type, '*'].includes(options.textType)) {
textContents[0].push(Buffer.from(node.body.data, 'base64'));
}
break;
case 'html':
textParts[1].push(node.partId);
if ([type, '*'].includes(options.textType)) {
textContents[1].push(Buffer.from(node.body.data, 'base64'));
}
break;
default:
textParts[2].push(node.partId);
if (['*'].includes(options.textType)) {
textContents[0].push(Buffer.from(node.body.data, 'base64'));
}
break;
}
}
Expand All @@ -254,15 +269,20 @@ class GmailClient {

walk(messageData.payload, false);

for (let i = 0; i < textContents.length; i++) {
textContents[i] = textContents[i].length ? Buffer.concat(textContents[i]) : null;
}

return {
attachments,
textId: msgpack.encode([messageData.id, textParts]).toString('base64url'),
encodedTextSize
encodedTextSize,
textContents
};
}

formatMessage(messageData, options) {
let { extended, path } = options || {};
let { extended, path, textType } = options || {};

let date = messageData.internalDate && !isNaN(messageData.internalDate) ? new Date(Number(messageData.internalDate)) : undefined;
if (date.toString() === 'Invalid Date') {
Expand Down Expand Up @@ -314,7 +334,7 @@ class GmailClient {
}
}

const { attachments, textId, encodedTextSize } = this.getAttachmentList(messageData);
const { attachments, textId, encodedTextSize, textContents } = this.getAttachmentList(messageData, { textType });

const result = {
id: messageData.id,
Expand Down Expand Up @@ -356,13 +376,23 @@ class GmailClient {
text: textId
? {
id: textId,
encodedSize: encodedTextSize
encodedSize: encodedTextSize,
plain: textContents?.[0]?.toString(),
html: textContents?.[1]?.toString(),
hasMore: textContents?.[0] || textContents?.[1] ? false : undefined
}
: undefined,

preview: messageData.snippet
};

for (let specialUseTag of ['\\Junk', '\\Sent', '\\Trash', '\\Inbox', '\\Drafts']) {
if (result.labels && result.labels.includes(specialUseTag)) {
result.messageSpecialUse = specialUseTag;
break;
}
}

return result;
}

Expand All @@ -383,7 +413,7 @@ class GmailClient {
.join('/')
.replace(/^INBOX(\/|$)/gi, 'INBOX');

let labelsResult = await this.oAuth2Client.request(accessToken, 'https://gmail.googleapis.com/gmail/v1/users/me/labels');
let labelsResult = await this.oAuth2Client.request(accessToken, `${GMAIL_API_BASE}/gmail/v1/users/me/labels`);
let label = labelsResult.labels.find(entry => entry.name === path || entry.id === path);
if (!label) {
return false;
Expand All @@ -392,17 +422,103 @@ class GmailClient {
}

let messageList = [];
let listingResult = await this.oAuth2Client.request(accessToken, 'https://gmail.googleapis.com/gmail/v1/users/me/messages', 'get', requestQuery);
let listingResult = await this.oAuth2Client.request(accessToken, `${GMAIL_API_BASE}/gmail/v1/users/me/messages`, 'get', requestQuery);

let promises = [];

let resolvePromises = async () => {
if (!promises.length) {
return;
}
let resultList = await Promise.allSettled(promises);
for (let entry of resultList) {
if (entry.status === 'rejected') {
throw entry.reason;
}
if (entry.value) {
messageList.push(this.formatMessage(entry.value, { path }));
}
}
promises = [];
};

for (let { id: message } of listingResult.messages) {
let messageData = await this.oAuth2Client.request(accessToken, `https://gmail.googleapis.com/gmail/v1/users/me/messages/${message}`);
//console.log(util.inspect(messageData, false, 22, true));
if (messageData) {
messageList.push(this.formatMessage(messageData, { path }));
promises.push(this.oAuth2Client.request(accessToken, `${GMAIL_API_BASE}/gmail/v1/users/me/messages/${message}`));
if (promises.length > LIST_BATCH_SIZE) {
await resolvePromises();
}
}
await resolvePromises();

return messageList;
}

async getRawMessage(messageId) {
await this.prepare();

const accessToken = await this.getToken();

const requestQuery = {};
const result = await this.oAuth2Client.request(
accessToken,
`${GMAIL_API_BASE}/gmail/v1/users/me/messages/${messageId}?format=raw`,
'get',
requestQuery
);

console.log(result);

return result?.raw ? Buffer.from(result?.raw, 'base64url') : null;
}

async getAttachmentContent(attachmentId) {
let sepPos = attachmentId.indexOf('_');
if (sepPos < 0) {
return null;
}
const messageId = attachmentId.substring(0, sepPos);
const id = attachmentId.substring(sepPos + 1);

await this.prepare();

const accessToken = await this.getToken();

const requestQuery = {};
const result = await this.oAuth2Client.request(
accessToken,
`${GMAIL_API_BASE}/gmail/v1/users/me/messages/${messageId}/attachments/${id}`,
'get',
requestQuery
);

console.log(result);

return result?.data ? Buffer.from(result?.data, 'base64url') : null;
}

async getMessage(messageId, options) {
options = options || {};
await this.prepare();

const accessToken = await this.getToken();

const requestQuery = {};
const messageData = await this.oAuth2Client.request(
accessToken,
`${GMAIL_API_BASE}/gmail/v1/users/me/messages/${messageId}?format=full`,
'get',
requestQuery
);

let result = this.formatMessage(messageData, { extended: true, textType: options.textType });

console.log('---MESSAGE----');
console.log(JSON.stringify(messageData));

console.log(result);

return result;
}
}

module.exports = { GmailClient };
Expand All @@ -417,6 +533,19 @@ let main = async () => {

let messages = await gmailClient.listMessages({ path: 'INBOX' });
console.log(JSON.stringify(messages, false, 2));
for (let msg of messages) {
if (msg.attachments && msg.attachments.length) {
let s = await gmailClient.getMessage(msg.id, { textType: '*' });

Check failure on line 538 in lib/api-client/gmail-client.js

View workflow job for this annotation

GitHub Actions / test (16.x, ubuntu-20.04) 

's' is assigned a value but never used

Check failure on line 538 in lib/api-client/gmail-client.js

View workflow job for this annotation

GitHub Actions / test (18.x, ubuntu-20.04) 

's' is assigned a value but never used

Check failure on line 538 in lib/api-client/gmail-client.js

View workflow job for this annotation

GitHub Actions / test (20.x, ubuntu-20.04) 

's' is assigned a value but never used

let raw = await gmailClient.getRawMessage(msg.id);
await require('fs').promises.writeFile(`/Users/andris/Desktop/${msg.id}.eml`, raw);

Check failure on line 541 in lib/api-client/gmail-client.js

View workflow job for this annotation

GitHub Actions / test (16.x, ubuntu-20.04) 

Unexpected require()

Check failure on line 541 in lib/api-client/gmail-client.js

View workflow job for this annotation

GitHub Actions / test (18.x, ubuntu-20.04) 

Unexpected require()

Check failure on line 541 in lib/api-client/gmail-client.js

View workflow job for this annotation

GitHub Actions / test (20.x, ubuntu-20.04) 

Unexpected require()
for (let a of msg.attachments) {
let attachment = await gmailClient.getAttachmentContent(a.id);
await require('fs').promises.writeFile(`/Users/andris/Desktop/${a.filename}`, attachment);

Check failure on line 544 in lib/api-client/gmail-client.js

View workflow job for this annotation

GitHub Actions / test (16.x, ubuntu-20.04) 

Unexpected require()

Check failure on line 544 in lib/api-client/gmail-client.js

View workflow job for this annotation

GitHub Actions / test (18.x, ubuntu-20.04) 

Unexpected require()

Check failure on line 544 in lib/api-client/gmail-client.js

View workflow job for this annotation

GitHub Actions / test (20.x, ubuntu-20.04) 

Unexpected require()
process.exit();
}
}
}
};

main()
Expand Down
3 changes: 2 additions & 1 deletion package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,8 @@
"licenses": "license-checker --excludePackages emailengine-app --json | node license-table.js > static/licenses.html",
"gettext": "find ./views -name \"*.hbs\" -print0 | xargs -0 xgettext-template -L Handlebars -o translations/messages.pot --force-po && jsxgettext lib/routes-ui.js workers/api.js lib/tools.js -j -o translations/messages.pot",
"prepare-docker": "echo \"EE_DOCKER_LEGACY=$EE_DOCKER_LEGACY\" >> system.env && cat system.env",
"update": "rm -rf node_modules package-lock.json && ncu -u && npm install && ./copy-static-files.sh && npm run licenses && npm run gettext"
"update": "rm -rf node_modules package-lock.json && ncu -u && npm install && ./copy-static-files.sh && npm run licenses && npm run gettext",
"test-gmail-api": "node lib/api-client/gmail-client.js --dbs.redis=redis://127.0.0.1/11"
},
"keywords": [
"IMAP",
Expand Down

0 comments on commit d771bad

Please sign in to comment.