From a67b04c22f81cd349586abcf0ba1ef5df63fba63 Mon Sep 17 00:00:00 2001 From: chime3 Date: Thu, 8 Aug 2024 20:33:11 -0700 Subject: [PATCH 001/182] feat: added Document schema --- src/schemas.ts | 18 ++++++++++++++++++ src/serverconfig.example.json | 3 ++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/src/schemas.ts b/src/schemas.ts index 157f08ad..ddbb9259 100644 --- a/src/schemas.ts +++ b/src/schemas.ts @@ -78,4 +78,22 @@ export enum ContentType { export interface SchemaFavourite extends SchemaRecord { favouriteType: FavouriteType contentType: ContentType +} + +export enum DocumentType { + TXT = "txt", + PDF = "pdf", + DOC = "doc", + DOCX = "docx", + XLS = "xls", + XLSX = "xlsx", + PPT = "ppt", + PPTX = "pptx" +} + +export interface SchemaDocument extends SchemaRecord { + type: DocumentType + size: number + contentText: string + contentRaw?: string } \ No newline at end of file diff --git a/src/serverconfig.example.json b/src/serverconfig.example.json index 80fc5338..31016e18 100644 --- a/src/serverconfig.example.json +++ b/src/serverconfig.example.json @@ -21,7 +21,8 @@ "FOLLOWING": "https://common.schemas.verida.io/social/following/v0.1.0/schema.json", "POST": "https://common.schemas.verida.io/social/post/v0.1.0/schema.json", "EMAIL": "https://common.schemas.verida.io/social/email/v0.1.0/schema.json", - "FAVOURITE": "https://common.schemas.verida.io/favourite/v0.1.0/schema.json" + "FAVOURITE": "https://common.schemas.verida.io/favourite/v0.1.0/schema.json", + "DOCUMENT": "https://common.schemas.verida.io/document/v0.1.0/schema.json" } }, "providers": { From f39a986d584bd7b02af92b26eab216c0638438ac Mon Sep 17 00:00:00 2001 From: chime3 Date: Thu, 8 Aug 2024 20:33:43 -0700 Subject: [PATCH 002/182] feat: added google driver helper --- src/providers/google/helpers.ts | 106 +++++++++++++++++++++++++++++++- 1 file changed, 105 insertions(+), 1 deletion(-) diff --git a/src/providers/google/helpers.ts b/src/providers/google/helpers.ts index 3a1978ba..0014fe65 100644 --- a/src/providers/google/helpers.ts +++ b/src/providers/google/helpers.ts @@ -1,4 +1,4 @@ -import { gmail_v1 } from "googleapis"; +import { gmail_v1, drive_v3 } from "googleapis"; import pdf from "pdf-parse"; export class GmailHelpers { @@ -170,3 +170,107 @@ export class GmailHelpers { return { name: "", email: "" }; } } + +export class GoogleDriveHelpers { + + static async getFile( + drive: drive_v3.Drive, + fileId: string + ): Promise { + try { + const res = await drive.files.get({ + fileId: fileId, + fields: 'id, name, mimeType, webViewLink, createdTime, modifiedTime, thumbnailLink' + }); + return res.data; + } catch (error) { + console.error("Error getting file:", error); + throw error; + } + } + + static async downloadFile( + drive: drive_v3.Drive, + fileId: string + ): Promise { + try { + const res = await drive.files.get( + { fileId: fileId, alt: 'media' }, + { responseType: 'arraybuffer' } + ); + return Buffer.from(res.data as ArrayBuffer); + } catch (error) { + console.error("Error downloading file:", error); + throw error; + } + } + + static async extractTextContent( + drive: drive_v3.Drive, + fileId: string, + mimeType: string + ): Promise { + let textContent = ''; + + if (mimeType === 'application/pdf') { + const fileBuffer = await this.downloadFile(drive, fileId); + textContent = await this.parsePdf(fileBuffer); + } else if (mimeType === 'application/vnd.google-apps.document') { + textContent = await this.extractGoogleDocsText(drive, fileId); + } else if (mimeType === 'text/plain') { + const fileBuffer = await this.downloadFile(drive, fileId); + textContent = fileBuffer.toString('utf8'); + } + + // Add more MIME types as needed (e.g., spreadsheets, presentations, etc.) + + return textContent; + } + + static async extractGoogleDocsText( + drive: drive_v3.Drive, + fileId: string + ): Promise { + try { + const res = await drive.files.export( + { fileId: fileId, mimeType: 'text/plain' }, + { responseType: 'arraybuffer' } + ); + return Buffer.from(res.data as ArrayBuffer).toString('utf8'); + } catch (error) { + console.error("Error extracting text from Google Docs:", error); + return ""; + } + } + + static async parsePdf(pdfBuffer: Buffer): Promise { + try { + const pdfData = await pdf(pdfBuffer); + return pdfData.text; + } catch (error) { + console.error("Error parsing PDF:", error); + return ""; + } + } + + static getFileMetadata( + file: drive_v3.Schema$File + ): { + id: string; + name: string; + mimeType: string; + webViewLink: string; + modifiedTime: string; + thumbnailLink?: string; + } { + return { + id: file.id || '', + name: file.name || 'Untitled', + mimeType: file.mimeType || 'Unknown', + webViewLink: file.webViewLink || '', + modifiedTime: file.modifiedTime || '', + thumbnailLink: file.thumbnailLink, + }; + } +} + From 78867fe70425be43a665a37edf126845c39e7712 Mon Sep 17 00:00:00 2001 From: chime3 Date: Thu, 8 Aug 2024 20:35:45 -0700 Subject: [PATCH 003/182] feat: added google drive document handler --- src/providers/google/gdrive-document.ts | 187 ++++++++++++++++++++++++ src/providers/google/index.ts | 5 +- 2 files changed, 191 insertions(+), 1 deletion(-) create mode 100644 src/providers/google/gdrive-document.ts diff --git a/src/providers/google/gdrive-document.ts b/src/providers/google/gdrive-document.ts new file mode 100644 index 00000000..03117a49 --- /dev/null +++ b/src/providers/google/gdrive-document.ts @@ -0,0 +1,187 @@ +import BaseSyncHandler from "../BaseSyncHandler"; +import CONFIG from "../../config"; + +import { + SyncResponse, + SyncHandlerPosition, + SyncHandlerStatus, +} from "../../interfaces"; +import { DocumentType, SchemaDocument } from "../../schemas"; +import { google, drive_v3 } from "googleapis"; +import { GaxiosResponse } from "gaxios"; +import { GoogleDriveHelpers } from "./helpers"; + +const _ = require("lodash"); + +export default class GoogleDriveDocument extends BaseSyncHandler { + + public getName(): string { + return "google-drive-documents"; + } + + public getSchemaUri(): string { + return CONFIG.verida.schemas.DOCUMENT; + } + + public getProviderApplicationUrl(): string { + return "https://drive.google.com"; + } + + public getGoogleDrive(): drive_v3.Drive { + const TOKEN = { + access_token: this.connection.accessToken, + refresh_token: this.connection.refreshToken, + scope: "https://www.googleapis.com/auth/drive.readonly", + token_type: "Bearer", + }; + + const redirectUrl = ""; + + const oAuth2Client = new google.auth.OAuth2( + this.config.clientId, + this.config.clientSecret, + redirectUrl + ); + + oAuth2Client.setCredentials(TOKEN); + + const drive = google.drive({ version: "v3", auth: oAuth2Client }); + return drive; + } + + public async _sync( + api: any, + syncPosition: SyncHandlerPosition + ): Promise { + const drive = this.getGoogleDrive(); + + const query: drive_v3.Params$Resource$Files$List = { + pageSize: this.config.batchSize, + fields: 'nextPageToken, files(id, name, mimeType, modifiedTime, webViewLink, thumbnailLink)', + q: "mimeType='application/vnd.google-apps.document' or mimeType='application/vnd.google-apps.spreadsheet' or mimeType='application/vnd.google-apps.presentation' or mimeType='application/pdf' or mimeType='application/vnd.openxmlformats-officedocument.wordprocessingml.document'" + }; + + if (syncPosition.thisRef) { + query.pageToken = syncPosition.thisRef; + } + + const serverResponse = await drive.files.list(query); + + if ( + !_.has(serverResponse, "data.files") || + !serverResponse.data.files.length + ) { + // No results found, so stop sync + syncPosition = this.stopSync(syncPosition); + + return { + position: syncPosition, + results: [], + }; + } + + const results = await this.buildResults( + drive, + serverResponse, + syncPosition.breakId, + _.has(this.config, "metadata.breakTimestamp") + ? this.config.metadata.breakTimestamp + : undefined + ); + + syncPosition = this.setNextPosition(syncPosition, serverResponse); + + if (results.length != this.config.batchSize) { + // Not a full page of results, so stop sync + syncPosition = this.stopSync(syncPosition); + } + + return { + results, + position: syncPosition, + }; + } + + protected stopSync(syncPosition: SyncHandlerPosition): SyncHandlerPosition { + if (syncPosition.status == SyncHandlerStatus.STOPPED) { + return syncPosition; + } + + syncPosition.status = SyncHandlerStatus.STOPPED; + syncPosition.thisRef = undefined; + syncPosition.breakId = syncPosition.futureBreakId; + syncPosition.futureBreakId = undefined; + + return syncPosition; + } + + protected setNextPosition( + syncPosition: SyncHandlerPosition, + serverResponse: GaxiosResponse + ): SyncHandlerPosition { + if (!syncPosition.futureBreakId && serverResponse.data.files.length) { + syncPosition.futureBreakId = `${this.connection.profile.id}-${serverResponse.data.files[0].id}`; + } + + if (_.has(serverResponse, "data.nextPageToken")) { + // Have more results, so set the next page ready for the next request + syncPosition.thisRef = serverResponse.data.nextPageToken; + } else { + syncPosition = this.stopSync(syncPosition); + } + + return syncPosition; + } + + protected async buildResults( + drive: drive_v3.Drive, + serverResponse: GaxiosResponse, + breakId: string, + breakTimestamp?: string + ): Promise { + const results: SchemaDocument[] = []; + for (const file of serverResponse.data.files) { + const fileId = `${this.connection.profile.id}-${file.id}`; + + if (fileId == breakId) { + break; + } + + const modifiedTime = file.modifiedTime || "Unknown"; + + if (breakTimestamp && modifiedTime < breakTimestamp) { + break; + } + + const title = file.name || "No title"; + const link = file.webViewLink || "No link"; + const mimeType = file.mimeType || "Unknown"; + const thumbnail = file.thumbnailLink || "No thumbnail"; + + const textContent = await GoogleDriveHelpers.extractTextContent(drive, fileId, mimeType); + + results.push({ + _id: `drive-${fileId}`, + name: title, + documentType: mimeType as DocumentType, + uri: link, + thumbnail, + content: textContent, + sourceData: file, + sourceAccountId: this.provider.getProviderId(), + sourceApplication: this.getProviderApplicationUrl(), + modifiedTimestamp: modifiedTime, + insertedAt: modifiedTime, + }); + } + + return results; + } + + private async getDocumentText(drive: drive_v3.Drive, file: drive_v3.Schema$File): Promise { + // Implement the logic to retrieve text content based on file type (Google Docs, PDF, etc.) + // Example: Download the file content and convert to text + let content = "Text content extraction logic goes here"; + return content; + } +} diff --git a/src/providers/google/index.ts b/src/providers/google/index.ts index 7ba063ab..cad84cad 100644 --- a/src/providers/google/index.ts +++ b/src/providers/google/index.ts @@ -5,6 +5,7 @@ import YouTubeFollowing from "./youtube-following"; import YouTubePost from "./youtube-post"; import { GoogleProviderConfig, GoogleProviderConnection } from "./interfaces"; import YouTubeFavourite from "./youtube-favourite"; +import GoogleDriveDocument from "./gdrive-document"; const passport = require("passport"); const GoogleStrategy = require("passport-google-oauth20"); @@ -30,7 +31,8 @@ export default class GoogleProvider extends Base { Gmail, YouTubeFollowing, YouTubePost, - YouTubeFavourite + YouTubeFavourite, + GoogleDriveDocument ]; } @@ -41,6 +43,7 @@ export default class GoogleProvider extends Base { "email", "https://www.googleapis.com/auth/gmail.readonly", "https://www.googleapis.com/auth/youtube.readonly", + "https://www.googleapis.com/auth/drive.readonly" ]; } From a3f9b22dc4ebf4741dc4dd15a1bc891719aa81f3 Mon Sep 17 00:00:00 2001 From: chime3 Date: Sun, 11 Aug 2024 23:01:12 -0700 Subject: [PATCH 004/182] feat: added limit file size 5MB --- src/providers/google/helpers.ts | 46 ++++++++++++++++++++++++++------- 1 file changed, 37 insertions(+), 9 deletions(-) diff --git a/src/providers/google/helpers.ts b/src/providers/google/helpers.ts index 0014fe65..b1c500fa 100644 --- a/src/providers/google/helpers.ts +++ b/src/providers/google/helpers.ts @@ -180,7 +180,7 @@ export class GoogleDriveHelpers { try { const res = await drive.files.get({ fileId: fileId, - fields: 'id, name, mimeType, webViewLink, createdTime, modifiedTime, thumbnailLink' + fields: 'id, name, mimeType, size, webViewLink, createdTime, modifiedTime, thumbnailLink' }); return res.data; } catch (error) { @@ -189,6 +189,27 @@ export class GoogleDriveHelpers { } } + static async getFileSize( + drive: drive_v3.Drive, + fileId: string + ): Promise { + const file = await this.getFile(drive, fileId); + + if (file.size) { + // For non-Google docs (like PDF, image) + return parseInt(file.size); + } else if (file.mimeType && file.mimeType.startsWith("application/vnd.google-apps.")) { + // For Google Docs, export the file as PDF to estimate size + const exportedFile = await drive.files.export( + { fileId: fileId, mimeType: "application/pdf" }, + { responseType: "arraybuffer" } + ); + return Buffer.byteLength(exportedFile.data as ArrayBuffer); + } else { + return undefined; + } + } + static async downloadFile( drive: drive_v3.Drive, fileId: string @@ -212,14 +233,21 @@ export class GoogleDriveHelpers { ): Promise { let textContent = ''; - if (mimeType === 'application/pdf') { - const fileBuffer = await this.downloadFile(drive, fileId); - textContent = await this.parsePdf(fileBuffer); - } else if (mimeType === 'application/vnd.google-apps.document') { - textContent = await this.extractGoogleDocsText(drive, fileId); - } else if (mimeType === 'text/plain') { - const fileBuffer = await this.downloadFile(drive, fileId); - textContent = fileBuffer.toString('utf8'); + // 5MB limit (5 * 1024 * 1024) + const sizeLimit = 5 * 1024 * 1024; + const fileSize = await this.getFileSize(drive, fileId); + + if (fileSize !== undefined && fileSize <= sizeLimit) { + if (mimeType === 'application/pdf') { + const fileBuffer = await this.downloadFile(drive, fileId); + textContent = await this.parsePdf(fileBuffer); + } else if (mimeType === 'application/vnd.google-apps.document') { + textContent = await this.extractGoogleDocsText(drive, fileId); + } else if (mimeType === 'text/plain') { + const fileBuffer = await this.downloadFile(drive, fileId); + textContent = fileBuffer.toString('utf8'); + } + } // Add more MIME types as needed (e.g., spreadsheets, presentations, etc.) From 840eebd1565073d7f914507b3dd087f128ed5e01 Mon Sep 17 00:00:00 2001 From: chime3 Date: Mon, 12 Aug 2024 00:06:40 -0700 Subject: [PATCH 005/182] fix: added size --- src/providers/google/gdrive-document.ts | 32 ++++++++++++++++--------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/src/providers/google/gdrive-document.ts b/src/providers/google/gdrive-document.ts index 03117a49..fbaafcb1 100644 --- a/src/providers/google/gdrive-document.ts +++ b/src/providers/google/gdrive-document.ts @@ -1,6 +1,6 @@ import BaseSyncHandler from "../BaseSyncHandler"; import CONFIG from "../../config"; - +import { SyncProviderLogEvent, SyncProviderLogLevel } from '../../interfaces' import { SyncResponse, SyncHandlerPosition, @@ -144,12 +144,22 @@ export default class GoogleDriveDocument extends BaseSyncHandler { const fileId = `${this.connection.profile.id}-${file.id}`; if (fileId == breakId) { + const logEvent: SyncProviderLogEvent = { + level: SyncProviderLogLevel.DEBUG, + message: `Break ID hit (${breakId})` + } + this.emit('log', logEvent) break; } - const modifiedTime = file.modifiedTime || "Unknown"; + const createdTime = file.createdTime || "Unkown"; - if (breakTimestamp && modifiedTime < breakTimestamp) { + if (breakTimestamp && createdTime < breakTimestamp) { + const logEvent: SyncProviderLogEvent = { + level: SyncProviderLogLevel.DEBUG, + message: `Break timestamp hit (${breakTimestamp})` + } + this.emit('log', logEvent) break; } @@ -157,21 +167,21 @@ export default class GoogleDriveDocument extends BaseSyncHandler { const link = file.webViewLink || "No link"; const mimeType = file.mimeType || "Unknown"; const thumbnail = file.thumbnailLink || "No thumbnail"; - - const textContent = await GoogleDriveHelpers.extractTextContent(drive, fileId, mimeType); + const size = await GoogleDriveHelpers.getFileSize(drive, file.id) + const textContent = await GoogleDriveHelpers.extractTextContent(drive, file.id, mimeType); results.push({ - _id: `drive-${fileId}`, + _id: this.buildItemId(fileId), name: title, - documentType: mimeType as DocumentType, + type: mimeType as DocumentType, + size: size, uri: link, - thumbnail, - content: textContent, + icon: thumbnail, + contentText: textContent, sourceData: file, sourceAccountId: this.provider.getProviderId(), sourceApplication: this.getProviderApplicationUrl(), - modifiedTimestamp: modifiedTime, - insertedAt: modifiedTime, + insertedAt: createdTime, }); } From 2783dcba4aeae11325652ac744b69a4cab753794 Mon Sep 17 00:00:00 2001 From: chime3 Date: Mon, 12 Aug 2024 21:23:32 -0700 Subject: [PATCH 006/182] fix: updated document type --- src/providers/google/gdrive-document.ts | 19 +++++++------------ src/providers/google/helpers.ts | 24 ++++++++++++++++++++++++ src/providers/google/index.ts | 8 ++++---- src/schemas.ts | 3 ++- 4 files changed, 37 insertions(+), 17 deletions(-) diff --git a/src/providers/google/gdrive-document.ts b/src/providers/google/gdrive-document.ts index fbaafcb1..594339cf 100644 --- a/src/providers/google/gdrive-document.ts +++ b/src/providers/google/gdrive-document.ts @@ -152,9 +152,9 @@ export default class GoogleDriveDocument extends BaseSyncHandler { break; } - const createdTime = file.createdTime || "Unkown"; + const modifiedTime = file.modifiedTime || "Unkown"; - if (breakTimestamp && createdTime < breakTimestamp) { + if (breakTimestamp && modifiedTime < breakTimestamp) { const logEvent: SyncProviderLogEvent = { level: SyncProviderLogLevel.DEBUG, message: `Break timestamp hit (${breakTimestamp})` @@ -166,32 +166,27 @@ export default class GoogleDriveDocument extends BaseSyncHandler { const title = file.name || "No title"; const link = file.webViewLink || "No link"; const mimeType = file.mimeType || "Unknown"; + const type = GoogleDriveHelpers.getDocumentTypeFromMimeType(mimeType); const thumbnail = file.thumbnailLink || "No thumbnail"; const size = await GoogleDriveHelpers.getFileSize(drive, file.id) const textContent = await GoogleDriveHelpers.extractTextContent(drive, file.id, mimeType); - + results.push({ _id: this.buildItemId(fileId), name: title, - type: mimeType as DocumentType, + type: type, size: size, uri: link, icon: thumbnail, contentText: textContent, + sourceId: file.id, sourceData: file, sourceAccountId: this.provider.getProviderId(), sourceApplication: this.getProviderApplicationUrl(), - insertedAt: createdTime, + insertedAt: modifiedTime, }); } return results; } - - private async getDocumentText(drive: drive_v3.Drive, file: drive_v3.Schema$File): Promise { - // Implement the logic to retrieve text content based on file type (Google Docs, PDF, etc.) - // Example: Download the file content and convert to text - let content = "Text content extraction logic goes here"; - return content; - } } diff --git a/src/providers/google/helpers.ts b/src/providers/google/helpers.ts index b1c500fa..0b95058d 100644 --- a/src/providers/google/helpers.ts +++ b/src/providers/google/helpers.ts @@ -1,5 +1,6 @@ import { gmail_v1, drive_v3 } from "googleapis"; import pdf from "pdf-parse"; +import { DocumentType } from "../../schemas"; export class GmailHelpers { static async getMessage( @@ -300,5 +301,28 @@ export class GoogleDriveHelpers { thumbnailLink: file.thumbnailLink, }; } + + static getDocumentTypeFromMimeType(mimeType: string): DocumentType { + switch (mimeType) { + case 'text/plain': + return DocumentType.TXT; + case 'application/pdf': + return DocumentType.PDF; + case 'application/msword': + return DocumentType.DOC; + case 'application/vnd.openxmlformats-officedocument.wordprocessingml.document': + return DocumentType.DOCX; + case 'application/vnd.ms-excel': + return DocumentType.XLS; + case 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': + return DocumentType.XLSX; + case 'application/vnd.ms-powerpoint': + return DocumentType.PPT; + case 'application/vnd.openxmlformats-officedocument.presentationml.presentation': + return DocumentType.PPTX; + default: + return DocumentType.OTHER; + } + } } diff --git a/src/providers/google/index.ts b/src/providers/google/index.ts index cad84cad..8311c1d7 100644 --- a/src/providers/google/index.ts +++ b/src/providers/google/index.ts @@ -28,10 +28,10 @@ export default class GoogleProvider extends Base { public syncHandlers(): any[] { return [ - Gmail, - YouTubeFollowing, - YouTubePost, - YouTubeFavourite, + // Gmail, + // YouTubeFollowing, + // YouTubePost, + // YouTubeFavourite, GoogleDriveDocument ]; } diff --git a/src/schemas.ts b/src/schemas.ts index c2e6c69e..c76a4d34 100644 --- a/src/schemas.ts +++ b/src/schemas.ts @@ -103,7 +103,8 @@ export enum DocumentType { XLS = "xls", XLSX = "xlsx", PPT = "ppt", - PPTX = "pptx" + PPTX = "pptx", + OTHER = "other" } export interface SchemaDocument extends SchemaRecord { From a0add401ec114d362b5724889cd596965d2b5922 Mon Sep 17 00:00:00 2001 From: chime3 Date: Mon, 12 Aug 2024 21:27:11 -0700 Subject: [PATCH 007/182] fix: updated id prefix --- src/providers/google/gdrive-document.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/providers/google/gdrive-document.ts b/src/providers/google/gdrive-document.ts index 594339cf..ec423f2c 100644 --- a/src/providers/google/gdrive-document.ts +++ b/src/providers/google/gdrive-document.ts @@ -6,7 +6,7 @@ import { SyncHandlerPosition, SyncHandlerStatus, } from "../../interfaces"; -import { DocumentType, SchemaDocument } from "../../schemas"; +import { SchemaDocument } from "../../schemas"; import { google, drive_v3 } from "googleapis"; import { GaxiosResponse } from "gaxios"; import { GoogleDriveHelpers } from "./helpers"; @@ -120,7 +120,7 @@ export default class GoogleDriveDocument extends BaseSyncHandler { serverResponse: GaxiosResponse ): SyncHandlerPosition { if (!syncPosition.futureBreakId && serverResponse.data.files.length) { - syncPosition.futureBreakId = `${this.connection.profile.id}-${serverResponse.data.files[0].id}`; + syncPosition.futureBreakId = serverResponse.data.files[0].id; } if (_.has(serverResponse, "data.nextPageToken")) { @@ -141,7 +141,7 @@ export default class GoogleDriveDocument extends BaseSyncHandler { ): Promise { const results: SchemaDocument[] = []; for (const file of serverResponse.data.files) { - const fileId = `${this.connection.profile.id}-${file.id}`; + const fileId = file.id; if (fileId == breakId) { const logEvent: SyncProviderLogEvent = { From 75623722d856d0d4e550bec3f0b85417cca07676 Mon Sep 17 00:00:00 2001 From: chime3 Date: Mon, 12 Aug 2024 21:41:55 -0700 Subject: [PATCH 008/182] feat: added unit test for google drive document --- src/providers/google/index.ts | 8 +- .../providers/google/gdrive-document.tests.ts | 129 ++++++++++++++++++ 2 files changed, 133 insertions(+), 4 deletions(-) create mode 100644 tests/providers/google/gdrive-document.tests.ts diff --git a/src/providers/google/index.ts b/src/providers/google/index.ts index 8311c1d7..cad84cad 100644 --- a/src/providers/google/index.ts +++ b/src/providers/google/index.ts @@ -28,10 +28,10 @@ export default class GoogleProvider extends Base { public syncHandlers(): any[] { return [ - // Gmail, - // YouTubeFollowing, - // YouTubePost, - // YouTubeFavourite, + Gmail, + YouTubeFollowing, + YouTubePost, + YouTubeFavourite, GoogleDriveDocument ]; } diff --git a/tests/providers/google/gdrive-document.tests.ts b/tests/providers/google/gdrive-document.tests.ts new file mode 100644 index 00000000..bb7d18fe --- /dev/null +++ b/tests/providers/google/gdrive-document.tests.ts @@ -0,0 +1,129 @@ +const assert = require("assert"); +import { + BaseProviderConfig, + Connection, + SyncHandlerStatus, + SyncHandlerPosition, + SyncSchemaPositionType, +} from "../../../src/interfaces"; +import Providers from "../../../src/providers"; +import CommonUtils, { NetworkInstance } from "../../common.utils"; + +import GoogleDriveDocument from "../../../src/providers/google/gdrive-document"; +import BaseProvider from "../../../src/providers/BaseProvider"; +import { CommonTests, GenericTestConfig } from "../../common.tests"; +import { SchemaFollowing } from "../../../src/schemas"; + +const providerName = "google"; +let network: NetworkInstance; +let connection: Connection; +let provider: BaseProvider; +let handlerName = "gdrive-document"; +let testConfig: GenericTestConfig; +let providerConfig: Omit = {}; + + +describe(`${providerName} GDrive Document Tests`, function () { + this.timeout(100000); + + this.beforeAll(async function () { + network = await CommonUtils.getNetwork(); + connection = await CommonUtils.getConnection(providerName); + provider = Providers(providerName, network.context, connection); + + testConfig = { + idPrefix: `${provider.getProviderName()}-${connection.profile.id}`, + timeOrderAttribute: "insertedAt", + batchSizeLimitAttribute: "batchSize", + }; + }); + + describe(`Fetch ${providerName} data`, () => { + + it(`Can pass basic tests: ${handlerName}`, async () => { + await CommonTests.runGenericTests( + providerName, + GoogleDriveDocument, + testConfig, + providerConfig, + connection + ); + }); + + it(`Can limit results by timestamp`, async () => { + const lastRecordHours = 2; + const lastRecordTimestamp = new Date( + Date.now() - lastRecordHours * 3600000 + ).toISOString(); + + const syncPosition: Omit = { + type: SyncSchemaPositionType.SYNC, + providerName, + providerId: provider.getProviderId(), + handlerName, + status: SyncHandlerStatus.ACTIVE, + }; + + providerConfig.batchSize = 5; + providerConfig.metadata = { + breakTimestamp: lastRecordTimestamp, + }; + + const syncResponse = await CommonTests.runSyncTest( + providerName, + GoogleDriveDocument, + connection, + testConfig, + syncPosition, + providerConfig + ); + assert.ok( + syncResponse.results && syncResponse.results.length, + "Have results (You may not have edited any document in the testing timeframe)" + ); + + const results = syncResponse.results; + assert.ok( + results[results.length - 1].insertedAt > lastRecordTimestamp, + "Last result is within expected date/time range" + ); + assert.ok( + results.length < providerConfig.batchSize, + `Results reached the expected timestamp within the current batch size (try increasing the test batch size or reducing the break timestamp)` + ); + }); + + it(`Can handle empty results`, async () => { + const syncPosition: Omit = { + type: SyncSchemaPositionType.SYNC, + providerName, + providerId: provider.getProviderId(), + handlerName, + status: SyncHandlerStatus.ACTIVE, + }; + + providerConfig.batchSize = 5; + providerConfig.metadata = { + breakTimestamp: new Date().toISOString(), + }; + + const syncResponse = await CommonTests.runSyncTest( + providerName, + GoogleDriveDocument, + connection, + testConfig, + syncPosition, + providerConfig + ); + assert.ok( + syncResponse.results.length === 0, + "No results should be returned for the future timestamp" + ); + }) + }); + + this.afterAll(async function () { + const { context } = await CommonUtils.getNetwork(); + await context.close(); + }); +}); From ff42579ebf418ac91b7eee8983e05b185dd091c2 Mon Sep 17 00:00:00 2001 From: chime3 Date: Tue, 13 Aug 2024 23:30:02 -0700 Subject: [PATCH 009/182] feat: extract indexable text from google spreadsheet and presentation --- src/providers/google/gdrive-document.ts | 19 +++-- src/providers/google/helpers.ts | 93 ++++++++++++++++++------- 2 files changed, 83 insertions(+), 29 deletions(-) diff --git a/src/providers/google/gdrive-document.ts b/src/providers/google/gdrive-document.ts index ec423f2c..3903e203 100644 --- a/src/providers/google/gdrive-document.ts +++ b/src/providers/google/gdrive-document.ts @@ -8,6 +8,7 @@ import { } from "../../interfaces"; import { SchemaDocument } from "../../schemas"; import { google, drive_v3 } from "googleapis"; +import { OAuth2Client } from 'google-auth-library'; import { GaxiosResponse } from "gaxios"; import { GoogleDriveHelpers } from "./helpers"; @@ -26,8 +27,8 @@ export default class GoogleDriveDocument extends BaseSyncHandler { public getProviderApplicationUrl(): string { return "https://drive.google.com"; } - - public getGoogleDrive(): drive_v3.Drive { + + public getGoogleAuth(): OAuth2Client { const TOKEN = { access_token: this.connection.accessToken, refresh_token: this.connection.refreshToken, @@ -45,7 +46,14 @@ export default class GoogleDriveDocument extends BaseSyncHandler { oAuth2Client.setCredentials(TOKEN); - const drive = google.drive({ version: "v3", auth: oAuth2Client }); + return oAuth2Client; + } + + public getGoogleDrive(): drive_v3.Drive { + + const auth = this.getGoogleAuth(); + + const drive = google.drive({ version: "v3", auth: auth }); return drive; } @@ -54,6 +62,7 @@ export default class GoogleDriveDocument extends BaseSyncHandler { syncPosition: SyncHandlerPosition ): Promise { const drive = this.getGoogleDrive(); + const auth = this.getGoogleAuth(); const query: drive_v3.Params$Resource$Files$List = { pageSize: this.config.batchSize, @@ -82,6 +91,7 @@ export default class GoogleDriveDocument extends BaseSyncHandler { const results = await this.buildResults( drive, + auth, serverResponse, syncPosition.breakId, _.has(this.config, "metadata.breakTimestamp") @@ -135,6 +145,7 @@ export default class GoogleDriveDocument extends BaseSyncHandler { protected async buildResults( drive: drive_v3.Drive, + auth: OAuth2Client, serverResponse: GaxiosResponse, breakId: string, breakTimestamp?: string @@ -169,7 +180,7 @@ export default class GoogleDriveDocument extends BaseSyncHandler { const type = GoogleDriveHelpers.getDocumentTypeFromMimeType(mimeType); const thumbnail = file.thumbnailLink || "No thumbnail"; const size = await GoogleDriveHelpers.getFileSize(drive, file.id) - const textContent = await GoogleDriveHelpers.extractTextContent(drive, file.id, mimeType); + const textContent = await GoogleDriveHelpers.extractIndexableText(drive, file.id, mimeType, auth); results.push({ _id: this.buildItemId(fileId), diff --git a/src/providers/google/helpers.ts b/src/providers/google/helpers.ts index 0b95058d..f14f42cf 100644 --- a/src/providers/google/helpers.ts +++ b/src/providers/google/helpers.ts @@ -1,4 +1,5 @@ -import { gmail_v1, drive_v3 } from "googleapis"; +import { drive_v3, gmail_v1, google } from 'googleapis'; +import { OAuth2Client } from 'google-auth-library'; import pdf from "pdf-parse"; import { DocumentType } from "../../schemas"; @@ -173,7 +174,6 @@ export class GmailHelpers { } export class GoogleDriveHelpers { - static async getFile( drive: drive_v3.Drive, fileId: string @@ -200,9 +200,9 @@ export class GoogleDriveHelpers { // For non-Google docs (like PDF, image) return parseInt(file.size); } else if (file.mimeType && file.mimeType.startsWith("application/vnd.google-apps.")) { - // For Google Docs, export the file as PDF to estimate size + // For Google Docs, export the file as plain text to estimate size const exportedFile = await drive.files.export( - { fileId: fileId, mimeType: "application/pdf" }, + { fileId: fileId, mimeType: "text/plain" }, { responseType: "arraybuffer" } ); return Buffer.byteLength(exportedFile.data as ArrayBuffer); @@ -211,26 +211,11 @@ export class GoogleDriveHelpers { } } - static async downloadFile( - drive: drive_v3.Drive, - fileId: string - ): Promise { - try { - const res = await drive.files.get( - { fileId: fileId, alt: 'media' }, - { responseType: 'arraybuffer' } - ); - return Buffer.from(res.data as ArrayBuffer); - } catch (error) { - console.error("Error downloading file:", error); - throw error; - } - } - - static async extractTextContent( + static async extractIndexableText( drive: drive_v3.Drive, fileId: string, - mimeType: string + mimeType: string, + auth: OAuth2Client ): Promise { let textContent = ''; @@ -244,15 +229,19 @@ export class GoogleDriveHelpers { textContent = await this.parsePdf(fileBuffer); } else if (mimeType === 'application/vnd.google-apps.document') { textContent = await this.extractGoogleDocsText(drive, fileId); + } else if (mimeType === 'application/vnd.google-apps.spreadsheet') { + textContent = await this.extractGoogleSheetsText(fileId, auth); + } else if (mimeType === 'application/vnd.google-apps.presentation') { + textContent = await this.extractGoogleSlidesText(fileId, auth); } else if (mimeType === 'text/plain') { const fileBuffer = await this.downloadFile(drive, fileId); textContent = fileBuffer.toString('utf8'); } + } else { + console.warn('File size exceeds the limit or unsupported file type.'); } - // Add more MIME types as needed (e.g., spreadsheets, presentations, etc.) - return textContent; } @@ -272,6 +261,61 @@ export class GoogleDriveHelpers { } } + static async extractGoogleSheetsText( + fileId: string, + auth: OAuth2Client + ): Promise { + try { + const sheets = google.sheets({ version: 'v4', auth: auth }); + const res = await sheets.spreadsheets.values.get({ + spreadsheetId: fileId, + range: 'A1:Z1000' // You can adjust the range as needed + }); + return res.data.values?.map(row => row.join('\t')).join('\n') || ''; + } catch (error) { + console.error("Error extracting text from Google Sheets:", error); + return ""; + } + } + + static async extractGoogleSlidesText( + fileId: string, + auth: OAuth2Client + ): Promise { + try { + + const slides = google.slides({ version: 'v1', auth: auth }); + const res = await slides.presentations.get({ + presentationId: fileId, + }); + const slidesContent = res.data.slides || []; + return slidesContent.map(slide => + slide.pageElements?.map(element => + element.shape?.text?.textElements?.map(te => te.textRun?.content).join('') + ).join('\n') + ).join('\n'); + } catch (error) { + console.error("Error extracting text from Google Slides:", error); + return ""; + } + } + + static async downloadFile( + drive: drive_v3.Drive, + fileId: string + ): Promise { + try { + const res = await drive.files.get( + { fileId: fileId, alt: 'media' }, + { responseType: 'arraybuffer' } + ); + return Buffer.from(res.data as ArrayBuffer); + } catch (error) { + console.error("Error downloading file:", error); + throw error; + } + } + static async parsePdf(pdfBuffer: Buffer): Promise { try { const pdfData = await pdf(pdfBuffer); @@ -325,4 +369,3 @@ export class GoogleDriveHelpers { } } } - From d69d23274be3fd6b8a7e8c5ce93134ed9eac45fd Mon Sep 17 00:00:00 2001 From: chime3 Date: Mon, 19 Aug 2024 21:44:18 -0700 Subject: [PATCH 010/182] feat: doc/docx/ppt/pptx/xls/xlsx parser --- package.json | 3 + src/providers/google/helpers.ts | 53 +- yarn.lock | 1254 ++++++++++++++++++++++++++++++- 3 files changed, 1292 insertions(+), 18 deletions(-) diff --git a/package.json b/package.json index 94301c1a..1fbe3817 100644 --- a/package.json +++ b/package.json @@ -61,6 +61,7 @@ "lodash": "^4.17.21", "log4js": "^6.4.1", "mailparser": "^3.7.1", + "mammoth": "^1.8.0", "memory-cache": "^0.2.0", "mocha": "^9.2.1", "nano": "^9.0.5", @@ -69,9 +70,11 @@ "passport-facebook": "^3.0.0", "passport-google-oauth20": "^2.0.0", "pdf-parse": "^1.1.1", + "pptx-parser": "^1.1.7-beta.9", "ts-mocha": "^9.0.2", "twitter-api-v2": "^1.14.0", "uuid": "^10.0.0", + "xlsx": "^0.18.5", "xoauth2": "^1.2.0" }, "devDependencies": { diff --git a/src/providers/google/helpers.ts b/src/providers/google/helpers.ts index f14f42cf..e49f4b49 100644 --- a/src/providers/google/helpers.ts +++ b/src/providers/google/helpers.ts @@ -218,11 +218,11 @@ export class GoogleDriveHelpers { auth: OAuth2Client ): Promise { let textContent = ''; - + // 5MB limit (5 * 1024 * 1024) const sizeLimit = 5 * 1024 * 1024; const fileSize = await this.getFileSize(drive, fileId); - + if (fileSize !== undefined && fileSize <= sizeLimit) { if (mimeType === 'application/pdf') { const fileBuffer = await this.downloadFile(drive, fileId); @@ -236,14 +236,59 @@ export class GoogleDriveHelpers { } else if (mimeType === 'text/plain') { const fileBuffer = await this.downloadFile(drive, fileId); textContent = fileBuffer.toString('utf8'); + } else if (mimeType === 'application/msword' || mimeType === 'application/vnd.openxmlformats-officedocument.wordprocessingml.document') { + const fileBuffer = await this.downloadFile(drive, fileId); + textContent = await this.parseDocx(fileBuffer); + } else if (mimeType === 'application/vnd.ms-excel' || mimeType === 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet') { + const fileBuffer = await this.downloadFile(drive, fileId); + textContent = await this.parseXlsx(fileBuffer); + } else if (mimeType === 'application/vnd.ms-powerpoint' || mimeType === 'application/vnd.openxmlformats-officedocument.presentationml.presentation') { + const fileBuffer = await this.downloadFile(drive, fileId); + textContent = await this.parsePptx(fileBuffer); } - } else { console.warn('File size exceeds the limit or unsupported file type.'); } - + return textContent; } + + static async parseDocx(docxBuffer: Buffer): Promise { + const mammoth = require('mammoth'); + try { + const result = await mammoth.extractRawText({ buffer: docxBuffer }); + return result.value; + } catch (error) { + console.error("Error parsing DOCX file:", error); + return ""; + } + } + + static async parseXlsx(xlsxBuffer: Buffer): Promise { + const XLSX = require('xlsx'); + try { + const workbook = XLSX.read(xlsxBuffer, { type: 'buffer' }); + const text = workbook.SheetNames.map((sheetName: string) => { + const sheet = workbook.Sheets[sheetName]; + return XLSX.utils.sheet_to_csv(sheet); + }).join('\n'); + return text; + } catch (error) { + console.error("Error parsing XLSX file:", error); + return ""; + } + } + + static async parsePptx(pptxBuffer: Buffer): Promise { + const PptxParser = require('pptx-parser'); + try { + const result = await PptxParser(pptxBuffer); + return result.text; + } catch (error) { + console.error("Error parsing PPTX file:", error); + return ""; + } + } static async extractGoogleDocsText( drive: drive_v3.Drive, diff --git a/yarn.lock b/yarn.lock index fe93b9fb..7299dc6e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -17,6 +17,147 @@ call-me-maybe "^1.0.1" js-yaml "^4.1.0" +"@babel/code-frame@^7.24.7": + version "7.24.7" + resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.24.7.tgz#882fd9e09e8ee324e496bd040401c6f046ef4465" + integrity sha512-BcYH1CVJBO9tvyIZ2jVeXgSIMvGZ2FDRvDdOIVQyuklNKSsx+eppDEBq/g47Ayw+RqNFE+URvOShmf+f/qwAlA== + dependencies: + "@babel/highlight" "^7.24.7" + picocolors "^1.0.0" + +"@babel/compat-data@^7.22.6", "@babel/compat-data@^7.25.2": + version "7.25.2" + resolved "https://registry.yarnpkg.com/@babel/compat-data/-/compat-data-7.25.2.tgz#e41928bd33475305c586f6acbbb7e3ade7a6f7f5" + integrity sha512-bYcppcpKBvX4znYaPEeFau03bp89ShqNMLs+rmdptMw+heSZh9+z84d2YG+K7cYLbWwzdjtDoW/uqZmPjulClQ== + +"@babel/generator@^7.25.0": + version "7.25.0" + resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.25.0.tgz#f858ddfa984350bc3d3b7f125073c9af6988f18e" + integrity sha512-3LEEcj3PVW8pW2R1SR1M89g/qrYk/m/mB/tLqn7dn4sbBUQyTqnlod+II2U4dqiGtUmkcnAmkMDralTFZttRiw== + dependencies: + "@babel/types" "^7.25.0" + "@jridgewell/gen-mapping" "^0.3.5" + "@jridgewell/trace-mapping" "^0.3.25" + jsesc "^2.5.1" + +"@babel/helper-compilation-targets@^7.22.6": + version "7.25.2" + resolved "https://registry.yarnpkg.com/@babel/helper-compilation-targets/-/helper-compilation-targets-7.25.2.tgz#e1d9410a90974a3a5a66e84ff55ef62e3c02d06c" + integrity sha512-U2U5LsSaZ7TAt3cfaymQ8WHh0pxvdHoEk6HVpaexxixjyEquMh0L0YNJNM6CTGKMXV1iksi0iZkGw4AcFkPaaw== + dependencies: + "@babel/compat-data" "^7.25.2" + "@babel/helper-validator-option" "^7.24.8" + browserslist "^4.23.1" + lru-cache "^5.1.1" + semver "^6.3.1" + +"@babel/helper-define-polyfill-provider@^0.6.2": + version "0.6.2" + resolved "https://registry.yarnpkg.com/@babel/helper-define-polyfill-provider/-/helper-define-polyfill-provider-0.6.2.tgz#18594f789c3594acb24cfdb4a7f7b7d2e8bd912d" + integrity sha512-LV76g+C502biUK6AyZ3LK10vDpDyCzZnhZFXkH1L75zHPj68+qc8Zfpx2th+gzwA2MzyK+1g/3EPl62yFnVttQ== + dependencies: + "@babel/helper-compilation-targets" "^7.22.6" + "@babel/helper-plugin-utils" "^7.22.5" + debug "^4.1.1" + lodash.debounce "^4.0.8" + resolve "^1.14.2" + +"@babel/helper-module-imports@^7.24.7": + version "7.24.7" + resolved "https://registry.yarnpkg.com/@babel/helper-module-imports/-/helper-module-imports-7.24.7.tgz#f2f980392de5b84c3328fc71d38bd81bbb83042b" + integrity sha512-8AyH3C+74cgCVVXow/myrynrAGv+nTVg5vKu2nZph9x7RcRwzmh0VFallJuFTZ9mx6u4eSdXZfcOzSqTUm0HCA== + dependencies: + "@babel/traverse" "^7.24.7" + "@babel/types" "^7.24.7" + +"@babel/helper-plugin-utils@^7.22.5", "@babel/helper-plugin-utils@^7.24.7": + version "7.24.8" + resolved "https://registry.yarnpkg.com/@babel/helper-plugin-utils/-/helper-plugin-utils-7.24.8.tgz#94ee67e8ec0e5d44ea7baeb51e571bd26af07878" + integrity sha512-FFWx5142D8h2Mgr/iPVGH5G7w6jDn4jUSpZTyDnQO0Yn7Ks2Kuz6Pci8H6MPCoUJegd/UZQ3tAvfLCxQSnWWwg== + +"@babel/helper-string-parser@^7.24.8": + version "7.24.8" + resolved "https://registry.yarnpkg.com/@babel/helper-string-parser/-/helper-string-parser-7.24.8.tgz#5b3329c9a58803d5df425e5785865881a81ca48d" + integrity sha512-pO9KhhRcuUyGnJWwyEgnRJTSIZHiT+vMD0kPeD+so0l7mxkMT19g3pjY9GTnHySck/hDzq+dtW/4VgnMkippsQ== + +"@babel/helper-validator-identifier@^7.24.7": + version "7.24.7" + resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.24.7.tgz#75b889cfaf9e35c2aaf42cf0d72c8e91719251db" + integrity sha512-rR+PBcQ1SMQDDyF6X0wxtG8QyLCgUB0eRAGguqRLfkCA87l7yAP7ehq8SNj96OOGTO8OBV70KhuFYcIkHXOg0w== + +"@babel/helper-validator-option@^7.24.8": + version "7.24.8" + resolved "https://registry.yarnpkg.com/@babel/helper-validator-option/-/helper-validator-option-7.24.8.tgz#3725cdeea8b480e86d34df15304806a06975e33d" + integrity sha512-xb8t9tD1MHLungh/AIoWYN+gVHaB9kwlu8gffXGSt3FFEIT7RjS+xWbc2vUD1UTZdIpKj/ab3rdqJ7ufngyi2Q== + +"@babel/highlight@^7.24.7": + version "7.24.7" + resolved "https://registry.yarnpkg.com/@babel/highlight/-/highlight-7.24.7.tgz#a05ab1df134b286558aae0ed41e6c5f731bf409d" + integrity sha512-EStJpq4OuY8xYfhGVXngigBJRWxftKX9ksiGDnmlY3o7B/V7KIAc9X4oiK87uPJSc/vs5L869bem5fhZa8caZw== + dependencies: + "@babel/helper-validator-identifier" "^7.24.7" + chalk "^2.4.2" + js-tokens "^4.0.0" + picocolors "^1.0.0" + +"@babel/parser@^7.25.0", "@babel/parser@^7.25.3": + version "7.25.3" + resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.25.3.tgz#91fb126768d944966263f0657ab222a642b82065" + integrity sha512-iLTJKDbJ4hMvFPgQwwsVoxtHyWpKKPBrxkANrSYewDPaPpT5py5yeVkgPIJ7XYXhndxJpaA3PyALSXQ7u8e/Dw== + dependencies: + "@babel/types" "^7.25.2" + +"@babel/plugin-transform-runtime@^7.7.6": + version "7.24.7" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-runtime/-/plugin-transform-runtime-7.24.7.tgz#00a5bfaf8c43cf5c8703a8a6e82b59d9c58f38ca" + integrity sha512-YqXjrk4C+a1kZjewqt+Mmu2UuV1s07y8kqcUf4qYLnoqemhR4gRQikhdAhSVJioMjVTu6Mo6pAbaypEA3jY6fw== + dependencies: + "@babel/helper-module-imports" "^7.24.7" + "@babel/helper-plugin-utils" "^7.24.7" + babel-plugin-polyfill-corejs2 "^0.4.10" + babel-plugin-polyfill-corejs3 "^0.10.1" + babel-plugin-polyfill-regenerator "^0.6.1" + semver "^6.3.1" + +"@babel/runtime-corejs3@^7.10.4", "@babel/runtime-corejs3@^7.7.7": + version "7.25.0" + resolved "https://registry.yarnpkg.com/@babel/runtime-corejs3/-/runtime-corejs3-7.25.0.tgz#0a318b66dfc765ad10562d829fea372ed7e1eb7d" + integrity sha512-BOehWE7MgQ8W8Qn0CQnMtg2tHPHPulcS/5AVpFvs2KCK1ET+0WqZqPvnpRpFN81gYoFopdIEJX9Sgjw3ZBccPg== + dependencies: + core-js-pure "^3.30.2" + regenerator-runtime "^0.14.0" + +"@babel/template@^7.25.0": + version "7.25.0" + resolved "https://registry.yarnpkg.com/@babel/template/-/template-7.25.0.tgz#e733dc3134b4fede528c15bc95e89cb98c52592a" + integrity sha512-aOOgh1/5XzKvg1jvVz7AVrx2piJ2XBi227DHmbY6y+bM9H2FlN+IfecYu4Xl0cNiiVejlsCri89LUsbj8vJD9Q== + dependencies: + "@babel/code-frame" "^7.24.7" + "@babel/parser" "^7.25.0" + "@babel/types" "^7.25.0" + +"@babel/traverse@^7.24.7": + version "7.25.3" + resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.25.3.tgz#f1b901951c83eda2f3e29450ce92743783373490" + integrity sha512-HefgyP1x754oGCsKmV5reSmtV7IXj/kpaE1XYY+D9G5PvKKoFfSbiS4M77MdjuwlZKDIKFCffq9rPU+H/s3ZdQ== + dependencies: + "@babel/code-frame" "^7.24.7" + "@babel/generator" "^7.25.0" + "@babel/parser" "^7.25.3" + "@babel/template" "^7.25.0" + "@babel/types" "^7.25.2" + debug "^4.3.1" + globals "^11.1.0" + +"@babel/types@^7.24.7", "@babel/types@^7.25.0", "@babel/types@^7.25.2": + version "7.25.2" + resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.25.2.tgz#55fb231f7dc958cd69ea141a4c2997e819646125" + integrity sha512-YTnYtra7W9e6/oAZEHj0bJehPRUlLH9/fbpT5LfB0NhQXyALCRkRs3zH9v07IYhkgpqX6Z78FnuccZr/l4Fs4Q== + dependencies: + "@babel/helper-string-parser" "^7.24.8" + "@babel/helper-validator-identifier" "^7.24.7" + to-fast-properties "^2.0.0" + "@discordjs/builders@^1.6.0": version "1.6.1" resolved "https://registry.yarnpkg.com/@discordjs/builders/-/builders-1.6.1.tgz#5b1447cfa493bc1306671ef18ce3aae13c0af0ba" @@ -403,6 +544,50 @@ "@ethersproject/properties" "^5.7.0" "@ethersproject/strings" "^5.7.0" +"@isaacs/cliui@^8.0.2": + version "8.0.2" + resolved "https://registry.yarnpkg.com/@isaacs/cliui/-/cliui-8.0.2.tgz#b37667b7bc181c168782259bab42474fbf52b550" + integrity sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA== + dependencies: + string-width "^5.1.2" + string-width-cjs "npm:string-width@^4.2.0" + strip-ansi "^7.0.1" + strip-ansi-cjs "npm:strip-ansi@^6.0.1" + wrap-ansi "^8.1.0" + wrap-ansi-cjs "npm:wrap-ansi@^7.0.0" + +"@jridgewell/gen-mapping@^0.3.5": + version "0.3.5" + resolved "https://registry.yarnpkg.com/@jridgewell/gen-mapping/-/gen-mapping-0.3.5.tgz#dcce6aff74bdf6dad1a95802b69b04a2fcb1fb36" + integrity sha512-IzL8ZoEDIBRWEzlCcRhOaCupYyN5gdIK+Q6fbFdPDg6HqX6jpkItn7DFIpW9LQzXG6Df9sA7+OKnq0qlz/GaQg== + dependencies: + "@jridgewell/set-array" "^1.2.1" + "@jridgewell/sourcemap-codec" "^1.4.10" + "@jridgewell/trace-mapping" "^0.3.24" + +"@jridgewell/resolve-uri@^3.1.0": + version "3.1.2" + resolved "https://registry.yarnpkg.com/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz#7a0ee601f60f99a20c7c7c5ff0c80388c1189bd6" + integrity sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw== + +"@jridgewell/set-array@^1.2.1": + version "1.2.1" + resolved "https://registry.yarnpkg.com/@jridgewell/set-array/-/set-array-1.2.1.tgz#558fb6472ed16a4c850b889530e6b36438c49280" + integrity sha512-R8gLRTZeyp03ymzP/6Lil/28tGeGEzhx1q2k703KGWRAI1VdvPIXdG70VJc2pAMw3NA6JKL5hhFu1sJX0Mnn/A== + +"@jridgewell/sourcemap-codec@^1.4.10", "@jridgewell/sourcemap-codec@^1.4.14": + version "1.5.0" + resolved "https://registry.yarnpkg.com/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.0.tgz#3188bcb273a414b0d215fd22a58540b989b9409a" + integrity sha512-gv3ZRaISU3fjPAgNsriBRqGWQL6quFx04YMPW/zD8XMLsU32mhCCbfbO6KZFLjvYpCZ8zyDEgqsgf+PwPaM7GQ== + +"@jridgewell/trace-mapping@^0.3.24", "@jridgewell/trace-mapping@^0.3.25": + version "0.3.25" + resolved "https://registry.yarnpkg.com/@jridgewell/trace-mapping/-/trace-mapping-0.3.25.tgz#15f190e98895f3fc23276ee14bc76b675c2e50f0" + integrity sha512-vNk6aEwybGtawWmy/PzwnGDOjCkLWSD2wqvjGGAgOAwCGWySYXfYoxt00IJkTF+8Lb57DwOb3Aa0o9CApepiYQ== + dependencies: + "@jridgewell/resolve-uri" "^3.1.0" + "@jridgewell/sourcemap-codec" "^1.4.14" + "@jsdevtools/ono@^7.1.3": version "7.1.3" resolved "https://registry.yarnpkg.com/@jsdevtools/ono/-/ono-7.1.3.tgz#9df03bbd7c696a5c58885c34aa06da41c8543796" @@ -443,6 +628,16 @@ resolved "https://registry.yarnpkg.com/@oauth-everything/profile/-/profile-1.0.0.tgz#0b5e78749415519fa312dc83347a677903f456ba" integrity sha512-OmCuBPhjaLHh9MST9P5jRuVBZaP0z7hBk8nH4Yt7Id5kNM1AXGd5uud6CP7W2zuhKl2nk0KsYmeMT7SkzN6VWg== +"@one-ini/wasm@0.1.1": + version "0.1.1" + resolved "https://registry.yarnpkg.com/@one-ini/wasm/-/wasm-0.1.1.tgz#6013659736c9dbfccc96e8a9c2b3de317df39323" + integrity sha512-XuySG1E38YScSJoMlqovLru4KTUNSjgVTIjyh7qMX6aNN5HY5Ct5LhRJdxO79JtTzKfzV/bnWpz+zquYrISsvw== + +"@pkgjs/parseargs@^0.11.0": + version "0.11.0" + resolved "https://registry.yarnpkg.com/@pkgjs/parseargs/-/parseargs-0.11.0.tgz#a77ea742fab25775145434eb1d2328cf5013ac33" + integrity sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg== + "@sapphire/async-queue@^1.5.0": version "1.5.0" resolved "https://registry.yarnpkg.com/@sapphire/async-queue/-/async-queue-1.5.0.tgz#2f255a3f186635c4fb5a2381e375d3dfbc5312d8" @@ -725,7 +920,7 @@ "@types/qs" "*" "@types/serve-static" "*" -"@types/json-schema@^7.0.6": +"@types/json-schema@^7.0.5", "@types/json-schema@^7.0.6", "@types/json-schema@^7.0.8": version "7.0.15" resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.15.tgz#596a1747233694d50f6ad8a7869fcb6f56cf5841" integrity sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA== @@ -1129,11 +1324,45 @@ axios "^1.2.3" ethers "^5.7.2" +"@vf.js/gui@^3.0.2": + version "3.0.3" + resolved "https://registry.yarnpkg.com/@vf.js/gui/-/gui-3.0.3.tgz#095b509676d5473e8af55ba221a59b5478954cab" + integrity sha512-ej5GKK7pe8+U1TmHov8L1MUoarDGPq8s8ThpBd/CMkwg8IrVyq11n/vEUpKZi71mqtFKLkDvIWb2frJUYyMY9A== + +"@vf.js/launcher@2.0.12": + version "2.0.12" + resolved "https://registry.yarnpkg.com/@vf.js/launcher/-/launcher-2.0.12.tgz#32cac047091faeb1dccb989dbde05c3aecfd69e5" + integrity sha512-5zXvkLfIM+VydHZy3qInhsdBY4Ydp6TueVZQNBZyk9rb4Go/nAh8ouKdjfdpPQ2iZU8xT106qpS28VChQqRoHQ== + dependencies: + "@vf.js/gui" "^3.0.2" + "@vf.js/player" "^2.0.4" + "@vf.js/vf" "^6.0.2-v63" + +"@vf.js/player@^2.0.4": + version "2.0.4" + resolved "https://registry.yarnpkg.com/@vf.js/player/-/player-2.0.4.tgz#f6daf3f125b343d0886f4bd85de4e7afa93b16ff" + integrity sha512-exIUcWt/G/E91ckK0LAiavG84+uxsqWngSO3FRNedGHhkU6uPj+sTI0MLLX78lkHRfcC2zpvRQKF5Q7e2qTsZg== + +"@vf.js/vf@^6.0.2-v63": + version "6.0.2-v65" + resolved "https://registry.yarnpkg.com/@vf.js/vf/-/vf-6.0.2-v65.tgz#1db0bb92cf5a7dfd8b97aad2e662aa7c57f63742" + integrity sha512-W1GtAFKeLIfM/GDO0gKBzteaGJc1+5ND5rpxdOsBLWNkKdqVrMn+pAbf908nkgxGtoNvDbsoFzpmb2/TNRQB9Q== + +"@xmldom/xmldom@^0.8.6": + version "0.8.10" + resolved "https://registry.yarnpkg.com/@xmldom/xmldom/-/xmldom-0.8.10.tgz#a1337ca426aa61cef9fe15b5b28e340a72f6fa99" + integrity sha512-2WALfTl4xo2SkGCYRt6rDTFfk9R1czmBvUQy12gK2KuRKIpWEhcbbzy8EZXtz/jkRqHX8bFEc6FC1HjX4TUWYw== + abbrev@1: version "1.1.1" resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-1.1.1.tgz#f8f2c887ad10bf67f634f005b6987fed3179aac8" integrity sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q== +abbrev@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-2.0.0.tgz#cf59829b8b4f03f89dda2771cb7f3653828c89bf" + integrity sha512-6/mh1E2u2YgEsCHdY0Yx5oW+61gZU+1vXaoiHHrpKeuRNNgFvS+/jrwHiQhB5apAf5oB7UB7E19ol2R2LKH8hQ== + abort-controller@3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/abort-controller/-/abort-controller-3.0.0.tgz#eaf54d53b62bae4138e809ca225c8439a6efb392" @@ -1141,6 +1370,11 @@ abort-controller@3.0.0: dependencies: event-target-shim "^5.0.0" +abs-svg-path@^0.1.1: + version "0.1.1" + resolved "https://registry.yarnpkg.com/abs-svg-path/-/abs-svg-path-0.1.1.tgz#df601c8e8d2ba10d4a76d625e236a9a39c2723bf" + integrity sha512-d8XPSGjfyzlXC3Xx891DJRyZfqk5JU0BJrDQcsWomFIV1/BIzPW5HDH5iDdWpqWaav0YVIEzT1RHTwWr0FFshA== + abstract-leveldown@^6.2.1: version "6.3.0" resolved "https://registry.yarnpkg.com/abstract-leveldown/-/abstract-leveldown-6.3.0.tgz#d25221d1e6612f820c35963ba4bd739928f6026a" @@ -1171,6 +1405,11 @@ accepts@~1.3.8: mime-types "~2.1.34" negotiator "0.6.3" +adler-32@~1.3.0: + version "1.3.1" + resolved "https://registry.yarnpkg.com/adler-32/-/adler-32-1.3.1.tgz#1dbf0b36dda0012189a32b3679061932df1821e2" + integrity sha512-ynZ4w/nUUv5rrsR8UUGoe1VC9hZj6V5hU9Qw1HlMDJGEJw5S7TfTErWTjMys6M7vr0YWcPqs3qAr4ss0nDfP+A== + aes-js@3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/aes-js/-/aes-js-3.0.0.tgz#e21df10ad6c2053295bcbb8dab40b09dbea87e4d" @@ -1202,7 +1441,12 @@ ajv-formats@^2.1.1: dependencies: ajv "^8.0.0" -ajv@^6.12.3: +ajv-keywords@^3.5.2: + version "3.5.2" + resolved "https://registry.yarnpkg.com/ajv-keywords/-/ajv-keywords-3.5.2.tgz#31f29da5ab6e00d1c2d329acf7b5929614d5014d" + integrity sha512-5p6WTN0DdTGVQk6VjcEju19IgaHudalcfabD7yhDGeA6bcQnmL+CpveLJq/3hvfwd1aof6L386Ougkx6RfyMIQ== + +ajv@^6.12.3, ajv@^6.12.4, ajv@^6.12.5: version "6.12.6" resolved "https://registry.yarnpkg.com/ajv/-/ajv-6.12.6.tgz#baf5a62e802b07d977034586f8c3baf5adf26df4" integrity sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g== @@ -1239,6 +1483,11 @@ ansi-regex@^5.0.1: resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-5.0.1.tgz#082cb2c89c9fe8659a311a53bd6a4dc5301db304" integrity sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ== +ansi-regex@^6.0.1: + version "6.0.1" + resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-6.0.1.tgz#3183e38fae9a65d7cb5e53945cd5897d0260a06a" + integrity sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA== + ansi-styles@^3.2.1: version "3.2.1" resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-3.2.1.tgz#41fbb20243e50b12be0f04b8dedbf07520ce841d" @@ -1253,6 +1502,11 @@ ansi-styles@^4.0.0, ansi-styles@^4.1.0: dependencies: color-convert "^2.0.1" +ansi-styles@^6.1.0: + version "6.2.1" + resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-6.2.1.tgz#0e62320cf99c21afff3b3012192546aacbfb05c5" + integrity sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug== + any-promise@^1.3.0: version "1.3.0" resolved "https://registry.yarnpkg.com/any-promise/-/any-promise-1.3.0.tgz#abc6afeedcea52e809cdc0376aed3ce39635d17f" @@ -1300,6 +1554,13 @@ argparse@^2.0.1: resolved "https://registry.yarnpkg.com/argparse/-/argparse-2.0.1.tgz#246f50f3ca78a3240f6c997e8a9bd1eac49e4b38" integrity sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q== +argparse@~1.0.3: + version "1.0.10" + resolved "https://registry.yarnpkg.com/argparse/-/argparse-1.0.10.tgz#bcd6791ea5ae09725e17e5ad988134cd40b3d911" + integrity sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg== + dependencies: + sprintf-js "~1.0.2" + argsarray@0.0.1: version "0.0.1" resolved "https://registry.yarnpkg.com/argsarray/-/argsarray-0.0.1.tgz#6e7207b4ecdb39b0af88303fa5ae22bda8df61cb" @@ -1431,6 +1692,30 @@ axios@^1.2.3, axios@^1.3.3, axios@^1.6.2, axios@^1.7.2: form-data "^4.0.0" proxy-from-env "^1.1.0" +babel-plugin-polyfill-corejs2@^0.4.10: + version "0.4.11" + resolved "https://registry.yarnpkg.com/babel-plugin-polyfill-corejs2/-/babel-plugin-polyfill-corejs2-0.4.11.tgz#30320dfe3ffe1a336c15afdcdafd6fd615b25e33" + integrity sha512-sMEJ27L0gRHShOh5G54uAAPaiCOygY/5ratXuiyb2G46FmlSpc9eFCzYVyDiPxfNbwzA7mYahmjQc5q+CZQ09Q== + dependencies: + "@babel/compat-data" "^7.22.6" + "@babel/helper-define-polyfill-provider" "^0.6.2" + semver "^6.3.1" + +babel-plugin-polyfill-corejs3@^0.10.1: + version "0.10.6" + resolved "https://registry.yarnpkg.com/babel-plugin-polyfill-corejs3/-/babel-plugin-polyfill-corejs3-0.10.6.tgz#2deda57caef50f59c525aeb4964d3b2f867710c7" + integrity sha512-b37+KR2i/khY5sKmWNVQAnitvquQbNdWy6lJdsr0kmquCKEEUgMKK4SboVM3HtfnZilfjr4MMQ7vY58FVWDtIA== + dependencies: + "@babel/helper-define-polyfill-provider" "^0.6.2" + core-js-compat "^3.38.0" + +babel-plugin-polyfill-regenerator@^0.6.1: + version "0.6.2" + resolved "https://registry.yarnpkg.com/babel-plugin-polyfill-regenerator/-/babel-plugin-polyfill-regenerator-0.6.2.tgz#addc47e240edd1da1058ebda03021f382bba785e" + integrity sha512-2R25rQZWP63nGwaAswvDazbPXfrM3HwVoBXK6HcqeKrSrL/JqcC/rDcf95l4r7LXLyxDXc8uQDa064GubtCABg== + dependencies: + "@babel/helper-define-polyfill-provider" "^0.6.2" + babel-runtime@^6.23.0: version "6.26.0" resolved "https://registry.yarnpkg.com/babel-runtime/-/babel-runtime-6.26.0.tgz#965c7058668e82b55d7bfe04ff2337bc8b5647fe" @@ -1456,7 +1741,7 @@ base-x@^4.0.0: resolved "https://registry.yarnpkg.com/base-x/-/base-x-4.0.0.tgz#d0e3b7753450c73f8ad2389b5c018a4af7b2224a" integrity sha512-FuwxlW4H5kh37X/oW59pwTzzTKRzfrrQwhmyspRM7swOEZcHtDZSCt45U6oKgtuFE+WYPblePMVIPR4RZrh/hw== -base64-js@^1.0.2, base64-js@^1.3.0, base64-js@^1.3.1: +base64-js@^1.0.2, base64-js@^1.3.0, base64-js@^1.3.1, base64-js@^1.5.1: version "1.5.1" resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.5.1.tgz#1b1b440160a5bf7ad40b650f095963481903930a" integrity sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA== @@ -1490,6 +1775,11 @@ bech32@^2.0.0: resolved "https://registry.yarnpkg.com/bech32/-/bech32-2.0.0.tgz#078d3686535075c8c79709f054b1b226a133b355" integrity sha512-LcknSilhIGatDAsY1ak2I8VtGaHNhgMSYVxFrGLXv+xLHytaKZKcaUJJUE7qmBr7h33o5YQwP55pMI0xmkpJwg== +big.js@^5.2.2: + version "5.2.2" + resolved "https://registry.yarnpkg.com/big.js/-/big.js-5.2.2.tgz#65f0af382f578bcdc742bd9c281e9cb2d7768328" + integrity sha512-vyL2OymJxmarO8gxMr0mhChsO9QGwhynfuu4+MHTAW6czfq9humCB7rKpUjDd9YUiDPU4mzpyupFSvOClAwbmQ== + bignumber.js@^9.0.0: version "9.1.2" resolved "https://registry.yarnpkg.com/bignumber.js/-/bignumber.js-9.1.2.tgz#b7c4242259c008903b13707983b5f4bbd31eda0c" @@ -1514,6 +1804,11 @@ bl@^4.0.3: inherits "^2.0.4" readable-stream "^3.4.0" +bluebird@~3.4.0: + version "3.4.7" + resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.4.7.tgz#f72d760be09b7f76d08ed8fae98b289a8d05fab3" + integrity sha512-iD3898SR7sWVRHbiQv+sHUtHnMvC1o3nW5rAcqnq3uOn07DSAppZYUkIGslDz6gXC7HfunPe7YVBgoEJASPcHA== + bn.js@^4.11.9: version "4.12.0" resolved "https://registry.yarnpkg.com/bn.js/-/bn.js-4.12.0.tgz#775b3f278efbb9718eec7361f483fb36fbbfea88" @@ -1564,6 +1859,13 @@ brace-expansion@^1.1.7: balanced-match "^1.0.0" concat-map "0.0.1" +brace-expansion@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-2.0.1.tgz#1edc459e0f0c548486ecf9fc99f2221364b9a0ae" + integrity sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA== + dependencies: + balanced-match "^1.0.0" + braces@~3.0.2: version "3.0.2" resolved "https://registry.yarnpkg.com/braces/-/braces-3.0.2.tgz#3454e1a462ee8d599e236df336cd9ea4f8afe107" @@ -1588,6 +1890,16 @@ browserify-zlib@^0.1.4: dependencies: pako "~0.2.0" +browserslist@^4.23.1, browserslist@^4.23.3: + version "4.23.3" + resolved "https://registry.yarnpkg.com/browserslist/-/browserslist-4.23.3.tgz#debb029d3c93ebc97ffbc8d9cbb03403e227c800" + integrity sha512-btwCFJVjI4YWDNfau8RhZ+B1Q/VLoUITrm3RlP6y1tYGWIOa+InuYiRGXUBXo8nA1qKmHMyLB/iVQg5TT4eFoA== + dependencies: + caniuse-lite "^1.0.30001646" + electron-to-chromium "^1.5.4" + node-releases "^2.0.18" + update-browserslist-db "^1.1.0" + bs58@^4.0.1: version "4.0.1" resolved "https://registry.yarnpkg.com/bs58/-/bs58-4.0.1.tgz#be161e76c354f6f788ae4071f63f34e8c4f0a42a" @@ -1683,11 +1995,21 @@ call-me-maybe@^1.0.1: resolved "https://registry.yarnpkg.com/call-me-maybe/-/call-me-maybe-1.0.2.tgz#03f964f19522ba643b1b0693acb9152fe2074baa" integrity sha512-HpX65o1Hnr9HH25ojC1YGs7HCQLq0GCOibSaWER0eNpgJ/Z1MZv2mTc7+xh6WOPxbRVcmgbv4hGU+uSQ/2xFZQ== +camelcase@^5.3.1: + version "5.3.1" + resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-5.3.1.tgz#e3c9b31569e106811df242f715725a1f4c494320" + integrity sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg== + camelcase@^6.0.0, camelcase@^6.2.0: version "6.3.0" resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-6.3.0.tgz#5685b95eb209ac9c0c177467778c9c84df58ba9a" integrity sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA== +caniuse-lite@^1.0.30001646: + version "1.0.30001651" + resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001651.tgz#52de59529e8b02b1aedcaaf5c05d9e23c0c28138" + integrity sha512-9Cf+Xv1jJNe1xPZLGuUXLNkE1BoDkqRqYyFJ9TDYSqhduqA4hu4oR9HluGoWYQC/aj8WHjsGVV+bwkh0+tegRg== + canonicalize@^1.0.5: version "1.0.8" resolved "https://registry.yarnpkg.com/canonicalize/-/canonicalize-1.0.8.tgz#24d1f1a00ed202faafd9bf8e63352cd4450c6df1" @@ -1703,6 +2025,14 @@ caseless@~0.12.0: resolved "https://registry.yarnpkg.com/caseless/-/caseless-0.12.0.tgz#1b681c21ff84033c826543090689420d187151dc" integrity sha512-4tYFyifaFfGacoiObjJegolkwSU4xQNGbVgUiNYVUxbQ2x2lUsFvY4hVgVzGiIe6WLOPqycWXA40l+PWsxthUw== +cfb@~1.2.1: + version "1.2.2" + resolved "https://registry.yarnpkg.com/cfb/-/cfb-1.2.2.tgz#94e687628c700e5155436dac05f74e08df23bc44" + integrity sha512-KfdUZsSOw19/ObEWasvBP/Ac4reZvAGauZhs6S/gqNhXhI7cKwvlH7ulj+dOEYnca4bm4SGo8C1bTAQvnTjgQA== + dependencies: + adler-32 "~1.3.0" + crc-32 "~1.2.0" + chalk@^2.4.2: version "2.4.2" resolved "https://registry.yarnpkg.com/chalk/-/chalk-2.4.2.tgz#cd42541677a54333cf541a49108c1432b44c9424" @@ -1798,6 +2128,11 @@ clone-response@^1.0.2: dependencies: mimic-response "^1.0.0" +codepage@~1.15.0: + version "1.15.0" + resolved "https://registry.yarnpkg.com/codepage/-/codepage-1.15.0.tgz#2e00519024b39424ec66eeb3ec07227e692618ab" + integrity sha512-3g6NUTPd/YtuuGrhMnOMRjFc+LJw/bnMp3+0r/Wcz3IXUuCosKRJvMphm5+Q+bvTVGcJJuRvVLuYba+WojaFaA== + color-convert@^1.9.0: version "1.9.3" resolved "https://registry.yarnpkg.com/color-convert/-/color-convert-1.9.3.tgz#bb71850690e1f136567de629d2d5471deda4c1e8" @@ -1822,6 +2157,11 @@ color-name@~1.1.4: resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.4.tgz#c2a09a87acbde69543de6f63fa3995c826c536a2" integrity sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA== +colors@^1.4.0: + version "1.4.0" + resolved "https://registry.yarnpkg.com/colors/-/colors-1.4.0.tgz#c50491479d4c1bdaed2c9ced32cf7c7dc2360f78" + integrity sha512-a+UqTh4kgZg/SlGvfbzDHpgRu7AAQOmmqRHJnxhRZICKFUT91brVhNNt58CMWU9PsBbv3PDCZUHbVxuDiH2mtA== + combined-stream@^1.0.6, combined-stream@^1.0.8, combined-stream@~1.0.6: version "1.0.8" resolved "https://registry.yarnpkg.com/combined-stream/-/combined-stream-1.0.8.tgz#c3d45a8b34fd730631a110a8a2520682b31d5a7f" @@ -1872,6 +2212,11 @@ command-line-usage@6.1.3: table-layout "^1.0.2" typical "^5.2.0" +commander@^10.0.0: + version "10.0.1" + resolved "https://registry.yarnpkg.com/commander/-/commander-10.0.1.tgz#881ee46b4f77d1c1dccc5823433aa39b022cbe06" + integrity sha512-y4Mg2tXshplEbSGzx7amzPwKKOCGuoSRP/CjEdwwk0FOGlUbq6lKuoyDZTNZkmxHdJtp54hdfY/JUrdL7Xfdug== + compress-commons@^2.1.1: version "2.1.1" resolved "https://registry.yarnpkg.com/compress-commons/-/compress-commons-2.1.1.tgz#9410d9a534cf8435e3fbbb7c6ce48de2dc2f0610" @@ -1887,6 +2232,14 @@ concat-map@0.0.1: resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b" integrity sha1-2Klr13/Wjfd5OnMDajug1UBdR3s= +config-chain@^1.1.13: + version "1.1.13" + resolved "https://registry.yarnpkg.com/config-chain/-/config-chain-1.1.13.tgz#fad0795aa6a6cdaff9ed1b68e9dff94372c232f4" + integrity sha512-qj+f8APARXHrM0hraqXYb2/bOVSV4PvJQlNZ/DVj0QrmNM2q2euizkeuVckQ57J+W0mRH6Hvi+k50M4Jul2VRQ== + dependencies: + ini "^1.3.4" + proto-list "~1.2.1" + configstore@^5.0.1: version "5.0.1" resolved "https://registry.yarnpkg.com/configstore/-/configstore-5.0.1.tgz#d365021b5df4b98cdd187d6a3b0e3f6a7cc5ed96" @@ -1944,11 +2297,28 @@ core-decorators@^0.17.0: resolved "https://registry.yarnpkg.com/core-decorators/-/core-decorators-0.17.0.tgz#3f43180a86d2ab0cc51069f46a1ec3e49e7cebd6" integrity sha1-P0MYCobSqwzFEGn0ah7D5J5869Y= +core-js-compat@^3.38.0: + version "3.38.0" + resolved "https://registry.yarnpkg.com/core-js-compat/-/core-js-compat-3.38.0.tgz#d93393b1aa346b6ee683377b0c31172ccfe607aa" + integrity sha512-75LAicdLa4OJVwFxFbQR3NdnZjNgX6ILpVcVzcC4T2smerB5lELMrJQQQoWV6TiuC/vlaFqgU2tKQx9w5s0e0A== + dependencies: + browserslist "^4.23.3" + +core-js-pure@^3.30.2: + version "3.38.0" + resolved "https://registry.yarnpkg.com/core-js-pure/-/core-js-pure-3.38.0.tgz#bc802cd152e33d5b0ec733b656c71cb847cac701" + integrity sha512-8balb/HAXo06aHP58mZMtXgD8vcnXz9tUDePgqBgJgKdmTlMt+jw3ujqniuBDQXMvTzxnMpxHFeuSM3g1jWQuQ== + core-js@^2.4.0: version "2.6.12" resolved "https://registry.yarnpkg.com/core-js/-/core-js-2.6.12.tgz#d9333dfa7b065e347cc5682219d6f690859cc2ec" integrity sha512-Kb2wC0fvsWfQrgk8HU5lW6U/Lcs8+9aaYcy4ZFc6DDlo4nZ7n70dEgE5rtR0oG6ufKDUnrwfWL1mXR5ljDatrQ== +core-js@^3.6.0: + version "3.38.0" + resolved "https://registry.yarnpkg.com/core-js/-/core-js-3.38.0.tgz#8acb7c050bf2ccbb35f938c0d040132f6110f636" + integrity sha512-XPpwqEodRljce9KswjZShh95qJ1URisBeKCjUdq27YdenkslVe7OO0ZJhlYXAChW7OhXaRLl8AAba7IBfoIHug== + core-util-is@1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.2.tgz#b5fd54220aa2bc5ab57aab7140c940754503c1a7" @@ -1967,6 +2337,11 @@ cors@^2.8.5: object-assign "^4" vary "^1" +crc-32@~1.2.0, crc-32@~1.2.1: + version "1.2.2" + resolved "https://registry.yarnpkg.com/crc-32/-/crc-32-1.2.2.tgz#3cad35a934b8bf71f25ca524b6da51fb7eace2ff" + integrity sha512-ROmzCKrTnOwybPcJApAA6WBWij23HVfGVNKqqrZpuyZOHqK2CwHSvpGuyt/UNNvaIjEd8X5IFGp4Mh+Ie1IHJQ== + crc32-stream@^3.0.1: version "3.0.1" resolved "https://registry.yarnpkg.com/crc32-stream/-/crc32-stream-3.0.1.tgz#cae6eeed003b0e44d739d279de5ae63b171b4e85" @@ -2005,6 +2380,15 @@ create-hmac@^1.1.4: safe-buffer "^5.0.1" sha.js "^2.4.8" +cross-spawn@^7.0.0: + version "7.0.3" + resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.3.tgz#f73a85b9d5d41d045551c177e2882d4ac85728a6" + integrity sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w== + dependencies: + path-key "^3.1.0" + shebang-command "^2.0.0" + which "^2.0.1" + "crypto-pouch@git+https://github.com/tahpot/crypto-pouch.git#feature/support-key-import": version "4.0.1" resolved "git+https://github.com/tahpot/crypto-pouch.git#7a712691b4404cfefb34ad3f58db1d83b55ed3bf" @@ -2017,6 +2401,30 @@ crypto-random-string@^2.0.0: resolved "https://registry.yarnpkg.com/crypto-random-string/-/crypto-random-string-2.0.0.tgz#ef2a7a966ec11083388369baa02ebead229b30d5" integrity sha512-v1plID3y9r/lPhviJ1wrXpLeyUIGAZ2SHNYTEapm7/8A9nLPoyvVp3RK/EPFqn5kEznyWgYZNsRtYYIWbuG8KA== +css-loader@^3.6.0: + version "3.6.0" + resolved "https://registry.yarnpkg.com/css-loader/-/css-loader-3.6.0.tgz#2e4b2c7e6e2d27f8c8f28f61bffcd2e6c91ef645" + integrity sha512-M5lSukoWi1If8dhQAUCvj4H8vUt3vOnwbQBH9DdTm/s4Ym2B/3dPMtYZeJmq7Q3S3Pa+I94DcZ7pc9bP14cWIQ== + dependencies: + camelcase "^5.3.1" + cssesc "^3.0.0" + icss-utils "^4.1.1" + loader-utils "^1.2.3" + normalize-path "^3.0.0" + postcss "^7.0.32" + postcss-modules-extract-imports "^2.0.0" + postcss-modules-local-by-default "^3.0.2" + postcss-modules-scope "^2.2.0" + postcss-modules-values "^3.0.0" + postcss-value-parser "^4.1.0" + schema-utils "^2.7.0" + semver "^6.3.0" + +cssesc@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/cssesc/-/cssesc-3.0.0.tgz#37741919903b868565e1c09ea747445cd18983ee" + integrity sha512-/Tb/JcjK111nNScGob5MNtsntNM1aCNUDipB/TkwZFhyDrrE47SOx/18wF2bbjgc3ZzCSKW1T5nt5EbFoAz/Vg== + dashdash@^1.12.0: version "1.14.1" resolved "https://registry.yarnpkg.com/dashdash/-/dashdash-1.14.1.tgz#853cfa0f7cbe2fed5de20326b8dd581035f6e2f0" @@ -2067,6 +2475,13 @@ debug@^3.1.0, debug@^3.2.7: dependencies: ms "^2.1.1" +debug@^4.1.1, debug@^4.3.1: + version "4.3.6" + resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.6.tgz#2ab2c38fbaffebf8aa95fdfe6d88438c7a13c52b" + integrity sha512-O/09Bd4Z1fBrU4VzkhFqVgpPzaGbw6Sm9FEkBT1A/YBXQFGuuSxa1dN2nxgxS34JmKXqYx8CZAwEVoJFImUXIg== + dependencies: + ms "2.1.2" + decamelize@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/decamelize/-/decamelize-4.0.0.tgz#aa472d7bf660eb15f3494efd531cab7f2a709837" @@ -2084,6 +2499,14 @@ deep-extend@^0.6.0, deep-extend@~0.6.0: resolved "https://registry.yarnpkg.com/deep-extend/-/deep-extend-0.6.0.tgz#c4fa7c95404a17a9c3e8ca7e1537312b736330ac" integrity sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA== +deep-rename-keys@^0.2.1: + version "0.2.1" + resolved "https://registry.yarnpkg.com/deep-rename-keys/-/deep-rename-keys-0.2.1.tgz#ede78537d7a66a2be61517e2af956d7f58a3f1d8" + integrity sha512-RHd9ABw4Fvk+gYDWqwOftG849x0bYOySl/RgX0tLI9i27ZIeSO91mLZJEp7oPHOMFqHvpgu21YptmDt0FYD/0A== + dependencies: + kind-of "^3.0.2" + rename-keys "^1.1.2" + deepcopy@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/deepcopy/-/deepcopy-2.1.0.tgz#2deb0dd52d079c2ecb7924b640a7c3abd4db1d6d" @@ -2091,7 +2514,7 @@ deepcopy@^2.1.0: dependencies: type-detect "^4.0.8" -deepmerge@^4.3.1: +deepmerge@^4.2.2, deepmerge@^4.3.1: version "4.3.1" resolved "https://registry.yarnpkg.com/deepmerge/-/deepmerge-4.3.1.tgz#44b5f2147cd3b00d4b56137685966f26fd25dd4a" integrity sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A== @@ -2205,6 +2628,11 @@ diff@^3.1.0: resolved "https://registry.yarnpkg.com/diff/-/diff-3.5.0.tgz#800c0dd1e0a8bfbc95835c202ad220fe317e5a12" integrity sha512-A46qtFgd+g7pDZinpnwiRJtxbC1hpgf0uzP3iG89scHk0AUC7A1TGxf5OiiOUv/JMZR8GOt8hL900hV0bOy5xA== +dingbat-to-unicode@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/dingbat-to-unicode/-/dingbat-to-unicode-1.0.1.tgz#5091dd673241453e6b5865e26e5a4452cdef5c83" + integrity sha512-98l0sW87ZT58pU4i61wa2OHwxbiYSbuxsCBozaVnYX2iCnr3bLM3fIes1/ej7h1YdOKuKt/MLs706TVnALA65w== + discord-api-types@^0.37.37: version "0.37.38" resolved "https://registry.yarnpkg.com/discord-api-types/-/discord-api-types-0.37.38.tgz#0599937f64aff63a2b534376563021f17537d166" @@ -2281,6 +2709,13 @@ double-ended-queue@2.1.0-0: resolved "https://registry.yarnpkg.com/double-ended-queue/-/double-ended-queue-2.1.0-0.tgz#103d3527fd31528f40188130c841efdd78264e5c" integrity sha512-+BNfZ+deCo8hMNpDqDnvT+c0XpJ5cUa6mqYq89bho2Ifze4URTqRkcwR399hWoTrTkbZ/XJYDgP6rc7pRgffEQ== +duck@^0.1.12: + version "0.1.12" + resolved "https://registry.yarnpkg.com/duck/-/duck-0.1.12.tgz#de7adf758421230b6d7aee799ce42670586b9efa" + integrity sha512-wkctla1O6VfP89gQ+J/yDesM0S7B7XLXjKGzXxMDVFg7uEn706niAtyYovKbyq1oT9YwDcly721/iUWoc8MVRg== + dependencies: + underscore "^1.13.1" + duplexer3@^0.1.4: version "0.1.4" resolved "https://registry.yarnpkg.com/duplexer3/-/duplexer3-0.1.4.tgz#ee01dd1cac0ed3cbc7fdbea37dc0a8f1ce002ce2" @@ -2296,6 +2731,11 @@ duplexify@^3.5.0, duplexify@^3.6.0: readable-stream "^2.0.0" stream-shift "^1.0.0" +eastasianwidth@^0.2.0: + version "0.2.0" + resolved "https://registry.yarnpkg.com/eastasianwidth/-/eastasianwidth-0.2.0.tgz#696ce2ec0aa0e6ea93a397ffcf24aa7840c827cb" + integrity sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA== + ecc-jsbn@~0.1.1: version "0.1.2" resolved "https://registry.yarnpkg.com/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz#3a83a904e54353287874c564b7549386849a98c9" @@ -2311,11 +2751,31 @@ ecdsa-sig-formatter@1.0.11, ecdsa-sig-formatter@^1.0.11: dependencies: safe-buffer "^5.0.1" +editorconfig@^1.0.4: + version "1.0.4" + resolved "https://registry.yarnpkg.com/editorconfig/-/editorconfig-1.0.4.tgz#040c9a8e9a6c5288388b87c2db07028aa89f53a3" + integrity sha512-L9Qe08KWTlqYMVvMcTIvMAdl1cDUubzRNYL+WfA4bLDMHe4nemKkpmYzkznE1FwLKu0EEmy6obgQKzMJrg4x9Q== + dependencies: + "@one-ini/wasm" "0.1.1" + commander "^10.0.0" + minimatch "9.0.1" + semver "^7.5.3" + ee-first@1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/ee-first/-/ee-first-1.1.1.tgz#590c61156b0ae2f4f0255732a158b266bc56b21d" integrity sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0= +electron-to-chromium@^1.5.4: + version "1.5.11" + resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.5.11.tgz#258077f1077a1c72f2925cd5b326c470a7f5adef" + integrity sha512-R1CccCDYqndR25CaXFd6hp/u9RaaMcftMkphmvuepXr5b1vfLkRml6aWVeBhXJ7rbevHkKEMJtz8XqPf7ffmew== + +element-to-path@^1.2.0: + version "1.2.1" + resolved "https://registry.yarnpkg.com/element-to-path/-/element-to-path-1.2.1.tgz#06afb439a50fa2870d11f846a9e876fe60c7e72c" + integrity sha512-JNFZS0yI3Myywn/ltFj/yTihHNzMTYk0ycHcgcjlvA/dYMUjMIGqvbezPZeXN3U1Klp/aiigr2mpmhVRfudtbg== + elliptic@6.5.4: version "6.5.4" resolved "https://registry.yarnpkg.com/elliptic/-/elliptic-6.5.4.tgz#da37cebd31e79a1367e941b592ed1fbebd58abbb" @@ -2347,6 +2807,16 @@ emoji-regex@^8.0.0: resolved "https://registry.yarnpkg.com/emoji-regex/-/emoji-regex-8.0.0.tgz#e818fd69ce5ccfcb404594f842963bf53164cc37" integrity sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A== +emoji-regex@^9.2.2: + version "9.2.2" + resolved "https://registry.yarnpkg.com/emoji-regex/-/emoji-regex-9.2.2.tgz#840c8803b0d8047f4ff0cf963176b32d4ef3ed72" + integrity sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg== + +emojis-list@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/emojis-list/-/emojis-list-3.0.0.tgz#5570662046ad29e2e916e71aae260abdff4f6a78" + integrity sha512-/kyM18EfinwXZbno9FyUGeFh87KC8HRQBQGildHZbEuRyWFOmv1U10o9BBp8XVZDVNNuQKyIGIu5ZYAAXJ0V2Q== + encodeurl@~1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/encodeurl/-/encodeurl-1.0.2.tgz#ad3ff4c86ec2d029322f5a02c3a9a606c95b3f59" @@ -2415,6 +2885,11 @@ escalade@^3.1.1: resolved "https://registry.yarnpkg.com/escalade/-/escalade-3.1.1.tgz#d8cfdc7000965c5a0174b4a82eaa5c0552742e40" integrity sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw== +escalade@^3.1.2: + version "3.1.2" + resolved "https://registry.yarnpkg.com/escalade/-/escalade-3.1.2.tgz#54076e9ab29ea5bf3d8f1ed62acffbb88272df27" + integrity sha512-ErCHMCae19vR8vQGe50xIsVomy19rg6gFu3+r3jkEO46suLMWBksvVyoGgQV+jOfl84ZSOSlmv6Gxa89PmTGmA== + escape-goat@^2.0.0: version "2.1.1" resolved "https://registry.yarnpkg.com/escape-goat/-/escape-goat-2.1.1.tgz#1b2dc77003676c457ec760b2dc68edb648188675" @@ -2494,6 +2969,11 @@ event-target-shim@^5.0.0: resolved "https://registry.yarnpkg.com/event-target-shim/-/event-target-shim-5.0.1.tgz#5d4d3ebdf9583d63a5333ce2deb7480ab2b05789" integrity sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ== +eventemitter3@^2.0.0: + version "2.0.3" + resolved "https://registry.yarnpkg.com/eventemitter3/-/eventemitter3-2.0.3.tgz#b5e1079b59fb5e1ba2771c0a993be060a58c99ba" + integrity sha512-jLN68Dx5kyFHaePoXWPsCGW5qdyZQtLYHkxkg02/Mz6g0kYpDx4FyP6XfArhQdlOC4b8Mv+EMxPo/8La7Tzghg== + events@1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/events/-/events-1.1.1.tgz#9ebdb7635ad099c70dcc4c2a1f5004288e8bd924" @@ -2668,6 +3148,14 @@ follow-redirects@^1.14.4, follow-redirects@^1.14.9, follow-redirects@^1.15.0, fo resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.6.tgz#7f815c0cda4249c74ff09e95ef97c23b5fd0399b" integrity sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA== +foreground-child@^3.1.0: + version "3.3.0" + resolved "https://registry.yarnpkg.com/foreground-child/-/foreground-child-3.3.0.tgz#0ac8644c06e431439f8561db8ecf29a7b5519c77" + integrity sha512-Ld2g8rrAyMYFXBhEqMz8ZAHBi4J4uS1i/CxGMDnjyFWddMXLVcDp051DZfu+t7+ab7Wv6SMqpWmyFIj5UbfFvg== + dependencies: + cross-spawn "^7.0.0" + signal-exit "^4.0.1" + forever-agent@~0.6.1: version "0.6.1" resolved "https://registry.yarnpkg.com/forever-agent/-/forever-agent-0.6.1.tgz#fbc71f0c41adeb37f96c577ad1ed42d8fdacca91" @@ -2696,6 +3184,11 @@ forwarded@0.2.0: resolved "https://registry.yarnpkg.com/forwarded/-/forwarded-0.2.0.tgz#2269936428aad4c15c7ebe9779a84bf0b2a81811" integrity sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow== +frac@~1.1.2: + version "1.1.2" + resolved "https://registry.yarnpkg.com/frac/-/frac-1.1.2.tgz#3d74f7f6478c88a1b5020306d747dc6313c74d0b" + integrity sha512-w/XBfkibaTl3YDqASwfDUqkna4Z2p9cFSr1aHDt0WoMTECnRfBOv2WArlZILlqgWlmdIlALXGpM2AOhEk5W3IA== + fresh@0.5.2: version "0.5.2" resolved "https://registry.yarnpkg.com/fresh/-/fresh-0.5.2.tgz#3d8cadd90d976569fa835ab1f8e4b23a105605a7" @@ -2724,6 +3217,15 @@ fs-extra@^6.0.1: jsonfile "^4.0.0" universalify "^0.1.0" +fs-extra@^8.1.0: + version "8.1.0" + resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-8.1.0.tgz#49d43c45a88cd9677668cb7be1b46efdb8d2e1c0" + integrity sha512-yhlQgA6mnOJUKOsRUFsgJdQCvkKhcz8tlZG5HBQfReYZy46OwLcY+Zia0mtdHsOo9y/hP+CxMN0TU9QxoOtG4g== + dependencies: + graceful-fs "^4.2.0" + jsonfile "^4.0.0" + universalify "^0.1.0" + fs.realpath@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f" @@ -2815,6 +3317,11 @@ get-stream@^5.1.0: dependencies: pump "^3.0.0" +get-value@^2.0.3: + version "2.0.6" + resolved "https://registry.yarnpkg.com/get-value/-/get-value-2.0.6.tgz#dc15ca1c672387ca76bd37ac0a395ba2042a2c28" + integrity sha512-Ln0UQDlxH1BapMu3GPtf7CuYNwRZf2gwCuPqbyG6pB8WfmFpzqcy4xtAaAMUhnNqjMKTiCPZG2oMT3YSx8U2NA== + getpass@^0.1.1: version "0.1.7" resolved "https://registry.yarnpkg.com/getpass/-/getpass-0.1.7.tgz#5eff8e3e684d569ae4cb2b1282604e8ba62149fa" @@ -2841,7 +3348,19 @@ glob@7.2.0: once "^1.3.0" path-is-absolute "^1.0.0" -glob@^7.0.5, glob@^7.1.2, glob@^7.1.4: +glob@^10.3.3: + version "10.4.5" + resolved "https://registry.yarnpkg.com/glob/-/glob-10.4.5.tgz#f4d9f0b90ffdbab09c9d77f5f29b4262517b0956" + integrity sha512-7Bv8RF0k6xjo7d4A/PxYLbUCfb6c+Vpd2/mB2yRDlew7Jb5hEXiCD9ibfO7wpk8i4sevK6DFny9h7EYbM3/sHg== + dependencies: + foreground-child "^3.1.0" + jackspeak "^3.1.2" + minimatch "^9.0.4" + minipass "^7.1.2" + package-json-from-dist "^1.0.0" + path-scurry "^1.11.1" + +glob@^7.0.5, glob@^7.1.2, glob@^7.1.4, glob@^7.1.6: version "7.2.3" resolved "https://registry.yarnpkg.com/glob/-/glob-7.2.3.tgz#b8df0fb802bbfa8e89bd1d938b4e16578ed44f2b" integrity sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q== @@ -2860,6 +3379,11 @@ global-dirs@^3.0.0: dependencies: ini "2.0.0" +globals@^11.1.0: + version "11.12.0" + resolved "https://registry.yarnpkg.com/globals/-/globals-11.12.0.tgz#ab8795338868a0babd8525758018c2a7eb95c42e" + integrity sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA== + google-auth-library@^9.0.0, google-auth-library@^9.7.0: version "9.11.0" resolved "https://registry.yarnpkg.com/google-auth-library/-/google-auth-library-9.11.0.tgz#bd6da364bcde4e0cc4ed70a0e0df5112b6a671dd" @@ -2986,6 +3510,20 @@ has-symbols@^1.0.1, has-symbols@^1.0.3: resolved "https://registry.yarnpkg.com/has-symbols/-/has-symbols-1.0.3.tgz#bb7b2c4349251dce87b125f7bdf874aa7c8b39f8" integrity sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A== +has-value@^0.3.1: + version "0.3.1" + resolved "https://registry.yarnpkg.com/has-value/-/has-value-0.3.1.tgz#7b1f58bada62ca827ec0a2078025654845995e1f" + integrity sha512-gpG936j8/MzaeID5Yif+577c17TxaDmhuyVgSwtnL/q8UUTySg8Mecb+8Cf1otgLoD7DDH75axp86ER7LFsf3Q== + dependencies: + get-value "^2.0.3" + has-values "^0.1.4" + isobject "^2.0.0" + +has-values@^0.1.4: + version "0.1.4" + resolved "https://registry.yarnpkg.com/has-values/-/has-values-0.1.4.tgz#6d61de95d91dfca9b9a02089ad384bff8f62b771" + integrity sha512-J8S0cEdWuQbqD9//tlZxiMuMNmxB8PlEwvYwuxsTmR1G5RXUePEX/SJn7aD0GMLieuZYSwNH0cQuJGwnYunXRQ== + has-yarn@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/has-yarn/-/has-yarn-2.1.0.tgz#137e11354a7b5bf11aa5cb649cf0c6f3ff2b2e77" @@ -3015,7 +3553,7 @@ hash.js@1.1.7, hash.js@^1.0.0, hash.js@^1.0.3: inherits "^2.0.3" minimalistic-assert "^1.0.1" -hasown@^2.0.0: +hasown@^2.0.0, hasown@^2.0.2: version "2.0.2" resolved "https://registry.yarnpkg.com/hasown/-/hasown-2.0.2.tgz#003eaf91be7adc372e84ec59dc37252cedb80003" integrity sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ== @@ -3112,6 +3650,13 @@ iconv-lite@0.6.3: dependencies: safer-buffer ">= 2.1.2 < 3.0.0" +icss-utils@^4.0.0, icss-utils@^4.1.1: + version "4.1.1" + resolved "https://registry.yarnpkg.com/icss-utils/-/icss-utils-4.1.1.tgz#21170b53789ee27447c2f47dd683081403f9a467" + integrity sha512-4aFq7wvWyMHKgxsH8QQtGpvbASCf+eM3wPRLI6R+MgAnTCZ6STYsRvttLvRWK0Nfif5piF394St3HeJDaljGPA== + dependencies: + postcss "^7.0.14" + ieee754@1.1.13: version "1.1.13" resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.1.13.tgz#ec168558e95aa181fd87d37f55c32bbcb6708b84" @@ -3132,6 +3677,11 @@ immediate@3.3.0, immediate@^3.2.3: resolved "https://registry.yarnpkg.com/immediate/-/immediate-3.3.0.tgz#1aef225517836bcdf7f2a2de2600c79ff0269266" integrity sha512-HR7EVodfFUdQCTIeySw+WDRFJlPcLOJbXfwwZ7Oom6tjsvZ3bOkCDJHehQC3nxJrv7+f9XecwazynjU8e4Vw3Q== +immediate@~3.0.5: + version "3.0.6" + resolved "https://registry.yarnpkg.com/immediate/-/immediate-3.0.6.tgz#9db1dbd0faf8de6fbe0f5dd5e56bb606280de69b" + integrity sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ== + import-lazy@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/import-lazy/-/import-lazy-2.1.0.tgz#05698e3d45c88e8d7e9d92cb0584e77f096f3e43" @@ -3160,7 +3710,7 @@ ini@2.0.0: resolved "https://registry.yarnpkg.com/ini/-/ini-2.0.0.tgz#e5fd556ecdd5726be978fa1001862eacb0a94bc5" integrity sha512-7PnF4oN3CvZF23ADhA5wRaYEQpJ8qygSkbtTXWBeXWXmEVRXK+1ITciHWwHhsjv1TmW0MgacIv6hEi5pX5NQdA== -ini@~1.3.0: +ini@^1.3.4, ini@~1.3.0: version "1.3.8" resolved "https://registry.yarnpkg.com/ini/-/ini-1.3.8.tgz#a29da425b48806f34767a4efce397269af28432c" integrity sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew== @@ -3177,6 +3727,11 @@ is-binary-path@~2.1.0: dependencies: binary-extensions "^2.0.0" +is-buffer@^1.1.5: + version "1.1.6" + resolved "https://registry.yarnpkg.com/is-buffer/-/is-buffer-1.1.6.tgz#efaa2ea9daa0d7ab2ea13a97b2b8ad51fefbe8be" + integrity sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w== + is-ci@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/is-ci/-/is-ci-2.0.0.tgz#6bc6334181810e04b5c22b3d589fdca55026404c" @@ -3184,6 +3739,13 @@ is-ci@^2.0.0: dependencies: ci-info "^2.0.0" +is-core-module@^2.13.0: + version "2.15.0" + resolved "https://registry.yarnpkg.com/is-core-module/-/is-core-module-2.15.0.tgz#71c72ec5442ace7e76b306e9d48db361f22699ea" + integrity sha512-Dd+Lb2/zvk9SKy1TGCt1wFJFo/MWBPMX5x7KcvLajWTGuomczdQX61PvY5yK6SVACwpoexWo81IfFyoKY2QnTA== + dependencies: + hasown "^2.0.2" + is-deflate@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/is-deflate/-/is-deflate-1.0.0.tgz#c862901c3c161fb09dac7cdc7e784f80e98f2f14" @@ -3249,6 +3811,13 @@ is-plain-obj@^2.1.0: resolved "https://registry.yarnpkg.com/is-plain-obj/-/is-plain-obj-2.1.0.tgz#45e42e37fccf1f40da8e5f76ee21515840c09287" integrity sha512-YWnfyRwxL/+SsrWYfOpUtz5b3YD+nyfkHvjbcanzk8zgyO4ASD67uVMRt8k5bM4lLMDnXfriRhOpemw+NfT1eA== +is-plain-object@^2.0.1: + version "2.0.4" + resolved "https://registry.yarnpkg.com/is-plain-object/-/is-plain-object-2.0.4.tgz#2c163b3fafb1b606d9d17928f05c2a1c38e07677" + integrity sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og== + dependencies: + isobject "^3.0.1" + is-redirect@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/is-redirect/-/is-redirect-1.0.0.tgz#1d03dded53bd8db0f30c26e4f95d36fc7c87dc24" @@ -3259,6 +3828,11 @@ is-stream@^2.0.0: resolved "https://registry.yarnpkg.com/is-stream/-/is-stream-2.0.1.tgz#fac1e3d53b97ad5a9d0ae9cef2389f5810a5c077" integrity sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg== +is-svg-path@^1.0.1: + version "1.0.2" + resolved "https://registry.yarnpkg.com/is-svg-path/-/is-svg-path-1.0.2.tgz#77ab590c12b3d20348e5c7a13d0040c87784dda0" + integrity sha512-Lj4vePmqpPR1ZnRctHv8ltSh1OrSxHkhUkd7wi+VQdcdP15/KvQFyk7LhNuM7ZW0EVbJz8kZLVmL9quLrfq4Kg== + is-typedarray@^1.0.0, is-typedarray@~1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/is-typedarray/-/is-typedarray-1.0.0.tgz#e479c80858df0c1b11ddda6940f96011fcda4a9a" @@ -3286,7 +3860,7 @@ isarray@0.0.1: resolved "https://registry.yarnpkg.com/isarray/-/isarray-0.0.1.tgz#8a18acfca9a8f4177e09abfc6038939b05d1eedf" integrity sha512-D2S+3GLxWH+uhrNEcoh/fnmYeP8E8/zHl644d/jdA0g2uyXvy3sb0qxotE+ne0LtccHknQzWwZEzhak7oJ0COQ== -isarray@^1.0.0, isarray@~1.0.0: +isarray@1.0.0, isarray@^1.0.0, isarray@~1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11" integrity sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE= @@ -3296,21 +3870,68 @@ isexe@^2.0.0: resolved "https://registry.yarnpkg.com/isexe/-/isexe-2.0.0.tgz#e8fbf374dc556ff8947a10dcb0572d633f2cfa10" integrity sha1-6PvzdNxVb/iUehDcsFctYz8s+hA= +isobject@^2.0.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/isobject/-/isobject-2.1.0.tgz#f065561096a3f1da2ef46272f815c840d87e0c89" + integrity sha512-+OUdGJlgjOBZDfxnDjYYG6zp487z0JGNQq3cYQYg5f5hKR+syHMsaztzGeml/4kGG55CSpKSpWTY+jYGgsHLgA== + dependencies: + isarray "1.0.0" + +isobject@^3.0.0, isobject@^3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/isobject/-/isobject-3.0.1.tgz#4e431e92b11a9731636aa1f9c8d1ccbcfdab78df" + integrity sha512-WhB9zCku7EGTj/HQQRz5aUQEUeoQZH2bWcltRErOpymJ4boYE6wL9Tbr23krRPSZ+C5zqNSrSw+Cc7sZZ4b7vg== + isstream@~0.1.2: version "0.1.2" resolved "https://registry.yarnpkg.com/isstream/-/isstream-0.1.2.tgz#47e63f7af55afa6f92e1500e690eb8b8529c099a" integrity sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo= +jackspeak@^3.1.2: + version "3.4.3" + resolved "https://registry.yarnpkg.com/jackspeak/-/jackspeak-3.4.3.tgz#8833a9d89ab4acde6188942bd1c53b6390ed5a8a" + integrity sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw== + dependencies: + "@isaacs/cliui" "^8.0.2" + optionalDependencies: + "@pkgjs/parseargs" "^0.11.0" + jmespath@0.16.0: version "0.16.0" resolved "https://registry.yarnpkg.com/jmespath/-/jmespath-0.16.0.tgz#b15b0a85dfd4d930d43e69ed605943c802785076" integrity sha512-9FzQjJ7MATs1tSpnco1K6ayiYE3figslrXA72G2HQ/n76RzvYlofyi5QM+iX4YRs/pu3yzxlVQSST23+dMDknw== +jquery@^3.5.1: + version "3.7.1" + resolved "https://registry.yarnpkg.com/jquery/-/jquery-3.7.1.tgz#083ef98927c9a6a74d05a6af02806566d16274de" + integrity sha512-m4avr8yL8kmFN8psrbFFFmB/If14iN5o9nw/NgnnM+kybDJpRsAynV2BsfpTYrTRysYUdADVD7CkUUizgkpLfg== + +js-beautify@^1.13.0: + version "1.15.1" + resolved "https://registry.yarnpkg.com/js-beautify/-/js-beautify-1.15.1.tgz#4695afb508c324e1084ee0b952a102023fc65b64" + integrity sha512-ESjNzSlt/sWE8sciZH8kBF8BPlwXPwhR6pWKAw8bw4Bwj+iZcnKW6ONWUutJ7eObuBZQpiIb8S7OYspWrKt7rA== + dependencies: + config-chain "^1.1.13" + editorconfig "^1.0.4" + glob "^10.3.3" + js-cookie "^3.0.5" + nopt "^7.2.0" + +js-cookie@^3.0.5: + version "3.0.5" + resolved "https://registry.yarnpkg.com/js-cookie/-/js-cookie-3.0.5.tgz#0b7e2fd0c01552c58ba86e0841f94dc2557dcdbc" + integrity sha512-cEiJEAEoIbWfCZYKWhVwFuvPX1gETRYPw6LlaTKoxD3s2AkXzkCjnp6h0V77ozyqj0jakteJ4YqDJT830+lVGw== + js-sha3@0.8.0, js-sha3@^0.8.0: version "0.8.0" resolved "https://registry.yarnpkg.com/js-sha3/-/js-sha3-0.8.0.tgz#b9b7a5da73afad7dedd0f8c463954cbde6818840" integrity sha512-gF1cRrHhIzNfToc802P800N8PpXS+evLLXfsVpowqmAFR9uwbi89WvXg2QspOmXL8QL86J4T1EpFu+yUkwJY3Q== +js-tokens@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/js-tokens/-/js-tokens-4.0.0.tgz#19203fb59991df98e3a287050d4647cdeaf32499" + integrity sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ== + js-yaml@4.1.0, js-yaml@^4.1.0: version "4.1.0" resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-4.1.0.tgz#c1fb65f8f5017901cdd2c951864ba18458a10602" @@ -3323,6 +3944,11 @@ jsbn@~0.1.0: resolved "https://registry.yarnpkg.com/jsbn/-/jsbn-0.1.1.tgz#a5e654c2e5a2deb5f201d96cefbca80c0ef2f513" integrity sha1-peZUwuWi3rXyAdls77yoDA7y9RM= +jsesc@^2.5.1: + version "2.5.2" + resolved "https://registry.yarnpkg.com/jsesc/-/jsesc-2.5.2.tgz#80564d2e483dacf6e8ef209650a67df3f0c283a4" + integrity sha512-OYu7XEzjkCQ3C5Ps3QIZsQfNpqoJyZZA99wd9aWd05NCtC5pWOkShK2mkL6HXQR6/Cy2lbNdPlZBpuQHXE63gA== + json-bigint@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/json-bigint/-/json-bigint-1.0.0.tgz#ae547823ac0cad8398667f8cd9ef4730f5b01ff1" @@ -3382,6 +4008,11 @@ json5@^1.0.1: dependencies: minimist "^1.2.0" +json5@^2.1.2: + version "2.2.3" + resolved "https://registry.yarnpkg.com/json5/-/json5-2.2.3.tgz#78cd6f1a19bdc12b73db5ad0c61efd66c1e29283" + integrity sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg== + jsonfile@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/jsonfile/-/jsonfile-4.0.0.tgz#8771aae0799b64076b76640fca058f9c10e33ecb" @@ -3408,6 +4039,28 @@ jsprim@^1.2.2: json-schema "0.4.0" verror "1.10.0" +jszip-utils@^0.1.0: + version "0.1.0" + resolved "https://registry.yarnpkg.com/jszip-utils/-/jszip-utils-0.1.0.tgz#8c04cdedcdb291e83f055f5b261b3a3188ceca0b" + integrity sha512-tBNe0o3HAf8vo0BrOYnLPnXNo5A3KsRMnkBFYjh20Y3GPYGfgyoclEMgvVchx0nnL+mherPi74yLPIusHUQpZg== + +jszip@2.6.1: + version "2.6.1" + resolved "https://registry.yarnpkg.com/jszip/-/jszip-2.6.1.tgz#b88f3a7b2e67a2a048152982c7a3756d9c4828f0" + integrity sha512-C4Z++nYQv+CudUkCWUdz+yKVhQiFJjuWSmRJ5Sg3d3/OzcJ6U4ooUYlmE3+rJXrVk89KWQaiJ9mPp/VLQ4D66g== + dependencies: + pako "~1.0.2" + +jszip@^3.7.1: + version "3.10.1" + resolved "https://registry.yarnpkg.com/jszip/-/jszip-3.10.1.tgz#34aee70eb18ea1faec2f589208a157d1feb091c2" + integrity sha512-xXDvecyTpGLrqFrvkrUSoxxfJI5AH7U8zxxtVclpsUtMCq4JQ290LY8AW5c7Ggnr/Y/oK+bQMbqK2qmtk3pN4g== + dependencies: + lie "~3.3.0" + pako "~1.0.2" + readable-stream "~2.3.6" + setimmediate "^1.0.5" + jwa@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/jwa/-/jwa-2.0.0.tgz#a7e9c3f29dae94027ebcaf49975c9345593410fc" @@ -3432,6 +4085,13 @@ keyv@^3.0.0: dependencies: json-buffer "3.0.0" +kind-of@^3.0.2: + version "3.2.2" + resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-3.2.2.tgz#31ea21a734bab9bbb0f32466d893aea51e4a3c64" + integrity sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ== + dependencies: + is-buffer "^1.1.5" + latest-version@^5.1.0: version "5.1.0" resolved "https://registry.yarnpkg.com/latest-version/-/latest-version-5.1.0.tgz#119dfe908fe38d15dfa43ecd13fa12ec8832face" @@ -3580,6 +4240,13 @@ libqp@2.1.0: resolved "https://registry.yarnpkg.com/libqp/-/libqp-2.1.0.tgz#ce84bffd86b76029032093bd866d316e12a3d3f5" integrity sha512-O6O6/fsG5jiUVbvdgT7YX3xY3uIadR6wEZ7+vy9u7PKHAlSEB6blvC1o5pHBjgsi95Uo0aiBBdkyFecj6jtb7A== +lie@~3.3.0: + version "3.3.0" + resolved "https://registry.yarnpkg.com/lie/-/lie-3.3.0.tgz#dcf82dee545f46074daf200c7c1c5a08e0f40f6a" + integrity sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ== + dependencies: + immediate "~3.0.5" + linkify-it@5.0.0: version "5.0.0" resolved "https://registry.yarnpkg.com/linkify-it/-/linkify-it-5.0.0.tgz#9ef238bfa6dc70bd8e7f9572b52d369af569b421" @@ -3587,6 +4254,24 @@ linkify-it@5.0.0: dependencies: uc.micro "^2.0.0" +loader-utils@^1.2.3: + version "1.4.2" + resolved "https://registry.yarnpkg.com/loader-utils/-/loader-utils-1.4.2.tgz#29a957f3a63973883eb684f10ffd3d151fec01a3" + integrity sha512-I5d00Pd/jwMD2QCduo657+YM/6L3KZu++pmX9VFncxaxvHcru9jx1lBaFft+r4Mt2jK0Yhp41XlRAihzPxHNCg== + dependencies: + big.js "^5.2.2" + emojis-list "^3.0.0" + json5 "^1.0.1" + +loader-utils@^2.0.0: + version "2.0.4" + resolved "https://registry.yarnpkg.com/loader-utils/-/loader-utils-2.0.4.tgz#8b5cb38b5c34a9a018ee1fc0e6a066d1dfcc528c" + integrity sha512-xXqpXoINfFhgua9xiqD8fPFHgkoq1mmmpE92WlDbm9rNRd/EbRb+Gqf908T2DMfuHjjJlksiK2RbHVOdD/MqSw== + dependencies: + big.js "^5.2.2" + emojis-list "^3.0.0" + json5 "^2.1.2" + locate-path@^6.0.0: version "6.0.0" resolved "https://registry.yarnpkg.com/locate-path/-/locate-path-6.0.0.tgz#55321eb309febbc59c4801d931a72452a681d286" @@ -3599,6 +4284,11 @@ lodash.camelcase@^4.3.0: resolved "https://registry.yarnpkg.com/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz#b28aa6288a2b9fc651035c7711f65ab6190331a6" integrity sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA== +lodash.debounce@^4.0.8: + version "4.0.8" + resolved "https://registry.yarnpkg.com/lodash.debounce/-/lodash.debounce-4.0.8.tgz#82d79bff30a67c4005ffd5e2515300ad9ca4d7af" + integrity sha512-FT1yDzDYEoYWhnSGnpE/4Kj1fLZkDFyqRb7fNt6FdYOSxlUWAtp42Eh6Wb0rGIv/m9Bgo7x4GhQbm5Ys4SG5ow== + lodash.defaults@^4.2.0: version "4.2.0" resolved "https://registry.yarnpkg.com/lodash.defaults/-/lodash.defaults-4.2.0.tgz#d09178716ffea4dde9e5fb7b37f6f0802274580c" @@ -3629,7 +4319,7 @@ lodash.union@^4.6.0: resolved "https://registry.yarnpkg.com/lodash.union/-/lodash.union-4.6.0.tgz#48bb5088409f16f1821666641c44dd1aaae3cd88" integrity sha1-SLtQiECfFvGCFmZkHETdGqrjzYg= -lodash@^4.14.0, lodash@^4.17.14, lodash@^4.17.21: +lodash@^4.14.0, lodash@^4.17.14, lodash@^4.17.19, lodash@^4.17.20, lodash@^4.17.21: version "4.17.21" resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c" integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg== @@ -3653,6 +4343,15 @@ log4js@^6.4.1: rfdc "^1.3.0" streamroller "^3.1.1" +lop@^0.4.1: + version "0.4.1" + resolved "https://registry.yarnpkg.com/lop/-/lop-0.4.1.tgz#744f1696ef480e68ce1947fe557b09db5af2a738" + integrity sha512-9xyho9why2A2tzm5aIcMWKvzqKsnxrf9B5I+8O30olh6lQU8PH978LqZoI4++37RBgS1Em5i54v1TFs/3wnmXQ== + dependencies: + duck "^0.1.12" + option "~0.2.1" + underscore "^1.13.1" + lowercase-keys@^1.0.0, lowercase-keys@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/lowercase-keys/-/lowercase-keys-1.0.1.tgz#6f9e30b47084d971a7c820ff15a6c5167b74c26f" @@ -3663,6 +4362,18 @@ lowercase-keys@^2.0.0: resolved "https://registry.yarnpkg.com/lowercase-keys/-/lowercase-keys-2.0.0.tgz#2603e78b7b4b0006cbca2fbcc8a3202558ac9479" integrity sha512-tqNXrS78oMOE73NMxK4EMLQsQowWf8jKooH9g7xPavRT706R6bkQJ6DY2Te7QukaZsulxa30wQ7bk0pm4XiHmA== +lru-cache@^10.2.0: + version "10.4.3" + resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-10.4.3.tgz#410fc8a17b70e598013df257c2446b7f3383f119" + integrity sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ== + +lru-cache@^5.1.1: + version "5.1.1" + resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-5.1.1.tgz#1da27e6710271947695daf6848e847f01d84b920" + integrity sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w== + dependencies: + yallist "^3.0.2" + lru-cache@^6.0.0: version "6.0.0" resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-6.0.0.tgz#6d6fe6570ebd96aaf90fcad1dafa3b2566db3a94" @@ -3712,6 +4423,22 @@ make-error@^1.1.1: resolved "https://registry.yarnpkg.com/make-error/-/make-error-1.3.6.tgz#2eb2e37ea9b67c4891f684a1394799af484cf7a2" integrity sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw== +mammoth@^1.8.0: + version "1.8.0" + resolved "https://registry.yarnpkg.com/mammoth/-/mammoth-1.8.0.tgz#d8f1b0d3a0355fda129270346e9dc853f223028f" + integrity sha512-pJNfxSk9IEGVpau+tsZFz22ofjUsl2mnA5eT8PjPs2n0BP+rhVte4Nez6FdgEuxv3IGI3afiV46ImKqTGDVlbA== + dependencies: + "@xmldom/xmldom" "^0.8.6" + argparse "~1.0.3" + base64-js "^1.5.1" + bluebird "~3.4.0" + dingbat-to-unicode "^1.0.1" + jszip "^3.7.1" + lop "^0.4.1" + path-is-absolute "^1.0.0" + underscore "^1.13.1" + xmlbuilder "^10.0.0" + md5.js@^1.3.4: version "1.3.5" resolved "https://registry.yarnpkg.com/md5.js/-/md5.js-1.3.5.tgz#b5d07b8e3216e3e27cd728d72f70d1e6a342005f" @@ -3746,7 +4473,7 @@ mime-db@1.52.0: resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.52.0.tgz#bbabcdc02859f4987301c856e3387ce5ec43bf70" integrity sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg== -mime-types@^2.1.12, mime-types@~2.1.19, mime-types@~2.1.24, mime-types@~2.1.34: +mime-types@^2.1.12, mime-types@^2.1.27, mime-types@~2.1.19, mime-types@~2.1.24, mime-types@~2.1.34: version "2.1.35" resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.35.tgz#381a871b62a734450660ae3deee44813f70d959a" integrity sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw== @@ -3790,6 +4517,13 @@ minimatch@4.2.1: dependencies: brace-expansion "^1.1.7" +minimatch@9.0.1: + version "9.0.1" + resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-9.0.1.tgz#8a555f541cf976c622daf078bb28f29fb927c253" + integrity sha512-0jWhJpD/MdhPXwPuiRkCbfYfSKp2qnn2eOc279qI7f+osl/l+prKSrvhg157zSYvx/1nmgn2NqdT6k2Z7zSH9w== + dependencies: + brace-expansion "^2.0.1" + minimatch@^3.0.3, minimatch@^3.0.4, minimatch@^3.1.1: version "3.1.2" resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.1.2.tgz#19cd194bfd3e428f049a70817c038d89ab4be35b" @@ -3797,11 +4531,23 @@ minimatch@^3.0.3, minimatch@^3.0.4, minimatch@^3.1.1: dependencies: brace-expansion "^1.1.7" +minimatch@^9.0.4: + version "9.0.5" + resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-9.0.5.tgz#d74f9dd6b57d83d8e98cfb82133b03978bc929e5" + integrity sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow== + dependencies: + brace-expansion "^2.0.1" + minimist@^1.2.0, minimist@^1.2.5, minimist@^1.2.6: version "1.2.6" resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.6.tgz#8637a5b759ea0d6e98702cfb3a9283323c93af44" integrity sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q== +"minipass@^5.0.0 || ^6.0.2 || ^7.0.0", minipass@^7.1.2: + version "7.1.2" + resolved "https://registry.yarnpkg.com/minipass/-/minipass-7.1.2.tgz#93a9626ce5e5e66bd4db86849e7515e92340a707" + integrity sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw== + mkdirp-classic@^0.5.2: version "0.5.3" resolved "https://registry.yarnpkg.com/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz#fa10c9115cc6d8865be221ba47ee9bed78601113" @@ -3927,6 +4673,11 @@ node-imap@^0.9.6: readable-stream "^3.6.0" utf7 "^1.0.2" +node-releases@^2.0.18: + version "2.0.18" + resolved "https://registry.yarnpkg.com/node-releases/-/node-releases-2.0.18.tgz#f010e8d35e2fe8d6b2944f03f70213ecedc4ca3f" + integrity sha512-d9VeXT4SJ7ZeOqGX6R5EM022wpL+eWPooLI+5UpWn2jCT1aosUQEhQP214x33Wkwx3JQMvIm+tIoVOdodFS40g== + nodemailer@6.9.13: version "6.9.13" resolved "https://registry.yarnpkg.com/nodemailer/-/nodemailer-6.9.13.tgz#5b292bf1e92645f4852ca872c56a6ba6c4a3d3d6" @@ -3956,6 +4707,13 @@ noms@0.0.0: inherits "^2.0.1" readable-stream "~1.0.31" +nopt@^7.2.0: + version "7.2.1" + resolved "https://registry.yarnpkg.com/nopt/-/nopt-7.2.1.tgz#1cac0eab9b8e97c9093338446eddd40b2c8ca1e7" + integrity sha512-taM24ViiimT/XntxbPyJQzCG+p4EKOpgD3mxFwW38mGjVUrfERQOeY4EDHjdnptttfHuHQXFx+lTP08Q+mLa/w== + dependencies: + abbrev "^2.0.0" + nopt@~1.0.10: version "1.0.10" resolved "https://registry.yarnpkg.com/nopt/-/nopt-1.0.10.tgz#6ddd21bd2a31417b92727dd585f8a6f37608ebee" @@ -3968,6 +4726,13 @@ normalize-path@^3.0.0, normalize-path@~3.0.0: resolved "https://registry.yarnpkg.com/normalize-path/-/normalize-path-3.0.0.tgz#0dcd69ff23a1c9b11fd0978316644a0388216a65" integrity sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA== +normalize-svg-path@^1.0.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/normalize-svg-path/-/normalize-svg-path-1.1.0.tgz#0e614eca23c39f0cffe821d6be6cd17e569a766c" + integrity sha512-r9KHKG2UUeB5LoTouwDzBy2VxXlHsiM6fyLQvnJa0S5hrhzqElH/CH7TUGhT1fVvIYBIKf3OpY4YJ4CK+iaqHg== + dependencies: + svg-arc-to-cubic-bezier "^3.0.0" + normalize-url@^4.1.0: version "4.5.1" resolved "https://registry.yarnpkg.com/normalize-url/-/normalize-url-4.5.1.tgz#0dd90cf1288ee1d1313b87081c9a5932ee48518a" @@ -4003,6 +4768,14 @@ oh-no-i-insist@^1.1.1: resolved "https://registry.yarnpkg.com/oh-no-i-insist/-/oh-no-i-insist-1.1.1.tgz#af6f12e2d43366839bae45f8c870b976a11eee35" integrity sha1-r28S4tQzZoObrkX4yHC5dqEe7jU= +omit-deep@0.3.0: + version "0.3.0" + resolved "https://registry.yarnpkg.com/omit-deep/-/omit-deep-0.3.0.tgz#21c8af3499bcadd29651a232cbcacbc52445ebec" + integrity sha512-Lbl/Ma59sss2b15DpnWnGmECBRL8cRl/PjPbPMVW+Y8zIQzRrwMaI65Oy6HvxyhYeILVKBJb2LWeG81bj5zbMg== + dependencies: + is-plain-object "^2.0.1" + unset-value "^0.1.1" + on-finished@2.4.1: version "2.4.1" resolved "https://registry.yarnpkg.com/on-finished/-/on-finished-2.4.1.tgz#58c8c44116e54845ad57f14ab10b03533184ac3f" @@ -4031,6 +4804,11 @@ open@^8.4.2: is-docker "^2.1.1" is-wsl "^2.2.0" +option@~0.2.1: + version "0.2.4" + resolved "https://registry.yarnpkg.com/option/-/option-0.2.4.tgz#fd475cdf98dcabb3cb397a3ba5284feb45edbfe4" + integrity sha512-pkEqbDyl8ou5cpq+VsnQbe/WlEy5qS7xPzMS1U55OCG9KPvwFD46zDbxQIj3egJSFc3D+XhYOPUzz49zQAVy7A== + p-cancelable@^1.0.0: version "1.1.0" resolved "https://registry.yarnpkg.com/p-cancelable/-/p-cancelable-1.1.0.tgz#d078d15a3af409220c886f1d9a0ca2e441ab26cc" @@ -4050,6 +4828,11 @@ p-locate@^5.0.0: dependencies: p-limit "^3.0.2" +package-json-from-dist@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/package-json-from-dist/-/package-json-from-dist-1.0.0.tgz#e501cd3094b278495eb4258d4c9f6d5ac3019f00" + integrity sha512-dATvCeZN/8wQsGywez1mzHtTlP22H8OEfPrVMLNr4/eGa+ijtLn/6M5f0dY8UKNrC2O9UCU6SSoG3qRKnt7STw== + package-json@^6.3.0: version "6.5.0" resolved "https://registry.yarnpkg.com/package-json/-/package-json-6.5.0.tgz#6feedaca35e75725876d0b0e64974697fed145b0" @@ -4065,6 +4848,16 @@ pako@~0.2.0: resolved "https://registry.yarnpkg.com/pako/-/pako-0.2.9.tgz#f3f7522f4ef782348da8161bad9ecfd51bf83a75" integrity sha1-8/dSL073gjSNqBYbrZ7P1Rv4OnU= +pako@~1.0.2: + version "1.0.11" + resolved "https://registry.yarnpkg.com/pako/-/pako-1.0.11.tgz#6c9599d340d54dfd3946380252a35705a6b992bf" + integrity sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw== + +parse-svg-path@^0.1.2: + version "0.1.2" + resolved "https://registry.yarnpkg.com/parse-svg-path/-/parse-svg-path-0.1.2.tgz#7a7ec0d1eb06fa5325c7d3e009b859a09b5d49eb" + integrity sha512-JyPSBnkTJ0AI8GGJLfMXvKq42cj5c006fnLz6fXy6zfoVjJizi8BNTpu8on8ziI1cKy9d9DGNuY17Ce7wuejpQ== + parseley@^0.12.0: version "0.12.1" resolved "https://registry.yarnpkg.com/parseley/-/parseley-0.12.1.tgz#4afd561d50215ebe259e3e7a853e62f600683aef" @@ -4137,6 +4930,24 @@ path-is-absolute@^1.0.0: resolved "https://registry.yarnpkg.com/path-is-absolute/-/path-is-absolute-1.0.1.tgz#174b9268735534ffbc7ace6bf53a5a9e1b5c5f5f" integrity sha1-F0uSaHNVNP+8es5r9TpanhtcX18= +path-key@^3.1.0: + version "3.1.1" + resolved "https://registry.yarnpkg.com/path-key/-/path-key-3.1.1.tgz#581f6ade658cbba65a0d3380de7753295054f375" + integrity sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q== + +path-parse@^1.0.7: + version "1.0.7" + resolved "https://registry.yarnpkg.com/path-parse/-/path-parse-1.0.7.tgz#fbc114b60ca42b30d9daf5858e4bd68bbedb6735" + integrity sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw== + +path-scurry@^1.11.1: + version "1.11.1" + resolved "https://registry.yarnpkg.com/path-scurry/-/path-scurry-1.11.1.tgz#7960a668888594a0720b12a911d1a742ab9f11d2" + integrity sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA== + dependencies: + lru-cache "^10.2.0" + minipass "^5.0.0 || ^6.0.2 || ^7.0.0" + path-to-regexp@0.1.7: version "0.1.7" resolved "https://registry.yarnpkg.com/path-to-regexp/-/path-to-regexp-0.1.7.tgz#df604178005f522f15eb4490e7247a1bfaa67f8c" @@ -4190,6 +5001,16 @@ performance-now@^2.1.0: resolved "https://registry.yarnpkg.com/performance-now/-/performance-now-2.1.0.tgz#6309f4e0e5fa913ec1c69307ae364b4b377c9e7b" integrity sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns= +picocolors@^0.2.1: + version "0.2.1" + resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-0.2.1.tgz#570670f793646851d1ba135996962abad587859f" + integrity sha512-cMlDqaLEqfSaW8Z7N5Jw+lyIW869EzT73/F5lhtY9cLGoVxSXznfgfXMO0Z5K0o0Q2TkTXq+0KFsdnSe3jDViA== + +picocolors@^1.0.0, picocolors@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-1.0.1.tgz#a8ad579b571952f0e5d25892de5445bcfe25aaa1" + integrity sha512-anP1Z8qwhkbmu7MFP5iTt+wQKXgwzf7zTyGlcdzabySa9vd0Xt392U0rVmz9poOaBj0uHJKyyo9/upk0HrEQew== + picomatch@^2.0.4, picomatch@^2.2.1: version "2.3.1" resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.3.1.tgz#3ba3833733646d9d3e4995946c1365a67fb07a42" @@ -4200,6 +5021,65 @@ pify@^5.0.0: resolved "https://registry.yarnpkg.com/pify/-/pify-5.0.0.tgz#1f5eca3f5e87ebec28cc6d54a0e4aaf00acc127f" integrity sha512-eW/gHNMlxdSP6dmG6uJip6FXN0EQBwm2clYYd8Wul42Cwu/DK8HEftzsapcNdYe2MfLiIwZqsDk2RDEsTE79hA== +polf@^0.0.3: + version "0.0.3" + resolved "https://registry.yarnpkg.com/polf/-/polf-0.0.3.tgz#340f97dccadefeb2d90264d7ae010a8864775a30" + integrity sha512-K8zUZu9VGKL+ldcvuU78EkMnzA0Oq5JglIKZNG0rA7aqo7kfW87nXoAK88u3aIjXkTa20i4sAZv0NvP6Qs80qw== + +postcss-modules-extract-imports@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/postcss-modules-extract-imports/-/postcss-modules-extract-imports-2.0.0.tgz#818719a1ae1da325f9832446b01136eeb493cd7e" + integrity sha512-LaYLDNS4SG8Q5WAWqIJgdHPJrDDr/Lv775rMBFUbgjTz6j34lUznACHcdRWroPvXANP2Vj7yNK57vp9eFqzLWQ== + dependencies: + postcss "^7.0.5" + +postcss-modules-local-by-default@^3.0.2: + version "3.0.3" + resolved "https://registry.yarnpkg.com/postcss-modules-local-by-default/-/postcss-modules-local-by-default-3.0.3.tgz#bb14e0cc78279d504dbdcbfd7e0ca28993ffbbb0" + integrity sha512-e3xDq+LotiGesympRlKNgaJ0PCzoUIdpH0dj47iWAui/kyTgh3CiAr1qP54uodmJhl6p9rN6BoNcdEDVJx9RDw== + dependencies: + icss-utils "^4.1.1" + postcss "^7.0.32" + postcss-selector-parser "^6.0.2" + postcss-value-parser "^4.1.0" + +postcss-modules-scope@^2.2.0: + version "2.2.0" + resolved "https://registry.yarnpkg.com/postcss-modules-scope/-/postcss-modules-scope-2.2.0.tgz#385cae013cc7743f5a7d7602d1073a89eaae62ee" + integrity sha512-YyEgsTMRpNd+HmyC7H/mh3y+MeFWevy7V1evVhJWewmMbjDHIbZbOXICC2y+m1xI1UVfIT1HMW/O04Hxyu9oXQ== + dependencies: + postcss "^7.0.6" + postcss-selector-parser "^6.0.0" + +postcss-modules-values@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/postcss-modules-values/-/postcss-modules-values-3.0.0.tgz#5b5000d6ebae29b4255301b4a3a54574423e7f10" + integrity sha512-1//E5jCBrZ9DmRX+zCtmQtRSV6PV42Ix7Bzj9GbwJceduuf7IqP8MgeTXuRDHOWj2m0VzZD5+roFWDuU8RQjcg== + dependencies: + icss-utils "^4.0.0" + postcss "^7.0.6" + +postcss-selector-parser@^6.0.0, postcss-selector-parser@^6.0.2: + version "6.1.2" + resolved "https://registry.yarnpkg.com/postcss-selector-parser/-/postcss-selector-parser-6.1.2.tgz#27ecb41fb0e3b6ba7a1ec84fff347f734c7929de" + integrity sha512-Q8qQfPiZ+THO/3ZrOrO0cJJKfpYCagtMUkXbnEfmgUjwXg6z/WBeOyS9APBBPCTSiDV+s4SwQGu8yFsiMRIudg== + dependencies: + cssesc "^3.0.0" + util-deprecate "^1.0.2" + +postcss-value-parser@^4.1.0: + version "4.2.0" + resolved "https://registry.yarnpkg.com/postcss-value-parser/-/postcss-value-parser-4.2.0.tgz#723c09920836ba6d3e5af019f92bc0971c02e514" + integrity sha512-1NNCs6uurfkVbeXG4S8JFT9t19m45ICnif8zWLd5oPSZ50QnwMfK+H3jv408d4jw/7Bttv5axS5IiHoLaVNHeQ== + +postcss@^7.0.14, postcss@^7.0.32, postcss@^7.0.5, postcss@^7.0.6: + version "7.0.39" + resolved "https://registry.yarnpkg.com/postcss/-/postcss-7.0.39.tgz#9624375d965630e2e1f2c02a935c82a59cb48309" + integrity sha512-yioayjNbHn6z1/Bywyb2Y4s3yvDAeXGOyxqD+LnVOinq6Mdmd++SW2wUNVzavyyHxd6+DxzWGIuosg6P1Rj8uA== + dependencies: + picocolors "^0.2.1" + source-map "^0.6.1" + pouchdb-abstract-mapreduce@7.3.1: version "7.3.1" resolved "https://registry.yarnpkg.com/pouchdb-abstract-mapreduce/-/pouchdb-abstract-mapreduce-7.3.1.tgz#96ff4a0f41cbe273f3f52fde003b719005a2093c" @@ -4331,6 +5211,30 @@ pouchdb@^7.2.2: uuid "8.3.2" vuvuzela "1.0.3" +pptx-parser@^1.1.7-beta.9: + version "1.1.7-beta.9" + resolved "https://registry.yarnpkg.com/pptx-parser/-/pptx-parser-1.1.7-beta.9.tgz#26648ba5c45c4c5c548b1de7ab03f04764399d37" + integrity sha512-xIjw65wRMVGNCmRLF91F3+f84gQ0uITDJEJp21bprsLkznurkSCG5sWGpvmnVZr81/hHqPy+kKsAU3/wgZQYLw== + dependencies: + "@babel/runtime-corejs3" "^7.10.4" + "@vf.js/launcher" "2.0.12" + css-loader "^3.6.0" + deepmerge "^4.2.2" + element-to-path "^1.2.0" + jquery "^3.5.1" + jszip "2.6.1" + jszip-utils "^0.1.0" + lodash "^4.17.19" + prst-shape-transform "^1.0.5-beta.0" + style-loader "^1.2.1" + svg-path-bbox "0.0.49" + svg-path-bounds "^1.0.1" + svgson "^4.1.0" + tinycolor2 "^1.4.1" + transformation-matrix "^2.5.0" + txml "3.1.3" + url-loader "^4.1.0" + prepend-http@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/prepend-http/-/prepend-http-2.0.0.tgz#e92434bfa5ea8c19f41cdfd401d741a3c819d897" @@ -4341,6 +5245,11 @@ process-nextick-args@~2.0.0: resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-2.0.1.tgz#7820d9b16120cc55ca9ae7792680ae7dba6d7fe2" integrity sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag== +proto-list@~1.2.1: + version "1.2.4" + resolved "https://registry.yarnpkg.com/proto-list/-/proto-list-1.2.4.tgz#212d5bfe1318306a420f6402b8e26ff39647a849" + integrity sha512-vtK/94akxsTMhe0/cbfpR+syPuszcuwhqVjJq26CuNDgFGj682oRBXOP5MJpv2r7JtE8MsiepGIqvvOTBwn2vA== + proxy-addr@~2.0.7: version "2.0.7" resolved "https://registry.yarnpkg.com/proxy-addr/-/proxy-addr-2.0.7.tgz#f19fe69ceab311eeb94b42e70e8c2070f9ba1025" @@ -4359,6 +5268,23 @@ prr@~1.0.1: resolved "https://registry.yarnpkg.com/prr/-/prr-1.0.1.tgz#d3fc114ba06995a45ec6893f484ceb1d78f5f476" integrity sha512-yPw4Sng1gWghHQWj0B3ZggWUm4qVbPwPFcRG8KyxiU7J2OHFSoEHKS+EZ3fv5l1t9CyCiop6l/ZYeWbrgoQejw== +prst-shape-transform@^1.0.5-beta.0: + version "1.0.5-beta.0" + resolved "https://registry.yarnpkg.com/prst-shape-transform/-/prst-shape-transform-1.0.5-beta.0.tgz#0e33043df63948d06729745a3fa14873866a9627" + integrity sha512-AsFdub+qDdqwEnF6CVOkbrVab4un/Ag1uc5uLTTBGlVCjan8wrQN1oNTtQC0+8PBs8DHGY11hiUNO2E9mC2k0w== + dependencies: + "@babel/plugin-transform-runtime" "^7.7.6" + "@babel/runtime-corejs3" "^7.7.7" + colors "^1.4.0" + core-js "^3.6.0" + fs-extra "^8.1.0" + glob "^7.1.6" + js-beautify "^1.13.0" + lodash "^4.17.20" + transformation-matrix "^2.4.0" + txml "^3.1.3" + xml-js "^1.6.11" + psl@^1.1.28, psl@^1.1.33: version "1.8.0" resolved "https://registry.yarnpkg.com/psl/-/psl-1.8.0.tgz#9326f8bcfb013adcc005fdff056acce020e51c24" @@ -4567,6 +5493,11 @@ regenerator-runtime@^0.11.0: resolved "https://registry.yarnpkg.com/regenerator-runtime/-/regenerator-runtime-0.11.1.tgz#be05ad7f9bf7d22e056f9726cee5017fbf19e2e9" integrity sha512-MguG95oij0fC3QV3URf4V2SDYGJhJnJGqvIIgdECeODCT98wSWDAJ94SSuVpYQUoTcGUIL6L4yNB7j1DFFHSBg== +regenerator-runtime@^0.14.0: + version "0.14.1" + resolved "https://registry.yarnpkg.com/regenerator-runtime/-/regenerator-runtime-0.14.1.tgz#356ade10263f685dda125100cd862c1db895327f" + integrity sha512-dYnhHh0nJoMfnkZs6GmmhFknAGRrLznOu5nc9ML+EJxGvrx6H7teuevqVqCuPcPK//3eDrrjQhehXVx9cnkGdw== + registry-auth-token@^4.0.0: version "4.2.1" resolved "https://registry.yarnpkg.com/registry-auth-token/-/registry-auth-token-4.2.1.tgz#6d7b4006441918972ccd5fedcd41dc322c79b250" @@ -4581,6 +5512,11 @@ registry-url@^5.0.0: dependencies: rc "^1.2.8" +rename-keys@^1.1.2: + version "1.2.0" + resolved "https://registry.yarnpkg.com/rename-keys/-/rename-keys-1.2.0.tgz#be602fb0b750476b513ebe85ba4465d03254f0a3" + integrity sha512-U7XpAktpbSgHTRSNRrjKSrjYkZKuhUukfoBlXWXUExCAqhzh1TU3BDRAfJmarcl5voKS+pbKU9MvyLWKZ4UEEg== + request@^2.81.0: version "2.88.2" resolved "https://registry.yarnpkg.com/request/-/request-2.88.2.tgz#d73c918731cb5a87da047e207234146f664d12b3" @@ -4622,6 +5558,15 @@ requires-port@^1.0.0: resolved "https://registry.yarnpkg.com/requires-port/-/requires-port-1.0.0.tgz#925d2601d39ac485e091cf0da5c6e694dc3dcaff" integrity sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ== +resolve@^1.14.2: + version "1.22.8" + resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.22.8.tgz#b6c87a9f2aa06dfab52e3d70ac8cde321fa5a48d" + integrity sha512-oKWePCxqpd6FlLvGV1VU0x7bkPmmCNolxzjMf4NczoDnQcIWrAF+cPtZn5i6n+RfD2d9i0tzpKnG6Yk168yIyw== + dependencies: + is-core-module "^2.13.0" + path-parse "^1.0.7" + supports-preserve-symlinks-flag "^1.0.0" + responselike@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/responselike/-/responselike-1.0.2.tgz#918720ef3b631c5642be068f15ade5a46f4ba1e7" @@ -4667,6 +5612,29 @@ sax@>=0.6.0: resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9" integrity sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw== +sax@^1.2.4: + version "1.4.1" + resolved "https://registry.yarnpkg.com/sax/-/sax-1.4.1.tgz#44cc8988377f126304d3b3fc1010c733b929ef0f" + integrity sha512-+aWOz7yVScEGoKNd4PA10LZ8sk0A/z5+nXQG5giUO5rprX9jgYsTdov9qCchZiPIZezbZH+jRut8nPodFAX4Jg== + +schema-utils@^2.7.0: + version "2.7.1" + resolved "https://registry.yarnpkg.com/schema-utils/-/schema-utils-2.7.1.tgz#1ca4f32d1b24c590c203b8e7a50bf0ea4cd394d7" + integrity sha512-SHiNtMOUGWBQJwzISiVYKu82GiV4QYGePp3odlY1tuKO7gPtphAT5R/py0fA6xtbgLL/RvtJZnU9b8s0F1q0Xg== + dependencies: + "@types/json-schema" "^7.0.5" + ajv "^6.12.4" + ajv-keywords "^3.5.2" + +schema-utils@^3.0.0: + version "3.3.0" + resolved "https://registry.yarnpkg.com/schema-utils/-/schema-utils-3.3.0.tgz#f50a88877c3c01652a15b622ae9e9795df7a60fe" + integrity sha512-pN/yOAvcC+5rQ5nERGuwrjLlYvLTbCibnZ1I7B1LaiAz9BRBlE9GMgE/eqV30P7aJQUf7Ddimy/RsbYO/GrVGg== + dependencies: + "@types/json-schema" "^7.0.8" + ajv "^6.12.5" + ajv-keywords "^3.5.2" + scrypt-js@3.0.1: version "3.0.1" resolved "https://registry.yarnpkg.com/scrypt-js/-/scrypt-js-3.0.1.tgz#d314a57c2aef69d1ad98a138a21fe9eafa9ee312" @@ -4696,6 +5664,11 @@ semver@^6.0.0, semver@^6.2.0, semver@^6.3.0: resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.0.tgz#ee0a64c8af5e8ceea67687b133761e1becbd1d3d" integrity sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw== +semver@^6.3.1: + version "6.3.1" + resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.1.tgz#556d2ef8689146e46dcea4bfdd095f3434dffcb4" + integrity sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA== + semver@^7.3.4: version "7.3.7" resolved "https://registry.yarnpkg.com/semver/-/semver-7.3.7.tgz#12c5b649afdbf9049707796e22a4028814ce523f" @@ -4703,6 +5676,11 @@ semver@^7.3.4: dependencies: lru-cache "^6.0.0" +semver@^7.5.3: + version "7.6.3" + resolved "https://registry.yarnpkg.com/semver/-/semver-7.6.3.tgz#980f7b5550bc175fb4dc09403085627f9eb33143" + integrity sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A== + semver@~5.3.0: version "5.3.0" resolved "https://registry.yarnpkg.com/semver/-/semver-5.3.0.tgz#9b2ce5d3de02d17c6012ad326aa6b4d0cf54f94f" @@ -4761,6 +5739,11 @@ set-function-length@^1.2.1: gopd "^1.0.1" has-property-descriptors "^1.0.2" +setimmediate@^1.0.5: + version "1.0.5" + resolved "https://registry.yarnpkg.com/setimmediate/-/setimmediate-1.0.5.tgz#290cbb232e306942d7d7ea9b83732ab7856f8285" + integrity sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA== + setprototypeof@1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/setprototypeof/-/setprototypeof-1.2.0.tgz#66c9a24a73f9fc28cbe66b09fed3d33dcaf1b424" @@ -4774,6 +5757,18 @@ sha.js@^2.4.0, sha.js@^2.4.8: inherits "^2.0.1" safe-buffer "^5.0.1" +shebang-command@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/shebang-command/-/shebang-command-2.0.0.tgz#ccd0af4f8835fbdc265b82461aaf0c36663f34ea" + integrity sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA== + dependencies: + shebang-regex "^3.0.0" + +shebang-regex@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/shebang-regex/-/shebang-regex-3.0.0.tgz#ae16f1644d873ecad843b0307b143362d4c42172" + integrity sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A== + side-channel@^1.0.4: version "1.0.4" resolved "https://registry.yarnpkg.com/side-channel/-/side-channel-1.0.4.tgz#efce5c8fdc104ee751b25c58d4290011fa5ea2cf" @@ -4798,6 +5793,11 @@ signal-exit@^3.0.2: resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.7.tgz#a9a1767f8af84155114eaabd73f99273c8f59ad9" integrity sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ== +signal-exit@^4.0.1: + version "4.1.0" + resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-4.1.0.tgz#952188c1cbd546070e2dd20d0f41c0ae0530cb04" + integrity sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw== + source-map-support@^0.5.6: version "0.5.21" resolved "https://registry.yarnpkg.com/source-map-support/-/source-map-support-0.5.21.tgz#04fe7c7f9e1ed2d662233c28cb2b35b9f63f6e4f" @@ -4806,7 +5806,7 @@ source-map-support@^0.5.6: buffer-from "^1.0.0" source-map "^0.6.0" -source-map@^0.6.0: +source-map@^0.6.0, source-map@^0.6.1: version "0.6.1" resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.6.1.tgz#74722af32e9614e9c287a8d0bbde48b5e2f1a263" integrity sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g== @@ -4816,6 +5816,18 @@ spark-md5@3.0.2: resolved "https://registry.yarnpkg.com/spark-md5/-/spark-md5-3.0.2.tgz#7952c4a30784347abcee73268e473b9c0167e3fc" integrity sha512-wcFzz9cDfbuqe0FZzfi2or1sgyIrsDwmPwfZC4hiNidPdPINjeUwNfv5kldczoEAcjl9Y1L3SM7Uz2PUEQzxQw== +sprintf-js@~1.0.2: + version "1.0.3" + resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c" + integrity sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g== + +ssf@~0.11.2: + version "0.11.2" + resolved "https://registry.yarnpkg.com/ssf/-/ssf-0.11.2.tgz#0b99698b237548d088fc43cdf2b70c1a7512c06c" + integrity sha512-+idbmIXoYET47hH+d7dfm2epdOMUDjqcB4648sTZ+t2JwoyBFL/insLfB/racrDmsKB3diwsDA696pZMieAC5g== + dependencies: + frac "~1.1.2" + sshpk@^1.7.0: version "1.17.0" resolved "https://registry.yarnpkg.com/sshpk/-/sshpk-1.17.0.tgz#578082d92d4fe612b13007496e543fa0fbcbe4c5" @@ -4860,6 +5872,15 @@ string-similarity@4.0.4: resolved "https://registry.yarnpkg.com/string-similarity/-/string-similarity-4.0.4.tgz#42d01ab0b34660ea8a018da8f56a3309bb8b2a5b" integrity sha512-/q/8Q4Bl4ZKAPjj8WerIBJWALKkaPRfrvhfF8k/B23i4nzrlRj2/go1m90In7nG/3XDSbOo0+pu6RvCTM9RGMQ== +"string-width-cjs@npm:string-width@^4.2.0": + version "4.2.3" + resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" + integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== + dependencies: + emoji-regex "^8.0.0" + is-fullwidth-code-point "^3.0.0" + strip-ansi "^6.0.1" + string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.2: version "4.2.3" resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" @@ -4869,6 +5890,15 @@ string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2 is-fullwidth-code-point "^3.0.0" strip-ansi "^6.0.1" +string-width@^5.0.1, string-width@^5.1.2: + version "5.1.2" + resolved "https://registry.yarnpkg.com/string-width/-/string-width-5.1.2.tgz#14f8daec6d81e7221d2a357e668cab73bdbca794" + integrity sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA== + dependencies: + eastasianwidth "^0.2.0" + emoji-regex "^9.2.2" + strip-ansi "^7.0.1" + string_decoder@^1.1.1: version "1.3.0" resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.3.0.tgz#42f114594a46cf1a8e30b0a84f56c78c3edac21e" @@ -4888,6 +5918,13 @@ string_decoder@~1.1.1: dependencies: safe-buffer "~5.1.0" +"strip-ansi-cjs@npm:strip-ansi@^6.0.1": + version "6.0.1" + resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" + integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== + dependencies: + ansi-regex "^5.0.1" + strip-ansi@^6.0.0, strip-ansi@^6.0.1: version "6.0.1" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" @@ -4895,6 +5932,13 @@ strip-ansi@^6.0.0, strip-ansi@^6.0.1: dependencies: ansi-regex "^5.0.1" +strip-ansi@^7.0.1: + version "7.1.0" + resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-7.1.0.tgz#d5b6568ca689d8561370b0707685d22434faff45" + integrity sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ== + dependencies: + ansi-regex "^6.0.1" + strip-bom@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/strip-bom/-/strip-bom-3.0.0.tgz#2334c18e9c759f7bdd56fdef7e9ae3d588e68ed3" @@ -4925,6 +5969,14 @@ strtok3@^7.0.0: "@tokenizer/token" "^0.3.0" peek-readable "^5.0.0" +style-loader@^1.2.1: + version "1.3.0" + resolved "https://registry.yarnpkg.com/style-loader/-/style-loader-1.3.0.tgz#828b4a3b3b7e7aa5847ce7bae9e874512114249e" + integrity sha512-V7TCORko8rs9rIqkSrlMfkqA63DfoGBBJmK1kKGCcSi+BWb4cqz0SRsnp4l6rU5iwOEd0/2ePv68SV22VXon4Q== + dependencies: + loader-utils "^2.0.0" + schema-utils "^2.7.0" + supports-color@8.1.1: version "8.1.1" resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-8.1.1.tgz#cd6fc17e28500cff56c1b86c0a7fd4a54a73005c" @@ -4946,6 +5998,48 @@ supports-color@^7.1.0: dependencies: has-flag "^4.0.0" +supports-preserve-symlinks-flag@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz#6eda4bd344a3c94aea376d4cc31bc77311039e09" + integrity sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w== + +svg-arc-to-cubic-bezier@^3.0.0: + version "3.2.0" + resolved "https://registry.yarnpkg.com/svg-arc-to-cubic-bezier/-/svg-arc-to-cubic-bezier-3.2.0.tgz#390c450035ae1c4a0104d90650304c3bc814abe6" + integrity sha512-djbJ/vZKZO+gPoSDThGNpKDO+o+bAeA4XQKovvkNCqnIS2t+S4qnLAGQhyyrulhCFRl1WWzAp0wUDV8PpTVU3g== + +svg-path-bbox@0.0.49: + version "0.0.49" + resolved "https://registry.yarnpkg.com/svg-path-bbox/-/svg-path-bbox-0.0.49.tgz#0d1c84b9b84341fe085a244f0d286ea2ecf95d15" + integrity sha512-QXKc4LhdZTlk3thjxR1jrWqoRGCMSjNjqqdgDTS1vqOSdYFTnTNQvGrKv3dvSCqB3VC1pyRBQGJKJdtXE3mp5Q== + dependencies: + polf "^0.0.3" + svgpath "^2.3.0" + +svg-path-bounds@^1.0.1: + version "1.0.2" + resolved "https://registry.yarnpkg.com/svg-path-bounds/-/svg-path-bounds-1.0.2.tgz#00312f672b08afc432a66ddfbd06db40cec8d0d0" + integrity sha512-H4/uAgLWrppIC0kHsb2/dWUYSmb4GE5UqH06uqWBcg6LBjX2fu0A8+JrO2/FJPZiSsNOKZAhyFFgsLTdYUvSqQ== + dependencies: + abs-svg-path "^0.1.1" + is-svg-path "^1.0.1" + normalize-svg-path "^1.0.0" + parse-svg-path "^0.1.2" + +svgpath@^2.3.0: + version "2.6.0" + resolved "https://registry.yarnpkg.com/svgpath/-/svgpath-2.6.0.tgz#5b160ef3d742b7dfd2d721bf90588d3450d7a90d" + integrity sha512-OIWR6bKzXvdXYyO4DK/UWa1VA1JeKq8E+0ug2DG98Y/vOmMpfZNj+TIG988HjfYSqtcy/hFOtZq/n/j5GSESNg== + +svgson@^4.1.0: + version "4.1.0" + resolved "https://registry.yarnpkg.com/svgson/-/svgson-4.1.0.tgz#eb70dac8d0075c61e5bfd45411a56014e2d3610e" + integrity sha512-DodISxHtdLKUghDYA+PGK4Qq350+CbBAkdvGLkBFSmWd9WKSg4dijgjB1IiRPTmsUCd+a7KYe+ILHtklYgQyzQ== + dependencies: + deep-rename-keys "^0.2.1" + omit-deep "0.3.0" + xml-reader "2.4.3" + table-layout@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/table-layout/-/table-layout-1.0.2.tgz#c4038a1853b0136d63365a734b6931cf4fad4a04" @@ -4977,7 +6071,7 @@ tar-stream@^2.1.0, tar-stream@^2.1.4: inherits "^2.0.3" readable-stream "^3.1.1" -through2@3.0.2: +through2@3.0.2, through2@^3.0.1: version "3.0.2" resolved "https://registry.yarnpkg.com/through2/-/through2-3.0.2.tgz#99f88931cfc761ec7678b41d5d7336b5b6a07bf4" integrity sha512-enaDQ4MUyP2W6ZyT6EsMzqBPZaM/avg8iuo+l2d3QCs0J+6RaqkHV/2/lOwDTueBHeJ/2LG9lrLW3d5rWPucuQ== @@ -4993,11 +6087,21 @@ through2@^2.0.1, through2@^2.0.3: readable-stream "~2.3.6" xtend "~4.0.1" +tinycolor2@^1.4.1: + version "1.6.0" + resolved "https://registry.yarnpkg.com/tinycolor2/-/tinycolor2-1.6.0.tgz#f98007460169b0263b97072c5ae92484ce02d09e" + integrity sha512-XPaBkWQJdsf3pLKJV9p4qN/S+fm2Oj8AIPo1BTUhg5oxkvm9+SVEGFdhyOz7tTdUTfvxMiAs4sp6/eZO2Ew+pw== + tlds@1.252.0: version "1.252.0" resolved "https://registry.yarnpkg.com/tlds/-/tlds-1.252.0.tgz#71d9617f4ef4cc7347843bee72428e71b8b0f419" integrity sha512-GA16+8HXvqtfEnw/DTcwB0UU354QE1n3+wh08oFjr6Znl7ZLAeUgYzCcK+/CCrOyE0vnHR8/pu3XXG3vDijXpQ== +to-fast-properties@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/to-fast-properties/-/to-fast-properties-2.0.0.tgz#dc5e698cbd079265bc73e0377681a4e4e83f616e" + integrity sha512-/OaKK0xYrs3DmxRYqL/yDc+FxFUVYhDlXMhRmv3z915w2HF1tnN1omB354j8VUGO/hbRzyD6Y3sA7v7GS/ceog== + to-readable-stream@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/to-readable-stream/-/to-readable-stream-1.0.0.tgz#ce0aa0c2f3df6adf852efb404a783e77c0475771" @@ -5069,6 +6173,11 @@ transform-pouch@^2.0.0: dependencies: pouchdb-wrappers "^5.0.0" +transformation-matrix@^2.4.0, transformation-matrix@^2.5.0: + version "2.16.1" + resolved "https://registry.yarnpkg.com/transformation-matrix/-/transformation-matrix-2.16.1.tgz#4a2de06331b94ae953193d1b9a5ba002ec5f658a" + integrity sha512-tdtC3wxVEuzU7X/ydL131Q3JU5cPMEn37oqVLITjRDSDsnSHVFzW2JiCLfZLIQEgWzZHdSy3J6bZzvKEN24jGA== + ts-mixer@^6.0.3: version "6.0.3" resolved "https://registry.yarnpkg.com/ts-mixer/-/ts-mixer-6.0.3.tgz#69bd50f406ff39daa369885b16c77a6194c7cae6" @@ -5144,6 +6253,20 @@ twitter-api-v2@^1.14.0: resolved "https://registry.yarnpkg.com/twitter-api-v2/-/twitter-api-v2-1.14.0.tgz#2aa186087aae58083dcbbafef8727b42cb703483" integrity sha512-kBc0X6hTl0qWYTSNH9Gp87S6d9wIM2qOxHg7jlWH054HnMPC8XMcttsrKyPwP8SVha28dt5DPQvlqhPxsWRC+Q== +txml@3.1.3: + version "3.1.3" + resolved "https://registry.yarnpkg.com/txml/-/txml-3.1.3.tgz#c2d890b18f1eb0685845c6e320cfe0e8f7aadeb7" + integrity sha512-JOXZxzZtdXqxczL3aYs6ZtJdHKbqrzdb/BOOj9M48kmL095RHmT8Ad+Ax+UVhE3t7XZLaCjaRsUo6qE4RYudIQ== + dependencies: + through2 "^3.0.1" + +txml@^3.1.3: + version "3.2.5" + resolved "https://registry.yarnpkg.com/txml/-/txml-3.2.5.tgz#607eeef7e021dba8c6dd173b3971b12443deb9ec" + integrity sha512-AtN8AgJLiDanttIXJaQlxH8/R0NOCNwto8kcO7BaxdLgsN9b7itM9lnTD7c2O3TadP+hHB9j7ra5XGFRPNnk/g== + dependencies: + through2 "^3.0.1" + type-detect@^4.0.8: version "4.1.0" resolved "https://registry.yarnpkg.com/type-detect/-/type-detect-4.1.0.tgz#deb2453e8f08dcae7ae98c626b13dddb0155906c" @@ -5213,6 +6336,11 @@ undefsafe@^2.0.5: resolved "https://registry.yarnpkg.com/undefsafe/-/undefsafe-2.0.5.tgz#38733b9327bdcd226db889fb723a6efd162e6e2c" integrity sha512-WxONCrssBM8TSPRqN5EmsjVrsv4A8X12J4ArBiiayv3DyyG3ZlIg6yysuuSYdZsVz3TKcTg2fd//Ujd4CHV1iA== +underscore@^1.13.1: + version "1.13.7" + resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.13.7.tgz#970e33963af9a7dda228f17ebe8399e5fbe63a10" + integrity sha512-GMXzWtsc57XAtguZgaQViUOzs0KTkk8ojr3/xAxXLITqf/3EMwxC0inyETfDFjH/Krbhuep0HNbbjI9i/q3F3g== + undici-types@~5.26.4: version "5.26.5" resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-5.26.5.tgz#bcd539893d00b56e964fd2657a4866b221a65617" @@ -5252,11 +6380,27 @@ unpipe@1.0.0, unpipe@~1.0.0: resolved "https://registry.yarnpkg.com/unpipe/-/unpipe-1.0.0.tgz#b2bf4ee8514aae6165b4817829d21b2ef49904ec" integrity sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw= +unset-value@^0.1.1: + version "0.1.2" + resolved "https://registry.yarnpkg.com/unset-value/-/unset-value-0.1.2.tgz#506810b867f27c2a5a6e9b04833631f6de58d310" + integrity sha512-yhv5I4TsldLdE3UcVQn0hD2T5sNCPv4+qm/CTUpRKIpwthYRIipsAPdsrNpOI79hPQa0rTTeW22Fq6JWRcTgNg== + dependencies: + has-value "^0.3.1" + isobject "^3.0.0" + untildify@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/untildify/-/untildify-4.0.0.tgz#2bc947b953652487e4600949fb091e3ae8cd919b" integrity sha512-KK8xQ1mkzZeg9inewmFVDNkg3l5LUhoq9kN6iWYB/CC9YMG8HA+c1Q8HwDe6dEX7kErrEVNVBO3fWsVq5iDgtw== +update-browserslist-db@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/update-browserslist-db/-/update-browserslist-db-1.1.0.tgz#7ca61c0d8650766090728046e416a8cde682859e" + integrity sha512-EdRAaAyk2cUE1wOf2DkEhzxqOQvFOoRJFNS6NeyJ01Gp2beMRpBAINjM2iDXE3KCuKhwnvHIQCJm6ThL2Z+HzQ== + dependencies: + escalade "^3.1.2" + picocolors "^1.0.1" + update-notifier@^5.1.0: version "5.1.0" resolved "https://registry.yarnpkg.com/update-notifier/-/update-notifier-5.1.0.tgz#4ab0d7c7f36a231dd7316cf7729313f0214d9ad9" @@ -5284,6 +6428,15 @@ uri-js@^4.2.2: dependencies: punycode "^2.1.0" +url-loader@^4.1.0: + version "4.1.1" + resolved "https://registry.yarnpkg.com/url-loader/-/url-loader-4.1.1.tgz#28505e905cae158cf07c92ca622d7f237e70a4e2" + integrity sha512-3BTV812+AVHHOJQO8O5MkWgZ5aosP7GnROJwvzLS9hWDj00lZ6Z0wNak423Lp9PBZN05N+Jk/N5Si8jRAlGyWA== + dependencies: + loader-utils "^2.0.0" + mime-types "^2.1.27" + schema-utils "^3.0.0" + url-parse-lax@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/url-parse-lax/-/url-parse-lax-3.0.0.tgz#16b5cafc07dbe3676c1b1999177823d6503acb0c" @@ -5327,7 +6480,7 @@ utf7@^1.0.2: dependencies: semver "~5.3.0" -util-deprecate@^1.0.1, util-deprecate@~1.0.1: +util-deprecate@^1.0.1, util-deprecate@^1.0.2, util-deprecate@~1.0.1: version "1.0.2" resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf" integrity sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8= @@ -5394,7 +6547,7 @@ whatwg-url@^5.0.0: tr46 "~0.0.3" webidl-conversions "^3.0.0" -which@2.0.2, which@^2.0.2: +which@2.0.2, which@^2.0.1, which@^2.0.2: version "2.0.2" resolved "https://registry.yarnpkg.com/which/-/which-2.0.2.tgz#7c6a8dd0a636a0327e10b59c9286eee93f3f51b1" integrity sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA== @@ -5408,6 +6561,16 @@ widest-line@^3.1.0: dependencies: string-width "^4.0.0" +wmf@~1.0.1: + version "1.0.2" + resolved "https://registry.yarnpkg.com/wmf/-/wmf-1.0.2.tgz#7d19d621071a08c2bdc6b7e688a9c435298cc2da" + integrity sha512-/p9K7bEh0Dj6WbXg4JG0xvLQmIadrner1bi45VMJTfnbVHsc7yIajZyoSoK60/dtVBs12Fm6WkUI5/3WAVsNMw== + +word@~0.3.0: + version "0.3.0" + resolved "https://registry.yarnpkg.com/word/-/word-0.3.0.tgz#8542157e4f8e849f4a363a288992d47612db9961" + integrity sha512-OELeY0Q61OXpdUfTp+oweA/vtLVg5VDOXh+3he3PNzLGG/y0oylSOC1xRVj0+l4vQ3tj/bB1HVHv1ocXkQceFA== + wordwrapjs@^4.0.0: version "4.0.1" resolved "https://registry.yarnpkg.com/wordwrapjs/-/wordwrapjs-4.0.1.tgz#d9790bccfb110a0fc7836b5ebce0937b37a8b98f" @@ -5421,6 +6584,15 @@ workerpool@6.2.0: resolved "https://registry.yarnpkg.com/workerpool/-/workerpool-6.2.0.tgz#827d93c9ba23ee2019c3ffaff5c27fccea289e8b" integrity sha512-Rsk5qQHJ9eowMH28Jwhe8HEbmdYDX4lwoMWshiCXugjtHqMD9ZbiqSDLxcsfdqsETPzVUtX5s1Z5kStiIM6l4A== +"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0": + version "7.0.0" + resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" + integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== + dependencies: + ansi-styles "^4.0.0" + string-width "^4.1.0" + strip-ansi "^6.0.0" + wrap-ansi@^7.0.0: version "7.0.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" @@ -5430,6 +6602,15 @@ wrap-ansi@^7.0.0: string-width "^4.1.0" strip-ansi "^6.0.0" +wrap-ansi@^8.1.0: + version "8.1.0" + resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-8.1.0.tgz#56dc22368ee570face1b49819975d9b9a5ead214" + integrity sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ== + dependencies: + ansi-styles "^6.1.0" + string-width "^5.0.1" + strip-ansi "^7.0.1" + wrappy@1: version "1.0.2" resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f" @@ -5472,6 +6653,41 @@ xdg-basedir@^4.0.0: resolved "https://registry.yarnpkg.com/xdg-basedir/-/xdg-basedir-4.0.0.tgz#4bc8d9984403696225ef83a1573cbbcb4e79db13" integrity sha512-PSNhEJDejZYV7h50BohL09Er9VaIefr2LMAf3OEmpCkjOi34eYyQYAXUTjEQtZJTKcF0E2UKTh+osDLsgNim9Q== +xlsx@^0.18.5: + version "0.18.5" + resolved "https://registry.yarnpkg.com/xlsx/-/xlsx-0.18.5.tgz#16711b9113c848076b8a177022799ad356eba7d0" + integrity sha512-dmg3LCjBPHZnQp5/F/+nnTa+miPJxUXB6vtk42YjBBKayDNagxGEeIdWApkYPOf3Z3pm3k62Knjzp7lMeTEtFQ== + dependencies: + adler-32 "~1.3.0" + cfb "~1.2.1" + codepage "~1.15.0" + crc-32 "~1.2.1" + ssf "~0.11.2" + wmf "~1.0.1" + word "~0.3.0" + +xml-js@^1.6.11: + version "1.6.11" + resolved "https://registry.yarnpkg.com/xml-js/-/xml-js-1.6.11.tgz#927d2f6947f7f1c19a316dd8eea3614e8b18f8e9" + integrity sha512-7rVi2KMfwfWFl+GpPg6m80IVMWXLRjO+PxTq7V2CDhoGak0wzYzFgUY2m4XJ47OGdXd8eLE8EmwfAmdjw7lC1g== + dependencies: + sax "^1.2.4" + +xml-lexer@^0.2.2: + version "0.2.2" + resolved "https://registry.yarnpkg.com/xml-lexer/-/xml-lexer-0.2.2.tgz#518193a4aa334d58fc7d248b549079b89907e046" + integrity sha512-G0i98epIwiUEiKmMcavmVdhtymW+pCAohMRgybyIME9ygfVu8QheIi+YoQh3ngiThsT0SQzJT4R0sKDEv8Ou0w== + dependencies: + eventemitter3 "^2.0.0" + +xml-reader@2.4.3: + version "2.4.3" + resolved "https://registry.yarnpkg.com/xml-reader/-/xml-reader-2.4.3.tgz#9f810caf7c425a5aafb848b1c45103c9e71d7530" + integrity sha512-xWldrIxjeAMAu6+HSf9t50ot1uL5M+BtOidRCWHXIeewvSeIpscWCsp4Zxjk8kHHhdqFBrfK8U0EJeCcnyQ/gA== + dependencies: + eventemitter3 "^2.0.0" + xml-lexer "^0.2.2" + xml2js@0.4.19: version "0.4.19" resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.19.tgz#686c20f213209e94abf0d1bcf1efaa291c7827a7" @@ -5480,6 +6696,11 @@ xml2js@0.4.19: sax ">=0.6.0" xmlbuilder "~9.0.1" +xmlbuilder@^10.0.0: + version "10.1.1" + resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-10.1.1.tgz#8cae6688cc9b38d850b7c8d3c0a4161dcaf475b0" + integrity sha512-OyzrcFLL/nb6fMGHbiRDuPup9ljBycsdCypwuyg5AAHvyWzGfChJpCXMG88AGTIMFhGZ9RccFN1e6lhg3hkwKg== + xmlbuilder@~9.0.1: version "9.0.7" resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-9.0.7.tgz#132ee63d2ec5565c557e20f4c22df9aca686b10d" @@ -5500,6 +6721,11 @@ y18n@^5.0.5: resolved "https://registry.yarnpkg.com/y18n/-/y18n-5.0.8.tgz#7f4934d0f7ca8c56f95314939ddcd2dd91ce1d55" integrity sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA== +yallist@^3.0.2: + version "3.1.1" + resolved "https://registry.yarnpkg.com/yallist/-/yallist-3.1.1.tgz#dbb7daf9bfd8bac9ab45ebf602b8cbad0d5d08fd" + integrity sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g== + yallist@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/yallist/-/yallist-4.0.0.tgz#9bb92790d9c0effec63be73519e11a35019a3a72" From 7d1afc4d12ab5a5afbace7b3e4e37fe735567fdc Mon Sep 17 00:00:00 2001 From: Chris Date: Wed, 21 Aug 2024 08:37:30 +0930 Subject: [PATCH 011/182] Support sync v force sync on connections page --- src/dashboard/public/connections/connections.js | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/dashboard/public/connections/connections.js b/src/dashboard/public/connections/connections.js index 54d3b0e8..1cd5a1ee 100644 --- a/src/dashboard/public/connections/connections.js +++ b/src/dashboard/public/connections/connections.js @@ -75,7 +75,17 @@ $(document).ready(function() { ${connection.syncStatus}
${formattedSyncTimes} ${handlers.map(handler => `[${handler.handlerName}] ${handler.syncMessage ? handler.syncMessage : ""} (${handler.status})
`).join('')} - +
+ + +
+ Force +
+
@@ -104,6 +114,7 @@ $(document).ready(function() { const providerId = $(this).data('provider-id'); $button.text('Syncing...') $button.prop('disabled', true); + const syncType = $(this).data('sync-type'); // Start tailing logs const eventSource = new EventSource(`/api/v1/logs?key=${veridaKey}`); @@ -138,7 +149,7 @@ $(document).ready(function() { }); // Initialize sync - $.getJSON(`/api/v1/sync?key=${veridaKey}&provider=${provider}&providerId=${providerId}&force=true`, function(response) { + $.getJSON(`/api/v1/sync?key=${veridaKey}&provider=${provider}&providerId=${providerId}&${syncType == 'force' ? 'force=true' : ''}`, function(response) { $button.prop('disabled', false); $button.text('Sync Now') From b78a6ddb7e2a93dc0b367d87d09a7941accb9bcf Mon Sep 17 00:00:00 2001 From: chime3 Date: Tue, 20 Aug 2024 22:45:33 -0700 Subject: [PATCH 012/182] fix: google unit test --- tests/providers/google/gmail.tests.ts | 4 +--- tests/providers/google/youtube-favourite.tests.ts | 7 ++----- tests/providers/google/youtube-following.tests.ts | 7 ++----- tests/providers/google/youtube-post.tests.ts | 7 ++----- 4 files changed, 7 insertions(+), 18 deletions(-) diff --git a/tests/providers/google/gmail.tests.ts b/tests/providers/google/gmail.tests.ts index 6b63b987..544e6f6f 100644 --- a/tests/providers/google/gmail.tests.ts +++ b/tests/providers/google/gmail.tests.ts @@ -4,7 +4,6 @@ import { Connection, SyncHandlerStatus, SyncHandlerPosition, - SyncSchemaPositionType, } from "../../../src/interfaces"; import Providers from "../../../src/providers"; import CommonUtils, { NetworkInstance } from "../../common.utils"; @@ -56,11 +55,10 @@ describe(`${providerName} Tests`, function () { ).toISOString(); const syncPosition: Omit = { - type: SyncSchemaPositionType.SYNC, providerName, providerId: provider.getProviderId(), handlerName, - status: SyncHandlerStatus.ACTIVE, + status: SyncHandlerStatus.ENABLED, }; providerConfig.batchSize = 10; diff --git a/tests/providers/google/youtube-favourite.tests.ts b/tests/providers/google/youtube-favourite.tests.ts index 615240e6..d07e8ec0 100644 --- a/tests/providers/google/youtube-favourite.tests.ts +++ b/tests/providers/google/youtube-favourite.tests.ts @@ -4,7 +4,6 @@ import { Connection, SyncHandlerStatus, SyncHandlerPosition, - SyncSchemaPositionType, } from "../../../src/interfaces"; import Providers from "../../../src/providers"; import CommonUtils, { NetworkInstance } from "../../common.utils"; @@ -55,11 +54,10 @@ describe(`${providerName} Youtube Favourite Tests`, function () { ).toISOString(); const syncPosition: Omit = { - type: SyncSchemaPositionType.SYNC, providerName, providerId: provider.getProviderId(), handlerName, - status: SyncHandlerStatus.ACTIVE, + status: SyncHandlerStatus.ENABLED, }; providerConfig.batchSize = 5; @@ -89,11 +87,10 @@ describe(`${providerName} Youtube Favourite Tests`, function () { it(`Can handle empty results`, async () => { const syncPosition: Omit = { - type: SyncSchemaPositionType.SYNC, providerName, providerId: provider.getProviderId(), handlerName, - status: SyncHandlerStatus.ACTIVE, + status: SyncHandlerStatus.ENABLED, }; providerConfig.batchSize = 5; diff --git a/tests/providers/google/youtube-following.tests.ts b/tests/providers/google/youtube-following.tests.ts index b11c4f6f..7819d802 100644 --- a/tests/providers/google/youtube-following.tests.ts +++ b/tests/providers/google/youtube-following.tests.ts @@ -4,7 +4,6 @@ import { Connection, SyncHandlerStatus, SyncHandlerPosition, - SyncSchemaPositionType, } from "../../../src/interfaces"; import Providers from "../../../src/providers"; import CommonUtils, { NetworkInstance } from "../../common.utils"; @@ -56,11 +55,10 @@ describe(`${providerName} Youtube Following Tests`, function () { ).toISOString(); const syncPosition: Omit = { - type: SyncSchemaPositionType.SYNC, providerName, providerId: provider.getProviderId(), handlerName, - status: SyncHandlerStatus.ACTIVE, + status: SyncHandlerStatus.ENABLED, }; providerConfig.batchSize = 5; @@ -94,11 +92,10 @@ describe(`${providerName} Youtube Following Tests`, function () { it(`Can handle empty results`, async () => { const syncPosition: Omit = { - type: SyncSchemaPositionType.SYNC, providerName, providerId: provider.getProviderId(), handlerName, - status: SyncHandlerStatus.ACTIVE, + status: SyncHandlerStatus.ENABLED, }; providerConfig.batchSize = 5; diff --git a/tests/providers/google/youtube-post.tests.ts b/tests/providers/google/youtube-post.tests.ts index f6be9323..8a77217a 100644 --- a/tests/providers/google/youtube-post.tests.ts +++ b/tests/providers/google/youtube-post.tests.ts @@ -4,7 +4,6 @@ import { Connection, SyncHandlerStatus, SyncHandlerPosition, - SyncSchemaPositionType, } from "../../../src/interfaces"; import Providers from "../../../src/providers"; import CommonUtils, { NetworkInstance } from "../../common.utils"; @@ -56,11 +55,10 @@ describe(`${providerName} Youtube Post Tests`, function () { ).toISOString(); const syncPosition: Omit = { - type: SyncSchemaPositionType.SYNC, providerName, providerId: provider.getProviderId(), handlerName, - status: SyncHandlerStatus.ACTIVE, + status: SyncHandlerStatus.ENABLED, }; providerConfig.batchSize = 5; @@ -94,11 +92,10 @@ describe(`${providerName} Youtube Post Tests`, function () { it(`Can handle empty results`, async () => { const syncPosition: Omit = { - type: SyncSchemaPositionType.SYNC, providerName, providerId: provider.getProviderId(), handlerName, - status: SyncHandlerStatus.ACTIVE, + status: SyncHandlerStatus.ENABLED, }; providerConfig.batchSize = 5; From eb25f8a88290e95881b900eb48a353282b2603f8 Mon Sep 17 00:00:00 2001 From: chime3 Date: Tue, 20 Aug 2024 22:54:14 -0700 Subject: [PATCH 013/182] fix: sync handler status --- src/providers/google/gdrive-document.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/providers/google/gdrive-document.ts b/src/providers/google/gdrive-document.ts index 3903e203..664a5d19 100644 --- a/src/providers/google/gdrive-document.ts +++ b/src/providers/google/gdrive-document.ts @@ -113,11 +113,11 @@ export default class GoogleDriveDocument extends BaseSyncHandler { } protected stopSync(syncPosition: SyncHandlerPosition): SyncHandlerPosition { - if (syncPosition.status == SyncHandlerStatus.STOPPED) { + if (syncPosition.status == SyncHandlerStatus.ENABLED) { return syncPosition; } - syncPosition.status = SyncHandlerStatus.STOPPED; + syncPosition.status = SyncHandlerStatus.ENABLED; syncPosition.thisRef = undefined; syncPosition.breakId = syncPosition.futureBreakId; syncPosition.futureBreakId = undefined; From ade7e18efa7e8cc5e8e5caa824d1a32bb392e243 Mon Sep 17 00:00:00 2001 From: chime3 Date: Tue, 20 Aug 2024 23:33:40 -0700 Subject: [PATCH 014/182] fix: gdrive unit test --- tests/providers/google/gdrive-document.tests.ts | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/tests/providers/google/gdrive-document.tests.ts b/tests/providers/google/gdrive-document.tests.ts index bb7d18fe..cdb507fb 100644 --- a/tests/providers/google/gdrive-document.tests.ts +++ b/tests/providers/google/gdrive-document.tests.ts @@ -4,7 +4,6 @@ import { Connection, SyncHandlerStatus, SyncHandlerPosition, - SyncSchemaPositionType, } from "../../../src/interfaces"; import Providers from "../../../src/providers"; import CommonUtils, { NetworkInstance } from "../../common.utils"; @@ -57,11 +56,10 @@ describe(`${providerName} GDrive Document Tests`, function () { ).toISOString(); const syncPosition: Omit = { - type: SyncSchemaPositionType.SYNC, providerName, providerId: provider.getProviderId(), handlerName, - status: SyncHandlerStatus.ACTIVE, + status: SyncHandlerStatus.ENABLED, }; providerConfig.batchSize = 5; @@ -95,11 +93,10 @@ describe(`${providerName} GDrive Document Tests`, function () { it(`Can handle empty results`, async () => { const syncPosition: Omit = { - type: SyncSchemaPositionType.SYNC, providerName, providerId: provider.getProviderId(), handlerName, - status: SyncHandlerStatus.ACTIVE, + status: SyncHandlerStatus.ENABLED, }; providerConfig.batchSize = 5; From fba7787715a535fff3a059a0ccceeb359accdaaa Mon Sep 17 00:00:00 2001 From: Chris Date: Wed, 21 Aug 2024 16:18:29 +0930 Subject: [PATCH 015/182] Refactor gmail to use items range tracker --- src/interfaces.ts | 10 +++ src/providers/google/gmail.ts | 161 ++++++++++++++++++++++------------ src/serverconfig.example.json | 7 +- 3 files changed, 119 insertions(+), 59 deletions(-) diff --git a/src/interfaces.ts b/src/interfaces.ts index 728e3840..03377f75 100644 --- a/src/interfaces.ts +++ b/src/interfaces.ts @@ -185,4 +185,14 @@ export interface SyncProviderLogEvent { export interface SyncHandlerResponse { syncPosition: SyncHandlerPosition syncResults: SchemaRecord[] +} + +export interface SyncItemsResult { + items: SchemaRecord[] + breakHit?: SyncItemsBreak +} + +export enum SyncItemsBreak { + ID = "id", + TIMESTAMP = "timestamp" } \ No newline at end of file diff --git a/src/providers/google/gmail.ts b/src/providers/google/gmail.ts index 04e58fa1..3482b4d2 100644 --- a/src/providers/google/gmail.ts +++ b/src/providers/google/gmail.ts @@ -1,22 +1,28 @@ import GoogleHandler from "./GoogleHandler"; import CONFIG from "../../config"; -import { SyncProviderLogEvent, SyncProviderLogLevel } from '../../interfaces' +import { SyncItemsBreak, SyncItemsResult, SyncProviderLogEvent, SyncProviderLogLevel } from '../../interfaces' import { google, gmail_v1 } from "googleapis"; import { GaxiosResponse } from "gaxios"; +import { ItemsRangeTracker } from "../../helpers/itemsRangeTracker" import { SyncResponse, - SyncHandlerPosition, SyncHandlerStatus, HandlerOption, ConnectionOptionType, } from "../../interfaces"; -import { SchemaEmail, SchemaEmailType } from "../../schemas"; +import { SchemaEmail, SchemaEmailType, SchemaRecord } from "../../schemas"; import { GmailHelpers } from "./helpers"; import { GmailSyncSchemaPosition } from "./interfaces"; const _ = require("lodash"); +const MAX_BATCH_SIZE = 500 + +export interface SyncEmailItemsResult extends SyncItemsResult { + items: SchemaEmail[] +} + export default class Gmail extends GoogleHandler { public getName(): string { @@ -52,85 +58,118 @@ export default class Gmail extends GoogleHandler { api: any, syncPosition: GmailSyncSchemaPosition ): Promise { - const gmail = this.getGmail(); + if (this.config.batchSize > MAX_BATCH_SIZE) { + throw new Error(`Batch size (${this.config.batchSize}) is larger than permitted (${MAX_BATCH_SIZE})`) + } - const query: gmail_v1.Params$Resource$Users$Messages$List = { - userId: "me", - maxResults: this.config.batchSize, // Google Docs: default = 100, max = 500 - }; + const gmail = this.getGmail(); + // Range tracker is used where completed startId = item ID, endId = pageToken + // And conversely, pending startId = page token, endId = item ID + const rangeTracker = new ItemsRangeTracker(syncPosition.thisRef) - if (syncPosition.thisRef) { - query.pageToken = syncPosition.thisRef; - } + let items: SchemaEmail[] = [] - const serverResponse = await gmail.users.messages.list(query); + /** + * Fetch any new items + */ + // Current range has `startId` = undefined, `endId` = breakId + let currentRange = rangeTracker.nextRange() - if ( - !_.has(serverResponse, "data.messages") || - !serverResponse.data.messages.length - ) { - syncPosition.syncMessage = `Stopping. No results found.` - syncPosition = this.stopSync(syncPosition); + let query: gmail_v1.Params$Resource$Users$Messages$List = { + userId: "me", + maxResults: this.config.batchSize, // default = 100, max = 500 + }; - return { - position: syncPosition, - results: [], - }; + if (currentRange.startId) { + query.pageToken = currentRange.startId } - const results = await this.buildResults( + const latestResponse = await gmail.users.messages.list(query); + const latestResult = await this.buildResults( gmail, - serverResponse, - syncPosition.breakId, + latestResponse, + currentRange.endId, SchemaEmailType.RECEIVE, _.has(this.config, "metadata.breakTimestamp") ? this.config.metadata.breakTimestamp : undefined ); - syncPosition = this.setNextPosition(syncPosition, serverResponse); + items = latestResult.items + + let nextPageToken = _.has(latestResponse, "data.nextPageToken") ? latestResponse.data.nextPageToken : undefined - if (results.length != this.config.batchSize) { - syncPosition.syncMessage = `Processed ${results.length} items. Stopping. No more results.` - syncPosition = this.stopSync(syncPosition); + if (items.length) { + rangeTracker.completedRange({ + startId: items[0].sourceId, + endId: nextPageToken + }, latestResult.breakHit == SyncItemsBreak.ID) + } else { + rangeTracker.completedRange({ + startId: undefined, + endId: undefined + }, false) // No results and first batch, so break ID couldn't have been hit } - return { - results, - position: syncPosition, - }; - } + if (items.length != this.config.batchSize) { + // Not enough items, fetch more from the next page of results + currentRange = rangeTracker.nextRange() - protected stopSync(syncPosition: SyncHandlerPosition): SyncHandlerPosition { - if (syncPosition.status == SyncHandlerStatus.ENABLED) { - return syncPosition; - } + query = { + userId: "me", + maxResults: this.config.batchSize - items.length, // only fetch enough items needed to complete the batch size + }; + + if (currentRange.startId) { + query.pageToken = currentRange.startId + } - syncPosition.status = SyncHandlerStatus.ENABLED; - syncPosition.thisRef = undefined; - syncPosition.breakId = syncPosition.futureBreakId; - syncPosition.futureBreakId = undefined; + const backfillResponse = await gmail.users.messages.list(query); + const backfillResult = await this.buildResults( + gmail, + backfillResponse, + currentRange.endId, + SchemaEmailType.RECEIVE, + _.has(this.config, "metadata.breakTimestamp") + ? this.config.metadata.breakTimestamp + : undefined + ); - return syncPosition; - } + items = items.concat(backfillResult.items) - protected setNextPosition( - syncPosition: SyncHandlerPosition, - serverResponse: GaxiosResponse - ): SyncHandlerPosition { - if (!syncPosition.futureBreakId && serverResponse.data.messages.length) { - syncPosition.futureBreakId = serverResponse.data.messages[0].id; + nextPageToken = _.has(backfillResponse, "data.nextPageToken") ? backfillResponse.data.nextPageToken : undefined + + if (backfillResult.items.length) { + rangeTracker.completedRange({ + startId: backfillResult.items[0].sourceId, + endId: nextPageToken + }, backfillResult.breakHit == SyncItemsBreak.ID) + } else { + rangeTracker.completedRange({ + startId: undefined, + endId: undefined + }, backfillResult.breakHit == SyncItemsBreak.ID) + } } - if (_.has(serverResponse, "data.nextPageToken")) { - syncPosition.syncMessage = `Batch complete (${this.config.batchSize}). More results pending.` - syncPosition.thisRef = serverResponse.data.nextPageToken; + if (!items.length) { + syncPosition.syncMessage = `Stopping. No results found.` + syncPosition.status = SyncHandlerStatus.ENABLED } else { - syncPosition.syncMessage = `Stopping. No more results.` - syncPosition = this.stopSync(syncPosition); + if (items.length != this.config.batchSize && !nextPageToken) { + syncPosition.syncMessage = `Processed ${items.length} items. Stopping. No more results.` + syncPosition.status = SyncHandlerStatus.ENABLED + } else { + syncPosition.syncMessage = `Batch complete (${this.config.batchSize}). More results pending.` + } } - return syncPosition; + syncPosition.thisRef = rangeTracker.export() + + return { + results: items, + position: syncPosition, + }; } protected async buildResults( @@ -139,8 +178,9 @@ export default class Gmail extends GoogleHandler { breakId: string, messageType: SchemaEmailType, breakTimestamp?: string - ): Promise { + ): Promise { const results: SchemaEmail[] = []; + let breakHit: SyncItemsBreak for (const message of serverResponse.data.messages) { const messageId = message.id; @@ -150,6 +190,7 @@ export default class Gmail extends GoogleHandler { message: `Break ID hit (${breakId})` } this.emit('log', logEvent) + breakHit = SyncItemsBreak.ID break; } @@ -164,6 +205,7 @@ export default class Gmail extends GoogleHandler { message: `Break timestamp hit (${breakTimestamp})` } this.emit('log', logEvent) + breakHit = SyncItemsBreak.TIMESTAMP break; } @@ -199,6 +241,9 @@ export default class Gmail extends GoogleHandler { }); } - return results; + return { + items: results, + breakHit + } } } diff --git a/src/serverconfig.example.json b/src/serverconfig.example.json index 7e3de88c..d1643fb6 100644 --- a/src/serverconfig.example.json +++ b/src/serverconfig.example.json @@ -63,7 +63,12 @@ }, "google": { "clientId": "", - "clientSecret": "" + "clientSecret": "", + "batchSize": 100, + "maxSyncLoops": 1, + "metadata": { + "breakTimestamp": "2000-07-21T12:07:11.000Z" + } }, "telegram": { "label": "Telegram", From 386493f932635d7036f2d2c3a3fde16a4a7e4f40 Mon Sep 17 00:00:00 2001 From: Chris Date: Wed, 21 Aug 2024 16:19:19 +0930 Subject: [PATCH 016/182] Refactor common tests to work within the items range tracker architecture. --- tests/common.tests.ts | 226 ++++++++++++++++++++++++++++-------------- 1 file changed, 150 insertions(+), 76 deletions(-) diff --git a/tests/common.tests.ts b/tests/common.tests.ts index c1abd87a..4ad72be2 100644 --- a/tests/common.tests.ts +++ b/tests/common.tests.ts @@ -4,9 +4,6 @@ import { SyncHandlerStatus, SyncResponse, SyncHandlerPosition, - SyncSchemaPositionType, - SyncStatus, - SyncProviderLogEntry, SyncProviderLogLevel, } from "../src/interfaces"; import providers from "../src/providers"; @@ -117,6 +114,12 @@ export class CommonTests { handler: BaseSyncHandler; provider: BaseProvider; }> { + // * - New items are processed + // * - Backfill items are processed + // * - Not enough new items? Process backfill + // * - Backfill twice doesn't process the same items + // * - No more backfill produces empty rangeTracker + // Set result limit to 3 results so page tests can work correctly providerConfig[testConfig.batchSizeLimitAttribute] = 3; @@ -134,16 +137,19 @@ export class CommonTests { try { const syncPosition: SyncHandlerPosition = { _id: `${providerName}-${schemaUri}`, - type: SyncSchemaPositionType.SYNC, providerName, handlerName: handler.getName(), providerId: provider.getProviderId(), - status: SyncHandlerStatus.ACTIVE, + status: SyncHandlerStatus.SYNCING, }; - // Snapshot: Page 1 + + // 1. Test new items are processed const response = await handler._sync(api, syncPosition); - const results = response.results; + + // console.log(response.position) + // console.log(CommonTests.outputItems(results, testConfig.timeOrderAttribute)) + assert.ok(results && results.length, "Have results returned"); assert.equal( providerConfig[testConfig.batchSizeLimitAttribute], @@ -159,43 +165,31 @@ export class CommonTests { ); } - assert.equal( - results[0].sourceApplication, - handler.getProviderApplicationUrl(), - "Items have correct source application" - ); - assert.equal( - results[0].sourceAccountId, - provider.getProviderId(), - "Items have correct source account / provider id" - ); - assert.ok(results[0].sourceId, "Items have sourceId set"); - assert.ok(results[0].sourceData, "Items have sourceData set"); + CommonTests.checkItem(results[0], handler, provider) assert.equal( - SyncHandlerStatus.ACTIVE, + SyncHandlerStatus.SYNCING, response.position.status, - "Sync is set to connected" - ); - assert.ok(response.position.thisRef, "Have a next page reference"); - assert.equal( - response.position.breakId, - undefined, - "Break ID is undefined" - ); - assert.equal( - `${idPrefix}-${response.position.futureBreakId}`, - results[0]._id, - "Future break ID matches the first result ID" + "Sync is active" ); + assert.ok(response.position.thisRef, "Have a defined processing range"); - // Snapshot: Page 2 - const response2 = await handler._sync(api, syncPosition); + const currentRangeParts = response.position.thisRef!.split(':') + assert.ok(currentRangeParts.length == 2, "Have correct number of parts for the processing range"); + assert.ok(currentRangeParts[0] == results[0]._id.replace(`${idPrefix}-`, ''), "Have correct break ID"); + assert.ok(currentRangeParts[1].length, "Have an end range"); + + // 2. Backfill items are processed + const syncPosition2 = response.position + const response2 = await handler._sync(api, syncPosition2); const results2 = response2.results; + + // console.log(response2.position) + // console.log(CommonTests.outputItems(results2, testConfig.timeOrderAttribute)) assert.ok( results2 && results2.length, - "Have second page of results returned" + "Have backfill results returned" ); assert.ok( results2 && @@ -217,59 +211,117 @@ export class CommonTests { } assert.equal( - response.position.status, - SyncHandlerStatus.ACTIVE, - "Sync is still active" - ); - assert.ok(response.position.thisRef, "Have a next page reference"); - assert.equal( - response.position.breakId, - undefined, - "Break ID is undefined" - ); - assert.equal( - results[0]._id, - `${idPrefix}-${response.position.futureBreakId}`, - "Future break ID matches the first result ID" + response2.position.status, + SyncHandlerStatus.SYNCING, + "Sync is active" ); - // Update: Page 1 (ensure 1 result only) - // Fetch the update set of results to confirm `position.pos` is correct - // Make sure we fetch the first post only, by setting the break to the second item - const position = response2.position; - position.thisRef = undefined; - // position.thisRefType = PostSyncRefTypes.Api - position.breakId = results[1]._id.replace(`${idPrefix}-`, ""); - position.futureBreakId = undefined; + assert.ok(response2.position.thisRef, "Have a defined processing range"); - const response3 = await handler._sync(api, position); + const currentRangeParts2 = response2.position.thisRef!.split(':') + assert.ok(currentRangeParts2.length == 2, "Have correct number of parts for the processing range"); + assert.ok(currentRangeParts2[0] == results[0]._id.replace(`${idPrefix}-`, ''), "Have correct break ID matching the very first result"); + assert.ok(currentRangeParts2[1].length, "Have an end range"); + assert.ok(results[0]._id != results2[0]._id, "Have different result IDs") + + // 3. Not enough new items? Process backfill + const syncPosition3 = response2.position + syncPosition3.thisRef = `${results[1].sourceId}:${currentRangeParts2[1]}` // Ensure the first item (only) is fetched + const response3 = await handler._sync(api, syncPosition3); const results3 = response3.results; - assert.equal(results3.length, 1, "1 result returned"); - assert.equal(results3[0]._id, results[0]._id, "Correct ID returned"); + + // console.log(response3.position) + // console.log(CommonTests.outputItems(results3, testConfig.timeOrderAttribute)) - assert.equal( - response.position.status, - SyncHandlerStatus.STOPPED, - "Sync is stopped" + assert.ok( + results3 && results3.length, + "Have results returned" ); - assert.equal( - response.position.thisRef, - undefined, - "No next page reference" + assert.ok( + results3 && + results3.length == providerConfig[testConfig.batchSizeLimitAttribute], + "Have correct number of results returned" ); - // assert.equal(PostSyncRefTypes.Api, response.position.thisRefType, 'This position reference type is API fetch') + assert.equal(results3[0]._id, results[0]._id, 'First result item matches the very first item') + assert.ok(results3[1]._id != results[1]._id, 'Second result item does not match the very first batch second item') + + if (testConfig.timeOrderAttribute) { + assert.ok( + results3[0][testConfig.timeOrderAttribute] > + results3[1][testConfig.timeOrderAttribute], + "Results are most recent first" + ); + // this will break? + assert.ok( + results3[2][testConfig.timeOrderAttribute] < + results[2][testConfig.timeOrderAttribute], + "Last item on return results have earlier timestamp than last item on first page" + ); + } + assert.equal( - response.position.breakId, - results3[0]._id.replace(`${idPrefix}-`, ""), - "Break ID is the first result" + response3.position.status, + SyncHandlerStatus.SYNCING, + "Sync is active" + ); + + assert.ok(response3.position.thisRef, "Have a defined processing range"); + + const currentRangeParts3 = response3.position.thisRef!.split(':') + assert.ok(currentRangeParts3.length == 2, "Have correct number of parts for the processing range"); + assert.ok(currentRangeParts3[0] == results3[0]._id.replace(`${idPrefix}-`, ''), "Have correct break ID matching the very first result"); + assert.ok(currentRangeParts3[1].length, "Have an end range"); + assert.ok(currentRangeParts3[1] != currentRangeParts2[1], "End range has changed between batches"); + + // - Backfill twice doesn't process the same items + const syncPosition4 = response3.position + const response4 = await handler._sync(api, syncPosition4); + const results4 = response4.results; + + // console.log(response4.position) + // console.log(CommonTests.outputItems(results4, testConfig.timeOrderAttribute)) + + assert.ok( + results4 && results4.length, + "Have results returned" ); + assert.ok( + results4 && + results4.length == providerConfig[testConfig.batchSizeLimitAttribute], + "Have correct number of results returned" + ); + + if (testConfig.timeOrderAttribute) { + assert.ok( + results4[0][testConfig.timeOrderAttribute] > + results4[1][testConfig.timeOrderAttribute], + "Results are most recent first" + ); + // this will break? + assert.ok( + results4[0][testConfig.timeOrderAttribute] < + results[2][testConfig.timeOrderAttribute], + "First item on return results have earlier timestamp than last item on first page" + ); + } + + assert.ok(results4[0]._id != results3[0]._id, "First items dont match between batches") + assert.equal( - response.position.futureBreakId, - undefined, - "Future break ID is undefined" + response4.position.status, + SyncHandlerStatus.SYNCING, + "Sync is active" ); + + assert.ok(response4.position.thisRef, "Have a defined processing range"); + const currentRangeParts4 = response4.position.thisRef!.split(':') + assert.ok(currentRangeParts4.length == 2, "Have correct number of parts for the processing range"); + assert.ok(currentRangeParts4[1].length, "Have an end range"); + + // No more backfill produces empty rangeTracker + + // Close the provider connection - console.log("closing provider"); await provider.close(); return { @@ -284,4 +336,26 @@ export class CommonTests { throw err; } } + + static checkItem(item: SchemaRecord, handler: BaseSyncHandler, provider: BaseProvider) { + assert.equal( + item.sourceApplication, + handler.getProviderApplicationUrl(), + "Items have correct source application" + ); + assert.equal( + item.sourceAccountId, + provider.getProviderId(), + "Items have correct source account / provider id" + ); + assert.ok(item.sourceId, "Items have sourceId set"); + assert.ok(item.sourceData, "Items have sourceData set"); + } + + // Helper method to output items to help with debugging + static outputItems(items: SchemaRecord[], timeAttribute?: string) { + for (const item of items) { + console.log(item._id, timeAttribute ? item[timeAttribute] : '', item.name) + } + } } From 5426c778a72940a914c6b93720fb59698369c1f1 Mon Sep 17 00:00:00 2001 From: Chris Date: Wed, 21 Aug 2024 16:19:56 +0930 Subject: [PATCH 017/182] Update gmail tests to pass using updated common tests --- tests/providers/google/gmail.tests.ts | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/tests/providers/google/gmail.tests.ts b/tests/providers/google/gmail.tests.ts index 6b63b987..1e4649c9 100644 --- a/tests/providers/google/gmail.tests.ts +++ b/tests/providers/google/gmail.tests.ts @@ -4,7 +4,6 @@ import { Connection, SyncHandlerStatus, SyncHandlerPosition, - SyncSchemaPositionType, } from "../../../src/interfaces"; import Providers from "../../../src/providers"; import CommonUtils, { NetworkInstance } from "../../common.utils"; @@ -22,6 +21,8 @@ let handlerName = "gmail"; let testConfig: GenericTestConfig; let providerConfig: Omit = {}; +console.log(`WARNING: Sometimes these tests fail because the Google API doesnt return inbox messages in the correct time order. This is a bug in the Google API and there's not much we can do about it.`) + describe(`${providerName} Tests`, function () { this.timeout(100000); @@ -40,6 +41,16 @@ describe(`${providerName} Tests`, function () { describe(`Fetch ${providerName} data`, () => { it(`Can pass basic tests: ${handlerName}`, async () => { + /** + * Things to test: + * + * - New items are processed + * - Backfill items are processed + * - Not enough new items? Process backfill + * - Backfill twice doesn't process the same items + * - No more backfill produces empty rangeTracker + */ + await CommonTests.runGenericTests( providerName, Gmail, @@ -50,20 +61,19 @@ describe(`${providerName} Tests`, function () { }); it(`Can limit results by timestamp`, async () => { - const lastRecordHours = 2; + const lastRecordHours = 1; const lastRecordTimestamp = new Date( Date.now() - lastRecordHours * 3600000 ).toISOString(); const syncPosition: Omit = { - type: SyncSchemaPositionType.SYNC, providerName, providerId: provider.getProviderId(), handlerName, - status: SyncHandlerStatus.ACTIVE, + status: SyncHandlerStatus.SYNCING, }; - providerConfig.batchSize = 10; + providerConfig.batchSize = 20; providerConfig.metadata = { breakTimestamp: lastRecordTimestamp, } @@ -86,9 +96,10 @@ describe(`${providerName} Tests`, function () { results[results.length - 1].sentAt > lastRecordTimestamp, "Last result is within expected date/time range" ); + assert.ok( results.length < providerConfig.batchSize, - `Results reached the expected timestamp within the current batch size (try increating the test batch size or reducing the break timetamp)` + `Results reached the expected timestamp within the current batch size (try increasing the test batch size or reducing the break timetamp)` ); }); }); From f0570c501867d0388debb544c19c97d27e99f94c Mon Sep 17 00:00:00 2001 From: chime3 Date: Tue, 20 Aug 2024 23:54:33 -0700 Subject: [PATCH 018/182] feat: added sync message --- src/providers/google/gdrive-document.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/providers/google/gdrive-document.ts b/src/providers/google/gdrive-document.ts index 664a5d19..9d7f20c4 100644 --- a/src/providers/google/gdrive-document.ts +++ b/src/providers/google/gdrive-document.ts @@ -81,6 +81,7 @@ export default class GoogleDriveDocument extends BaseSyncHandler { !serverResponse.data.files.length ) { // No results found, so stop sync + syncPosition.syncMessage = "Stopping. No results found."; syncPosition = this.stopSync(syncPosition); return { @@ -103,6 +104,7 @@ export default class GoogleDriveDocument extends BaseSyncHandler { if (results.length != this.config.batchSize) { // Not a full page of results, so stop sync + syncPosition.syncMessage = `Processed ${results.length} items. Stopping. No more results.`; syncPosition = this.stopSync(syncPosition); } @@ -135,8 +137,10 @@ export default class GoogleDriveDocument extends BaseSyncHandler { if (_.has(serverResponse, "data.nextPageToken")) { // Have more results, so set the next page ready for the next request + syncPosition.syncMessage = `Batch complete (${this.config.batchSize}). More results pending.`; syncPosition.thisRef = serverResponse.data.nextPageToken; } else { + syncPosition.syncMessage = "Stopping. No more results."; syncPosition = this.stopSync(syncPosition); } From dcaa3811a6f53ece2a54a30219c16eb9e9e6596e Mon Sep 17 00:00:00 2001 From: Chris Date: Wed, 21 Aug 2024 16:28:02 +0930 Subject: [PATCH 019/182] Fix incorrect sync status in common tests --- tests/common.tests.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/common.tests.ts b/tests/common.tests.ts index 5931aab5..7bee5138 100644 --- a/tests/common.tests.ts +++ b/tests/common.tests.ts @@ -140,7 +140,7 @@ export class CommonTests { providerName, handlerName: handler.getName(), providerId: provider.getProviderId(), - status: SyncHandlerStatus.ENABLED, + status: SyncHandlerStatus.SYNCING, }; // 1. Test new items are processed @@ -168,7 +168,7 @@ export class CommonTests { CommonTests.checkItem(results[0], handler, provider) assert.equal( - SyncHandlerStatus.ENABLED, + SyncHandlerStatus.SYNCING, response.position.status, "Sync is active" ); @@ -212,7 +212,7 @@ export class CommonTests { assert.equal( response2.position.status, - SyncHandlerStatus.ENABLED, + SyncHandlerStatus.SYNCING, "Sync is active" ); @@ -261,7 +261,7 @@ export class CommonTests { assert.equal( response3.position.status, - SyncHandlerStatus.ENABLED, + SyncHandlerStatus.SYNCING, "Sync is active" ); @@ -318,7 +318,7 @@ export class CommonTests { assert.ok(currentRangeParts4.length == 2, "Have correct number of parts for the processing range"); assert.ok(currentRangeParts4[1].length, "Have an end range"); - // No more backfill produces empty rangeTracker + // @todo: No more backfill produces empty rangeTracker and SyncHandlerStatus.CONNECTED // Close the provider connection From 83a4f078a21951190fc7c5fc2fcc122287549130 Mon Sep 17 00:00:00 2001 From: chime3 Date: Thu, 22 Aug 2024 05:09:29 -0700 Subject: [PATCH 020/182] refactor: optimized youtube sync --- src/providers/google/README.md | 6 +- src/providers/google/youtube-favourite.ts | 206 +++++++++++++--------- src/providers/google/youtube-following.ts | 187 +++++++++++--------- src/providers/google/youtube-post.ts | 199 ++++++++++++--------- 4 files changed, 346 insertions(+), 252 deletions(-) diff --git a/src/providers/google/README.md b/src/providers/google/README.md index 1a9d5cb8..b02263e9 100644 --- a/src/providers/google/README.md +++ b/src/providers/google/README.md @@ -26,9 +26,9 @@ Before running the unit tests, ensure that you have the following set up: 1. **YouTube Account**: A YouTube account with some activity is required. This activity includes uploaded videos, subscriptions, and liked videos. 2. **YouTube Data**: Make sure your YouTube account has: - - **Favorites**: At least 7 videos you have liked. - - **Following**: At least 7 Channels you have subscribed to. - - **Posts**: At least 7 videos you have uploaded. + - **Favorites**: At least 14 videos you have liked. + - **Following**: At least 14 Channels you have subscribed to. + - **Posts**: At least 14 videos you have uploaded. 3. **Activity**: Make sure you have made activities within the last 24 hours. ## Running the Tests diff --git a/src/providers/google/youtube-favourite.ts b/src/providers/google/youtube-favourite.ts index 7b18cb35..444a4e2d 100644 --- a/src/providers/google/youtube-favourite.ts +++ b/src/providers/google/youtube-favourite.ts @@ -1,17 +1,25 @@ import GoogleHandler from "./GoogleHandler"; import CONFIG from "../../config"; -import { SyncProviderLogEvent, SyncProviderLogLevel } from '../../interfaces' +import { SyncHandlerPosition, SyncItemsBreak, SyncItemsResult, SyncProviderLogEvent, SyncProviderLogLevel } from '../../interfaces'; import { SyncResponse, - SyncHandlerPosition, SyncHandlerStatus, + HandlerOption, + ConnectionOptionType, } from "../../interfaces"; import { SchemaFavouriteContentType, SchemaFavouriteType, SchemaFavourite } from "../../schemas"; import { google, youtube_v3 } from "googleapis"; import { GaxiosResponse } from "gaxios"; +import { ItemsRangeTracker } from "../../helpers/itemsRangeTracker"; const _ = require("lodash"); +const MAX_BATCH_SIZE = 50; + +export interface SyncFavouriteItemsResult extends SyncItemsResult { + items: SchemaFavourite[]; +} + export default class YouTubeFavourite extends GoogleHandler { public getName(): string { @@ -27,143 +35,173 @@ export default class YouTubeFavourite extends GoogleHandler { } public getYouTube(): youtube_v3.Youtube { - const oAuth2Client = this.getGoogleAuth() + const oAuth2Client = this.getGoogleAuth(); const youtube = google.youtube({ version: "v3", auth: oAuth2Client }); return youtube; } + public getOptions(): HandlerOption[] { + return [{ + name: 'backdate', + label: 'Backdate history', + type: ConnectionOptionType.ENUM, + enumOptions: ['1 month', '3 months', '6 months', '12 months'], + defaultValue: '3 months' + }]; + } + public async _sync( api: any, syncPosition: SyncHandlerPosition ): Promise { + if (this.config.batchSize > MAX_BATCH_SIZE) { + throw new Error(`Batch size (${this.config.batchSize}) is larger than permitted (${MAX_BATCH_SIZE})`); + } + const youtube = this.getYouTube(); - - const query: youtube_v3.Params$Resource$Videos$List = { + const rangeTracker = new ItemsRangeTracker(syncPosition.thisRef); + + let items: SchemaFavourite[] = []; + + // Fetch any new items + let currentRange = rangeTracker.nextRange(); + let query: youtube_v3.Params$Resource$Videos$List = { part: ["snippet", "contentDetails"], myRating: "like", - maxResults: this.config.batchSize, // Google Docs: default = 5, max = 50 + maxResults: this.config.batchSize, }; - - if (syncPosition.thisRef) { - query.pageToken = syncPosition.thisRef; - } - - const serverResponse = await youtube.videos.list(query); - - if ( - !_.has(serverResponse, "data.items") || - !serverResponse.data.items.length - ) { - // No results found, so stop sync - syncPosition.syncMessage = "Stopping. No results found."; - syncPosition = this.stopSync(syncPosition); - - return { - position: syncPosition, - results: [], - }; + + if (currentRange.startId) { + query.pageToken = currentRange.startId; } - const results = await this.buildResults( - serverResponse, - syncPosition.breakId, + const latestResponse = await youtube.videos.list(query); + const latestResult = await this.buildResults( + latestResponse, + currentRange.endId, _.has(this.config, "metadata.breakTimestamp") ? this.config.metadata.breakTimestamp : undefined ); - - syncPosition = this.setNextPosition(syncPosition, serverResponse); - - if (results.length != this.config.batchSize) { - // Not a full page of results, so stop sync - syncPosition.syncMessage = `Processed ${results.length} items. Stopping. No more results.`; - syncPosition = this.stopSync(syncPosition); - } - - return { - results, - position: syncPosition, - }; - } - - protected stopSync(syncPosition: SyncHandlerPosition): SyncHandlerPosition { - if (syncPosition.status == SyncHandlerStatus.ENABLED) { - return syncPosition; + + items = latestResult.items; + + let nextPageToken = _.has(latestResponse, "data.nextPageToken") ? latestResponse.data.nextPageToken : undefined; + + if (items.length) { + rangeTracker.completedRange({ + startId: items[0].sourceId, + endId: nextPageToken + }, latestResult.breakHit == SyncItemsBreak.ID); + } else { + rangeTracker.completedRange({ + startId: undefined, + endId: undefined + }, false); } - syncPosition.status = SyncHandlerStatus.ENABLED; - syncPosition.thisRef = undefined; - syncPosition.breakId = syncPosition.futureBreakId; - syncPosition.futureBreakId = undefined; + if (items.length != this.config.batchSize) { + currentRange = rangeTracker.nextRange(); + query = { + part: ["snippet", "contentDetails"], + myRating: "like", + maxResults: this.config.batchSize - items.length, + }; - return syncPosition; - } + if (currentRange.startId) { + query.pageToken = currentRange.startId; + } + + const backfillResponse = await youtube.videos.list(query); + const backfillResult = await this.buildResults( + backfillResponse, + currentRange.endId, + _.has(this.config, "metadata.breakTimestamp") + ? this.config.metadata.breakTimestamp + : undefined + ); + + items = items.concat(backfillResult.items); + nextPageToken = _.has(backfillResponse, "data.nextPageToken") ? backfillResponse.data.nextPageToken : undefined; - protected setNextPosition( - syncPosition: SyncHandlerPosition, - serverResponse: GaxiosResponse - ): SyncHandlerPosition { - if (!syncPosition.futureBreakId && serverResponse.data.items.length) { - syncPosition.futureBreakId = serverResponse.data.items[0].id; + if (backfillResult.items.length) { + rangeTracker.completedRange({ + startId: backfillResult.items[0].sourceId, + endId: nextPageToken + }, backfillResult.breakHit == SyncItemsBreak.ID); + } else { + rangeTracker.completedRange({ + startId: undefined, + endId: undefined + }, backfillResult.breakHit == SyncItemsBreak.ID); + } } - if (_.has(serverResponse, "data.nextPageToken")) { - // Have more results, so set the next page ready for the next request - syncPosition.syncMessage = `Batch complete (${this.config.batchSize}). More results pending.`; - syncPosition.thisRef = serverResponse.data.nextPageToken; + if (!items.length) { + syncPosition.syncMessage = `Stopping. No results found.`; + syncPosition.status = SyncHandlerStatus.ENABLED; } else { - syncPosition.syncMessage = "Stopping. No more results."; - syncPosition = this.stopSync(syncPosition); + if (items.length != this.config.batchSize && !nextPageToken) { + syncPosition.syncMessage = `Processed ${items.length} items. Stopping. No more results.`; + syncPosition.status = SyncHandlerStatus.ENABLED; + } else { + syncPosition.syncMessage = `Batch complete (${this.config.batchSize}). More results pending.`; + } } - return syncPosition; + syncPosition.thisRef = rangeTracker.export(); + + return { + results: items, + position: syncPosition, + }; } protected async buildResults( serverResponse: GaxiosResponse, breakId: string, breakTimestamp?: string - ): Promise { + ): Promise { const results: SchemaFavourite[] = []; - - const videos = serverResponse.data.items; - - for (const video of videos) { + let breakHit: SyncItemsBreak; + + for (const video of serverResponse.data.items) { const videoId = video.id; const favouriteId = videoId; - + if (favouriteId == breakId) { const logEvent: SyncProviderLogEvent = { level: SyncProviderLogLevel.DEBUG, message: `Break ID hit (${breakId})` - } - this.emit('log', logEvent) + }; + this.emit('log', logEvent); + breakHit = SyncItemsBreak.ID; break; } - + const snippet = video.snippet; const insertedAt = snippet.publishedAt || "Unknown"; - + if (breakTimestamp && insertedAt < breakTimestamp) { const logEvent: SyncProviderLogEvent = { level: SyncProviderLogLevel.DEBUG, message: `Break timestamp hit (${breakTimestamp})` - } - this.emit('log', logEvent) + }; + this.emit('log', logEvent); + breakHit = SyncItemsBreak.TIMESTAMP; break; } - + const title = snippet.title || "No title"; const description = snippet.description || "No description"; const iconUri = snippet.thumbnails.default.url; const activityUri = `https://www.youtube.com/watch?v=${videoId}`; - + results.push({ _id: this.buildItemId(favouriteId), name: title, icon: iconUri, uri: activityUri, - // summary: description.substring(0, 256), description: description, favouriteType: SchemaFavouriteType.LIKE, contentType: SchemaFavouriteContentType.VIDEO, @@ -174,8 +212,10 @@ export default class YouTubeFavourite extends GoogleHandler { insertedAt: insertedAt, }); } - - return results; + + return { + items: results, + breakHit, + }; } - } diff --git a/src/providers/google/youtube-following.ts b/src/providers/google/youtube-following.ts index dae55d52..1c94fed7 100644 --- a/src/providers/google/youtube-following.ts +++ b/src/providers/google/youtube-following.ts @@ -1,20 +1,17 @@ import GoogleHandler from "./GoogleHandler"; import CONFIG from "../../config"; -import { SyncProviderLogEvent, SyncProviderLogLevel } from '../../interfaces' - -import { - SyncResponse, - SyncHandlerPosition, - SyncHandlerStatus, -} from "../../interfaces"; +import { SyncProviderLogEvent, SyncProviderLogLevel, SyncHandlerPosition, SyncResponse, SyncHandlerStatus, SyncItemsBreak, HandlerOption, ConnectionOptionType } from "../../interfaces"; import { SchemaFollowing } from "../../schemas"; import { google, youtube_v3 } from "googleapis"; import { GaxiosResponse } from "gaxios"; +import { ItemsRangeTracker } from "../../helpers/itemsRangeTracker"; const _ = require("lodash"); +const MAX_BATCH_SIZE = 50; + export default class YouTubeFollowing extends GoogleHandler { - + public getName(): string { return "youtube-following"; } @@ -28,106 +25,135 @@ export default class YouTubeFollowing extends GoogleHandler { } public getYouTube(): youtube_v3.Youtube { - const oAuth2Client = this.getGoogleAuth() - const youtube = google.youtube({ version: "v3", auth: oAuth2Client }); - return youtube; + const oAuth2Client = this.getGoogleAuth(); + return google.youtube({ version: "v3", auth: oAuth2Client }); + } + + public getOptions(): HandlerOption[] { + return [{ + name: 'backdate', + label: 'Backdate history', + type: ConnectionOptionType.ENUM, + enumOptions: ['1 month', '3 months', '6 months', '12 months'], + defaultValue: '3 months' + }]; } public async _sync( api: any, syncPosition: SyncHandlerPosition ): Promise { + if (this.config.batchSize > MAX_BATCH_SIZE) { + throw new Error(`Batch size (${this.config.batchSize}) is larger than permitted (${MAX_BATCH_SIZE})`); + } + const youtube = this.getYouTube(); + const rangeTracker = new ItemsRangeTracker(syncPosition.thisRef); - const query: youtube_v3.Params$Resource$Subscriptions$List = { + let items: SchemaFollowing[] = []; + + // Fetch any new items + let currentRange = rangeTracker.nextRange(); + let query: youtube_v3.Params$Resource$Subscriptions$List = { part: ["snippet"], mine: true, maxResults: this.config.batchSize, - }; - - if (syncPosition.thisRef) { - query.pageToken = syncPosition.thisRef; - } - - const serverResponse = await youtube.subscriptions.list(query); - - if ( - !_.has(serverResponse, "data.items") || - !serverResponse.data.items.length - ) { - // No results found, so stop sync - syncPosition.syncMessage = "Stopping. No more results."; - syncPosition = this.stopSync(syncPosition); + }; - return { - position: syncPosition, - results: [], - }; + if (currentRange.startId) { + query.pageToken = currentRange.startId; } - const results = await this.buildResults( - youtube, - serverResponse, - syncPosition.breakId, + const latestResponse = await youtube.subscriptions.list(query); + const latestResult = await this.buildResults( + latestResponse, + currentRange.endId, _.has(this.config, "metadata.breakTimestamp") ? this.config.metadata.breakTimestamp : undefined ); - syncPosition = this.setNextPosition(syncPosition, serverResponse); - - if (results.length != this.config.batchSize) { - // Not a full page of results, so stop sync - syncPosition.syncMessage = `Processed ${results.length} items. Stopping. No more results.`; - syncPosition = this.stopSync(syncPosition); - } + items = latestResult.items; - return { - results, - position: syncPosition, - }; - } + let nextPageToken = _.has(latestResponse, "data.nextPageToken") ? latestResponse.data.nextPageToken : undefined; - protected stopSync(syncPosition: SyncHandlerPosition): SyncHandlerPosition { - if (syncPosition.status == SyncHandlerStatus.ENABLED) { - return syncPosition; + if (items.length) { + rangeTracker.completedRange({ + startId: items[0].sourceId, + endId: nextPageToken + }, latestResult.breakHit == SyncItemsBreak.ID); + } else { + rangeTracker.completedRange({ + startId: undefined, + endId: undefined + }, false); } - syncPosition.status = SyncHandlerStatus.ENABLED; - syncPosition.thisRef = undefined; - syncPosition.breakId = syncPosition.futureBreakId; - syncPosition.futureBreakId = undefined; + if (items.length != this.config.batchSize) { + currentRange = rangeTracker.nextRange(); + query = { + part: ["snippet"], + mine: true, + maxResults: this.config.batchSize - items.length, + }; - return syncPosition; - } + if (currentRange.startId) { + query.pageToken = currentRange.startId; + } - protected setNextPosition( - syncPosition: SyncHandlerPosition, - serverResponse: GaxiosResponse - ): SyncHandlerPosition { - if (!syncPosition.futureBreakId && serverResponse.data.items.length) { - syncPosition.futureBreakId = serverResponse.data.items[0].id; + const backfillResponse = await youtube.subscriptions.list(query); + const backfillResult = await this.buildResults( + backfillResponse, + currentRange.endId, + _.has(this.config, "metadata.breakTimestamp") + ? this.config.metadata.breakTimestamp + : undefined + ); + + items = items.concat(backfillResult.items); + nextPageToken = _.has(backfillResponse, "data.nextPageToken") ? backfillResponse.data.nextPageToken : undefined; + + if (backfillResult.items.length) { + rangeTracker.completedRange({ + startId: backfillResult.items[0].sourceId, + endId: nextPageToken + }, backfillResult.breakHit == SyncItemsBreak.ID); + } else { + rangeTracker.completedRange({ + startId: undefined, + endId: undefined + }, backfillResult.breakHit == SyncItemsBreak.ID); + } } - if (_.has(serverResponse, "data.nextPageToken")) { - // Have more results, so set the next page ready for the next request - syncPosition.syncMessage = `Batch complete (${this.config.batchSize}). More results pending.`; - syncPosition.thisRef = serverResponse.data.nextPageToken; + if (!items.length) { + syncPosition.syncMessage = `Stopping. No results found.`; + syncPosition.status = SyncHandlerStatus.ENABLED; } else { - syncPosition.syncMessage = "Stopping. No more results."; - syncPosition = this.stopSync(syncPosition); + if (items.length != this.config.batchSize && !nextPageToken) { + syncPosition.syncMessage = `Processed ${items.length} items. Stopping. No more results.`; + syncPosition.status = SyncHandlerStatus.ENABLED; + } else { + syncPosition.syncMessage = `Batch complete (${this.config.batchSize}). More results pending.`; + } } - return syncPosition; + syncPosition.thisRef = rangeTracker.export(); + + return { + results: items, + position: syncPosition, + }; } protected async buildResults( - youtube: youtube_v3.Youtube, serverResponse: GaxiosResponse, breakId: string, breakTimestamp?: string - ): Promise { + ): Promise<{ items: SchemaFollowing[], breakHit?: SyncItemsBreak }> { const results: SchemaFollowing[] = []; + let breakHit: SyncItemsBreak; + for (const item of serverResponse.data.items) { const itemId = item.id; @@ -135,23 +161,25 @@ export default class YouTubeFollowing extends GoogleHandler { const logEvent: SyncProviderLogEvent = { level: SyncProviderLogLevel.DEBUG, message: `Break ID hit (${breakId})` - } - this.emit('log', logEvent) - break + }; + this.emit('log', logEvent); + breakHit = SyncItemsBreak.ID; + break; } const snippet = item.snippet; const insertedAt = snippet.publishedAt || "Unknown"; - + if (breakTimestamp && insertedAt < breakTimestamp) { const logEvent: SyncProviderLogEvent = { level: SyncProviderLogLevel.DEBUG, message: `Break timestamp hit (${breakTimestamp})` - } - this.emit('log', logEvent) + }; + this.emit('log', logEvent); + breakHit = SyncItemsBreak.TIMESTAMP; break; } - + const title = snippet.title || "No title"; const description = snippet.description || "No description"; const uri = "https://www.youtube.com/channel/" + snippet.resourceId.channelId; @@ -162,7 +190,6 @@ export default class YouTubeFollowing extends GoogleHandler { name: title, icon: icon, uri: uri, - // summary: description.substring(0, 256), description: description, sourceId: item.id, sourceData: snippet, @@ -173,6 +200,6 @@ export default class YouTubeFollowing extends GoogleHandler { }); } - return results; + return { items: results, breakHit }; } } diff --git a/src/providers/google/youtube-post.ts b/src/providers/google/youtube-post.ts index e80b80df..8a12dae3 100644 --- a/src/providers/google/youtube-post.ts +++ b/src/providers/google/youtube-post.ts @@ -1,18 +1,25 @@ import GoogleHandler from "./GoogleHandler"; import CONFIG from "../../config"; -import { SyncProviderLogEvent, SyncProviderLogLevel } from '../../interfaces' +import { ConnectionOptionType, SyncHandlerPosition, SyncItemsBreak, SyncItemsResult, SyncProviderLogEvent, SyncProviderLogLevel } from '../../interfaces'; import { SyncResponse, - SyncHandlerPosition, SyncHandlerStatus, + HandlerOption, } from "../../interfaces"; import { SchemaPostType, SchemaPost } from "../../schemas"; import { google, youtube_v3 } from "googleapis"; import { GaxiosResponse } from "gaxios"; import { YoutubeActivityType } from "./interfaces"; +import { ItemsRangeTracker } from "../../helpers/itemsRangeTracker"; const _ = require("lodash"); +const MAX_BATCH_SIZE = 50; + +export interface SyncPostItemsResult extends SyncItemsResult { + items: SchemaPost[]; +} + export default class YouTubePost extends GoogleHandler { public getName(): string { @@ -28,110 +35,139 @@ export default class YouTubePost extends GoogleHandler { } public getYouTube(): youtube_v3.Youtube { - const oAuth2Client = this.getGoogleAuth() + const oAuth2Client = this.getGoogleAuth(); const youtube = google.youtube({ version: "v3", auth: oAuth2Client }); return youtube; } + public getOptions(): HandlerOption[] { + return [{ + name: 'backdate', + label: 'Backdate history', + type: ConnectionOptionType.ENUM, + enumOptions: ['1 month', '3 months', '6 months', '12 months'], + defaultValue: '3 months' + }]; + } + public async _sync( api: any, syncPosition: SyncHandlerPosition ): Promise { + if (this.config.batchSize > MAX_BATCH_SIZE) { + throw new Error(`Batch size (${this.config.batchSize}) is larger than permitted (${MAX_BATCH_SIZE})`); + } + const youtube = this.getYouTube(); + const rangeTracker = new ItemsRangeTracker(syncPosition.thisRef); - const query: youtube_v3.Params$Resource$Activities$List = { + let items: SchemaPost[] = []; + + // Fetch any new items + let currentRange = rangeTracker.nextRange(); + let query: youtube_v3.Params$Resource$Activities$List = { part: ["snippet", "contentDetails"], mine: true, - maxResults: this.config.batchSize, // Google Docs: default = 5, max = 50 + maxResults: this.config.batchSize, }; - if (syncPosition.thisRef) { - query.pageToken = syncPosition.thisRef; - } - - const serverResponse = await youtube.activities.list(query); - - if ( - !_.has(serverResponse, "data.items") || - !serverResponse.data.items.length - ) { - // No results found, so stop sync - syncPosition.syncMessage = "Stopping. No more results."; - syncPosition = this.stopSync(syncPosition); - - return { - position: syncPosition, - results: [], - }; + if (currentRange.startId) { + query.pageToken = currentRange.startId; } - const results = await this.buildResults( - youtube, - serverResponse, - syncPosition.breakId, + const latestResponse = await youtube.activities.list(query); + const latestResult = await this.buildResults( + latestResponse, + currentRange.endId, _.has(this.config, "metadata.breakTimestamp") ? this.config.metadata.breakTimestamp : undefined ); - syncPosition = this.setNextPosition(syncPosition, serverResponse); + items = latestResult.items; - if (results.length != this.config.batchSize) { - // Not a full page of results, so stop sync - syncPosition.syncMessage = `Processed ${results.length} items. Stopping. No more results.`; - syncPosition = this.stopSync(syncPosition); - } + let nextPageToken = _.has(latestResponse, "data.nextPageToken") ? latestResponse.data.nextPageToken : undefined; - return { - results, - position: syncPosition, - }; - } - - protected stopSync(syncPosition: SyncHandlerPosition): SyncHandlerPosition { - if (syncPosition.status == SyncHandlerStatus.ENABLED) { - return syncPosition; + if (items.length) { + rangeTracker.completedRange({ + startId: items[0].sourceId, + endId: nextPageToken + }, latestResult.breakHit == SyncItemsBreak.ID); + } else { + rangeTracker.completedRange({ + startId: undefined, + endId: undefined + }, false); } - syncPosition.status = SyncHandlerStatus.ENABLED; - syncPosition.thisRef = undefined; - syncPosition.breakId = syncPosition.futureBreakId; - syncPosition.futureBreakId = undefined; + if (items.length != this.config.batchSize) { + currentRange = rangeTracker.nextRange(); + query = { + part: ["snippet", "contentDetails"], + mine: true, + maxResults: this.config.batchSize - items.length, + }; - return syncPosition; - } + if (currentRange.startId) { + query.pageToken = currentRange.startId; + } - protected setNextPosition( - syncPosition: SyncHandlerPosition, - serverResponse: GaxiosResponse - ): SyncHandlerPosition { - if (!syncPosition.futureBreakId && serverResponse.data.items.length) { - syncPosition.futureBreakId = serverResponse.data.items[0].id; + const backfillResponse = await youtube.activities.list(query); + const backfillResult = await this.buildResults( + backfillResponse, + currentRange.endId, + _.has(this.config, "metadata.breakTimestamp") + ? this.config.metadata.breakTimestamp + : undefined + ); + + items = items.concat(backfillResult.items); + nextPageToken = _.has(backfillResponse, "data.nextPageToken") ? backfillResponse.data.nextPageToken : undefined; + + if (backfillResult.items.length) { + rangeTracker.completedRange({ + startId: backfillResult.items[0].sourceId, + endId: nextPageToken + }, backfillResult.breakHit == SyncItemsBreak.ID); + } else { + rangeTracker.completedRange({ + startId: undefined, + endId: undefined + }, backfillResult.breakHit == SyncItemsBreak.ID); + } } - if (_.has(serverResponse, "data.nextPageToken")) { - // Have more results, so set the next page ready for the next request - syncPosition.syncMessage = `Batch complete (${this.config.batchSize}). More results pending.`; - syncPosition.thisRef = serverResponse.data.nextPageToken; + if (!items.length) { + syncPosition.syncMessage = `Stopping. No results found.`; + syncPosition.status = SyncHandlerStatus.ENABLED; } else { - syncPosition.syncMessage = "Stopping. No more results."; - syncPosition = this.stopSync(syncPosition); + if (items.length != this.config.batchSize && !nextPageToken) { + syncPosition.syncMessage = `Processed ${items.length} items. Stopping. No more results.`; + syncPosition.status = SyncHandlerStatus.ENABLED; + } else { + syncPosition.syncMessage = `Batch complete (${this.config.batchSize}). More results pending.`; + } } - return syncPosition; + syncPosition.thisRef = rangeTracker.export(); + + return { + results: items, + position: syncPosition, + }; } protected async buildResults( - youtube: youtube_v3.Youtube, serverResponse: GaxiosResponse, breakId: string, breakTimestamp?: string - ): Promise { + ): Promise { const results: SchemaPost[] = []; + let breakHit: SyncItemsBreak; const activities = serverResponse.data.items; - // filter post(upload) - const posts = activities.filter(activity => activity.snippet.type == YoutubeActivityType.UPLOAD) + const posts = activities.filter(activity => activity.snippet.type == YoutubeActivityType.UPLOAD); + for (const post of posts) { const postId = post.id; @@ -139,8 +175,9 @@ export default class YouTubePost extends GoogleHandler { const logEvent: SyncProviderLogEvent = { level: SyncProviderLogLevel.DEBUG, message: `Break ID hit (${breakId})` - } - this.emit('log', logEvent) + }; + this.emit('log', logEvent); + breakHit = SyncItemsBreak.ID; break; } @@ -151,29 +188,17 @@ export default class YouTubePost extends GoogleHandler { const logEvent: SyncProviderLogEvent = { level: SyncProviderLogLevel.DEBUG, message: `Break timestamp hit (${breakTimestamp})` - } - this.emit('log', logEvent) + }; + this.emit('log', logEvent); + breakHit = SyncItemsBreak.TIMESTAMP; break; } const title = snippet.title || "No title"; const description = snippet.description || "No description"; - const contentDetails = post.contentDetails; - - const activityType = snippet.type; const iconUri = snippet.thumbnails.default.url; - // extract activity URI - let activityUri = ""; - switch (activityType) { - case YoutubeActivityType.UPLOAD: - var videoId = contentDetails.upload.videoId; - activityUri = 'https://www.youtube.com/watch?v=' + videoId; - break; - default: - activityUri = 'Unknown activity type'; - break; - } - + const videoId = post.contentDetails.upload.videoId; + const activityUri = `https://www.youtube.com/watch?v=${videoId}`; results.push({ _id: this.buildItemId(postId), @@ -181,7 +206,6 @@ export default class YouTubePost extends GoogleHandler { icon: iconUri, uri: activityUri, type: SchemaPostType.VIDEO, - // summary: description.substring(0, 256), content: description, sourceId: post.id, sourceData: snippet, @@ -191,6 +215,9 @@ export default class YouTubePost extends GoogleHandler { }); } - return results; + return { + items: results, + breakHit, + }; } } From a079e2456aab250b640efedbb9cd11ed6c6eac59 Mon Sep 17 00:00:00 2001 From: chime3 Date: Thu, 22 Aug 2024 06:18:04 -0700 Subject: [PATCH 021/182] refactor: optimize sync and fields --- src/providers/google/gdrive-document.ts | 220 ++++++++++++++---------- 1 file changed, 128 insertions(+), 92 deletions(-) diff --git a/src/providers/google/gdrive-document.ts b/src/providers/google/gdrive-document.ts index 9d7f20c4..eccf691f 100644 --- a/src/providers/google/gdrive-document.ts +++ b/src/providers/google/gdrive-document.ts @@ -1,19 +1,21 @@ import BaseSyncHandler from "../BaseSyncHandler"; import CONFIG from "../../config"; -import { SyncProviderLogEvent, SyncProviderLogLevel } from '../../interfaces' -import { - SyncResponse, - SyncHandlerPosition, - SyncHandlerStatus, -} from "../../interfaces"; +import { SyncProviderLogEvent, SyncProviderLogLevel, SyncHandlerPosition, SyncItemsBreak, SyncResponse, SyncHandlerStatus, SyncItemsResult, HandlerOption, ConnectionOptionType } from '../../interfaces'; import { SchemaDocument } from "../../schemas"; import { google, drive_v3 } from "googleapis"; import { OAuth2Client } from 'google-auth-library'; import { GaxiosResponse } from "gaxios"; import { GoogleDriveHelpers } from "./helpers"; +import { ItemsRangeTracker } from "../../helpers/itemsRangeTracker"; const _ = require("lodash"); +const MAX_BATCH_SIZE = 50; + +export interface SyncDocumentItemsResult extends SyncItemsResult { + items: SchemaDocument[]; +} + export default class GoogleDriveDocument extends BaseSyncHandler { public getName(): string { @@ -27,7 +29,7 @@ export default class GoogleDriveDocument extends BaseSyncHandler { public getProviderApplicationUrl(): string { return "https://drive.google.com"; } - + public getGoogleAuth(): OAuth2Client { const TOKEN = { access_token: this.connection.accessToken, @@ -50,147 +52,180 @@ export default class GoogleDriveDocument extends BaseSyncHandler { } public getGoogleDrive(): drive_v3.Drive { - const auth = this.getGoogleAuth(); + return google.drive({ version: "v3", auth }); + } - const drive = google.drive({ version: "v3", auth: auth }); - return drive; + public getOptions(): HandlerOption[] { + return [{ + name: 'backdate', + label: 'Backdate history', + type: ConnectionOptionType.ENUM, + enumOptions: ['1 month', '3 months', '6 months', '12 months'], + defaultValue: '3 months' + }]; } public async _sync( api: any, syncPosition: SyncHandlerPosition ): Promise { + if (this.config.batchSize > MAX_BATCH_SIZE) { + throw new Error(`Batch size (${this.config.batchSize}) is larger than permitted (${MAX_BATCH_SIZE})`); + } + const drive = this.getGoogleDrive(); - const auth = this.getGoogleAuth(); + const rangeTracker = new ItemsRangeTracker(syncPosition.thisRef); - const query: drive_v3.Params$Resource$Files$List = { + let items: SchemaDocument[] = []; + let currentRange = rangeTracker.nextRange(); + let query: drive_v3.Params$Resource$Files$List = { pageSize: this.config.batchSize, fields: 'nextPageToken, files(id, name, mimeType, modifiedTime, webViewLink, thumbnailLink)', q: "mimeType='application/vnd.google-apps.document' or mimeType='application/vnd.google-apps.spreadsheet' or mimeType='application/vnd.google-apps.presentation' or mimeType='application/pdf' or mimeType='application/vnd.openxmlformats-officedocument.wordprocessingml.document'" }; - if (syncPosition.thisRef) { - query.pageToken = syncPosition.thisRef; - } - - const serverResponse = await drive.files.list(query); - - if ( - !_.has(serverResponse, "data.files") || - !serverResponse.data.files.length - ) { - // No results found, so stop sync - syncPosition.syncMessage = "Stopping. No results found."; - syncPosition = this.stopSync(syncPosition); - - return { - position: syncPosition, - results: [], - }; + if (currentRange.startId) { + query.pageToken = currentRange.startId; } - const results = await this.buildResults( + const latestResponse = await drive.files.list(query); + const latestResult = await this.buildResults( drive, - auth, - serverResponse, - syncPosition.breakId, + latestResponse, + currentRange.endId, _.has(this.config, "metadata.breakTimestamp") ? this.config.metadata.breakTimestamp : undefined ); - syncPosition = this.setNextPosition(syncPosition, serverResponse); - - if (results.length != this.config.batchSize) { - // Not a full page of results, so stop sync - syncPosition.syncMessage = `Processed ${results.length} items. Stopping. No more results.`; - syncPosition = this.stopSync(syncPosition); - } + items = latestResult.items; + let nextPageToken = _.get(latestResponse, "data.nextPageToken"); - return { - results, - position: syncPosition, - }; - } - - protected stopSync(syncPosition: SyncHandlerPosition): SyncHandlerPosition { - if (syncPosition.status == SyncHandlerStatus.ENABLED) { - return syncPosition; + if (items.length) { + rangeTracker.completedRange({ + startId: items[0].sourceId, + endId: nextPageToken + }, latestResult.breakHit === SyncItemsBreak.ID); + } else { + rangeTracker.completedRange({ + startId: undefined, + endId: undefined + }, false); } - syncPosition.status = SyncHandlerStatus.ENABLED; - syncPosition.thisRef = undefined; - syncPosition.breakId = syncPosition.futureBreakId; - syncPosition.futureBreakId = undefined; + if (items.length != this.config.batchSize) { + currentRange = rangeTracker.nextRange(); + query = { + ...query, + pageSize: this.config.batchSize - items.length, + }; - return syncPosition; - } + if (currentRange.startId) { + query.pageToken = currentRange.startId; + } - protected setNextPosition( - syncPosition: SyncHandlerPosition, - serverResponse: GaxiosResponse - ): SyncHandlerPosition { - if (!syncPosition.futureBreakId && serverResponse.data.files.length) { - syncPosition.futureBreakId = serverResponse.data.files[0].id; + const backfillResponse = await drive.files.list(query); + const backfillResult = await this.buildResults( + drive, + backfillResponse, + currentRange.endId, + _.has(this.config, "metadata.breakTimestamp") + ? this.config.metadata.breakTimestamp + : undefined + ); + + items = items.concat(backfillResult.items); + nextPageToken = _.get(backfillResponse, "data.nextPageToken"); + + if (backfillResult.items.length) { + rangeTracker.completedRange({ + startId: backfillResult.items[0].sourceId, + endId: nextPageToken + }, backfillResult.breakHit === SyncItemsBreak.ID); + } else { + rangeTracker.completedRange({ + startId: undefined, + endId: undefined + }, backfillResult.breakHit === SyncItemsBreak.ID); + } } - if (_.has(serverResponse, "data.nextPageToken")) { - // Have more results, so set the next page ready for the next request - syncPosition.syncMessage = `Batch complete (${this.config.batchSize}). More results pending.`; - syncPosition.thisRef = serverResponse.data.nextPageToken; + if (!items.length) { + syncPosition.syncMessage = `Stopping. No results found.`; + syncPosition.status = SyncHandlerStatus.ENABLED; } else { - syncPosition.syncMessage = "Stopping. No more results."; - syncPosition = this.stopSync(syncPosition); + syncPosition.syncMessage = items.length != this.config.batchSize && !nextPageToken + ? `Processed ${items.length} items. Stopping. No more results.` + : `Batch complete (${this.config.batchSize}). More results pending.`; } - return syncPosition; + syncPosition.thisRef = rangeTracker.export(); + + return { + results: items, + position: syncPosition, + }; } protected async buildResults( drive: drive_v3.Drive, - auth: OAuth2Client, serverResponse: GaxiosResponse, breakId: string, breakTimestamp?: string - ): Promise { + ): Promise { const results: SchemaDocument[] = []; + let breakHit: SyncItemsBreak; + for (const file of serverResponse.data.files) { const fileId = file.id; - if (fileId == breakId) { + if (fileId === breakId) { + this.emit('log', { level: SyncProviderLogLevel.DEBUG, message: `Break ID hit (${breakId})` }); + breakHit = SyncItemsBreak.ID; + break; + } + + const createdTime = file.createdTime || new Date().toISOString(); + const modifiedTime = file.modifiedTime || new Date().toISOString(); + + if (breakTimestamp && modifiedTime < breakTimestamp) { + this.emit('log', { level: SyncProviderLogLevel.DEBUG, message: `Break timestamp hit (${breakTimestamp})` }); + breakHit = SyncItemsBreak.TIMESTAMP; + break; + } + + const title = file.name || "No title"; + const link = file.webViewLink; + if (!link) { const logEvent: SyncProviderLogEvent = { level: SyncProviderLogLevel.DEBUG, - message: `Break ID hit (${breakId})` + message: `No link available for file ${fileId}. Ignoring this file.` } - this.emit('log', logEvent) - break; + this.emit('log', logEvent); + continue; } - const modifiedTime = file.modifiedTime || "Unkown"; - - if (breakTimestamp && modifiedTime < breakTimestamp) { + const mimeType = file.mimeType; + if (!mimeType) { const logEvent: SyncProviderLogEvent = { level: SyncProviderLogLevel.DEBUG, - message: `Break timestamp hit (${breakTimestamp})` + message: `No mimeType available for file ${fileId}. Ignoring this file.` } - this.emit('log', logEvent) - break; + this.emit('log', logEvent); + continue; } - const title = file.name || "No title"; - const link = file.webViewLink || "No link"; - const mimeType = file.mimeType || "Unknown"; const type = GoogleDriveHelpers.getDocumentTypeFromMimeType(mimeType); - const thumbnail = file.thumbnailLink || "No thumbnail"; - const size = await GoogleDriveHelpers.getFileSize(drive, file.id) - const textContent = await GoogleDriveHelpers.extractIndexableText(drive, file.id, mimeType, auth); - + const thumbnail = file.thumbnailLink || undefined; + const size = await GoogleDriveHelpers.getFileSize(drive, file.id); + const textContent = await GoogleDriveHelpers.extractIndexableText(drive, file.id, mimeType, this.getGoogleAuth()); + results.push({ _id: this.buildItemId(fileId), name: title, - type: type, - size: size, + type, + size, uri: link, icon: thumbnail, contentText: textContent, @@ -198,10 +233,11 @@ export default class GoogleDriveDocument extends BaseSyncHandler { sourceData: file, sourceAccountId: this.provider.getProviderId(), sourceApplication: this.getProviderApplicationUrl(), - insertedAt: modifiedTime, + insertedAt: createdTime, + modifiedAt: modifiedTime, }); } - return results; + return { items: results, breakHit }; } } From 255b286537ffe46d1279f33606dee4248f756074 Mon Sep 17 00:00:00 2001 From: Chris Date: Fri, 23 Aug 2024 07:26:05 +0930 Subject: [PATCH 022/182] Support enable / disable of private data on AI page --- src/dashboard/public/ai/ai.js | 11 ++++++++++- src/dashboard/public/ai/index.html | 12 ++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/src/dashboard/public/ai/ai.js b/src/dashboard/public/ai/ai.js index 67b39971..7752bd7f 100644 --- a/src/dashboard/public/ai/ai.js +++ b/src/dashboard/public/ai/ai.js @@ -41,10 +41,19 @@ $(document).ready(function() { addMessage(prompt, 'user'); showTypingIndicator(); + const userInput = $('#privateData-input').val(); + + let urlType = "prompt" + if (userInput == "on") { + urlType = "personal" + } + + console.log(urlType) + const body = { prompt: prompt, key: veridaKey }; $.ajax({ - url: `/api/v1/llm/personal?key=${veridaKey}`, + url: `/api/v1/llm/${urlType}?key=${veridaKey}`, method: 'POST', contentType: 'application/json', data: JSON.stringify(body), diff --git a/src/dashboard/public/ai/index.html b/src/dashboard/public/ai/index.html index ef20498f..3efee59b 100644 --- a/src/dashboard/public/ai/index.html +++ b/src/dashboard/public/ai/index.html @@ -164,6 +164,18 @@ +
+ + +
+
+ + +
From e85b5a8cc286e0f6f29c237f41ba7a324943a1dd Mon Sep 17 00:00:00 2001 From: chime3 Date: Thu, 22 Aug 2024 17:53:31 -0700 Subject: [PATCH 023/182] fix: use parent googleauth --- src/providers/google/gdrive-document.ts | 26 ++----------------------- 1 file changed, 2 insertions(+), 24 deletions(-) diff --git a/src/providers/google/gdrive-document.ts b/src/providers/google/gdrive-document.ts index eccf691f..042cb641 100644 --- a/src/providers/google/gdrive-document.ts +++ b/src/providers/google/gdrive-document.ts @@ -1,12 +1,11 @@ -import BaseSyncHandler from "../BaseSyncHandler"; import CONFIG from "../../config"; import { SyncProviderLogEvent, SyncProviderLogLevel, SyncHandlerPosition, SyncItemsBreak, SyncResponse, SyncHandlerStatus, SyncItemsResult, HandlerOption, ConnectionOptionType } from '../../interfaces'; import { SchemaDocument } from "../../schemas"; import { google, drive_v3 } from "googleapis"; -import { OAuth2Client } from 'google-auth-library'; import { GaxiosResponse } from "gaxios"; import { GoogleDriveHelpers } from "./helpers"; import { ItemsRangeTracker } from "../../helpers/itemsRangeTracker"; +import GoogleHandler from "./GoogleHandler"; const _ = require("lodash"); @@ -16,7 +15,7 @@ export interface SyncDocumentItemsResult extends SyncItemsResult { items: SchemaDocument[]; } -export default class GoogleDriveDocument extends BaseSyncHandler { +export default class GoogleDriveDocument extends GoogleHandler { public getName(): string { return "google-drive-documents"; @@ -30,27 +29,6 @@ export default class GoogleDriveDocument extends BaseSyncHandler { return "https://drive.google.com"; } - public getGoogleAuth(): OAuth2Client { - const TOKEN = { - access_token: this.connection.accessToken, - refresh_token: this.connection.refreshToken, - scope: "https://www.googleapis.com/auth/drive.readonly", - token_type: "Bearer", - }; - - const redirectUrl = ""; - - const oAuth2Client = new google.auth.OAuth2( - this.config.clientId, - this.config.clientSecret, - redirectUrl - ); - - oAuth2Client.setCredentials(TOKEN); - - return oAuth2Client; - } - public getGoogleDrive(): drive_v3.Drive { const auth = this.getGoogleAuth(); return google.drive({ version: "v3", auth }); From 07bbf341a6e75864d4e1c1904ca233fcc012be4d Mon Sep 17 00:00:00 2001 From: Chris Date: Fri, 23 Aug 2024 11:42:46 +0930 Subject: [PATCH 024/182] Refactor how static web pages are served. Start splitting UI into developers and users. --- package.json | 4 ++-- src/server-app.ts | 3 +-- src/{dashboard/public => web/developer}/api/api.js | 0 .../public => web/developer}/api/index.html | 0 .../public => web/developer}/data/data.js | 0 .../public => web/developer}/data/index.html | 0 src/{dashboard/public => web/developer}/index.html | 4 ++-- src/web/index.html | 13 +++++++++++++ src/{ => web}/provider/telegram/index.html | 0 src/{ => web}/provider/telegram/script.js | 0 src/{dashboard/public => web/user}/about/index.html | 0 src/{dashboard/public => web/user}/ai/ai.js | 0 src/{dashboard/public => web/user}/ai/index.html | 0 .../public => web/user}/connections/connections.js | 0 .../public => web/user}/connections/index.html | 0 src/web/user/index.html | 13 +++++++++++++ src/{dashboard/public => web/user}/styles.css | 0 .../user}/welcome/download-verida-wallet.svg | 0 .../public => web/user}/welcome/index.html | 0 19 files changed, 31 insertions(+), 6 deletions(-) rename src/{dashboard/public => web/developer}/api/api.js (100%) rename src/{dashboard/public => web/developer}/api/index.html (100%) rename src/{dashboard/public => web/developer}/data/data.js (100%) rename src/{dashboard/public => web/developer}/data/index.html (100%) rename src/{dashboard/public => web/developer}/index.html (60%) create mode 100644 src/web/index.html rename src/{ => web}/provider/telegram/index.html (100%) rename src/{ => web}/provider/telegram/script.js (100%) rename src/{dashboard/public => web/user}/about/index.html (100%) rename src/{dashboard/public => web/user}/ai/ai.js (100%) rename src/{dashboard/public => web/user}/ai/index.html (100%) rename src/{dashboard/public => web/user}/connections/connections.js (100%) rename src/{dashboard/public => web/user}/connections/index.html (100%) create mode 100644 src/web/user/index.html rename src/{dashboard/public => web/user}/styles.css (100%) rename src/{dashboard/public => web/user}/welcome/download-verida-wallet.svg (100%) rename src/{dashboard/public => web/user}/welcome/index.html (100%) diff --git a/package.json b/package.json index 296e34ba..9e09e3c1 100644 --- a/package.json +++ b/package.json @@ -7,8 +7,8 @@ "cli": "node dist/cli/exec.js", "core-cli": "node node_modules/@verida/cli-tools/dist/exec.js", "dev": "yarn build && yarn symlink-static && nodemon src/server.js", - "symlink-static": "rm -rf dist/dashboard && ln -s ../src/dashboard/public dist/dashboard && rm -rf dist/custom && ln -s ../src/custom dist/custom", - "build": "rm -rf dist && tsc && rm -rf dist/assets && cp -R assets dist/ && cp -R src/dashboard/public dist/dashboard && cp -R src/provider dist/provider", + "symlink-static": "rm -rf dist/web && ln -s ../src/web dist/web", + "build": "rm -rf dist && tsc && rm -rf dist/assets && cp -R assets dist/ && cp -R src/web dist/web", "prestart": "yarn build", "start": "node dist/server.js", "tests": "ts-mocha './tests/**/*.ts'", diff --git a/src/server-app.ts b/src/server-app.ts index 15274f3c..b8365b7e 100644 --- a/src/server-app.ts +++ b/src/server-app.ts @@ -25,8 +25,7 @@ const app = express(); app.use(requestIdMiddleware) app.use('/assets', express.static(path.join(__dirname, 'assets'))) -app.use('/dashboard', express.static(path.join(__dirname, 'dashboard'))) -app.use('/provider', express.static(path.join(__dirname, 'provider'))) +app.use('/', express.static(path.join(__dirname, 'web'))) app.use(session({ secret: CONFIG.verida.sessionSecret, resave: false, diff --git a/src/dashboard/public/api/api.js b/src/web/developer/api/api.js similarity index 100% rename from src/dashboard/public/api/api.js rename to src/web/developer/api/api.js diff --git a/src/dashboard/public/api/index.html b/src/web/developer/api/index.html similarity index 100% rename from src/dashboard/public/api/index.html rename to src/web/developer/api/index.html diff --git a/src/dashboard/public/data/data.js b/src/web/developer/data/data.js similarity index 100% rename from src/dashboard/public/data/data.js rename to src/web/developer/data/data.js diff --git a/src/dashboard/public/data/index.html b/src/web/developer/data/index.html similarity index 100% rename from src/dashboard/public/data/index.html rename to src/web/developer/data/index.html diff --git a/src/dashboard/public/index.html b/src/web/developer/index.html similarity index 60% rename from src/dashboard/public/index.html rename to src/web/developer/index.html index 73a35893..469ba763 100644 --- a/src/dashboard/public/index.html +++ b/src/web/developer/index.html @@ -4,10 +4,10 @@ Redirecting... -

If you are not redirected, click here.

+

If you are not redirected, click here.

\ No newline at end of file diff --git a/src/web/index.html b/src/web/index.html new file mode 100644 index 00000000..5bdf5f1f --- /dev/null +++ b/src/web/index.html @@ -0,0 +1,13 @@ + + + + + Redirecting... + + + +

If you are not redirected, click here.

+ + \ No newline at end of file diff --git a/src/provider/telegram/index.html b/src/web/provider/telegram/index.html similarity index 100% rename from src/provider/telegram/index.html rename to src/web/provider/telegram/index.html diff --git a/src/provider/telegram/script.js b/src/web/provider/telegram/script.js similarity index 100% rename from src/provider/telegram/script.js rename to src/web/provider/telegram/script.js diff --git a/src/dashboard/public/about/index.html b/src/web/user/about/index.html similarity index 100% rename from src/dashboard/public/about/index.html rename to src/web/user/about/index.html diff --git a/src/dashboard/public/ai/ai.js b/src/web/user/ai/ai.js similarity index 100% rename from src/dashboard/public/ai/ai.js rename to src/web/user/ai/ai.js diff --git a/src/dashboard/public/ai/index.html b/src/web/user/ai/index.html similarity index 100% rename from src/dashboard/public/ai/index.html rename to src/web/user/ai/index.html diff --git a/src/dashboard/public/connections/connections.js b/src/web/user/connections/connections.js similarity index 100% rename from src/dashboard/public/connections/connections.js rename to src/web/user/connections/connections.js diff --git a/src/dashboard/public/connections/index.html b/src/web/user/connections/index.html similarity index 100% rename from src/dashboard/public/connections/index.html rename to src/web/user/connections/index.html diff --git a/src/web/user/index.html b/src/web/user/index.html new file mode 100644 index 00000000..5bdf5f1f --- /dev/null +++ b/src/web/user/index.html @@ -0,0 +1,13 @@ + + + + + Redirecting... + + + +

If you are not redirected, click here.

+ + \ No newline at end of file diff --git a/src/dashboard/public/styles.css b/src/web/user/styles.css similarity index 100% rename from src/dashboard/public/styles.css rename to src/web/user/styles.css diff --git a/src/dashboard/public/welcome/download-verida-wallet.svg b/src/web/user/welcome/download-verida-wallet.svg similarity index 100% rename from src/dashboard/public/welcome/download-verida-wallet.svg rename to src/web/user/welcome/download-verida-wallet.svg diff --git a/src/dashboard/public/welcome/index.html b/src/web/user/welcome/index.html similarity index 100% rename from src/dashboard/public/welcome/index.html rename to src/web/user/welcome/index.html From 14612c7d9ac65e3d2701a9e3c2223c9e978f5740 Mon Sep 17 00:00:00 2001 From: Chris Date: Fri, 23 Aug 2024 11:52:19 +0930 Subject: [PATCH 025/182] Fix navigation between user and developer interfaces --- src/web/developer/api/index.html | 19 ++++---- src/web/developer/data/index.html | 19 ++++---- src/web/developer/welcome/index.html | 51 +++++++++++++++++++++ src/web/user/ai/index.html | 18 ++++---- src/web/user/connections/index.html | 18 ++++---- src/web/user/{about => security}/index.html | 18 ++++---- src/web/user/welcome/index.html | 18 ++++---- 7 files changed, 103 insertions(+), 58 deletions(-) create mode 100644 src/web/developer/welcome/index.html rename src/web/user/{about => security}/index.html (77%) diff --git a/src/web/developer/api/index.html b/src/web/developer/api/index.html index 93366faa..98efd20f 100644 --- a/src/web/developer/api/index.html +++ b/src/web/developer/api/index.html @@ -30,29 +30,26 @@
- -
From 83912ec723bbedea4623251a82f03dfb4e017d1d Mon Sep 17 00:00:00 2001 From: Chris Date: Sat, 24 Aug 2024 22:59:10 +0930 Subject: [PATCH 030/182] Support Groq. Refactor to easily support multiple LLM services. --- package.json | 1 + src/api/v1/llm/controller.ts | 12 +-- src/services/assistants/search.ts | 127 +++++++++++++++++++++++ src/services/llm.ts | 160 +++++++++++++++++++++++++---- src/services/prompt.ts | 96 ----------------- src/services/search.ts | 98 ++++++++++++++---- src/services/tools/promptSearch.ts | 44 ++++++++ yarn.lock | 77 +++++++++++++- 8 files changed, 470 insertions(+), 145 deletions(-) create mode 100644 src/services/assistants/search.ts delete mode 100644 src/services/prompt.ts create mode 100644 src/services/tools/promptSearch.ts diff --git a/package.json b/package.json index 9e09e3c1..e400ede8 100644 --- a/package.json +++ b/package.json @@ -60,6 +60,7 @@ "fs": "^0.0.1-security", "gaxios": "^6.7.0", "googleapis": "^140.0.1", + "groq-sdk": "^0.5.0", "lodash": "^4.17.21", "log4js": "^6.4.1", "mailparser": "^3.7.1", diff --git a/src/api/v1/llm/controller.ts b/src/api/v1/llm/controller.ts index 4387db94..b8a6000d 100644 --- a/src/api/v1/llm/controller.ts +++ b/src/api/v1/llm/controller.ts @@ -1,11 +1,11 @@ import { Request, Response } from "express"; -import { LLMServices } from '../../../services/llm' -import { PromptService } from '../../../services/prompt' +import { bedrock } from '../../../services/llm' +import { PromptSearchService } from '../../../services/assistants/search' import { Utils } from "../../../utils"; const _ = require('lodash') -const llm = LLMServices.bedrock +const llm = bedrock /** * @@ -15,7 +15,7 @@ export class LLMController { public async prompt(req: Request, res: Response) { try { const prompt = req.body.prompt - const serverResponse = await llm(prompt) + const serverResponse = await llm.prompt(prompt) return res.json({ result: serverResponse @@ -32,8 +32,8 @@ export class LLMController { const did = await account.did() const prompt = req.body.prompt - const promptService = new PromptService(did, context) - const promptResult = await promptService.personalPrompt(prompt) + const promptService = new PromptSearchService(did, context) + const promptResult = await promptService.prompt(prompt) return res.json(promptResult) } catch (error) { diff --git a/src/services/assistants/search.ts b/src/services/assistants/search.ts new file mode 100644 index 00000000..dfc89ba0 --- /dev/null +++ b/src/services/assistants/search.ts @@ -0,0 +1,127 @@ +import Axios from 'axios' +const _ = require('lodash') +import { defaultModel } from "../llm" +import { PromptSearch, PromptSearchLLMResponse } from "../tools/promptSearch" +import { ChatThreadResult, SearchService, SearchSortType } from "../search" +import { VeridaService } from '../veridaService' +import { SchemaEmail, SchemaEmailType, SchemaSocialChatMessage } from '../../schemas' + +const llm = defaultModel + +const MAX_EMAIL_LENGTH = 500 +const MAX_ATTACHMENT_LENGTH = 1000 +const MAX_CONTEXT_LENGTH = 20000 + +// "You are a personal assistant with the ability to search the following categories; emails, chat_history and documents. You receive a prompt and generate a JSON response (with no other text) that provides search queries that will source useful information to help answer the prompt. Search queries for each category should contain three properties; \"terms\" (an array of 10 individual words), \"beforeDate\" (results must be before this date), \"afterDate\" (results must be after this date), \"resultType\" (either \"count\" to count results or \"results\" to return the search results), \"filter\" (an array of key, value pairs of fields to filter the results). Categories can be empty if not relevant to the prompt. The current date is 2024-08-12.\n\nHere is an example JSON response:\n{\"email\": {\"terms\": [\"golf\", \"tennis\", \"soccer\"], \"beforeDate\": \"2024-06-01\", \"afterDate\": \"2024-01-10\" \"filter\": {\"from\": \"dave\"}, \"resultType\": \"results}}\n\nHere is the prompt:\nWhat subscriptions do I currently pay for?" + +export class PromptSearchService extends VeridaService { + + public async prompt(prompt: string): Promise<{ + result: string, + duration: number, + promptSearchResult: PromptSearchLLMResponse + }> { + const start = Date.now() + // // Get queries that can help answer the prompt + // //const queryPrompt = `Generate 10 lucene search queries, include reasonable synonyms, to find relevant emails to help respond to this prompt:\n${prompt}\nYou have the following searchable fields: subject,messageText,fromName,fromEmail.\nYour response must only contain a single JSON list of search strings, no other commentary and no formatting.` + // const keywordPrompt = `Generate 10 individual words that could help search for relevant emails realated to this prompt:\n${prompt}\nYour response must only contain a single JSON list of search strings, no other commentary and no formatting.` + // const keywordResponse = await llm(keywordPrompt) + // const entityPrompt = `Extract any individual or organization names mentioned in this prompt:\n${prompt}\nYour response must only contain a single JSON list of search strings, no other commentary and no formatting.` + // const entityResponse = await llm(entityPrompt) + + // console.log(keywordResponse) + // console.log(entityResponse) + // const keywords = JSON.parse(keywordResponse) + // let entities = [] + // try { + // entities = JSON.parse(entityResponse) + // } catch (err) { + // // do nothing + // } + + // console.log(keywords) + // console.log(entities) + const promptSearch = new PromptSearch(llm) + const promptSearchResult = await promptSearch.search(prompt) + + let chatThreads: ChatThreadResult[] = [] + let emails: SchemaEmail[] = [] + + const searchService = new SearchService(this.did, this.context) + + let maxAgeSeconds = undefined + const dayInSeconds = 60*60*24 + switch (promptSearchResult.timeframe) { + case "day": + maxAgeSeconds = dayInSeconds + break + case "week": + maxAgeSeconds = dayInSeconds*7 + break + case "month": + maxAgeSeconds = dayInSeconds*30 + break + case "quarter": + maxAgeSeconds = dayInSeconds*90 + break + case "half-year": + maxAgeSeconds = dayInSeconds*180 + break + case "full-year": + maxAgeSeconds = dayInSeconds*365 + break + } + const maxDatetime = new Date((new Date()).getTime() - maxAgeSeconds * 1000); + const sortType = promptSearchResult.sort == "keyword_rank" ? SearchSortType.RECENT : promptSearchResult.sort + + emails = await searchService.emailsByKeywords(promptSearchResult.keywords, 20) + chatThreads = await searchService.chatThreadsByKeywords(promptSearchResult.keywords, 10, 10) + + let finalPrompt = `Answer this prompt:\n${prompt}\nHere are some recent messages that may help you provide a relevant answer.\n` + let contextString = '' + + let maxChatMessages = 50 + for (const chatThread of chatThreads) { + for (const chatMessage of chatThread.messages) { + contextString += `From: ${chatMessage.fromName} <${chatMessage.fromHandle}> (${chatMessage.groupName})\nBody: ${chatMessage.messageText}\n\n` + + if (maxChatMessages-- <= 0) { + break + } + } + } + + for (const email of emails) { + let extraContext = "" + let body = email.messageText.substring(0, MAX_EMAIL_LENGTH) + if (email.attachments) { + for (const attachment of email.attachments) { + body += attachment.textContent.substring(0, MAX_ATTACHMENT_LENGTH) + } + } + + extraContext = `From: ${email.fromName} <${email.fromEmail}> (${email.name})\nBody: ${body}\n\n` + if ((extraContext.length + contextString.length + finalPrompt.length) > MAX_CONTEXT_LENGTH) { + break + } + + contextString += extraContext + } + + const now = (new Date()).toISOString() + finalPrompt += `${contextString}\nThe current time is: ${now}` + + console.log('Running final prompt', finalPrompt.length) + const finalResponse = await llm.prompt(finalPrompt, undefined, false) + const duration = Date.now() - start + + console.log(contextString) + + return { + result: finalResponse.choices[0].message.content, + duration, + promptSearchResult + } + } + +} \ No newline at end of file diff --git a/src/services/llm.ts b/src/services/llm.ts index 7325a8b0..54930616 100644 --- a/src/services/llm.ts +++ b/src/services/llm.ts @@ -1,4 +1,4 @@ -// import Groq from "groq-sdk" +import Groq from "groq-sdk" import Axios from 'axios' import CONFIG from "../config" @@ -8,27 +8,145 @@ const enum BedrockModels { MIXTRAL_8_7B = "mistral.mixtral-8x7b-instruct-v0:1" } +const enum GroqModels { + LLAMA3_70B = "llama3-70b-8192", + LLAMA3_8B = "llama3-8b-8192", + LLAMA31_70B = "llama-3.1-70b-versatile" +} + const BEDROCK_KEY = CONFIG.verida.llms.bedrockKey const BEDROCK_ENDPOINT = CONFIG.verida.llms.bedrockEndpoint -export class LLMServices { - - public static async bedrock(prompt: string, model: BedrockModels = BedrockModels.LLAMA3_70B) { - const response = await Axios.post(BEDROCK_ENDPOINT, { - messages: [ - { - role: "user", - content: prompt, - }, - ], - model, - }, { - headers: { - 'Content-Type': 'application/json', - 'Authorization': `Bearer ${BEDROCK_KEY}` - } - }); - - return response.data.choices[0].message.content +// const GROQ_MODEL = 'llama3-70b-8192' // 'llama3-8b-8192' // 'llama3-70b-8192' // 'llama3-8b-8192' //'llama-3.1-8b-instant' //'llama3-70b-8192' // 'llama3-8b-8192' +const GROQ_KEY = CONFIG.verida.llms.groqKey + +export interface OpenAIConfig { + endpoint: string + key?: string +} + +export const LLMS: Record = { + BEDROCK: { + endpoint: BEDROCK_ENDPOINT, + key: BEDROCK_KEY + } +} + +export interface LLMResponse { + agent: string +} + +export interface LLM { + prompt(userPrompt: string, systemPrompt?: string, format?: boolean, model?: string): Promise +} + +// export interface OpenAIChatResponse { +// id: string; +// object: string; +// created: number; +// model: string; +// choices: { +// index: number; +// message: { +// role: string; +// content: string; +// }; +// finish_reason: string; +// }[]; +// usage?: { +// prompt_tokens: number; +// completion_tokens: number; +// total_tokens: number; +// }; +// }; +export interface OpenAIChatResponse extends Groq.Chat.ChatCompletion {} + +export class GroqLLM implements LLM { + private groq: Groq + private defaultModel: string + + constructor(defaultModel: string) { + this.defaultModel = defaultModel + this.groq = new Groq({ apiKey: GROQ_KEY }); + } + + public async prompt(userPrompt: string, systemPrompt?: string, jsonFormat: boolean = true, model: string = this.defaultModel): Promise { + const messages = [ + { + role: "user", + content: userPrompt, + } + ]; + + if (systemPrompt) { + messages.push({ + role: "system", + content: systemPrompt, + }) + } + + const response = await this.groq.chat.completions.create({ + response_format: jsonFormat ? {type: "json_object"} : undefined, + // @ts-ignore + messages, + model, + temperature: 1, + top_p: 1 + }); + console.log(JSON.stringify(response, null, 2)) + return response +} +} + +export class OpenAILLM implements LLM { + + private config: OpenAIConfig + private defaultModel: string + + constructor(config: OpenAIConfig, defaultModel: string) { + this.config = config + this.defaultModel = defaultModel + } + + public async prompt(userPrompt: string, systemPrompt?: string, jsonFormat: boolean = true, model: string = this.defaultModel): Promise { + const headers: Record = { + 'Content-Type': 'application/json', + } + + if (this.config.key) { + headers['Authorization'] = `Bearer ${this.config.key}` + } + + const messages = [ + { + role: "user", + content: userPrompt, + } + ]; + + if (systemPrompt) { + messages.push({ + role: "system", + content: systemPrompt, + }) } -} \ No newline at end of file + + const response = await Axios.post(this.config.endpoint, { + format: jsonFormat ? "json" : undefined, + messages, + model, + temperature: 1, + top_p: 1 + }, { + headers + }); + + // return response.data.choices[0].message.content + console.log(JSON.stringify(response.data, null, 2)) + return response.data + } +} + +export const bedrock = new OpenAILLM(LLMS.BEDROCK, BedrockModels.LLAMA3_70B) +export const groq = new GroqLLM(GroqModels.LLAMA3_70B) +export const defaultModel = groq \ No newline at end of file diff --git a/src/services/prompt.ts b/src/services/prompt.ts deleted file mode 100644 index 0bd0f6a3..00000000 --- a/src/services/prompt.ts +++ /dev/null @@ -1,96 +0,0 @@ -import Axios from 'axios' -const _ = require('lodash') -import { LLMServices } from "../services/llm" -import { ChatThreadResult, SearchService, SearchTypes } from "../services/search" -import { VeridaService } from './veridaService' -import { SchemaEmail, SchemaEmailType, SchemaSocialChatMessage } from '../schemas' - -const llm = LLMServices.bedrock - -const MAX_EMAIL_LENGTH = 500 -const MAX_ATTACHMENT_LENGTH = 1000 -const MAX_CONTEXT_LENGTH = 20000 - -// "You are a personal assistant with the ability to search the following categories; emails, chat_history and documents. You receive a prompt and generate a JSON response (with no other text) that provides search queries that will source useful information to help answer the prompt. Search queries for each category should contain three properties; \"terms\" (an array of 10 individual words), \"beforeDate\" (results must be before this date), \"afterDate\" (results must be after this date), \"resultType\" (either \"count\" to count results or \"results\" to return the search results), \"filter\" (an array of key, value pairs of fields to filter the results). Categories can be empty if not relevant to the prompt. The current date is 2024-08-12.\n\nHere is an example JSON response:\n{\"email\": {\"terms\": [\"golf\", \"tennis\", \"soccer\"], \"beforeDate\": \"2024-06-01\", \"afterDate\": \"2024-01-10\" \"filter\": {\"from\": \"dave\"}, \"resultType\": \"results}}\n\nHere is the prompt:\nWhat subscriptions do I currently pay for?" - -export class PromptService extends VeridaService { - - public async personalPrompt(prompt: string): Promise<{ - result: any[], - duration: number, - keywords: string[], - entities: string[] - }> { - const start = Date.now() - // Get queries that can help answer the prompt - //const queryPrompt = `Generate 10 lucene search queries, include reasonable synonyms, to find relevant emails to help respond to this prompt:\n${prompt}\nYou have the following searchable fields: subject,messageText,fromName,fromEmail.\nYour response must only contain a single JSON list of search strings, no other commentary and no formatting.` - const keywordPrompt = `Generate 10 individual words that could help search for relevant emails realated to this prompt:\n${prompt}\nYour response must only contain a single JSON list of search strings, no other commentary and no formatting.` - const keywordResponse = await llm(keywordPrompt) - const entityPrompt = `Extract any individual or organization names mentioned in this prompt:\n${prompt}\nYour response must only contain a single JSON list of search strings, no other commentary and no formatting.` - const entityResponse = await llm(entityPrompt) - - console.log(keywordResponse) - console.log(entityResponse) - const keywords = JSON.parse(keywordResponse) - let entities = [] - try { - entities = JSON.parse(entityResponse) - } catch (err) { - // do nothing - } - - console.log(keywords) - console.log(entities) - - const searchService = new SearchService(this.did, this.context) - const messages = await searchService.emails(keywords.concat(entities), 20) - const chatThreads = await searchService.chatThreads(keywords.concat(entities), 10, 10) - - let finalPrompt = `Answer this prompt:\n${prompt}\nHere are some recent messages that may help you provide a relevant answer.\n` - let contextString = '' - - let maxChatMessages = 50 - for (const chatThread of chatThreads) { - for (const chatMessage of chatThread.messages) { - contextString += `From: ${chatMessage.fromName} <${chatMessage.fromHandle}> (${chatMessage.groupName})\nBody: ${chatMessage.messageText}\n\n` - - if (maxChatMessages-- <= 0) { - break - } - } - } - - for (const message of messages) { - let extraContext = "" - const email = message - let body = email.messageText.substring(0, MAX_EMAIL_LENGTH) - if (email.attachments) { - for (const attachment of email.attachments) { - body += attachment.textContent.substring(0, MAX_ATTACHMENT_LENGTH) - } - } - - extraContext = `From: ${email.fromName} <${email.fromEmail}> (${email.name})\nBody: ${body}\n\n` - if ((extraContext.length + contextString.length + finalPrompt.length) > MAX_CONTEXT_LENGTH) { - break - } - - contextString += extraContext - } - - finalPrompt += contextString - console.log('Running final prompt', finalPrompt.length) - const finalResponse = await llm(finalPrompt) - const duration = Date.now() - start - - console.log(contextString) - - return { - result: finalResponse, - duration, - keywords, - entities - } - } - -} \ No newline at end of file diff --git a/src/services/search.ts b/src/services/search.ts index c16d65de..5b437c72 100644 --- a/src/services/search.ts +++ b/src/services/search.ts @@ -1,7 +1,8 @@ import { DataService } from "./data" import { VeridaService } from "./veridaService" -import { SchemaRecord, SchemaSocialChatGroup, SchemaSocialChatMessage } from "../schemas" +import { SchemaEmail, SchemaRecord, SchemaSocialChatGroup, SchemaSocialChatMessage } from "../schemas" import { IDatastore } from "@verida/types" +import { threadId } from "worker_threads" const _ = require('lodash') export interface MinisearchResult { @@ -13,7 +14,7 @@ export interface MinisearchResult { } export interface SearchServiceSchemaResult { - schemaUri: string + searchType: SearchType rows: MinisearchResult[] } @@ -23,9 +24,27 @@ export interface SortedResult { score: number } -export enum SearchTypes { - CHAT_MESSAGES = "https://common.schemas.verida.io/social/chat/message/v0.1.0/schema.json", - EMAILS = "https://common.schemas.verida.io/social/email/v0.1.0/schema.json" +export enum SearchSortType { + RECENT = "recent", + OLDEST = "oldest" +} + +export enum SearchType { + // CHAT_THREADS = "chat-threads", + CHAT_MESSAGES = "chat-messages", + EMAILS = "emails", + FAVORITES = "favorites", + FOLLOWING = "following", + POSTS = "posts" +} + +export const SearchTypeSchemas: Record = { + // [SearchType.CHAT_THREADS]: "https://common.schemas.verida.io/social/chat/message/v0.1.0/schema.json", + [SearchType.CHAT_MESSAGES]: "https://common.schemas.verida.io/social/chat/message/v0.1.0/schema.json", + [SearchType.EMAILS]: "https://common.schemas.verida.io/social/email/v0.1.0/schema.json", + [SearchType.FAVORITES]: "https://common.schemas.verida.io/favourite/v0.1.0/schema.json", + [SearchType.POSTS]: "https://common.schemas.verida.io/social/post/v0.1.0/schema.json", + [SearchType.FOLLOWING]: "https://common.schemas.verida.io/favourite/v0.1.0/schema.json", } export interface ChatThreadResult { @@ -57,7 +76,8 @@ export class SearchService extends VeridaService { } } - datastores.push(await this.context.openDatastore(schemaResult.schemaUri)) + const schemaUri = SearchTypeSchemas[schemaResult.searchType] + datastores.push(await this.context.openDatastore(schemaUri)) } const unsortedResultCount = Object.values(unsortedResults).length @@ -76,6 +96,9 @@ export class SearchService extends VeridaService { const results = [] for (let i = 0; i < limit; i++) { const result = queuedResults[i] + if (!result) { + continue + } const datastore = datastores[result.schemaId] const row = await datastore.get(result.id, {}) row._score = result.score @@ -85,24 +108,49 @@ export class SearchService extends VeridaService { return results } - public async emails(keywordsList: string[], limit: number = 20): Promise { + public async emailsByKeywords(keywordsList: string[], limit: number = 20): Promise { const query = keywordsList.join(' ') - const schemaUri = "https://common.schemas.verida.io/social/email/v0.1.0/schema.json" + const searchType = SearchType.EMAILS + const schemaUri = SearchTypeSchemas[searchType] const dataService = new DataService(this.did, this.context) const miniSearchIndex = await dataService.getIndex(schemaUri) console.log('Emails: searching for', query) const searchResults = await miniSearchIndex.search(query) - return this.rankAndMergeResults([{ - schemaUri, + return await this.rankAndMergeResults([{ + searchType, rows: searchResults }], limit) } - public async chatHistory(keywordsList: string[], limit: number = 20): Promise { + public async emailsByDateRange(maxDatetime: Date, sortType: SearchSortType, limit: number = 20): Promise { + const searchType = SearchType.EMAILS + const schemaUri = SearchTypeSchemas[searchType] + const dataService = new DataService(this.did, this.context) + const emailDatastore = await dataService.getDatastore(schemaUri) + const filter = { + sentAt: { + "$gte": maxDatetime.toISOString() + } + } + const options = { + limit, + sort: [ + { + sentAt: sortType == SearchSortType.OLDEST ? "asc" : "desc" + } + ] + } + + console.log('Emails: searching for', filter, options) + return await emailDatastore.getMany(filter, options) + } + + public async chatHistoryByKeywords(keywordsList: string[], limit: number = 20): Promise { + const searchType = SearchType.CHAT_MESSAGES + const schemaUri = SearchTypeSchemas[searchType] const query = keywordsList.join(' ') - const schemaUri = "https://common.schemas.verida.io/social/chat/message/v0.1.0/schema.json" const dataService = new DataService(this.did, this.context) const miniSearchIndex = await dataService.getIndex(schemaUri) @@ -110,7 +158,7 @@ export class SearchService extends VeridaService { const searchResults = await miniSearchIndex.search(query) return this.rankAndMergeResults([{ - schemaUri, + searchType, rows: searchResults }], limit) } @@ -126,7 +174,7 @@ export class SearchService extends VeridaService { * @param mergeOverlaps If there is an overlap of messages within the same chat group, they will be merged into a single thread. * @returns */ - public async chatThreads(keywordsList: string[], threadSize: 10, limit: number = 20, mergeOverlaps: boolean = true): Promise { + public async chatThreadsByKeywords(keywordsList: string[], threadSize: 10, limit: number = 20, mergeOverlaps: boolean = true): Promise { const query = keywordsList.join(' ') const messageSchemaUri = "https://common.schemas.verida.io/social/chat/message/v0.1.0/schema.json" const groupSchemaUri = "https://common.schemas.verida.io/social/chat/group/v0.1.0/schema.json" @@ -216,18 +264,30 @@ export class SearchService extends VeridaService { } - public async multi(searchTypes: SearchTypes[], keywordsList: string[], limit: number = 20, minResultsPerType: number = 10) { + public async multiByKeywords(searchTypes: SearchType[], keywordsList: string[], limit: number = 20, minResultsPerType: number = 10) { const query = keywordsList.join(' ') const dataService = new DataService(this.did, this.context) console.log('Multi: searching for', query) const searchResults = [] - for (const schemaUri of searchTypes) { - const miniSearchIndex = await dataService.getIndex(schemaUri) - const queryResult = await miniSearchIndex.search(query) + for (const searchType of searchTypes) { + // let queryResult: SchemaRecord[] | ChatThreadResult[] = [] + // if (searchType == SearchType.CHAT_THREADS) { + // const threadSize = 10 + // queryResult = await this.chatThreadsByKeywords(keywordsList, threadSize, limit, true) + // } else { + const schemaUri = SearchTypeSchemas[searchType] + if (!schemaUri) { + // Invalid search type, ignore + continue + } + const miniSearchIndex = await dataService.getIndex(schemaUri) + const queryResult = await miniSearchIndex.search(query) + // } + searchResults.push({ - schemaUri, + searchType, rows: queryResult }) } diff --git a/src/services/tools/promptSearch.ts b/src/services/tools/promptSearch.ts new file mode 100644 index 00000000..6f555dab --- /dev/null +++ b/src/services/tools/promptSearch.ts @@ -0,0 +1,44 @@ +import { LLM } from "../llm" + +const systemPrompt = `You are an expert data analyst. When I give you a prompt, you must generate search metadata that will be used to extract relevant information to help answer the prompt. +You must generate a JSON response containing the following information: +- search_type: keywords (search for specific keywords), all (search with filters, no keywords required) +- keywords: array of single word terms to search on that match the underlying objective of the search. extract entity names. aim for 5-10 terms. +- timeframe: one of; day, week, month, quarter, half-year, full-year, all +- databases: an array of databases to search; emails, chat_messages, files, favourites, web_history, calendar +- sort: keyword_rank, recent, oldest +- output_type: The amount of detail in the output of each search result to provide meaningful context. full_content, summary, name +- profile_information; Array of these options only; name, contactInfo, demographics, lifestyle, preferences, habits, financial, health, personality, employment, education, skills, language, interests + +JSON only, no explanation.` + +export interface PromptSearchLLMResponse { + search_type: "keywords" | "all"; + keywords?: string[]; // Array of single word terms, required if search_type is "keywords" + timeframe: "day" | "week" | "month" | "quarter" | "half-year" | "full-year" | "all"; + databases: Array<"emails" | "chat_messages" | "files" | "favourites" | "web_history" | "calendar">; + sort: "keyword_rank" | "recent" | "oldest"; + output_type: "full_content" | "summary" | "name"; + profile_information: Array< + "name" | "contactInfo" | "demographics" | "lifestyle" | "preferences" | "habits" | + "financial" | "health" | "personality" | "employment" | "education" | "skills" | + "language" | "interests" + >; + } + +export class PromptSearch { + + private llm: LLM + + constructor(llm: LLM) { + this.llm = llm + } + + public async search(userPrompt: string): Promise { + const response = await this.llm.prompt(userPrompt, systemPrompt) + console.log(response.choices[0]) + return JSON.parse(response.choices[0].message.content) + + } + +} \ No newline at end of file diff --git a/yarn.lock b/yarn.lock index 0e463575..4e8542e1 100644 --- a/yarn.lock +++ b/yarn.lock @@ -790,6 +790,14 @@ resolved "https://registry.yarnpkg.com/@types/ms/-/ms-0.7.34.tgz#10964ba0dee6ac4cd462e2795b6bebd407303433" integrity sha512-nG96G3Wp6acyAgJqGasjODb+acrI7KltPiRxzHPXnP3NgI28bpQDRv53olbqGXbfcgF5aiiHmO3xpwEpS5Ld9g== +"@types/node-fetch@^2.6.4": + version "2.6.11" + resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.6.11.tgz#9b39b78665dae0e82a08f02f4967d62c66f95d24" + integrity sha512-24xFj9R5+rfQJLRyM56qh+wnVSYhyXC2tkoBndtY0U+vubqNsYXGjufB2nn8Q6gt0LrARwL6UBtMCSVCwl4B1g== + dependencies: + "@types/node" "*" + form-data "^4.0.0" + "@types/node@*": version "17.0.35" resolved "https://registry.yarnpkg.com/@types/node/-/node-17.0.35.tgz#635b7586086d51fb40de0a2ec9d1014a5283ba4a" @@ -800,6 +808,13 @@ resolved "https://registry.yarnpkg.com/@types/node/-/node-18.15.13.tgz#f64277c341150c979e42b00e4ac289290c9df469" integrity sha512-N+0kuo9KgrUQ1Sn/ifDXsvg0TTleP7rIy4zOBGECxAljqvqfqpTfzx0Q1NUedOixRMBfe2Whhb056a42cWs26Q== +"@types/node@^18.11.18": + version "18.19.45" + resolved "https://registry.yarnpkg.com/@types/node/-/node-18.19.45.tgz#a9ebfe4c316a356be7ca11f753ecb2feda6d6bdf" + integrity sha512-VZxPKNNhjKmaC1SUYowuXSRSMGyQGmQjvvA1xE4QZ0xce2kLtEhPDS+kqpCPBZYgqblCLQ2DAjSzmgCM5auvhA== + dependencies: + undici-types "~5.26.4" + "@types/node@^18.15.11": version "18.19.42" resolved "https://registry.yarnpkg.com/@types/node/-/node-18.19.42.tgz#b54ed4752c85427906aab40917b0f7f3d724bf72" @@ -1174,7 +1189,7 @@ abbrev@1: resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-1.1.1.tgz#f8f2c887ad10bf67f634f005b6987fed3179aac8" integrity sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q== -abort-controller@3.0.0: +abort-controller@3.0.0, abort-controller@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/abort-controller/-/abort-controller-3.0.0.tgz#eaf54d53b62bae4138e809ca225c8439a6efb392" integrity sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg== @@ -1235,6 +1250,13 @@ agent-base@^7.0.2: dependencies: debug "^4.3.4" +agentkeepalive@^4.2.1: + version "4.5.0" + resolved "https://registry.yarnpkg.com/agentkeepalive/-/agentkeepalive-4.5.0.tgz#2673ad1389b3c418c5a20c5d7364f93ca04be923" + integrity sha512-5GG/5IbQQpC9FpkRGsSvZI5QYeSCzlJHdpBQntCsuTOxhKD8lqKhrleg2Yi7yvMIf82Ycmmqln9U8V9qwEiJew== + dependencies: + humanize-ms "^1.2.1" + ajv-formats@^2.1.1: version "2.1.1" resolved "https://registry.yarnpkg.com/ajv-formats/-/ajv-formats-2.1.1.tgz#6e669400659eb74973bbf2e33327180a0996b520" @@ -2725,6 +2747,11 @@ forever-agent@~0.6.1: resolved "https://registry.yarnpkg.com/forever-agent/-/forever-agent-0.6.1.tgz#fbc71f0c41adeb37f96c577ad1ed42d8fdacca91" integrity sha1-+8cfDEGt6zf5bFd60e1C2P2sypE= +form-data-encoder@1.7.2: + version "1.7.2" + resolved "https://registry.yarnpkg.com/form-data-encoder/-/form-data-encoder-1.7.2.tgz#1f1ae3dccf58ed4690b86d87e4f57c654fbab040" + integrity sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A== + form-data@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/form-data/-/form-data-4.0.0.tgz#93919daeaf361ee529584b9b31664dc12c9fa452" @@ -2743,6 +2770,14 @@ form-data@~2.3.2: combined-stream "^1.0.6" mime-types "^2.1.12" +formdata-node@^4.3.2: + version "4.4.1" + resolved "https://registry.yarnpkg.com/formdata-node/-/formdata-node-4.4.1.tgz#23f6a5cb9cb55315912cbec4ff7b0f59bbd191e2" + integrity sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ== + dependencies: + node-domexception "1.0.0" + web-streams-polyfill "4.0.0-beta.3" + forwarded@0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/forwarded/-/forwarded-0.2.0.tgz#2269936428aad4c15c7ebe9779a84bf0b2a81811" @@ -2978,6 +3013,20 @@ graceful-fs@^4.1.2, graceful-fs@^4.1.6, graceful-fs@^4.2.0: resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.10.tgz#147d3a006da4ca3ce14728c7aefc287c367d7a6c" integrity sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA== +groq-sdk@^0.5.0: + version "0.5.0" + resolved "https://registry.yarnpkg.com/groq-sdk/-/groq-sdk-0.5.0.tgz#8eefea81c3709e815c96dffa941200e85a50cf19" + integrity sha512-RVmhW7qZ+XZoy5fIuSdx/LGQJONpL8MHgZEW7dFwTdgkzStub2XQx6OKv28CHogijdwH41J+Npj/z2jBPu3vmw== + dependencies: + "@types/node" "^18.11.18" + "@types/node-fetch" "^2.6.4" + abort-controller "^3.0.0" + agentkeepalive "^4.2.1" + form-data-encoder "1.7.2" + formdata-node "^4.3.2" + node-fetch "^2.6.7" + web-streams-polyfill "^3.2.1" + growl@1.10.5: version "1.10.5" resolved "https://registry.yarnpkg.com/growl/-/growl-1.10.5.tgz#f2735dc2283674fa67478b10181059355c369e5e" @@ -3155,6 +3204,13 @@ https-proxy-agent@^7.0.1: agent-base "^7.0.2" debug "4" +humanize-ms@^1.2.1: + version "1.2.1" + resolved "https://registry.yarnpkg.com/humanize-ms/-/humanize-ms-1.2.1.tgz#c46e3159a293f6b896da29316d8b6fe8bb79bbed" + integrity sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ== + dependencies: + ms "^2.0.0" + iconv-lite@0.4.24: version "0.4.24" resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.24.tgz#2022b4b25fbddc21d2f524974a474aafe733908b" @@ -3921,7 +3977,7 @@ ms@2.1.2: resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.2.tgz#d09d1f357b443f493382a8eb3ccd183872ae6009" integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w== -ms@2.1.3, ms@^2.1.1: +ms@2.1.3, ms@^2.0.0, ms@^2.1.1: version "2.1.3" resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2" integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA== @@ -3967,6 +4023,11 @@ node-addon-api@^8.1.0: resolved "https://registry.yarnpkg.com/node-addon-api/-/node-addon-api-8.1.0.tgz#55a573685dd4bd053f189cffa4e6332d2b1f1645" integrity sha512-yBY+qqWSv3dWKGODD6OGE6GnTX7Q2r+4+DfpqxHSHh8x0B4EKP9+wVGLS6U/AM1vxSNNmUEuIV5EGhYwPpfOwQ== +node-domexception@1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/node-domexception/-/node-domexception-1.0.0.tgz#6888db46a1f71c0b76b3f7555016b63fe64766e5" + integrity sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ== + node-ensure@^0.0.0: version "0.0.0" resolved "https://registry.yarnpkg.com/node-ensure/-/node-ensure-0.0.0.tgz#ecae764150de99861ec5c810fd5d096b183932a7" @@ -3979,7 +4040,7 @@ node-fetch@2.6.7: dependencies: whatwg-url "^5.0.0" -node-fetch@^2.6.9: +node-fetch@^2.6.7, node-fetch@^2.6.9: version "2.7.0" resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.7.0.tgz#d0f0fa6e3e2dc1d27efcd8ad99d550bda94d187d" integrity sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A== @@ -5495,6 +5556,16 @@ vuvuzela@1.0.3: resolved "https://registry.yarnpkg.com/vuvuzela/-/vuvuzela-1.0.3.tgz#3be145e58271c73ca55279dd851f12a682114b0b" integrity sha512-Tm7jR1xTzBbPW+6y1tknKiEhz04Wf/1iZkcTJjSFcpNko43+dFW6+OOeQe9taJIug3NdfUAjFKgUSyQrIKaDvQ== +web-streams-polyfill@4.0.0-beta.3: + version "4.0.0-beta.3" + resolved "https://registry.yarnpkg.com/web-streams-polyfill/-/web-streams-polyfill-4.0.0-beta.3.tgz#2898486b74f5156095e473efe989dcf185047a38" + integrity sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug== + +web-streams-polyfill@^3.2.1: + version "3.3.3" + resolved "https://registry.yarnpkg.com/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz#2073b91a2fdb1fbfbd401e7de0ac9f8214cecb4b" + integrity sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw== + webidl-conversions@^3.0.0: version "3.0.1" resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-3.0.1.tgz#24534275e2a7bc6be7bc86611cc16ae0a5654871" From b213caa640dc5bef27e64e174bcbe6ac5eb93655 Mon Sep 17 00:00:00 2001 From: Chris Date: Sat, 24 Aug 2024 22:59:55 +0930 Subject: [PATCH 031/182] Improve API endpoints. Add universal search. --- src/api/v1/db/controller.ts | 5 +++-- src/api/v1/search/controller.ts | 34 +++++++++++++++++++++++++++++++-- src/api/v1/search/routes.ts | 1 + src/services/data.ts | 13 +++++++++++-- 4 files changed, 47 insertions(+), 6 deletions(-) diff --git a/src/api/v1/db/controller.ts b/src/api/v1/db/controller.ts index 85c14fb0..00808e4e 100644 --- a/src/api/v1/db/controller.ts +++ b/src/api/v1/db/controller.ts @@ -60,11 +60,12 @@ export class DbController { permissions }) - const selector = req.body.query + const filter = req.body.query || {} const options = req.body.options || {} - const results = await (await db).getMany(selector, options) + const results = await db.getMany(filter, options) res.json(results) } catch (error) { + console.log(error) res.status(500).send(error.message); } } diff --git a/src/api/v1/search/controller.ts b/src/api/v1/search/controller.ts index c84c1e76..4b128ff4 100644 --- a/src/api/v1/search/controller.ts +++ b/src/api/v1/search/controller.ts @@ -1,6 +1,6 @@ import { Request, Response } from "express"; import { Utils } from "../../../utils"; -import { SearchService } from "../../../services/search" +import { SearchService, SearchType } from "../../../services/search" class SearchController { @@ -17,7 +17,7 @@ class SearchController { const mergeOverlaps = true const searchService = new SearchService(did, context) - const results = await searchService.chatThreads(keywords, threadSize, limit, mergeOverlaps) + const results = await searchService.chatThreadsByKeywords(keywords, threadSize, limit, mergeOverlaps) return res.json({ keywords, @@ -30,6 +30,36 @@ class SearchController { } } + public async universal(req: Request, res: Response) { + try { + const { context, account } = await Utils.getNetworkFromRequest(req) + const did = await account.did() + const keywordString = req.body.keywords ? req.body.keywords.toString() : "" + const keywords = keywordString.split(' ') + + const options = req.body.options || {} + const searchTypes = req.body.searchTypes ? req.body.searchTypes.split(',') : [ + SearchType.EMAILS, + SearchType.CHAT_MESSAGES + ] + const limit = req.body.limit ? req.body.limit : 20 + const minResultsPerType = req.body.minResultsPerType ? req.body.minResultsPerType : 5 + + const searchService = new SearchService(did, context) + const results = await searchService.multiByKeywords(searchTypes, keywords, limit, minResultsPerType) + + return res.json({ + keywords, + results + }) + + } catch (error) { + console.log(error) + res.status(500).send(error.message); + } + + } + public async email() { } diff --git a/src/api/v1/search/routes.ts b/src/api/v1/search/routes.ts index 262d9c82..2cda88c4 100644 --- a/src/api/v1/search/routes.ts +++ b/src/api/v1/search/routes.ts @@ -7,6 +7,7 @@ const router = express.Router() router.get("/email", controller.email) router.get("/chatHistory", controller.chatHistory) router.get("/chatThreads", controller.chatThreads) +router.post("/universal", controller.universal) router.get("/hotload", controller.hotLoad) diff --git a/src/services/data.ts b/src/services/data.ts index 34b8f69e..a058be82 100644 --- a/src/services/data.ts +++ b/src/services/data.ts @@ -1,4 +1,4 @@ -import { IContext } from '@verida/types'; +import { IContext, IDatastore } from '@verida/types'; import * as CryptoJS from 'crypto-js'; import { EventEmitter } from 'events' import MiniSearch from 'minisearch'; @@ -41,9 +41,14 @@ const schemas: Record = { indexFields: ['name','fromName','fromEmail','messageText','attachments_0.textContent','attachments_1.textContent','attachments_2.textContent', 'indexableText', 'sentAt'] }, "https://common.schemas.verida.io/social/chat/message/v0.1.0/schema.json": { - label: "Chat History", + label: "Chat Message", storeFields: ['_id', 'groupId'], indexFields: ['messageText', 'fromHandle', 'fromName', 'groupName', 'indexableText', 'sentAt'] + }, + "https://common.schemas.verida.io/favourite/v0.1.0/schema.json": { + label: "Favourite", + storeFields: ['_id'], + indexFields: ['name', 'favouriteType', 'contentTYpe', 'summary'] } } @@ -76,6 +81,10 @@ export class DataService extends EventEmitter { this.emit('progress', progress) } + public async getDatastore(schemaUri: string): Promise { + return this.context.openDatastore(schemaUri) + } + public async getIndex(schemaUri: string): Promise> { const schemaConfig = schemas[schemaUri] const indexFields = schemaConfig.indexFields From acdcd6057c7ad942f2af3d3a6b9755e096221206 Mon Sep 17 00:00:00 2001 From: Chris Date: Sat, 24 Aug 2024 23:00:24 +0930 Subject: [PATCH 032/182] Improve API dashboard. --- src/web/developer/api/api.js | 300 ++++++++++++++++++----------- src/web/developer/api/endpoints.js | 215 +++++++++++++++++++++ src/web/developer/api/index.html | 50 ++++- src/web/developer/styles.css | 27 +++ 4 files changed, 481 insertions(+), 111 deletions(-) create mode 100644 src/web/developer/api/endpoints.js create mode 100644 src/web/developer/styles.css diff --git a/src/web/developer/api/api.js b/src/web/developer/api/api.js index ca32d07d..3172122c 100644 --- a/src/web/developer/api/api.js +++ b/src/web/developer/api/api.js @@ -1,76 +1,48 @@ -const commonParams = { - "provider": { - "type": "string", - "required": true, - "documentation": "The name of the provider to connect to, ie: `google`", - "default": "google" - }, - "providerId": { - "type": "string", - "required": false, - "documentation": "The unique provider ID to use. For example, if you have two Google accounts connected, you can specify which account. The provider ID is listed in the /dashboard/connections table.", - }, + +function saveState() { + const state = { + selectedEndpoint: $('#endpointSelect').val(), + urlVariables: {}, + queryParams: {}, + selectedLanguage: $('#codeExampleTabs .nav-link.active').attr('id').replace('-tab', '') + }; + + // Save URL variables + $('.url-variable').each(function() { + state.urlVariables[$(this).attr('id')] = $(this).val(); + }); + + // Save query parameters + $('#endpointOptions input, #endpointOptions textarea').each(function() { + state.queryParams[$(this).attr('name')] = $(this).val(); + }); + + localStorage.setItem('apiTestState', JSON.stringify(state)); } -// Global JSON object with endpoint configurations -const apiEndpoints = { - "/db/get": { - "method": "GET", - "path": "/api/v1/db/get", - "documentation": "Retrieves a list of available providers." - }, - "/db/query": { - "method": "POST", - "path": "/api/v1/db/query", - "documentation": "Query a database", - "params": { - "query": { - "type": "string", - "required": false, - "documentation": "A pouchdb style filter ie: {name: \"John\"}" - }, - "options": { - "type": "string", - "required": false, - "documentation": "Additional options provided as JSON. Available options are; sort, limit, skip as per the pouchdb documentation.", - "default": JSON.stringify({ - sort: [{ - _id: "desc" - }], - limit: 20 - }) - } +function loadState() { + const savedState = localStorage.getItem('apiTestState'); + if (savedState) { + const state = JSON.parse(savedState); + + // Set selected endpoint + $('#endpointSelect').val(state.selectedEndpoint).change(); + + // Set URL variables + for (let id in state.urlVariables) { + $(`#${id}`).val(state.urlVariables[id]); } - }, - "/providers": { - "method": "GET", - "path": "/api/v1/providers", - "documentation": "Retrieves a list of available providers." - }, - "/sync": { - "method": "GET", - "path": "/api/v1/sync", - "documentation": "Start syncing data for a given provider", - "params": { - "provider": commonParams.provider, - "providerId": commonParams.providerId, - "force": { - "type": "boolean", - "required": false, - "documentation": "Force the sync to occur, ignoring the current status of the connection." - } + + // Set query parameters + for (let name in state.queryParams) { + $(`#endpointOptions [name="${name}"]`).val(state.queryParams[name]); } - }, - "/syncStatus": { - "method": "GET", - "path": "/api/v1/syncStatus", - "params": { - "provider": commonParams.provider, - "providerId": commonParams.providerId, - }, - "documentation": "Get the status of the current sync connection for a provider." + + // Set selected language + $(`.code-example`).removeClass('active'); + $(`#codeExampleTabs a[href="#${state.selectedLanguage}"]`).tab('show'); } -}; +} $(document).ready(function() { // Load the private key from local storage @@ -91,21 +63,13 @@ $(document).ready(function() { })); } - // Update endpoint options when selection changes - $('#endpointSelect').change(function() { - updateEndpointOptions($(this).val()); - }); - - // Initial update of endpoint options - updateEndpointOptions($('#endpointSelect').val()); - // Run endpoint button click handler $('#runEndpoint').click(function() { runEndpoint(); }); // Update code examples when any input changes - $(document).on('input', 'input, select', function() { + $(document).on('input', 'input, select, textarea', function() { updateCodeExamples(); }); @@ -118,46 +82,143 @@ $(document).ready(function() { if (!$('#privateKey').val()) { $('#settingsPanel').show(); } + + // Add event listeners for code example toggles + $('#codeExampleTabs a[data-toggle="tab"]').on('shown.bs.tab', function (e) { + console.log('Tab clicked:', $(e.target).attr('href')); + saveState(); + }); + + $('#showPrivateKey').change(function() { + updateCodeExamples(); + }); + + // Load saved state + loadState(); + + // Save state when inputs change + $(document).on('input', 'input, select, textarea', function() { + saveState(); + updateCodeExamples(); + }); + + // Save state when endpoint selection changes + $('#endpointSelect').change(function() { + updateEndpointOptions($(this).val()); + saveState(); + }); + + // Save state when language selection changes + $(document).on('click', '.code-example-toggle', function(e) { + e.preventDefault(); + console.log('toggle'); + $('.code-example').removeClass('active'); + $($(this).attr('href')).addClass('active'); + saveState(); + }); + + // Initial update of endpoint options + updateEndpointOptions($('#endpointSelect').val()); }); function updateEndpointOptions(endpoint) { const endpointConfig = apiEndpoints[endpoint]; + let urlVariablesHtml = ''; let optionsHtml = ''; // Add endpoint documentation - $('#endpointDocumentation').html(`

Documentation: ${endpointConfig.documentation}

`); + $('#endpointDocumentation').html(` + + `); + + $('#docContent').html(marked.parse(endpointConfig.documentation)); + + // Handle URL variables + if (endpointConfig.urlVariables) { + // urlVariablesHtml += '

URL Variables:

'; + for (let variable in endpointConfig.urlVariables) { + const variableConfig = endpointConfig.urlVariables[variable]; + urlVariablesHtml += ` +
+
+ + +
+
+ ${marked.parse(variableConfig.documentation)} +
+
+ `; + } + } + // Handle regular parameters if (endpointConfig.params) { for (let param in endpointConfig.params) { const paramConfig = endpointConfig.params[param]; optionsHtml += ` -
- - - ${paramConfig.documentation} +
+
+ + ${paramConfig.type === 'object' ? + `` : + `` + } +
+
+ ${marked.parse(paramConfig.documentation)} +
`; } } + $('#urlVariables').html(urlVariablesHtml); $('#endpointOptions').html(optionsHtml); + + // Initialize JSON formatting for the 'options' parameter + if (endpointConfig.params && endpointConfig.params.options) { + const optionsTextArea = $('#options'); + optionsTextArea.val(JSON.stringify(JSON.parse(optionsTextArea.val()), null, 2)); + } + updateCodeExamples(); } function updateCodeExamples() { const endpoint = $('#endpointSelect').val(); - const privateKey = ''// $('#privateKey').val(); + const privateKey = $('#showPrivateKey').is(':checked') ? $('#privateKey').val() : ''; const baseUrl = $('#baseUrl').val(); const endpointConfig = apiEndpoints[endpoint]; - const url = `${baseUrl}${endpointConfig.path}`; + let url = `${baseUrl}${endpointConfig.path}`; const method = endpointConfig.method; const data = {}; - $('#endpointOptions input').each(function() { + // Handle URL variables + $('.url-variable').each(function() { + const variableName = $(this).attr('name'); + const variableValue = $(this).val() || `{${variableName}}`; + url = url.replace(`{${variableName}}`, variableValue); + }); + + $('#endpointOptions input, #endpointOptions textarea').each(function() { const paramName = $(this).attr('name'); - const paramValue = $(this).val() || (endpointConfig.params && endpointConfig.params[paramName] ? endpointConfig.params[paramName].default : ''); + let paramValue = $(this).val() || (endpointConfig.params && endpointConfig.params[paramName] ? endpointConfig.params[paramName].default : ''); + + if (paramName === 'options') { + try { + paramValue = JSON.parse(paramValue); + } catch (e) { + console.error('Invalid JSON in options field'); + } + } + if (paramValue) { data[paramName] = paramValue; } @@ -166,7 +227,7 @@ function updateCodeExamples() { // Update cURL example let curlCommand = `curl -X ${method} `; if (privateKey) { - curlCommand += `-H "Authorization: Bearer ${privateKey}" `; + curlCommand += `-H "key: ${privateKey}" `; } curlCommand += `"${url}`; if (method === 'GET' && Object.keys(data).length > 0) { @@ -190,7 +251,7 @@ async function makeRequest() { url: '${url}', ${method === 'GET' ? `params: ${JSON.stringify(data)}` : `data: ${JSON.stringify(data)}`}, headers: { - ${privateKey ? `'Authorization': 'Bearer ${privateKey}',` : ''} + ${privateKey ? `'key': '${privateKey}',` : ''} 'Content-Type': 'application/json' } }); @@ -210,7 +271,7 @@ makeRequest();`; ${method === 'GET' ? `data: ${JSON.stringify(data)},` : `data: JSON.stringify(${JSON.stringify(data)}),`} contentType: 'application/json', headers: { - ${privateKey ? `'Authorization': 'Bearer ${privateKey}'` : ''} + ${privateKey ? `'key': '${privateKey}'` : ''} }, success: function(response) { console.log(response); @@ -236,7 +297,7 @@ curl_setopt_array($curl, array( ${method === 'POST' ? `CURLOPT_POSTFIELDS => '${JSON.stringify(data)}',` : ''} CURLOPT_HTTPHEADER => array( "content-type: application/json"${privateKey ? `, - "authorization: Bearer ${privateKey}"` : ''} + "key: ${privateKey}"` : ''} ), )); @@ -258,7 +319,7 @@ if ($err) { url = "${url}" headers = { "Content-Type": "application/json"${privateKey ? `, - "Authorization": "Bearer ${privateKey}"` : ''} + "key": "${privateKey}"` : ''} } ${method === 'GET' ? `params = ${JSON.stringify(data)}` : `data = ${JSON.stringify(data)}`} @@ -273,40 +334,61 @@ function runEndpoint() { const privateKey = $('#privateKey').val(); const baseUrl = $('#baseUrl').val(); const endpointConfig = apiEndpoints[endpoint]; - const url = `${baseUrl}${endpointConfig.path}`; + let url = `${baseUrl}${endpointConfig.path}`; const method = endpointConfig.method; const data = {}; - $('#endpointOptions input').each(function() { + // Handle URL variables + if (endpointConfig.urlVariables) { + for (let variable in endpointConfig.urlVariables) { + let inputValue = $(`#${variable}`).val(); + if (inputValue) { + if (endpointConfig.urlVariables[variable].preProcessing) { + inputValue = endpointConfig.urlVariables[variable].preProcessing(inputValue) + } + + url = url.replace(`{${variable}}`, encodeURIComponent(inputValue)); + } + } + } + + // Handle other parameters + $('#endpointOptions input, #endpointOptions textarea').each(function() { const paramName = $(this).attr('name'); - const paramValue = $(this).val() || (endpointConfig.params && endpointConfig.params[paramName] ? endpointConfig.params[paramName].default : ''); + let paramValue = $(this).val() || (endpointConfig.params && endpointConfig.params[paramName] ? endpointConfig.params[paramName].default : ''); + + if ($(this).is("textarea")) { + try { + paramValue = paramValue ? JSON.parse(paramValue) : {} + } catch (e) { + console.error(`Invalid JSON in ${paramName} field`); + } + } + if (paramValue) { data[paramName] = paramValue; } }); - data.key = privateKey + const headers = { + key: privateKey + } + + $('#result').text('Request sent... waiting...') $.ajax({ url: url, method: method, data: method === 'GET' ? data : JSON.stringify(data), + headers, contentType: 'application/json', - /*headers: { - 'Authorization': `Bearer ${privateKey}` - },*/ success: function(response) { - $('#result').text(JSON.stringify(response, null, 2)); + const $customElement = $(`${JSON.stringify(response)}`); + $('#result').empty() + $('#result').append($customElement) }, error: function(xhr, status, error) { $('#result').text(`Error: ${error}\n\nResponse: ${xhr.responseText}`); } }); -} - -// Check private key on input change -$('#privateKey').on('input', function() { - if (!$(this).val()) { - $('#settingsPanel').show(); - } -}); \ No newline at end of file +} \ No newline at end of file diff --git a/src/web/developer/api/endpoints.js b/src/web/developer/api/endpoints.js new file mode 100644 index 00000000..1059b061 --- /dev/null +++ b/src/web/developer/api/endpoints.js @@ -0,0 +1,215 @@ +const apiPrefix = `/api/v1` + +const commonParams = { + "provider": { + "type": "string", + "required": true, + "documentation": "The name of the provider to connect to, ie: `google`", + "default": "google" + }, + "providerId": { + "type": "string", + "required": false, + "documentation": "The unique provider ID to use. For example, if you have two Google accounts connected, you can specify which account. The provider ID is listed in the /dashboard/connections table.", + }, + "query": { + "type": "object", + "required": false, + "documentation": `A pouchdb style filter. + +**Example:** + +\`\`\` +{ + category: "sport", + insertedAt: { + "$gte": "2020-01-01" + } +} +\`\`\` +` + }, + "options": { + "type": "object", + "required": false, + "documentation": `Additional options provided as JSON. Available options are; sort, limit, skip as per the pouchdb documentation. + +**Example:** + +\`\`\` +{ + sort: [{ + _id: "desc" + }], + limit: 20 +} +\`\`\` +`, + "default": JSON.stringify({ + sort: [{ + _id: "desc" + }], + limit: 20 + }) + } +} + +const commonUrlVariables = { + "databaseName": { + "type": "string", + "required": true, + "documentation": "The name of the database (ie: `social_chat_group`)." + }, + "schemaUrl": { + "type": "string", + "required": true, + "documentation": "The base64 encoded URL of the datastore schema (ie: `https://common.schemas.verida.io/social/chat/group/v0.1.0/schema.json` is encoded to `aHR0cHM6Ly9jb21tb24uc2NoZW1hcy52ZXJpZGEuaW8vc29jaWFsL2NoYXQvZ3JvdXAvdjAuMS4wL3NjaGVtYS5qc29u`).\n\nEnter the schema URL in the input box and it will be automatically converted to base64.", + "preProcessing": (value) => btoa(value), + "default": "https://common.schemas.verida.io/social/chat/group/v0.1.0/schema.json" + } +} + +// Global JSON object with endpoint configurations +const apiEndpoints = { + "/ds/query/{schemaUrl}": { + "method": "POST", + "path": `${apiPrefix}/ds/query/{schemaUrl}`, + "documentation": "Query a datastore", + "urlVariables": { + "schemaUrl": commonUrlVariables.schemaUrl + }, + "params": { + "query": commonParams.query, + "options": { + "type": "object", + "required": false, + "documentation": "Additional options provided as JSON. Available options are; sort, limit, skip as per the pouchdb documentation.", + "default": JSON.stringify({ + sort: [{ + _id: "desc" + }], + limit: 20 + }) + } + } + }, + "/ds/get/{schemaUrl}/{recordId}": { + "method": "GET", + "path": `${apiPrefix}/get/{schemaUrl}/{recordId}`, + "documentation": "Retrieves a record from a datastore.", + "urlVariables": { + "schemaUrl": commonUrlVariables.schemaUrl, + "recordId": { + "type": "string", + "required": true, + "documentation": "The unique ID of the record to fetch." + } + } + }, + "/db/query/{databaseName}": { + "method": "POST", + "path": `${apiPrefix}/db/query/{databaseName}`, + "documentation": "Query a database", + "urlVariables": { + "databaseName": commonUrlVariables.databaseName + }, + "params": { + "query": commonParams.query, + "options": commonParams.options + } + }, + "/db/get/{databaseName}/{recordId}": { + "method": "GET", + "path": `${apiPrefix}/get/{databaseName}/{recordId}`, + "documentation": "Retrieves a record from a database.", + "urlVariables": { + "databaseName": commonUrlVariables.databaseName, + "recordId": { + "type": "string", + "required": true, + "documentation": "The unique ID of the record to fetch." + } + } + }, + "/search/universal": { + "method": "POST", + "path": `${apiPrefix}/search/universal`, + "documentation": "Universal keyword search across multiple datastores", + "params": { + "keywords": { + "type": "string", + "documentation": "List of keywords to search for", + "default": "robert gray", + "required": true + }, + "limit": { + "type": "number", + "documentation": "Limit results. Defaults to `20`.", + "default": 5 + }, + "minResultsPerType": { + "type": "number", + "documentation": "Minimum number of results per type (ie: `emails`). Defaults to `5`.", + "default": 5 + }, + "searchTypes": { + "type": "string", + "documentation": `Comma separated list of record types to search: + +- chat-messages: Individual chat messages +- emails: Individual emails +- favorites: Individual favorites +- following: Individual social media accounts followed +- posts: Individual social media posts + +Defaults to \`"emails,chat-messages"\`. +`, + "default": `emails,chat-messages` + } + } + }, + "/providers": { + "method": "GET", + "path": `${apiPrefix}/providers`, + "documentation": "Retrieves a list of available providers." + }, + "/sync": { + "method": "GET", + "path": "/api/v1/sync", + "documentation": "Start syncing data for a given provider", + "params": { + "provider": commonParams.provider, + "providerId": commonParams.providerId, + "force": { + "type": "boolean", + "required": false, + "documentation": "Force the sync to occur, ignoring the current status of the connection." + } + } + }, + "/syncStatus": { + "method": "GET", + "path": "/api/v1/syncStatus", + "params": { + "provider": commonParams.provider, + "providerId": commonParams.providerId, + }, + "documentation": "Get the status of the current sync connection for a provider." + }, + "/admin/memory": { + "method": "GET", + "path": `${apiPrefix}/admin/memory`, + "documentation": `Memory usage of the server. + +**rss (Resident Set Size):** This represents the total memory allocated for the process, including code, stack, and heap. It is the overall memory usage by the server (in bytes), which includes all allocations by the operating system, not just the memory allocated by the JavaScript engine (V8). + +**heapTotal:** This is the total size of the heap that V8 (the JavaScript engine) has reserved for the server. It indicates the amount of memory allocated for the JavaScript objects and functions that your application may use. + +**heapUsed:** This represents the actual memory used by JavaScript objects and functions within the total allocated heap (heapTotal). It shows how much of the heap is currently occupied by active data. + +**external:** This refers to the memory used by C++ objects bound to JavaScript objects. It is memory outside of V8's JavaScript heap but managed by native code and typically allocated for objects that V8 doesn’t directly manage. + +**arrayBuffers:** This is the memory allocated for ArrayBuffer and SharedArrayBuffer instances in JavaScript. It indicates how much memory is consumed specifically by array buffer-backed objects. +` + } +}; \ No newline at end of file diff --git a/src/web/developer/api/index.html b/src/web/developer/api/index.html index 98efd20f..8aafb661 100644 --- a/src/web/developer/api/index.html +++ b/src/web/developer/api/index.html @@ -23,6 +23,34 @@ padding: 15px; margin-bottom: 20px; } + .param-row { + display: flex; + margin-bottom: 15px; + } + .param-input { + flex: 1; + padding-right: 15px; + } + .param-docs { + flex: 1; + margin-top: 25px; + } + .endpoint-docs { + margin-top: 20px; + margin-bottom: 20px; + } + #codeExampleTabs .nav-tabs .nav-link.active { + background-color: #f8f9fa; + } + .json-pre { + white-space: pre-wrap; /* Wrap long lines */ + word-break: break-word; + overflow-x: auto; + font-family: monospace; /* Monospace font for better JSON readability */ + } + #result { + min-height: 400px; + } @@ -78,12 +106,27 @@

Developer API Interface

+

Code Examples:

+ +
+ + +
-
+ 

                 

@@ -122,13 +165,16 @@ 
Code Examples:

         
         

             
Result:

-            

+            

         

     

 
     
     
     
+    
+    
+    
     
 
 
\ No newline at end of file
diff --git a/src/web/developer/styles.css b/src/web/developer/styles.css
new file mode 100644
index 00000000..ee4b0d9b
--- /dev/null
+++ b/src/web/developer/styles.css
@@ -0,0 +1,27 @@
+.filter-panel {
+    margin-bottom: 20px;
+    border: 1px solid #ddd;
+    padding: 10px;
+    border-radius: 5px;
+    display: none; /* Initially hidden */
+}
+.filter-panel input {
+    margin-bottom: 5px;
+}
+
+.nav-link.active {
+    border-bottom: 2px solid;
+}
+
+.modal-dialog {
+    max-height: 80vh; /* Adjust as needed */
+    margin: 30px auto;
+}
+.modal-content {
+    height: 100%;
+    display: flex;
+    flex-direction: column;
+}
+.modal-body {
+    overflow-y: auto;
+}
\ No newline at end of file

From 474201aac4793ef0d4efbeb81a28ec69ab3b1fe7 Mon Sep 17 00:00:00 2001
From: Chris 
Date: Sat, 24 Aug 2024 23:00:52 +0930
Subject: [PATCH 033/182] Close event source connection to server when log
 modal is closed

---
 src/web/user/ai/ai.js | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/web/user/ai/ai.js b/src/web/user/ai/ai.js
index 7752bd7f..bd55694a 100644
--- a/src/web/user/ai/ai.js
+++ b/src/web/user/ai/ai.js
@@ -48,8 +48,6 @@ $(document).ready(function() {
             urlType = "personal"
         }
 
-        console.log(urlType)
-
         const body = { prompt: prompt, key: veridaKey };
 
         $.ajax({
@@ -101,6 +99,7 @@ $(document).ready(function() {
         if (data.status === 'Load Complete' && data.totalProgress >= 1) {
             loadComplete = true
             $('#loading-overlay').fadeOut();
+            eventSource.close()
         }
     };
 

From 6aec966176145833056d64fa8f7f1d5c4da9d50b Mon Sep 17 00:00:00 2001
From: Chris 
Date: Sun, 25 Aug 2024 08:54:12 +0930
Subject: [PATCH 034/182] Add search endpoints to API playground. Make
 improvements to search endpoints.

---
 src/api/v1/minisearch/controller.ts |  5 +-
 src/api/v1/search/controller.ts     | 15 +++---
 src/api/v1/search/routes.ts         |  2 +-
 src/services/minisearch.ts          | 10 ++--
 src/services/search.ts              |  2 +-
 src/web/developer/api/api.js        | 53 ++++++++++++++++---
 src/web/developer/api/endpoints.js  | 82 ++++++++++++++++++++++++++++-
 src/web/developer/api/index.html    | 77 ++++++++++-----------------
 8 files changed, 170 insertions(+), 76 deletions(-)

diff --git a/src/api/v1/minisearch/controller.ts b/src/api/v1/minisearch/controller.ts
index bd27f5bf..a19bbd85 100644
--- a/src/api/v1/minisearch/controller.ts
+++ b/src/api/v1/minisearch/controller.ts
@@ -16,13 +16,14 @@ export class DsController {
             const did = await account.did()
 
             const schemaName = Utils.getSchemaFromParams(req.params[0])
-            const query = req.query.q.toString()
+            const query = req.query.keywords.toString()
+            const searchOptions = req.query.options ? JSON.parse(req.query.options.toString()) : {}
             const indexFields = req.query.fields ? req.query.fields.toString().split(',') : []
             let storeFields = req.query.store ? req.query.store.toString().split(',') : []
             const permissions = Utils.buildPermissions(req)
             const limit = req.query.limit ? parseInt(req.query.limit.toString()) : MAX_RESULTS
 
-            const result = await MinisearchService.searchDs(context, did, schemaName, query, indexFields, storeFields, limit, permissions)
+            const result = await MinisearchService.searchDs(context, did, schemaName, query, searchOptions, indexFields, storeFields, limit, permissions)
             return res.json(result)
         } catch (error) {
             console.log(error)
diff --git a/src/api/v1/search/controller.ts b/src/api/v1/search/controller.ts
index 4b128ff4..de841ea7 100644
--- a/src/api/v1/search/controller.ts
+++ b/src/api/v1/search/controller.ts
@@ -12,9 +12,9 @@ class SearchController {
             const keywordString = req.query.keywords ? req.query.keywords.toString() : ""
             const keywords = keywordString.split(' ')
 
-            const threadSize = 10
-            const limit = 10
-            const mergeOverlaps = true
+            const threadSize = req.query.threadSize ? parseInt(req.query.threadSize.toString()) : 10
+            const limit = req.query.limit ? parseInt(req.query.limit.toString()) : 20
+            const mergeOverlaps = req.query.limit ? req.query.merge.toString() == 'true' : true
 
             const searchService = new SearchService(did, context)
             const results = await searchService.chatThreadsByKeywords(keywords, threadSize, limit, mergeOverlaps)
@@ -34,16 +34,15 @@ class SearchController {
         try {
             const { context, account } = await Utils.getNetworkFromRequest(req)
             const did = await account.did()
-            const keywordString = req.body.keywords ? req.body.keywords.toString() : ""
+            const keywordString = req.query.keywords ? req.query.keywords.toString() : ""
             const keywords = keywordString.split(' ')
 
-            const options = req.body.options || {}
-            const searchTypes = req.body.searchTypes ? req.body.searchTypes.split(',') : [
+            const searchTypes = req.query.searchTypes ?  req.query.searchTypes.toString().split(',') : [
                 SearchType.EMAILS,
                 SearchType.CHAT_MESSAGES
             ]
-            const limit = req.body.limit ? req.body.limit : 20
-            const minResultsPerType = req.body.minResultsPerType ? req.body.minResultsPerType : 5
+            const limit = req.query.limit ? parseInt(req.query.limit.toString()) : 20
+            const minResultsPerType = req.query.minResultsPerType ? parseInt(req.query.minResultsPerType.toString()) : 5
 
             const searchService = new SearchService(did, context)
             const results = await searchService.multiByKeywords(searchTypes, keywords, limit, minResultsPerType)
diff --git a/src/api/v1/search/routes.ts b/src/api/v1/search/routes.ts
index 2cda88c4..49c5fedf 100644
--- a/src/api/v1/search/routes.ts
+++ b/src/api/v1/search/routes.ts
@@ -7,7 +7,7 @@ const router = express.Router()
 router.get("/email", controller.email)
 router.get("/chatHistory", controller.chatHistory)
 router.get("/chatThreads", controller.chatThreads)
-router.post("/universal", controller.universal)
+router.get("/universal", controller.universal)
 router.get("/hotload", controller.hotLoad)
 
 
diff --git a/src/services/minisearch.ts b/src/services/minisearch.ts
index b58c0e00..5aac1c13 100644
--- a/src/services/minisearch.ts
+++ b/src/services/minisearch.ts
@@ -19,6 +19,7 @@ export class MinisearchService {
     did: string,
     schemaName: string,
     query: string,
+    options: object = {},
     indexFields: string[],
     storeFields: string[] = [],
     limit: number = 20,
@@ -96,11 +97,6 @@ export class MinisearchService {
               }
 
               // @todo: Make sure the original field isn't stored (`arrayProperty`)
-
-              console.log(
-                arrayItem.filename,
-                arrayItem.textContent.substring(0, 100)
-              );
               i++;
             }
           }
@@ -129,8 +125,8 @@ export class MinisearchService {
       console.log("cache match!");
     }
 
-    console.log("Searching...");
-    const results = indexCache[cacheKey].search(query);
+    console.log("Searching...", query, options);
+    const results = indexCache[cacheKey].search(query, options);
 
     return {
       results: results.slice(0, limit),
diff --git a/src/services/search.ts b/src/services/search.ts
index 5b437c72..c0ce610b 100644
--- a/src/services/search.ts
+++ b/src/services/search.ts
@@ -174,7 +174,7 @@ export class SearchService extends VeridaService {
      * @param mergeOverlaps If there is an overlap of messages within the same chat group, they will be merged into a single thread.
      * @returns 
      */
-    public async chatThreadsByKeywords(keywordsList: string[], threadSize: 10, limit: number = 20, mergeOverlaps: boolean = true): Promise {
+    public async chatThreadsByKeywords(keywordsList: string[], threadSize: number = 10, limit: number = 20, mergeOverlaps: boolean = true): Promise {
         const query = keywordsList.join(' ')
         const messageSchemaUri = "https://common.schemas.verida.io/social/chat/message/v0.1.0/schema.json"
         const groupSchemaUri = "https://common.schemas.verida.io/social/chat/group/v0.1.0/schema.json"
diff --git a/src/web/developer/api/api.js b/src/web/developer/api/api.js
index 3172122c..9792327b 100644
--- a/src/web/developer/api/api.js
+++ b/src/web/developer/api/api.js
@@ -4,7 +4,7 @@ function saveState() {
         selectedEndpoint: $('#endpointSelect').val(),
         urlVariables: {},
         queryParams: {},
-        selectedLanguage: $('#codeExampleTabs .nav-link.active').attr('id').replace('-tab', '')
+        selectedLanguage: $('#codeExampleDropdown').text().trim().toLowerCase()
     };
 
     // Save URL variables
@@ -39,8 +39,11 @@ function loadState() {
         }
 
         // Set selected language
-        $(`.code-example`).removeClass('active');
-        $(`#codeExampleTabs a[href="#${state.selectedLanguage}"]`).tab('show');
+        const $dropdownItem = $(`.dropdown-item[data-language="${state.selectedLanguage}"]`);
+        if ($dropdownItem.length) {
+            $('#codeExampleDropdown').text($dropdownItem.text());
+            updateCodeExample(state.selectedLanguage);
+        }
     }
 }
 
@@ -84,11 +87,15 @@ $(document).ready(function() {
     }
 
     // Add event listeners for code example toggles
-    $('#codeExampleTabs a[data-toggle="tab"]').on('shown.bs.tab', function (e) {
-        console.log('Tab clicked:', $(e.target).attr('href'));
+    $('.code-examples .dropdown-item').on('click', function (e) {
+        e.preventDefault();
+        const selectedLanguage = $(this).data('language');
+        $('#codeExampleDropdown').text($(this).text());
+        updateCodeExamples();
         saveState();
     });
 
+
     $('#showPrivateKey').change(function() {
         updateCodeExamples();
     });
@@ -111,7 +118,6 @@ $(document).ready(function() {
     // Save state when language selection changes
     $(document).on('click', '.code-example-toggle', function(e) {
         e.preventDefault();
-        console.log('toggle');
         $('.code-example').removeClass('active');
         $($(this).attr('href')).addClass('active');
         saveState();
@@ -119,6 +125,16 @@ $(document).ready(function() {
 
     // Initial update of endpoint options
     updateEndpointOptions($('#endpointSelect').val());
+
+    // Function to initialize the code examples
+    function initializeCodeExamples() {
+        // Set initial selection to cURL
+        $('#codeExampleDropdown').text('cURL');
+        updateCodeExample('curl');
+    }
+
+    // Call the initialization function
+    initializeCodeExamples();
 });
 
 function updateEndpointOptions(endpoint) {
@@ -191,6 +207,28 @@ function updateEndpointOptions(endpoint) {
     updateCodeExamples();
 }
 
+function updateCodeExample(language) {
+    let codeContent = '';
+    switch (language) {
+        case 'curl':
+            codeContent = $('#curlCommand').text();
+            break;
+        case 'node.js':
+            codeContent = $('#nodejsCode').text();
+            break;
+        case 'jquery':
+            codeContent = $('#jqueryCode').text();
+            break;
+        case 'php':
+            codeContent = $('#phpCode').text();
+            break;
+        case 'python':
+            codeContent = $('#pythonCode').text();
+            break;
+    }
+    $('#codeExample').text(codeContent);
+}
+
 function updateCodeExamples() {
     const endpoint = $('#endpointSelect').val();
     const privateKey = $('#showPrivateKey').is(':checked') ? $('#privateKey').val() : '';
@@ -327,6 +365,9 @@ response = requests.${method.toLowerCase()}(url, ${method === 'GET' ? 'params=pa
 
 print(response.json())`;
     $('#pythonCode').text(pythonCode);
+
+    const currentLanguage = $('#codeExampleDropdown').text().trim().toLowerCase();
+    updateCodeExample(currentLanguage);
 }
 
 function runEndpoint() {
diff --git a/src/web/developer/api/endpoints.js b/src/web/developer/api/endpoints.js
index 1059b061..e03d7104 100644
--- a/src/web/developer/api/endpoints.js
+++ b/src/web/developer/api/endpoints.js
@@ -132,7 +132,7 @@ const apiEndpoints = {
         }
     },
     "/search/universal": {
-        "method": "POST",
+        "method": "GET",
         "path": `${apiPrefix}/search/universal`,
         "documentation": "Universal keyword search across multiple datastores",
         "params": {
@@ -168,6 +168,86 @@ Defaults to \`"emails,chat-messages"\`.
             }
         }
     },
+    "/search/chatThreads": {
+        "method": "GET",
+        "path": `${apiPrefix}/search/chatThreads`,
+        "documentation": `Search chat messages by keyword and return matching chat threads to ensure the full message context is available.
+
+Each result contains the chat group and an array of messages.`,
+        "params": {
+            "keywords": {
+                "type": "string",
+                "documentation": "List of keywords to search for",
+                "default": "robert gray",
+                "required": true
+            },
+            "limit": {
+                "type": "number",
+                "documentation": "Limit how many chat threads to return. Defaults to `20`.",
+                "default": 5
+            },
+            "merge": {
+                "type": "boolean",
+                "documentation": "Merge overlapping threads. If two messages match within the same chat group, they are merged to produce a single chat group.",
+                "default": true
+            }
+        }
+    },
+    "/minisearch/ds/{schemaUrl}": {
+        "method": "GET",
+        "path": `${apiPrefix}/minisearch/ds/{schemaUrl}`,
+        "documentation": `Execute a keyword search on a datastore.
+
+It's possible to define the fields to index on and the fields to be stored in the search index which is then returned with results.
+
+The index is cached for the user, until the user cache times out.
+
+Requests with the exact same list of indexed and stored fields will re-use the same index. If there is any difference in the indexed or stored fields, a new index is created, which increases the memory footprint.`,
+        "urlVariables": {
+            "schemaUrl": commonUrlVariables.schemaUrl,
+        },
+        "params": {
+            "keywords": {
+                "type": "string",
+                "documentation": "List of keywords to search for",
+                "default": "robert gray",
+                "required": true
+            },
+            "options": {
+                "type": "object",
+                "documentation": `Search options that match the options availalbe from the [minisearch documentation](https://www.npmjs.com/package/minisearch).
+
+**Example:**
+
+\`\`\`
+{
+    fields: ['title', 'text'],
+    searchOptions: {
+        boost: { title: 2 },
+        fuzzy: 0.2
+    }
+}
+\`\`\`,
+`,
+                "default": "{}"
+            },
+            "limit": {
+                "type": "number",
+                "documentation": "Limit how many chat threads to return. Defaults to `20`.",
+                "default": 20
+            },
+            "fields": {
+                "type": "string",
+                "documentation": "Comma separated list of fields to inlude in search index (ie: `name,description`)",
+                "default": "name,description"
+            },
+            "store": {
+                "type": "string",
+                "documentation": "Comma separated list of fields to store in the index and return with results (ie: `name,description`)",
+                "default": "name,description"
+            }
+        }
+    },
     "/providers": {
         "method": "GET",
         "path": `${apiPrefix}/providers`,
diff --git a/src/web/developer/api/index.html b/src/web/developer/api/index.html
index 8aafb661..0ea4b6e7 100644
--- a/src/web/developer/api/index.html
+++ b/src/web/developer/api/index.html
@@ -100,7 +100,6 @@ 
Developer API Interface

         

         
         

-            
             
         

         
@@ -112,55 +111,34 @@ 
Developer API Interface

         
         
         

-            
Code Examples:

-            
-            

-                
-                
-            

-            
-            

-                

-                    

+            

+                

+                    
+                    

+                        cURL
+                        Node.js
+                        jQuery
+                        PHP
+                        Python
+                    

                 

-                

-                    

-                

-                

-                    

-                

-                

-                    

-                

-                

-                    

+                

+                    
+                    
                 

             

+            

+                

+                
+                
+                
+                
+                
+            

         

         
         

@@ -169,9 +147,8 @@ 
Result:

         

     

 
-    
-    
-    
+    
+    
     
     
     

From 7d7944e10fe31886787f0683c62c064e8570be24 Mon Sep 17 00:00:00 2001
From: Chris 
Date: Sun, 25 Aug 2024 17:00:49 +0930
Subject: [PATCH 035/182] Fix API state loading

---
 src/web/developer/api/api.js | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/web/developer/api/api.js b/src/web/developer/api/api.js
index 9792327b..5783b81d 100644
--- a/src/web/developer/api/api.js
+++ b/src/web/developer/api/api.js
@@ -100,9 +100,6 @@ $(document).ready(function() {
         updateCodeExamples();
     });
 
-    // Load saved state
-    loadState();
-
     // Save state when inputs change
     $(document).on('input', 'input, select, textarea', function() {
         saveState();
@@ -135,6 +132,9 @@ $(document).ready(function() {
 
     // Call the initialization function
     initializeCodeExamples();
+
+    // Load saved state
+    loadState();
 });
 
 function updateEndpointOptions(endpoint) {

From 177e3c5d2ed4717edeb418ae84a77613a076fe2e Mon Sep 17 00:00:00 2001
From: Chris 
Date: Sun, 25 Aug 2024 21:36:19 +0930
Subject: [PATCH 036/182] Make routes and results more consistent. Fix minor UI
 issues. Remove debug output.

---
 src/api/v1/db/controller.ts         | 21 +++++----
 src/api/v1/ds/controller.ts         | 19 +++++---
 src/api/v1/minisearch/controller.ts | 62 --------------------------
 src/api/v1/minisearch/routes.ts     | 11 -----
 src/api/v1/routes.ts                |  2 -
 src/api/v1/search/controller.ts     | 68 +++++++++++++++++++++--------
 src/api/v1/search/routes.ts         |  8 +---
 src/services/minisearch.ts          | 10 ++++-
 src/services/search.ts              | 49 +++++++++------------
 src/web/developer/api/api.js        | 15 ++-----
 src/web/developer/api/endpoints.js  | 66 +++++++++++++++-------------
 src/web/developer/api/index.html    |  2 +-
 12 files changed, 147 insertions(+), 186 deletions(-)
 delete mode 100644 src/api/v1/minisearch/controller.ts
 delete mode 100644 src/api/v1/minisearch/routes.ts

diff --git a/src/api/v1/db/controller.ts b/src/api/v1/db/controller.ts
index 00808e4e..cfc4eed4 100644
--- a/src/api/v1/db/controller.ts
+++ b/src/api/v1/db/controller.ts
@@ -16,8 +16,10 @@ export class DbController {
                 // @ts-ignore
                 permissions
             })
-            const results = await (await db).getMany()
-            res.json(results)
+            const items = await db.getMany()
+            res.json({
+                items
+            })
         } catch (error) {
             let message = error.message
             if (error.message.match('invalid encoding')) {
@@ -35,15 +37,14 @@ export class DbController {
             const rowId = req.params[1]
             const permissions = Utils.buildPermissions(req)
 
-            console.log(rowId)
-            console.log(req)
-
             const db = await context.openDatabase(dbName, {
                 // @ts-ignore
                 permissions
             })
-            const results = await (await db).get(rowId)
-            res.json(results)
+            const item = await db.get(rowId)
+            res.json({
+                item
+            })
         } catch (error) {
             res.status(500).send(error.message);
         }
@@ -62,8 +63,10 @@ export class DbController {
 
             const filter = req.body.query || {}
             const options = req.body.options || {}
-            const results = await db.getMany(filter, options)
-            res.json(results)
+            const items = await db.getMany(filter, options)
+            res.json({
+                items
+            })
         } catch (error) {
             console.log(error)
             res.status(500).send(error.message);
diff --git a/src/api/v1/ds/controller.ts b/src/api/v1/ds/controller.ts
index 1d757f62..6e7e8a45 100644
--- a/src/api/v1/ds/controller.ts
+++ b/src/api/v1/ds/controller.ts
@@ -18,8 +18,10 @@ export class DsController {
                 // @ts-ignore
                 permissions
             })
-            const results = await (await ds).getMany()
-            res.json(results)
+            const items = await (await ds).getMany()
+            res.json({
+                items
+            })
         } catch (error) {
             let message = error.message
             if (error.message.match('invalid encoding')) {
@@ -41,8 +43,10 @@ export class DsController {
                 permissions
             })
         
-            const results = await (await ds).get(rowId, {})
-            res.json(results)
+            const item = await ds.get(rowId, {})
+            res.json({
+                item: item
+            })
         } catch (error) {
             res.status(500).send(error.message);
         }
@@ -54,7 +58,6 @@ export class DsController {
             const permissions = Utils.buildPermissions(req)
             const schemaName = Utils.getSchemaFromParams(req.params[0])
 
-            console.log(schemaName, permissions)
             const ds = await context.openDatastore(schemaName, {
                 // @ts-ignore
                 permissions
@@ -62,8 +65,10 @@ export class DsController {
         
             const selector = req.body.query
             const options = req.body.options || {}
-            const results = await (await ds).getMany(selector, options)
-            res.json(results)
+            const items = await ds.getMany(selector, options)
+            res.json({
+                items
+            })
         } catch (error) {
             res.status(500).send(error.message);
         }
diff --git a/src/api/v1/minisearch/controller.ts b/src/api/v1/minisearch/controller.ts
deleted file mode 100644
index a19bbd85..00000000
--- a/src/api/v1/minisearch/controller.ts
+++ /dev/null
@@ -1,62 +0,0 @@
-import { Request, Response } from "express";
-import { DataService, HotLoadProgress } from '../../../services/data';
-import { MinisearchService } from "../../../services/minisearch"
-
-import { Utils } from '../../../utils';
-const MAX_RESULTS = 20
-
-/**
- * 
- */
-export class DsController {
-
-    public async searchDs(req: Request, res: Response) {
-        try {
-            const { context, account } = await Utils.getNetworkFromRequest(req)
-            const did = await account.did()
-
-            const schemaName = Utils.getSchemaFromParams(req.params[0])
-            const query = req.query.keywords.toString()
-            const searchOptions = req.query.options ? JSON.parse(req.query.options.toString()) : {}
-            const indexFields = req.query.fields ? req.query.fields.toString().split(',') : []
-            let storeFields = req.query.store ? req.query.store.toString().split(',') : []
-            const permissions = Utils.buildPermissions(req)
-            const limit = req.query.limit ? parseInt(req.query.limit.toString()) : MAX_RESULTS
-
-            const result = await MinisearchService.searchDs(context, did, schemaName, query, searchOptions, indexFields, storeFields, limit, permissions)
-            return res.json(result)
-        } catch (error) {
-            console.log(error)
-            res.status(500).send(error.message);
-        }
-    }
-
-    public async hotLoad(req: Request, res: Response) {
-        try {
-            const { context, account } = await Utils.getNetworkFromRequest(req)
-            const did = await account.did()
-            const data = new DataService(did, context)
-
-            data.on('progress', (progress: HotLoadProgress) => {
-                res.write(`data: ${JSON.stringify(progress)}\n\n`)
-            })
-
-            // Set-up event source response
-            res.setHeader('Content-Type', 'text/event-stream');
-            res.setHeader('Cache-Control', 'no-cache');
-            res.setHeader('Connection', 'keep-alive');
-            res.flushHeaders()
-            // Tell the client to retry every 10 seconds if connectivity is lost
-            res.write('retry: 10000\n\n')
-
-            await data.hotLoad()
-            res.end()
-        } catch (error) {
-            console.log(error)
-            res.status(500).send(error.message);
-        }
-    }
-
-}
-
-export const controller = new DsController()
\ No newline at end of file
diff --git a/src/api/v1/minisearch/routes.ts b/src/api/v1/minisearch/routes.ts
deleted file mode 100644
index 1b9ceb24..00000000
--- a/src/api/v1/minisearch/routes.ts
+++ /dev/null
@@ -1,11 +0,0 @@
-import express from 'express'
-import { controller } from './controller'
-
-const router = express.Router()
-
-// router.get(/db\/(.*)$/, controller.searchDb)
-router.get(/ds\/(.*)$/, controller.searchDs)
-router.get(/hotload/, controller.hotLoad)
-
-
-export default router
\ No newline at end of file
diff --git a/src/api/v1/routes.ts b/src/api/v1/routes.ts
index 9de11173..bb085f55 100644
--- a/src/api/v1/routes.ts
+++ b/src/api/v1/routes.ts
@@ -4,7 +4,6 @@ import DbRoutes from './db/routes'
 import DsRoutes from './ds/routes'
 import AdminRoutes from './admin/routes'
 import LLMRoutes from './llm/routes'
-import MiniSearchRoutes from './minisearch/routes'
 import BaseRoutes from './base/routes'
 import TelegramRoutes from './telegram/routes'
 import Search from "./search/routes"
@@ -16,7 +15,6 @@ router.use('/db', DbRoutes)
 router.use('/ds', DsRoutes)
 router.use('/admin', AdminRoutes)
 router.use('/llm', LLMRoutes)
-router.use('/minisearch', MiniSearchRoutes)
 router.use('/search', Search)
 
 router.use('/telegram', TelegramRoutes)
diff --git a/src/api/v1/search/controller.ts b/src/api/v1/search/controller.ts
index de841ea7..3fea0fa6 100644
--- a/src/api/v1/search/controller.ts
+++ b/src/api/v1/search/controller.ts
@@ -1,7 +1,19 @@
 import { Request, Response } from "express";
 import { Utils } from "../../../utils";
 import { SearchService, SearchType } from "../../../services/search"
+import { MinisearchService, SearchResultItem } from "../../../services/minisearch";
+import { SchemaRecord } from "../../../schemas";
 
+const DEFAULT_LIMIT = 20
+
+export interface SchemaRecordSearchResult extends SchemaRecord {
+    _match: SearchResultItem
+}
+
+export interface SearchResult {
+    total: number
+    items: SchemaRecordSearchResult
+}
 
 class SearchController {
 
@@ -13,15 +25,14 @@ class SearchController {
             const keywords = keywordString.split(' ')
 
             const threadSize = req.query.threadSize ? parseInt(req.query.threadSize.toString()) : 10
-            const limit = req.query.limit ? parseInt(req.query.limit.toString()) : 20
+            const limit = req.query.limit ? parseInt(req.query.limit.toString()) : DEFAULT_LIMIT
             const mergeOverlaps = req.query.limit ? req.query.merge.toString() == 'true' : true
 
             const searchService = new SearchService(did, context)
-            const results = await searchService.chatThreadsByKeywords(keywords, threadSize, limit, mergeOverlaps)
+            const items = await searchService.chatThreadsByKeywords(keywords, threadSize, limit, mergeOverlaps)
 
             return res.json({
-                keywords,
-                results
+                items
             })
 
         } catch (error) {
@@ -41,36 +52,59 @@ class SearchController {
                 SearchType.EMAILS,
                 SearchType.CHAT_MESSAGES
             ]
-            const limit = req.query.limit ? parseInt(req.query.limit.toString()) : 20
+            const limit = req.query.limit ? parseInt(req.query.limit.toString()) : DEFAULT_LIMIT
             const minResultsPerType = req.query.minResultsPerType ? parseInt(req.query.minResultsPerType.toString()) : 5
 
             const searchService = new SearchService(did, context)
-            const results = await searchService.multiByKeywords(searchTypes, keywords, limit, minResultsPerType)
+            const items = await searchService.multiByKeywords(searchTypes, keywords, limit, minResultsPerType)
 
             return res.json({
-                keywords,
-                results
+                items
             })
 
         } catch (error) {
             console.log(error)
-            res.status(500).send(error.message);
+            return res.status(500).send(error.message);
         }
-
     }
 
-    public async email() {
-
+    public async ds(req: Request, res: Response) {
+        return res.json({hello: 'world'})
     }
 
-    public async chatHistory() {
-
-    }
+    public async datastore(req: Request, res: Response) {
+        try {
+            const { context, account } = await Utils.getNetworkFromRequest(req)
+            const did = await account.did()
 
-    public async hotLoad() {
+            const schemaName = Utils.getSchemaFromParams(req.params[0])
+            const query = req.query.keywords.toString()
+            const searchOptions = req.query.options ? JSON.parse(req.query.options.toString()) : {}
+            const indexFields = req.query.fields ? req.query.fields.toString().split(',') : []
+            let storeFields = req.query.store ? req.query.store.toString().split(',') : []
+            const permissions = Utils.buildPermissions(req)
+            const limit = req.query.limit ? parseInt(req.query.limit.toString()) : DEFAULT_LIMIT
+
+            const searchResults = await MinisearchService.searchDs(context, did, schemaName, query, searchOptions, indexFields, storeFields, limit, permissions)
+            const datastore = await context.openDatastore(schemaName)
+            const items: SchemaRecordSearchResult[] = []
+            for (const searchResult of searchResults.results) {
+                const item = await datastore.get(searchResult.id, {})
+                items.push({
+                    ...item,
+                    _match: searchResult
+                })
+            }
 
+            return res.json({
+                total: searchResults.count,
+                items
+            })
+        } catch (error) {
+            console.log(error)
+            return res.status(500).send(error.message);
+        }
     }
-
 }
 
 export const controller = new SearchController()
\ No newline at end of file
diff --git a/src/api/v1/search/routes.ts b/src/api/v1/search/routes.ts
index 49c5fedf..33f3d576 100644
--- a/src/api/v1/search/routes.ts
+++ b/src/api/v1/search/routes.ts
@@ -1,14 +1,10 @@
-
 import express from 'express'
 import { controller } from './controller'
 
 const router = express.Router()
 
-router.get("/email", controller.email)
-router.get("/chatHistory", controller.chatHistory)
-router.get("/chatThreads", controller.chatThreads)
 router.get("/universal", controller.universal)
-router.get("/hotload", controller.hotLoad)
-
+router.get("/chatThreads", controller.chatThreads)
+router.get(/datastore\/(.*)$/, controller.datastore)
 
 export default router
\ No newline at end of file
diff --git a/src/services/minisearch.ts b/src/services/minisearch.ts
index 5aac1c13..cf8e3fbb 100644
--- a/src/services/minisearch.ts
+++ b/src/services/minisearch.ts
@@ -6,8 +6,16 @@ import { indexCache } from "./data";
 import { IContext } from "@verida/types";
 import { Utils } from "../utils";
 
+export interface SearchResultItem {
+  id: string
+  terms: string[]
+  queryTerms: string[]
+  match: object
+  score: number
+}
+
 export interface MinisearchServiceSearchResult {
-  results: object[];
+  results: SearchResultItem[];
   count: number;
 }
 
diff --git a/src/services/search.ts b/src/services/search.ts
index c0ce610b..a30de84c 100644
--- a/src/services/search.ts
+++ b/src/services/search.ts
@@ -7,6 +7,7 @@ const _ = require('lodash')
 
 export interface MinisearchResult {
     id: string
+    schemaUrl?: string
     score: number
     terms: string[]
     queryTerms: string[]
@@ -18,11 +19,6 @@ export interface SearchServiceSchemaResult {
     rows: MinisearchResult[]
 }
 
-export interface SortedResult {
-    id: string
-    schemaId: number
-    score: number
-}
 
 export enum SearchSortType {
     RECENT = "recent",
@@ -55,18 +51,17 @@ export interface ChatThreadResult {
 export class SearchService extends VeridaService {
 
     protected async rankAndMergeResults(schemaResults: SearchServiceSchemaResult[], limit: number, minResultsPerType: number = 10): Promise {
-        const unsortedResults: Record = {}
-        const guaranteedResults: Record = {}
+        const unsortedResults: Record = {}
+        const guaranteedResults: Record = {}
 
-        const datastores: IDatastore[] = []
+        const datastores: Record = {}
         for (const schemaResult of schemaResults) {
             let schemaResultCount = 0
             for (const row of schemaResult.rows) {
                 // console.log(row.id, row.score)
                 const result = {
-                    id: row.id,
-                    schemaId: datastores.length,
-                    score: row.score
+                    ...row,
+                    schemaUrl: SearchTypeSchemas[schemaResult.searchType]
                 }
 
                 if (schemaResultCount++ < minResultsPerType) {
@@ -77,7 +72,7 @@ export class SearchService extends VeridaService {
             }
 
             const schemaUri = SearchTypeSchemas[schemaResult.searchType]
-            datastores.push(await this.context.openDatastore(schemaUri))
+            datastores[schemaUri] = await this.context.openDatastore(schemaUri)
         }
 
         const unsortedResultCount = Object.values(unsortedResults).length
@@ -99,10 +94,14 @@ export class SearchService extends VeridaService {
             if (!result) {
                 continue
             }
-            const datastore = datastores[result.schemaId]
+
+            const datastore = datastores[result.schemaUrl]
             const row = await datastore.get(result.id, {})
-            row._score = result.score
-            results.push(row)
+            delete result['schemaUrl']
+            results.push({
+                ...row,
+                _match: result
+            })
         }
 
         return results
@@ -272,19 +271,13 @@ export class SearchService extends VeridaService {
 
         const searchResults = []
         for (const searchType of searchTypes) {
-            // let queryResult: SchemaRecord[] | ChatThreadResult[] = []
-            // if (searchType == SearchType.CHAT_THREADS) {
-            //     const threadSize = 10
-            //     queryResult = await this.chatThreadsByKeywords(keywordsList, threadSize, limit, true)
-            // } else {
-                const schemaUri = SearchTypeSchemas[searchType]
-                if (!schemaUri) {
-                    // Invalid search type, ignore
-                    continue
-                }
-                const miniSearchIndex = await dataService.getIndex(schemaUri)
-                const queryResult =  await miniSearchIndex.search(query)
-            // }
+            const schemaUri = SearchTypeSchemas[searchType]
+            if (!schemaUri) {
+                // Invalid search type, ignore
+                continue
+            }
+            const miniSearchIndex = await dataService.getIndex(schemaUri)
+            const queryResult =  await miniSearchIndex.search(query)
 
             searchResults.push({
                 searchType,
diff --git a/src/web/developer/api/api.js b/src/web/developer/api/api.js
index 5783b81d..981f0f76 100644
--- a/src/web/developer/api/api.js
+++ b/src/web/developer/api/api.js
@@ -71,8 +71,9 @@ $(document).ready(function() {
         runEndpoint();
     });
 
-    // Update code examples when any input changes
+    // Update code examples and save state when any input changes
     $(document).on('input', 'input, select, textarea', function() {
+        saveState()
         updateCodeExamples();
     });
 
@@ -95,21 +96,11 @@ $(document).ready(function() {
         saveState();
     });
 
-
-    $('#showPrivateKey').change(function() {
-        updateCodeExamples();
-    });
-
-    // Save state when inputs change
-    $(document).on('input', 'input, select, textarea', function() {
-        saveState();
-        updateCodeExamples();
-    });
-
     // Save state when endpoint selection changes
     $('#endpointSelect').change(function() {
         updateEndpointOptions($(this).val());
         saveState();
+        $('#result').empty()
     });
 
     // Save state when language selection changes
diff --git a/src/web/developer/api/endpoints.js b/src/web/developer/api/endpoints.js
index e03d7104..78205bb5 100644
--- a/src/web/developer/api/endpoints.js
+++ b/src/web/developer/api/endpoints.js
@@ -95,7 +95,7 @@ const apiEndpoints = {
     },
     "/ds/get/{schemaUrl}/{recordId}": {
         "method": "GET",
-        "path": `${apiPrefix}/get/{schemaUrl}/{recordId}`,
+        "path": `${apiPrefix}/ds/get/{schemaUrl}/{recordId}`,
         "documentation": "Retrieves a record from a datastore.",
         "urlVariables": {
             "schemaUrl": commonUrlVariables.schemaUrl,
@@ -120,7 +120,7 @@ const apiEndpoints = {
     },
     "/db/get/{databaseName}/{recordId}": {
         "method": "GET",
-        "path": `${apiPrefix}/get/{databaseName}/{recordId}`,
+        "path": `${apiPrefix}/db/get/{databaseName}/{recordId}`,
         "documentation": "Retrieves a record from a database.",
         "urlVariables": {
             "databaseName": commonUrlVariables.databaseName,
@@ -168,41 +168,22 @@ Defaults to \`"emails,chat-messages"\`.
             }
         }
     },
-    "/search/chatThreads": {
+    "/search/datastore/{schemaUrl}": {
         "method": "GET",
-        "path": `${apiPrefix}/search/chatThreads`,
-        "documentation": `Search chat messages by keyword and return matching chat threads to ensure the full message context is available.
-
-Each result contains the chat group and an array of messages.`,
-        "params": {
-            "keywords": {
-                "type": "string",
-                "documentation": "List of keywords to search for",
-                "default": "robert gray",
-                "required": true
-            },
-            "limit": {
-                "type": "number",
-                "documentation": "Limit how many chat threads to return. Defaults to `20`.",
-                "default": 5
-            },
-            "merge": {
-                "type": "boolean",
-                "documentation": "Merge overlapping threads. If two messages match within the same chat group, they are merged to produce a single chat group.",
-                "default": true
-            }
-        }
-    },
-    "/minisearch/ds/{schemaUrl}": {
-        "method": "GET",
-        "path": `${apiPrefix}/minisearch/ds/{schemaUrl}`,
+        "path": `${apiPrefix}/search/datastore/{schemaUrl}`,
         "documentation": `Execute a keyword search on a datastore.
 
 It's possible to define the fields to index on and the fields to be stored in the search index which is then returned with results.
 
 The index is cached for the user, until the user cache times out.
 
-Requests with the exact same list of indexed and stored fields will re-use the same index. If there is any difference in the indexed or stored fields, a new index is created, which increases the memory footprint.`,
+Requests with the exact same list of indexed and stored fields will re-use the same index. If there is any difference in the indexed or stored fields, a new index is created, which increases the memory footprint.
+
+Returns:
+
+- \`total\` - Total number of search results found in the search index
+- \`items\` - Array of item results
+`,
         "urlVariables": {
             "schemaUrl": commonUrlVariables.schemaUrl,
         },
@@ -248,6 +229,31 @@ Requests with the exact same list of indexed and stored fields will re-use the s
             }
         }
     },
+    "/search/chatThreads": {
+        "method": "GET",
+        "path": `${apiPrefix}/search/chatThreads`,
+        "documentation": `Search chat messages by keyword and return matching chat threads to ensure the full message context is available.
+
+Each result contains the chat group and an array of messages.`,
+        "params": {
+            "keywords": {
+                "type": "string",
+                "documentation": "List of keywords to search for",
+                "default": "robert gray",
+                "required": true
+            },
+            "limit": {
+                "type": "number",
+                "documentation": "Limit how many chat threads to return. Defaults to `20`.",
+                "default": 5
+            },
+            "merge": {
+                "type": "boolean",
+                "documentation": "Merge overlapping threads. If two messages match within the same chat group, they are merged to produce a single chat group.",
+                "default": true
+            }
+        }
+    },
     "/providers": {
         "method": "GET",
         "path": `${apiPrefix}/providers`,
diff --git a/src/web/developer/api/index.html b/src/web/developer/api/index.html
index 0ea4b6e7..6ce6699b 100644
--- a/src/web/developer/api/index.html
+++ b/src/web/developer/api/index.html
@@ -127,7 +127,7 @@ 
Developer API Interface

                 

                     
                     
                 

             


From 92672b84a68869e6cbec50973ba3751fbe72b2cf Mon Sep 17 00:00:00 2001
From: Chris 
Date: Sun, 25 Aug 2024 21:49:37 +0930
Subject: [PATCH 037/182] Adding LLM endpoints to API playground

---
 src/api/v1/llm/controller.ts       | 34 ++++++++++++++++++++++++
 src/api/v1/llm/routes.ts           |  1 +
 src/services/assistants/search.ts  |  8 +++---
 src/web/developer/api/endpoints.js | 42 ++++++++++++++++++++++++++++++
 4 files changed, 81 insertions(+), 4 deletions(-)

diff --git a/src/api/v1/llm/controller.ts b/src/api/v1/llm/controller.ts
index b8a6000d..85420920 100644
--- a/src/api/v1/llm/controller.ts
+++ b/src/api/v1/llm/controller.ts
@@ -2,6 +2,8 @@ import { Request, Response } from "express";
 import { bedrock } from '../../../services/llm'
 import { PromptSearchService } from '../../../services/assistants/search'
 import { Utils } from "../../../utils";
+import { HotLoadProgress } from "../../../services/data";
+import { DataService } from "../../../services/data";
 const _ = require('lodash')
 
 
@@ -41,6 +43,38 @@ export class LLMController {
             res.status(500).send(error.message);
         }
     }
+
+    /**
+     * Hotload the data necessary to power the AI search capabilities
+     * 
+     * @param req 
+     * @param res 
+     */
+    public async hotLoad(req: Request, res: Response) {
+        try {
+            const { context, account } = await Utils.getNetworkFromRequest(req)
+            const did = await account.did()
+            const data = new DataService(did, context)
+
+            data.on('progress', (progress: HotLoadProgress) => {
+                res.write(`data: ${JSON.stringify(progress)}\n\n`)
+            })
+
+            // Set-up event source response
+            res.setHeader('Content-Type', 'text/event-stream');
+            res.setHeader('Cache-Control', 'no-cache');
+            res.setHeader('Connection', 'keep-alive');
+            res.flushHeaders()
+            // Tell the client to retry every 10 seconds if connectivity is lost
+            res.write('retry: 10000\n\n')
+
+            await data.hotLoad()
+            res.end()
+        } catch (error) {
+            console.log(error)
+            res.status(500).send(error.message);
+        }
+    }
 }
 
 export const controller = new LLMController()
\ No newline at end of file
diff --git a/src/api/v1/llm/routes.ts b/src/api/v1/llm/routes.ts
index ea983dbf..675b9fb8 100644
--- a/src/api/v1/llm/routes.ts
+++ b/src/api/v1/llm/routes.ts
@@ -4,5 +4,6 @@ import { controller } from './controller'
 const router = express.Router()
 router.post('/prompt', controller.prompt)
 router.post('/personal', controller.personalPrompt)
+router.get('/hotload', controller.hotLoad)
 
 export default router
\ No newline at end of file
diff --git a/src/services/assistants/search.ts b/src/services/assistants/search.ts
index dfc89ba0..cb420d91 100644
--- a/src/services/assistants/search.ts
+++ b/src/services/assistants/search.ts
@@ -19,7 +19,7 @@ export class PromptSearchService extends VeridaService {
     public async prompt(prompt: string): Promise<{
         result: string,
         duration: number,
-        promptSearchResult: PromptSearchLLMResponse
+        process: PromptSearchLLMResponse
     }> {
         const start = Date.now()
         // // Get queries that can help answer the prompt
@@ -111,16 +111,16 @@ export class PromptSearchService extends VeridaService {
         const now = (new Date()).toISOString()
         finalPrompt += `${contextString}\nThe current time is: ${now}`
 
-        console.log('Running final prompt', finalPrompt.length)
+        // console.log('Running final prompt', finalPrompt.length)
         const finalResponse = await llm.prompt(finalPrompt, undefined, false)
         const duration = Date.now() - start
 
-        console.log(contextString)
+        // console.log(contextString)
 
         return {
             result: finalResponse.choices[0].message.content,
             duration,
-            promptSearchResult
+            process: promptSearchResult
         }
     }
 
diff --git a/src/web/developer/api/endpoints.js b/src/web/developer/api/endpoints.js
index 78205bb5..4ca57eb5 100644
--- a/src/web/developer/api/endpoints.js
+++ b/src/web/developer/api/endpoints.js
@@ -297,5 +297,47 @@ Each result contains the chat group and an array of messages.`,
 
 **arrayBuffers:** This is the memory allocated for ArrayBuffer and SharedArrayBuffer instances in JavaScript. It indicates how much memory is consumed specifically by array buffer-backed objects.
 `
+    },
+    "/llm/hotload": {
+        "method": "GET",
+        "path": `${apiPrefix}/llm/hotload`,
+        "documentation": `Hot load into memory all the data necessary for fast presonal LLM requests.
+
+This is not a typical HTTP request, it uses EventSource to stream the loading progress.
+
+**Example code:**
+
+\`\`\`
+const eventSource = new EventSource(\`/api/v1/llm/hotload?key=\`);
+    
+eventSource.onmessage = function(event) {
+    console.log(event)
+}
+\`\`\`
+`
+    },
+    "/llm/prompt": {
+        "method": "POST",
+        "path": `${apiPrefix}/llm/prompt`,
+        "documentation": `Send a LLM prompt request to a pre-configured LLM.`,
+        "params": {
+            "prompt": {
+                "type": "string",
+                "required": true,
+                "documentation": `User prompt (ie: \`Who hosted the 2000 olympics?\`)`
+            }
+        }
+    },
+    "/llm/personal": {
+        "method": "POST",
+        "path": `${apiPrefix}/llm/personal`,
+        "documentation": `Send a LLM prompt request to the built-in personal AI LLM.`,
+        "params": {
+            "prompt": {
+                "type": "string",
+                "required": true,
+                "documentation": `User prompt (ie: \`How much have I spent on software this quarter?\`)`
+            }
+        }
     }
 };
\ No newline at end of file

From de784fc37194c81ae1c7d52e7d0b06997f39da6e Mon Sep 17 00:00:00 2001
From: Chris 
Date: Sun, 25 Aug 2024 22:48:59 +0930
Subject: [PATCH 038/182] Sipmlify sync manager and sync endpoints.

---
 src/api/v1/base/controller.ts      | 84 ++++--------------------------
 src/api/v1/base/routes.ts          |  9 ++--
 src/api/v1/ds/controller.ts        |  2 -
 src/cli/commands/connections.ts    |  3 +-
 src/cli/commands/resetProvider.ts  |  3 +-
 src/cli/commands/sync.ts           |  3 +-
 src/sync-manager.ts                | 32 ++----------
 src/web/developer/api/endpoints.js | 32 +++++++++---
 8 files changed, 46 insertions(+), 122 deletions(-)

diff --git a/src/api/v1/base/controller.ts b/src/api/v1/base/controller.ts
index 5565dc95..28f17dac 100644
--- a/src/api/v1/base/controller.ts
+++ b/src/api/v1/base/controller.ts
@@ -82,7 +82,9 @@ export default class Controller {
             const key = req.session.key
             const redirect = req.session.redirect
 
-            const syncManager = new SyncManager(did, key, req.requestId)
+            const networkInstance = await Utils.getNetwork(key, req.requestId)
+
+            const syncManager = new SyncManager(networkInstance.context, req.requestId)
             await syncManager.saveProvider(providerName, connectionResponse.accessToken, connectionResponse.refreshToken, connectionResponse.profile)
 
             if (redirect) {
@@ -181,17 +183,14 @@ export default class Controller {
      */
     public static async sync(req: UniqueRequest, res: Response, next: any) {
         const query = req.query
-        const vaultSeedPhrase = query.key.toString()
-        const did = await Utils.getDidFromKey(vaultSeedPhrase)
         const providerName = query.provider ? query.provider.toString() : undefined
         const providerId = query.providerId ? query.providerId.toString() : undefined
         const forceSync = query.force ? query.force == 'true' : undefined
 
-        const syncManager = new SyncManager(did, vaultSeedPhrase, req.requestId)
+        const networkInstance = await Utils.getNetworkFromRequest(req)
+        const syncManager = new SyncManager(networkInstance.context, req.requestId)
         const connections = await syncManager.sync(providerName, providerId, forceSync)
 
-        Utils.closeConnection(did, req.requestId)
-
         // @todo: catch and send errors
         return res.send({
             connection: connections[0],
@@ -233,12 +232,11 @@ export default class Controller {
     public static async syncStatus(req: UniqueRequest, res: Response, next: any) {
         try {
             const query = req.query
-            const vaultSeedPhrase = query.key.toString()
-            const did = await Utils.getDidFromKey(vaultSeedPhrase)
             const providerName = query.provider ? query.provider.toString() : undefined
             const providerId = query.providerId ? query.providerId.toString() : undefined
 
-            const syncManager = new SyncManager(did, vaultSeedPhrase, req.requestId)
+            const networkInstance = await Utils.getNetworkFromRequest(req)
+            const syncManager = new SyncManager(networkInstance.context, req.requestId)
             const connections = await syncManager.getProviders(providerName, providerId)
 
             const result: Record = {};
-            const [ sortField, sortDirection ] = sortParams.split(':')
-            sort[sortField] = sortDirection ? sortDirection : 'asc'
-
-            const networkInstance = await Utils.getNetwork(privateKey, req.requestId)
-
-            const filter: Record = {}
-            if (filterParams) {
-                const filterAttributes = filterParams ? filterParams.split(",") : [];
-                for (const attribute of filterAttributes) {
-                    const [key, value] = attribute.split(':')
-                    filter[key] = value
-                }
-            }
-
-            const options = {
-                sort: [sort],
-                limit: parseInt(limit),
-                skip: parseInt(offset)
-              }
-
-            const datastore = await networkInstance.context.openDatastore(schema)
-            const results =  await datastore.getMany(
-                filter,
-                options
-              )
-            
-              Utils.closeConnection(networkInstance.did, req.requestId)
-
-            return res.send({
-                results,
-                filter,
-                options: {
-                    sort,
-                    limit: options.limit,
-                    offset: options.skip
-                }
-            })
-        } catch (error) {
-            console.log(error)
-            res.status(400).send({
-                error: error.message
-            });
-        }
-    }
-
     public static async logs(req: UniqueRequest, res: Response, next: any) {
         try {
             res.setHeader('Content-Type', 'text/event-stream');
@@ -371,8 +308,7 @@ export default class Controller {
             res.write('retry: 10000\n\n')
 
             const query = req.query
-            const privateKey = query.key.toString()
-            const networkInstance = await Utils.getNetwork(privateKey, req.requestId)
+            const networkInstance = await Utils.getNetworkFromRequest(req)
 
             const logsDs = await networkInstance.context.openDatastore(SCHEMA_SYNC_LOG)
             const logsDb = await logsDs.getDb()
diff --git a/src/api/v1/base/routes.ts b/src/api/v1/base/routes.ts
index 707d1683..55a2db45 100644
--- a/src/api/v1/base/routes.ts
+++ b/src/api/v1/base/routes.ts
@@ -3,15 +3,14 @@ import Controller from './controller'
 
 const router = express.Router()
 
+router.get('/providers', Controller.providers)
+
 router.get('/connect/:provider', Controller.connect)
 router.get('/disconnect/:provider', Controller.disconnect)
 router.get('/callback/:provider', Controller.callback)
 
+router.get('/sync/status', Controller.syncStatus)
+router.get('/sync/logs', Controller.logs)
 router.get('/sync', Controller.sync)
-router.get('/syncStatus', Controller.syncStatus)
-
-router.get('/providers', Controller.providers)
-router.get('/data', Controller.data)
-router.get('/logs', Controller.logs)
 
 export default router
\ No newline at end of file
diff --git a/src/api/v1/ds/controller.ts b/src/api/v1/ds/controller.ts
index 6e7e8a45..89922a7c 100644
--- a/src/api/v1/ds/controller.ts
+++ b/src/api/v1/ds/controller.ts
@@ -1,6 +1,4 @@
 import { Request, Response } from "express";
-import Common from "../common";
-import { IContext, IDatabase, IDatastore } from "@verida/types";
 import { Utils } from "../../../utils";
 
 /**
diff --git a/src/cli/commands/connections.ts b/src/cli/commands/connections.ts
index 73dca1e1..ce43035d 100644
--- a/src/cli/commands/connections.ts
+++ b/src/cli/commands/connections.ts
@@ -56,8 +56,7 @@ export const Connections: Command = {
     const vault = networkInstance.context;
 
     const syncManager = new SyncManager(
-      await networkInstance.account.did(),
-      options.key
+      vault
     );
 
     const providers = await syncManager.getProviders(
diff --git a/src/cli/commands/resetProvider.ts b/src/cli/commands/resetProvider.ts
index b12abe08..70480b46 100644
--- a/src/cli/commands/resetProvider.ts
+++ b/src/cli/commands/resetProvider.ts
@@ -50,8 +50,7 @@ export const ResetProvider: Command = {
     const vault = networkInstance.context;
 
     const syncManager = new SyncManager(
-      await networkInstance.account.did(),
-      options.key
+      networkInstance.context
     );
 
     const providers = await syncManager.getProviders(
diff --git a/src/cli/commands/sync.ts b/src/cli/commands/sync.ts
index f7baf600..89911488 100644
--- a/src/cli/commands/sync.ts
+++ b/src/cli/commands/sync.ts
@@ -56,8 +56,7 @@ export const Sync: Command = {
     }, {});
 
     const syncManager = new SyncManager(
-      await networkInstance.account.did(),
-      options.key
+      networkInstance.context
     );
 
     const providers = await syncManager.getProviders(
diff --git a/src/sync-manager.ts b/src/sync-manager.ts
index d7d76be8..ad1bf493 100644
--- a/src/sync-manager.ts
+++ b/src/sync-manager.ts
@@ -20,9 +20,7 @@ const delay = async (ms: number) => {
  */
 export default class SyncManager {
 
-    private vault?: IContext
-    private did: string
-    private seedPhrase: string
+    private vault: IContext
     private requestId: string
 
     private connectionDatastore?: IDatastore
@@ -30,9 +28,8 @@ export default class SyncManager {
 
     private status: SyncStatus = SyncStatus.CONNECTED
 
-    public constructor(did: string, seedPhrase: string, requestId: string = 'none') {
-        this.did = did
-        this.seedPhrase = seedPhrase
+    public constructor(vaultContext: IContext, requestId: string = 'none') {
+        this.vault = vaultContext
         this.requestId = requestId
     }
 
@@ -76,21 +73,6 @@ export default class SyncManager {
 
     }
 
-    public async getVault(): Promise {
-        if (this.vault) {
-            return this.vault
-        }
-
-        try {
-            const { context } = await Utils.getNetwork(this.seedPhrase, this.requestId)
-
-            this.vault =  context
-            return this.vault
-        } catch (err) {
-            console.log(err)
-        }
-    }
-
     public async getProviders(providerName?: string, providerId?: string): Promise {
         if (this.connections) {
             if (providerName) {
@@ -112,8 +94,6 @@ export default class SyncManager {
             return this.connections
         }
 
-        const vault = await this.getVault()
-
         const datastore = await this.getConnectionDatastore()
         const allProviders = providerName ? [providerName] : Object.keys(CONFIG.providers)
         const userConnections = []
@@ -131,7 +111,7 @@ export default class SyncManager {
 
                 const connections =  await datastore.getMany(filter, {})
                 for (const connection of connections) {
-                    const provider = Providers(providerName, vault, connection)
+                    const provider = Providers(providerName, this.vault, connection)
                     userConnections.push(provider)
                 }
                 
@@ -155,9 +135,7 @@ export default class SyncManager {
             return this.connectionDatastore
         }
 
-        const vault = await this.getVault()
-
-        this.connectionDatastore = await vault.openDatastore(
+        this.connectionDatastore = await this.vault.openDatastore(
             DATA_CONNECTION_SCHEMA
         )
 
diff --git a/src/web/developer/api/endpoints.js b/src/web/developer/api/endpoints.js
index 4ca57eb5..9ac75e42 100644
--- a/src/web/developer/api/endpoints.js
+++ b/src/web/developer/api/endpoints.js
@@ -41,7 +41,8 @@ const commonParams = {
     sort: [{
         _id: "desc"
     }],
-    limit: 20
+    limit: 20,
+    skip: 0
 }
 \`\`\`
 `,
@@ -49,7 +50,8 @@ const commonParams = {
             sort: [{
                 _id: "desc"
             }],
-            limit: 20
+            limit: 20,
+            skip: 0
         })
     }
 }
@@ -273,13 +275,27 @@ Each result contains the chat group and an array of messages.`,
             }
         }
     },
-    "/syncStatus": {
+    "/sync/status": {
         "method": "GET",
-        "path": "/api/v1/syncStatus",
-        "params": {
-            "provider": commonParams.provider,
-            "providerId": commonParams.providerId,
-        },
+        "path": "/api/v1/sync/status",
+        "documentation": `Live stream of the sync logs.
+
+This is not a typical HTTP request, it uses EventSource to stream the loading progress.
+
+**Example code:**
+
+\`\`\`
+const eventSource = new EventSource(\`/api/v1/llm/hotload?key=\`);
+    
+eventSource.onmessage = function(event) {
+    console.log(event)
+}
+\`\`\`
+`
+    },
+    "/sync/logs": {
+        "method": "GET",
+        "path": "/api/v1/sync/logs",
         "documentation": "Get the status of the current sync connection for a provider."
     },
     "/admin/memory": {

From 1afb619b166c820ca1f0d278ff2eb7be7ca01209 Mon Sep 17 00:00:00 2001
From: Chris 
Date: Mon, 26 Aug 2024 07:49:00 +0930
Subject: [PATCH 039/182] Fix updated endpoint paths

---
 src/api/v1/base/routes.ts               |  4 ++--
 src/web/developer/data/data.js          |  4 ++--
 src/web/user/ai/ai.js                   |  2 +-
 src/web/user/connections/connections.js | 10 +++++-----
 4 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/src/api/v1/base/routes.ts b/src/api/v1/base/routes.ts
index 55a2db45..dc1524a0 100644
--- a/src/api/v1/base/routes.ts
+++ b/src/api/v1/base/routes.ts
@@ -5,8 +5,8 @@ const router = express.Router()
 
 router.get('/providers', Controller.providers)
 
-router.get('/connect/:provider', Controller.connect)
-router.get('/disconnect/:provider', Controller.disconnect)
+router.get('/provider/connect/:provider', Controller.connect)
+router.get('/provider/disconnect/:provider', Controller.disconnect)
 router.get('/callback/:provider', Controller.callback)
 
 router.get('/sync/status', Controller.syncStatus)
diff --git a/src/web/developer/data/data.js b/src/web/developer/data/data.js
index a3efacd4..51f58f27 100644
--- a/src/web/developer/data/data.js
+++ b/src/web/developer/data/data.js
@@ -1,6 +1,6 @@
 $(document).ready(function() {
     let offset = 0;
-    const apiUrl = '/api/v1/data';
+    const apiUrl = '/api/v1/ds/query';
     let currentSortField = '';
     let currentSortDirection = 'asc';
     let currentFilters = {};
@@ -54,7 +54,7 @@ $(document).ready(function() {
         $('.alert').hide(); // Hide previous error messages
 
         $.ajax({
-            url: apiUrl,
+            url: `${apiUrl}/${btoa(schema)}`,
             data: {
                 key: veridaKey,
                 schema: schema,
diff --git a/src/web/user/ai/ai.js b/src/web/user/ai/ai.js
index bd55694a..5fb2bbc6 100644
--- a/src/web/user/ai/ai.js
+++ b/src/web/user/ai/ai.js
@@ -81,7 +81,7 @@ $(document).ready(function() {
     });
 
     // Hotload data
-    const eventSource = new EventSource(`/api/v1/minisearch/hotload?key=${savedVeridaKey}`);
+    const eventSource = new EventSource(`/api/v1/llm/hotload?key=${savedVeridaKey}`);
     
     let loadComplete = false
     eventSource.onmessage = function(event) {
diff --git a/src/web/user/connections/connections.js b/src/web/user/connections/connections.js
index 1cd5a1ee..aa1a037f 100644
--- a/src/web/user/connections/connections.js
+++ b/src/web/user/connections/connections.js
@@ -54,7 +54,7 @@ $(document).ready(function() {
         $('#loadingIndicator').show();
         $('#loadBtn').prop('disabled', true);
 
-        $.getJSON(`/api/v1/syncStatus?key=${veridaKey}`, function(syncStatusResponse) {
+        $.getJSON(`/api/v1/sync/status?key=${veridaKey}`, function(syncStatusResponse) {
             $.each(syncStatusResponse.result, function(key, value) {
                 const connection = value.connection;
                 const handlers = value.handlers;
@@ -97,13 +97,13 @@ $(document).ready(function() {
             $('.logs-btn').click(function() {
                 const provider = $(this).data('provider');
                 const providerId = $(this).data('provider-id');
-                window.open(`/dashboard/data?limit=50&filter=providerName:${provider},providerId:${providerId}&schema=https://vault.schemas.verida.io/data-connections/activity-log/v0.1.0/schema.json&sort=insertedAt:desc`, '_blank');
+                window.open(`/developer/data?limit=50&filter=providerName:${provider},providerId:${providerId}&schema=https://vault.schemas.verida.io/data-connections/activity-log/v0.1.0/schema.json&sort=insertedAt:desc`, '_blank');
             });
 
             $('.disconnect-btn').click(function() {
                 const provider = $(this).data('provider');
                 const providerId = $(this).data('provider-id');
-                $.getJSON(`/api/v1/disconnect/${provider}?key=${veridaKey}&providerId=${providerId}`, function(response) {
+                $.getJSON(`/api/v1/provider/disconnect/${provider}?key=${veridaKey}&providerId=${providerId}`, function(response) {
                     console.log(response.data)
                 })
             });
@@ -117,7 +117,7 @@ $(document).ready(function() {
                 const syncType = $(this).data('sync-type');
 
                 // Start tailing logs
-                const eventSource = new EventSource(`/api/v1/logs?key=${veridaKey}`);
+                const eventSource = new EventSource(`/api/v1/sync/logs?key=${veridaKey}`);
 
                 const tableBody = $('#eventTableBody');
                 tableBody.empty()
@@ -177,7 +177,7 @@ $(document).ready(function() {
         $dropdown.empty();
         $.each(providersData, function(key, provider) {
             if (provider.name === 'mock') return; // Skip 'mock' provider
-            $dropdown.append(`
+            $dropdown.append(`
                 ${provider.label}
                 ${provider.label}
             `);

From 308bf1af2612f7571a8d6d7bb2f9b3f9bcca62c2 Mon Sep 17 00:00:00 2001
From: chime3 
Date: Sun, 25 Aug 2024 20:16:02 -0700
Subject: [PATCH 040/182] fix: getExtension function

---
 src/providers/google/helpers.ts | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/src/providers/google/helpers.ts b/src/providers/google/helpers.ts
index fd4edcbc..8236f255 100644
--- a/src/providers/google/helpers.ts
+++ b/src/providers/google/helpers.ts
@@ -495,8 +495,15 @@ export class GoogleDriveHelpers {
         const fileName = response.data.name;
         const mimeType = response.data.mimeType;
 
+        // Check if the file name contains a period (.)
+        const lastDotIndex = fileName.lastIndexOf(".");
+        if (lastDotIndex === -1) {
+            // No period found, so no extension
+            return undefined;
+        }
+
         // Determine extension from file name
-        const extension = fileName.slice((fileName.lastIndexOf(".") - 1 >>> 0) + 2);
+        const extension = fileName.slice(lastDotIndex + 1);
         
         // Fallback if extension is not in file name (based on MIME type)
         if (!extension) {
@@ -509,6 +516,6 @@ export class GoogleDriveHelpers {
         console.error('Error retrieving file metadata:', error);
         throw error;
     }
-}
+  }
 
 }

From de265b6a91ce3a3c4a562841fcc2fa9be7a4ad0c Mon Sep 17 00:00:00 2001
From: chime3 
Date: Sun, 25 Aug 2024 20:19:48 -0700
Subject: [PATCH 041/182] fix: added nullish coalescing

---
 src/providers/google/gdrive-document.ts | 65 +++++++++++++++----------
 1 file changed, 39 insertions(+), 26 deletions(-)

diff --git a/src/providers/google/gdrive-document.ts b/src/providers/google/gdrive-document.ts
index a0442143..b1cc379c 100644
--- a/src/providers/google/gdrive-document.ts
+++ b/src/providers/google/gdrive-document.ts
@@ -155,60 +155,69 @@ export default class GoogleDriveDocument extends GoogleHandler {
     ): Promise {
         const results: SchemaFile[] = [];
         let breakHit: SyncItemsBreak;
-
-        for (const file of serverResponse.data.files) {
-            const fileId = file.id;
-
+    
+        for (const file of serverResponse.data.files ?? []) {
+            const fileId = file.id ?? '';
+            
             if (fileId === breakId) {
-                this.emit('log', { level: SyncProviderLogLevel.DEBUG, message: `Break ID hit (${breakId})` });
+                const logEvent: SyncProviderLogEvent = {
+                    level: SyncProviderLogLevel.DEBUG,
+                    message: `Break ID hit (${breakId})`,
+                };
+                this.emit('log', logEvent);
                 breakHit = SyncItemsBreak.ID;
                 break;
             }
-
-            const createdTime = file.createdTime || new Date().toISOString();
-            const modifiedTime = file.modifiedTime || new Date().toISOString();
-
+    
+            const createdTime = file.createdTime ?? new Date().toISOString();
+            const modifiedTime = file.modifiedTime ?? new Date().toISOString();
+    
             if (breakTimestamp && modifiedTime < breakTimestamp) {
-                this.emit('log', { level: SyncProviderLogLevel.DEBUG, message: `Break timestamp hit (${breakTimestamp})` });
+                const logEvent: SyncProviderLogEvent = {
+                    level: SyncProviderLogLevel.DEBUG,
+                    message: `Break timestamp hit (${breakTimestamp})`,
+                };
+                this.emit('log', logEvent);
                 breakHit = SyncItemsBreak.TIMESTAMP;
                 break;
             }
-
-            const title = file.name || "No title";
+    
+            const title = file.name ?? 'No title';
             const link = file.webViewLink;
             if (!link) {
                 const logEvent: SyncProviderLogEvent = {
                     level: SyncProviderLogLevel.DEBUG,
-                    message: `No link available for file ${fileId}. Ignoring this file.`
-                }
+                    message: `No link available for file ${fileId}. Ignoring this file.`,
+                };
                 this.emit('log', logEvent);
                 continue;
             }
-
+    
             const mimeType = file.mimeType;
             if (!mimeType) {
                 const logEvent: SyncProviderLogEvent = {
                     level: SyncProviderLogLevel.DEBUG,
-                    message: `No mimeType available for file ${fileId}. Ignoring this file.`
-                }
+                    message: `No mimeType available for file ${fileId}. Ignoring this file.`,
+                };
                 this.emit('log', logEvent);
                 continue;
             }
-            const extension = await GoogleDriveHelpers.getFileExtension(this.getGoogleDrive(), fileId)
-            const thumbnail = file.thumbnailLink || undefined;
-            const size = await GoogleDriveHelpers.getFileSize(drive, file.id);
-            const textContent = await GoogleDriveHelpers.extractIndexableText(drive, file.id, mimeType, this.getGoogleAuth());
-
+    
+            const extension = await GoogleDriveHelpers.getFileExtension(this.getGoogleDrive(), fileId);
+            const thumbnail = file.thumbnailLink ?? '';
+            const size = await GoogleDriveHelpers.getFileSize(drive, fileId);
+            const textContent = await GoogleDriveHelpers.extractIndexableText(drive, fileId, mimeType, this.getGoogleAuth());
+    
             results.push({
                 _id: this.buildItemId(fileId),
                 name: title,
                 mimeType: mimeType,
                 extension: extension,
-                size,
+                size: size,
                 uri: link,
                 icon: thumbnail,
                 contentText: textContent,
-                sourceId: file.id,
+                sourceId: fileId,
                 sourceData: file,
                 sourceAccountId: this.provider.getProviderId(),
                 sourceApplication: this.getProviderApplicationUrl(),
@@ -216,7 +225,11 @@ export default class GoogleDriveDocument extends GoogleHandler {
                 modifiedAt: modifiedTime,
             });
         }
-
-        return { items: results, breakHit };
+    
+        return {
+            items: results,
+            breakHit,
+        };
     }
+    
 }

From d07652cc6cb577f5d90ead94587662dc39a5250a Mon Sep 17 00:00:00 2001
From: chime3 
Date: Sun, 25 Aug 2024 22:34:19 -0700
Subject: [PATCH 042/182] feat: added indexable text

---
 src/dashboard/public/data/data.js       |   1 +
 src/providers/google/gdrive-document.ts |  15 +++-
 src/providers/google/helpers.ts         | 109 ++++++++++++++++--------
 3 files changed, 89 insertions(+), 36 deletions(-)

diff --git a/src/dashboard/public/data/data.js b/src/dashboard/public/data/data.js
index a3efacd4..d71674dd 100644
--- a/src/dashboard/public/data/data.js
+++ b/src/dashboard/public/data/data.js
@@ -209,6 +209,7 @@ $(document).ready(function() {
             "Social Post": "https://common.schemas.verida.io/social/post/v0.1.0/schema.json",
             "Favourites": "https://common.schemas.verida.io/favourite/v0.1.0/schema.json",
             "Email": "https://common.schemas.verida.io/social/email/v0.1.0/schema.json",
+            "FILE": "https://common.schemas.verida.io/file/v0.1.0/schema.json",
             "Chat Group": "https://common.schemas.verida.io/social/chat/group/v0.1.0/schema.json",
             "Chat Message": "https://common.schemas.verida.io/social/chat/message/v0.1.0/schema.json"
         };
diff --git a/src/providers/google/gdrive-document.ts b/src/providers/google/gdrive-document.ts
index b1cc379c..6ba2e679 100644
--- a/src/providers/google/gdrive-document.ts
+++ b/src/providers/google/gdrive-document.ts
@@ -157,6 +157,9 @@ export default class GoogleDriveDocument extends GoogleHandler {
         let breakHit: SyncItemsBreak;
     
         for (const file of serverResponse.data.files ?? []) {
+            console.log("==========")
+            console.log(file)
+            
             const fileId = file.id ?? '';
             
             if (fileId === breakId) {
@@ -206,7 +209,16 @@ export default class GoogleDriveDocument extends GoogleHandler {
             const extension = await GoogleDriveHelpers.getFileExtension(this.getGoogleDrive(), fileId);
             const thumbnail = file.thumbnailLink ?? '';
             const size = await GoogleDriveHelpers.getFileSize(drive, fileId);
-            const textContent = await GoogleDriveHelpers.extractIndexableText(drive, fileId, mimeType, this.getGoogleAuth());
+            if (!size) {
+                const logEvent: SyncProviderLogEvent = {
+                    level: SyncProviderLogLevel.DEBUG,
+                    message: `No size for file ${fileId}. Ignoring this file.`,
+                };
+                this.emit('log', logEvent);
+                continue;
+            }
+
+            const textContent = await GoogleDriveHelpers.extractTextContent(drive, fileId, mimeType, this.getGoogleAuth());
     
             results.push({
                 _id: this.buildItemId(fileId),
@@ -217,6 +229,7 @@ export default class GoogleDriveDocument extends GoogleHandler {
                 uri: link,
                 icon: thumbnail,
                 contentText: textContent,
+                fileDataId: undefined,
                 sourceId: fileId,
                 sourceData: file,
                 sourceAccountId: this.provider.getProviderId(),
diff --git a/src/providers/google/helpers.ts b/src/providers/google/helpers.ts
index 8236f255..92a46417 100644
--- a/src/providers/google/helpers.ts
+++ b/src/providers/google/helpers.ts
@@ -304,45 +304,84 @@ export class GoogleDriveHelpers {
     }
   }
 
-  static async extractIndexableText(
+  static async extractTextContent(
     drive: drive_v3.Drive,
     fileId: string,
     mimeType: string,
     auth: OAuth2Client
   ): Promise {
-    let textContent = '';
-  
-    // 5MB limit (5 * 1024 * 1024)
-    const sizeLimit = 5 * 1024 * 1024;
-    const fileSize = await this.getFileSize(drive, fileId);
-  
-    if (fileSize !== undefined && fileSize <= sizeLimit) {
-      if (mimeType === 'application/pdf') {
-        const fileBuffer = await this.downloadFile(drive, fileId);
-        textContent = await this.parsePdf(fileBuffer);
-      } else if (mimeType === 'application/vnd.google-apps.document') {
-        textContent = await this.extractGoogleDocsText(drive, fileId);
-      } else if (mimeType === 'application/vnd.google-apps.spreadsheet') {
-        textContent = await this.extractGoogleSheetsText(fileId, auth);
-      } else if (mimeType === 'application/vnd.google-apps.presentation') {
-        textContent = await this.extractGoogleSlidesText(fileId, auth);
-      } else if (mimeType === 'text/plain') {
-        const fileBuffer = await this.downloadFile(drive, fileId);
-        textContent = fileBuffer.toString('utf8');
-      } else if (mimeType === 'application/msword' || mimeType === 'application/vnd.openxmlformats-officedocument.wordprocessingml.document') {
-        const fileBuffer = await this.downloadFile(drive, fileId);
-        textContent = await this.parseDocx(fileBuffer);
-      } else if (mimeType === 'application/vnd.ms-excel' || mimeType === 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet') {
-        const fileBuffer = await this.downloadFile(drive, fileId);
-        textContent = await this.parseXlsx(fileBuffer);
-      } else if (mimeType === 'application/vnd.ms-powerpoint' || mimeType === 'application/vnd.openxmlformats-officedocument.presentationml.presentation') {
-        const fileBuffer = await this.downloadFile(drive, fileId);
-        textContent = await this.parsePptx(fileBuffer);
+    let textContent = "";
+
+    try {
+      // Attempt to fetch the indexable text from contentHints
+      const response = await drive.files.get({
+        fileId: fileId,
+        fields: "contentHints/indexableText, size",
+      });
+
+      // Check if indexable text is available
+      textContent = response.data.contentHints?.indexableText || "";
+
+      // If no indexable text is found, proceed with original extraction methods
+      if (!textContent) {
+        console.warn("No indexable text found, using fallback method.");
+
+        // 10MB limit (5 * 1024 * 1024)
+        const sizeLimit = 10 * 1024 * 1024;
+        const fileSize = response.data.size
+          ? parseInt(response.data.size)
+          : undefined;
+
+        if (fileSize !== undefined && fileSize <= sizeLimit) {
+          if (mimeType === "application/pdf") {
+            const fileBuffer = await this.downloadFile(drive, fileId);
+            textContent = await this.parsePdf(fileBuffer);
+          } else if (mimeType === "application/vnd.google-apps.document") {
+            textContent = await this.extractGoogleDocsText(drive, fileId);
+          } else if (mimeType === "application/vnd.google-apps.spreadsheet") {
+            textContent = await this.extractGoogleSheetsText(fileId, auth);
+          } else if (mimeType === "application/vnd.google-apps.presentation") {
+            textContent = await this.extractGoogleSlidesText(fileId, auth);
+          } else if (mimeType === "text/plain") {
+            const fileBuffer = await this.downloadFile(drive, fileId);
+            textContent = fileBuffer.toString("utf8");
+          } else if (
+            mimeType === "application/msword" ||
+            mimeType ===
+              "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
+          ) {
+            const fileBuffer = await this.downloadFile(drive, fileId);
+            textContent = await this.parseDocx(fileBuffer);
+          } else if (
+            mimeType === "application/vnd.ms-excel" ||
+            mimeType ===
+              "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
+          ) {
+            const fileBuffer = await this.downloadFile(drive, fileId);
+            textContent = await this.parseXlsx(fileBuffer);
+          } else if (
+            mimeType === "application/vnd.ms-powerpoint" ||
+            mimeType ===
+              "application/vnd.openxmlformats-officedocument.presentationml.presentation"
+          ) {
+            const fileBuffer = await this.downloadFile(drive, fileId);
+            textContent = await this.parsePptx(fileBuffer);
+          } else {
+            console.warn(
+              "Unsupported MIME type or file size exceeds the limit."
+            );
+          }
+        } else {
+          console.warn("File size exceeds the limit or unsupported file type.");
+        }
+      } else {
+        console.log("Indexable text extracted successfully from contentHints.");
       }
-    } else {
-      console.warn('File size exceeds the limit or unsupported file type.');
+    } catch (error) {
+      console.error("Error extracting indexable text:", error);
+      throw error;
     }
-  
+
     return textContent;
   }
   
@@ -484,7 +523,7 @@ export class GoogleDriveHelpers {
     };
   }
 
-  static async getFileExtension(drive: drive_v3.Drive, fileId: string): Promise {
+  static async getFileExtension(drive: drive_v3.Drive, fileId: string): Promise {
 
     try {
         const response = await drive.files.get({
@@ -499,7 +538,7 @@ export class GoogleDriveHelpers {
         const lastDotIndex = fileName.lastIndexOf(".");
         if (lastDotIndex === -1) {
             // No period found, so no extension
-            return undefined;
+            return 'Unknown';
         }
 
         // Determine extension from file name
@@ -508,7 +547,7 @@ export class GoogleDriveHelpers {
         // Fallback if extension is not in file name (based on MIME type)
         if (!extension) {
             
-            return mimeExtensions[mimeType] || '';
+            return mimeExtensions[mimeType] || 'Unknown';
         }
 
         return extension;

From 6242a924cd7b7abdf8012db328c912c54a495057 Mon Sep 17 00:00:00 2001
From: chime3 
Date: Sun, 25 Aug 2024 22:35:40 -0700
Subject: [PATCH 043/182] chore: remove console

---
 src/providers/google/gdrive-document.ts | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/providers/google/gdrive-document.ts b/src/providers/google/gdrive-document.ts
index 6ba2e679..f9c3e496 100644
--- a/src/providers/google/gdrive-document.ts
+++ b/src/providers/google/gdrive-document.ts
@@ -157,8 +157,6 @@ export default class GoogleDriveDocument extends GoogleHandler {
         let breakHit: SyncItemsBreak;
     
         for (const file of serverResponse.data.files ?? []) {
-            console.log("==========")
-            console.log(file)
             
             const fileId = file.id ?? '';
             

From ae0e36556db8e16a9bdc741ed5c7106cff5c3fe4 Mon Sep 17 00:00:00 2001
From: Chris 
Date: Tue, 27 Aug 2024 06:59:33 +0930
Subject: [PATCH 044/182] Fix force sync button

---
 src/web/user/connections/connections.js | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/web/user/connections/connections.js b/src/web/user/connections/connections.js
index aa1a037f..f8a94358 100644
--- a/src/web/user/connections/connections.js
+++ b/src/web/user/connections/connections.js
@@ -83,7 +83,7 @@ $(document).ready(function() {
                                     
                                 
                                 

-                                    Force
+                                    Force
                                 

                             

                             
@@ -109,6 +109,7 @@ $(document).ready(function() {
             });
 
             $('.sync-btn').click(function() {
+                console.log('clicked!')
                 const $button = $(this)
                 const provider = $(this).data('provider');
                 const providerId = $(this).data('provider-id');

From 064229ce4cfa88ba54e96b96c50dda0b62bbb96a Mon Sep 17 00:00:00 2001
From: Chris 
Date: Tue, 27 Aug 2024 07:07:22 +0930
Subject: [PATCH 045/182] Remove console.log

---
 src/web/user/connections/connections.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/web/user/connections/connections.js b/src/web/user/connections/connections.js
index f8a94358..132c94de 100644
--- a/src/web/user/connections/connections.js
+++ b/src/web/user/connections/connections.js
@@ -1,6 +1,6 @@
 $(document).ready(function() {
     // Load the private key from local storage
-    const savedVeridaKey = localStorage.getItem('veridaKey');
+    let savedVeridaKey = localStorage.getItem('veridaKey');
     $('#veridaKey').val(savedVeridaKey);
     handleButtonStates();
 
@@ -18,6 +18,7 @@ $(document).ready(function() {
         $('#loadBtn').prop('disabled', !veridaKey);
         $('#generateIdentityBtn').toggle(!veridaKey);
         $('#clearBtn').toggle(!!veridaKey);
+        savedVeridaKey = veridaKey
     }
 
     $('#loadBtn').click(function() {
@@ -109,7 +110,6 @@ $(document).ready(function() {
             });
 
             $('.sync-btn').click(function() {
-                console.log('clicked!')
                 const $button = $(this)
                 const provider = $(this).data('provider');
                 const providerId = $(this).data('provider-id');

From 2167d7d25f0a6e53e061e76911aa6aee8c8dff8a Mon Sep 17 00:00:00 2001
From: Chris 
Date: Tue, 27 Aug 2024 07:37:00 +0930
Subject: [PATCH 046/182] Refactor to support timeframe restrictions on AI
 prompts

---
 src/api/v1/search/controller.ts    |  7 ++-
 src/helpers/interfaces.ts          | 10 ++++
 src/services/assistants/search.ts  | 82 +++++++++++-------------------
 src/services/data.ts               |  6 +--
 src/services/helpers.ts            | 26 ++++++++++
 src/services/search.ts             | 43 ++++++++++------
 src/services/tools/promptSearch.ts | 21 ++++++--
 7 files changed, 119 insertions(+), 76 deletions(-)
 create mode 100644 src/services/helpers.ts

diff --git a/src/api/v1/search/controller.ts b/src/api/v1/search/controller.ts
index 3fea0fa6..3ca19195 100644
--- a/src/api/v1/search/controller.ts
+++ b/src/api/v1/search/controller.ts
@@ -3,6 +3,7 @@ import { Utils } from "../../../utils";
 import { SearchService, SearchType } from "../../../services/search"
 import { MinisearchService, SearchResultItem } from "../../../services/minisearch";
 import { SchemaRecord } from "../../../schemas";
+import { KeywordSearchTimeframe } from "../../../helpers/interfaces";
 
 const DEFAULT_LIMIT = 20
 
@@ -27,9 +28,10 @@ class SearchController {
             const threadSize = req.query.threadSize ? parseInt(req.query.threadSize.toString()) : 10
             const limit = req.query.limit ? parseInt(req.query.limit.toString()) : DEFAULT_LIMIT
             const mergeOverlaps = req.query.limit ? req.query.merge.toString() == 'true' : true
+            const timeframe: KeywordSearchTimeframe = req.query.timeframe ?  req.query.timeframe.toString() : undefined
 
             const searchService = new SearchService(did, context)
-            const items = await searchService.chatThreadsByKeywords(keywords, threadSize, limit, mergeOverlaps)
+            const items = await searchService.chatThreadsByKeywords(keywords, timeframe, threadSize, limit, mergeOverlaps)
 
             return res.json({
                 items
@@ -47,6 +49,7 @@ class SearchController {
             const did = await account.did()
             const keywordString = req.query.keywords ? req.query.keywords.toString() : ""
             const keywords = keywordString.split(' ')
+            const timeframe: KeywordSearchTimeframe = req.query.timeframe ?  req.query.timeframe.toString() : undefined
 
             const searchTypes = req.query.searchTypes ?  req.query.searchTypes.toString().split(',') : [
                 SearchType.EMAILS,
@@ -56,7 +59,7 @@ class SearchController {
             const minResultsPerType = req.query.minResultsPerType ? parseInt(req.query.minResultsPerType.toString()) : 5
 
             const searchService = new SearchService(did, context)
-            const items = await searchService.multiByKeywords(searchTypes, keywords, limit, minResultsPerType)
+            const items = await searchService.multiByKeywords(searchTypes, keywords, timeframe, limit, minResultsPerType)
 
             return res.json({
                 items
diff --git a/src/helpers/interfaces.ts b/src/helpers/interfaces.ts
index 64da2b00..4a70a7c6 100644
--- a/src/helpers/interfaces.ts
+++ b/src/helpers/interfaces.ts
@@ -7,4 +7,14 @@ export interface ItemsRange {
 export enum ItemsRangeStatus {
     NEW = "new",
     BACKFILL = "backfill"
+}
+
+export enum KeywordSearchTimeframe {
+    DAY = "day",
+    WEEK = "week",
+    MONTH = "month",
+    QUARTER = "quarter",
+    HALF_YEAR = "half-year",
+    FULL_YEAR = "full-year",
+    ALL = "all"
 }
\ No newline at end of file
diff --git a/src/services/assistants/search.ts b/src/services/assistants/search.ts
index cb420d91..6ee7f0ab 100644
--- a/src/services/assistants/search.ts
+++ b/src/services/assistants/search.ts
@@ -1,16 +1,19 @@
-import Axios from 'axios'
 const _ = require('lodash')
 import { defaultModel } from "../llm"
-import { PromptSearch, PromptSearchLLMResponse } from "../tools/promptSearch"
-import { ChatThreadResult, SearchService, SearchSortType } from "../search"
+import { PromptSearch, PromptSearchLLMResponse, PromptSearchSort } from "../tools/promptSearch"
+import { ChatThreadResult, SearchService, SearchSortType, SearchType } from "../search"
 import { VeridaService } from '../veridaService'
-import { SchemaEmail, SchemaEmailType, SchemaSocialChatMessage } from '../../schemas'
+import { SchemaEmail, SchemaSocialChatMessage } from '../../schemas'
+import { Helpers } from "../helpers"
 
 const llm = defaultModel
 
 const MAX_EMAIL_LENGTH = 500
 const MAX_ATTACHMENT_LENGTH = 1000
-const MAX_CONTEXT_LENGTH = 20000
+const MAX_CONTEXT_LENGTH = 20000 // (~5000 tokens)
+
+const MAX_DATERANGE_EMAILS = 40
+const MAX_DATERANGE_CHAT_MESSAGES = 100
 
 // "You are a personal assistant with the ability to search the following categories; emails, chat_history and documents. You receive a prompt and generate a JSON response (with no other text) that provides search queries that will source useful information to help answer the prompt. Search queries for each category should contain three properties; \"terms\" (an array of 10 individual words), \"beforeDate\" (results must be before this date), \"afterDate\" (results must be after this date), \"resultType\" (either \"count\" to count results or \"results\" to return the search results), \"filter\" (an array of key, value pairs of fields to filter the results). Categories can be empty if not relevant to the prompt. The current date is 2024-08-12.\n\nHere is an example JSON response:\n{\"email\": {\"terms\": [\"golf\", \"tennis\", \"soccer\"], \"beforeDate\": \"2024-06-01\", \"afterDate\": \"2024-01-10\" \"filter\": {\"from\": \"dave\"}, \"resultType\": \"results}}\n\nHere is the prompt:\nWhat subscriptions do I currently pay for?"
 
@@ -22,65 +25,31 @@ export class PromptSearchService extends VeridaService {
         process: PromptSearchLLMResponse
     }> {
         const start = Date.now()
-        // // Get queries that can help answer the prompt
-        // //const queryPrompt = `Generate 10 lucene search queries, include reasonable synonyms, to find relevant emails to help respond to this prompt:\n${prompt}\nYou have the following searchable fields: subject,messageText,fromName,fromEmail.\nYour response must only contain a single JSON list of search strings, no other commentary and no formatting.`
-        // const keywordPrompt = `Generate 10 individual words that could help search for relevant emails realated to this prompt:\n${prompt}\nYour response must only contain a single JSON list of search strings, no other commentary and no formatting.`
-        // const keywordResponse = await llm(keywordPrompt)
-        // const entityPrompt = `Extract any individual or organization names mentioned in this prompt:\n${prompt}\nYour response must only contain a single JSON list of search strings, no other commentary and no formatting.`
-        // const entityResponse = await llm(entityPrompt)
-
-        // console.log(keywordResponse)
-        // console.log(entityResponse)
-        // const keywords = JSON.parse(keywordResponse)
-        // let entities = []
-        // try {
-        //     entities = JSON.parse(entityResponse)
-        // } catch (err) {
-        //     // do nothing
-        // }
-
-        // console.log(keywords)
-        // console.log(entities)
         const promptSearch = new PromptSearch(llm)
         const promptSearchResult = await promptSearch.search(prompt)
 
+        console.log(promptSearchResult)
+
         let chatThreads: ChatThreadResult[] = []
         let emails: SchemaEmail[] = []
+        let chatMessages: SchemaSocialChatMessage[] = []
 
         const searchService = new SearchService(this.did, this.context)
 
-        let maxAgeSeconds = undefined
-        const dayInSeconds = 60*60*24
-        switch (promptSearchResult.timeframe) {
-            case "day":
-                maxAgeSeconds = dayInSeconds
-                break
-            case "week":
-                maxAgeSeconds = dayInSeconds*7
-                break
-            case "month":
-                maxAgeSeconds = dayInSeconds*30
-                break
-            case "quarter":
-                maxAgeSeconds = dayInSeconds*90
-                break
-            case "half-year":
-                maxAgeSeconds = dayInSeconds*180
-                break
-            case "full-year":
-                maxAgeSeconds = dayInSeconds*365
-                break
+        if (promptSearchResult.sort == PromptSearchSort.KEYWORD_RANK) {
+            emails = await searchService.emailsByKeywords(promptSearchResult.keywords, promptSearchResult.timeframe, 20)
+            chatThreads = await searchService.chatThreadsByKeywords(promptSearchResult.keywords, promptSearchResult.timeframe, 10, 10)
+        } else {
+            const maxDatetime = Helpers.keywordTimeframeToDate(promptSearchResult.timeframe)
+            const sort = promptSearchResult.sort == PromptSearchSort.RECENT ? SearchSortType.RECENT : SearchSortType.OLDEST
+            emails =  await searchService.schemaByDateRange(SearchType.EMAILS, maxDatetime, sort, MAX_DATERANGE_EMAILS)
+            chatMessages =  await searchService.schemaByDateRange(SearchType.CHAT_MESSAGES, maxDatetime, sort, MAX_DATERANGE_CHAT_MESSAGES)
         }
-        const maxDatetime = new Date((new Date()).getTime() - maxAgeSeconds * 1000);
-        const sortType = promptSearchResult.sort == "keyword_rank" ? SearchSortType.RECENT :  promptSearchResult.sort
-
-        emails = await searchService.emailsByKeywords(promptSearchResult.keywords, 20)
-        chatThreads = await searchService.chatThreadsByKeywords(promptSearchResult.keywords, 10, 10)
 
         let finalPrompt = `Answer this prompt:\n${prompt}\nHere are some recent messages that may help you provide a relevant answer.\n`
         let contextString = ''
 
-        let maxChatMessages = 50
+        let maxChatMessages = MAX_DATERANGE_CHAT_MESSAGES
         for (const chatThread of chatThreads) {
             for (const chatMessage of chatThread.messages) {
                 contextString += `From: ${chatMessage.fromName} <${chatMessage.fromHandle}> (${chatMessage.groupName})\nBody: ${chatMessage.messageText}\n\n`
@@ -91,6 +60,13 @@ export class PromptSearchService extends VeridaService {
             }
         }
 
+        for (const chatMessage of chatMessages) {
+            contextString += `From: ${chatMessage.fromName} <${chatMessage.fromHandle}> (${chatMessage.groupName})\nBody: ${chatMessage.messageText}\n\n`
+        }
+
+        // console.log('pre-email context string: ', contextString.length)
+
+        let emailCount = 0
         for (const email of emails) {
             let extraContext = ""
             let body = email.messageText.substring(0, MAX_EMAIL_LENGTH)
@@ -101,13 +77,17 @@ export class PromptSearchService extends VeridaService {
             }
 
             extraContext = `From: ${email.fromName} <${email.fromEmail}> (${email.name})\nBody: ${body}\n\n`
+            // console.log(email.fromName, email.fromEmail, email.name, body.length, email.messageText.length)
             if ((extraContext.length + contextString.length + finalPrompt.length) > MAX_CONTEXT_LENGTH) {
                 break
             }
             
             contextString += extraContext
+            emailCount++
         }
 
+        // console.log('email count', emailCount)
+
         const now = (new Date()).toISOString()
         finalPrompt += `${contextString}\nThe current time is: ${now}`
 
diff --git a/src/services/data.ts b/src/services/data.ts
index a058be82..5f98741a 100644
--- a/src/services/data.ts
+++ b/src/services/data.ts
@@ -37,17 +37,17 @@ const schemas: Record = {
     },
     "https://common.schemas.verida.io/social/email/v0.1.0/schema.json": {
         label: "Email",
-        storeFields: ['_id'],
+        storeFields: ['_id', 'sentAt'],
         indexFields: ['name','fromName','fromEmail','messageText','attachments_0.textContent','attachments_1.textContent','attachments_2.textContent', 'indexableText', 'sentAt']
     },
     "https://common.schemas.verida.io/social/chat/message/v0.1.0/schema.json": {
         label: "Chat Message",
-        storeFields: ['_id', 'groupId'],
+        storeFields: ['_id', 'groupId', 'sentAt'],
         indexFields: ['messageText', 'fromHandle', 'fromName', 'groupName', 'indexableText', 'sentAt']
     },
     "https://common.schemas.verida.io/favourite/v0.1.0/schema.json": {
         label: "Favourite",
-        storeFields: ['_id'],
+        storeFields: ['_id', 'insertedAt'],
         indexFields: ['name', 'favouriteType', 'contentTYpe', 'summary']
     }
 }
diff --git a/src/services/helpers.ts b/src/services/helpers.ts
new file mode 100644
index 00000000..20b17f5b
--- /dev/null
+++ b/src/services/helpers.ts
@@ -0,0 +1,26 @@
+import { KeywordSearchTimeframe } from "../helpers/interfaces"
+
+const dayInSeconds = 60*60*24
+
+const TIMEFRAME_SECONDS: Record = {
+    [KeywordSearchTimeframe.DAY]: dayInSeconds,
+    [KeywordSearchTimeframe.WEEK]: dayInSeconds * 7,
+    [KeywordSearchTimeframe.MONTH]: dayInSeconds * 30,
+    [KeywordSearchTimeframe.QUARTER]: dayInSeconds * 90,
+    [KeywordSearchTimeframe.HALF_YEAR]: dayInSeconds * 180,
+    [KeywordSearchTimeframe.FULL_YEAR]: dayInSeconds * 365,
+    [KeywordSearchTimeframe.ALL]: undefined
+}
+
+export class Helpers {
+
+    public static keywordTimeframeToDate(timeframe: KeywordSearchTimeframe) {
+        const maxAgeSeconds = TIMEFRAME_SECONDS[timeframe]
+        if (!maxAgeSeconds) {
+            return new Date("1900-01-01")
+        }
+
+        return new Date((new Date()).getTime() - maxAgeSeconds * 1000);
+    }
+
+}
\ No newline at end of file
diff --git a/src/services/search.ts b/src/services/search.ts
index a30de84c..336a697b 100644
--- a/src/services/search.ts
+++ b/src/services/search.ts
@@ -2,7 +2,8 @@ import { DataService } from "./data"
 import { VeridaService } from "./veridaService"
 import { SchemaEmail, SchemaRecord, SchemaSocialChatGroup, SchemaSocialChatMessage } from "../schemas"
 import { IDatastore } from "@verida/types"
-import { threadId } from "worker_threads"
+import { KeywordSearchTimeframe } from "../helpers/interfaces"
+import { Helpers } from "./helpers"
 const _ = require('lodash')
 
 export interface MinisearchResult {
@@ -107,27 +108,29 @@ export class SearchService extends VeridaService {
         return results
     }
 
-    public async emailsByKeywords(keywordsList: string[], limit: number = 20): Promise {
+    public async emailsByKeywords(keywordsList: string[], timeframe: KeywordSearchTimeframe, limit: number = 20): Promise {
         const query = keywordsList.join(' ')
         const searchType = SearchType.EMAILS
         const schemaUri = SearchTypeSchemas[searchType]
         const dataService = new DataService(this.did, this.context)
         const miniSearchIndex = await dataService.getIndex(schemaUri)
 
-        console.log('Emails: searching for', query)
+        const maxDatetime = Helpers.keywordTimeframeToDate(timeframe)
+        console.log('Emails: searching for', query, timeframe)
         
-        const searchResults = await miniSearchIndex.search(query)
+        const searchResults = await miniSearchIndex.search(query, {
+            filter: (result: any) => maxDatetime ? result.sentAt > maxDatetime.toISOString() : true
+        })
         return  await this.rankAndMergeResults([{
             searchType,
             rows: searchResults
         }], limit)
     }
 
-    public async emailsByDateRange(maxDatetime: Date, sortType: SearchSortType, limit: number = 20): Promise {
-        const searchType = SearchType.EMAILS
+    public async schemaByDateRange(searchType: SearchType, maxDatetime: Date, sortType: SearchSortType, limit: number = 20): Promise {
         const schemaUri = SearchTypeSchemas[searchType]
         const dataService = new DataService(this.did, this.context)
-        const emailDatastore = await dataService.getDatastore(schemaUri)
+        const datastore = await dataService.getDatastore(schemaUri)
         const filter = {
             sentAt: {
                 "$gte": maxDatetime.toISOString()
@@ -142,20 +145,24 @@ export class SearchService extends VeridaService {
             ]
         }
 
-        console.log('Emails: searching for', filter, options)
-        return  await emailDatastore.getMany(filter, options)
+        console.log(searchType, ': searching for', filter, options)
+        return  await datastore.getMany(filter, options)
     }
 
-    public async chatHistoryByKeywords(keywordsList: string[], limit: number = 20): Promise {
+    public async chatHistoryByKeywords(keywordsList: string[], timeframe: KeywordSearchTimeframe, limit: number = 20): Promise {
         const searchType = SearchType.CHAT_MESSAGES
         const schemaUri = SearchTypeSchemas[searchType]
         const query = keywordsList.join(' ')
         const dataService = new DataService(this.did, this.context)
         const miniSearchIndex = await dataService.getIndex(schemaUri)
 
-        console.log('Chat history: searching for', query)
+        const maxDatetime = Helpers.keywordTimeframeToDate(timeframe)
+        console.log('Chat history: searching for', query, timeframe, maxDatetime)
         
-        const searchResults = await miniSearchIndex.search(query)
+        const searchResults = await miniSearchIndex.search(query, {
+            filter: (result: any) => maxDatetime ? result.sentAt > maxDatetime.toISOString() : true
+        })
+
         return this.rankAndMergeResults([{
             searchType,
             rows: searchResults
@@ -173,16 +180,19 @@ export class SearchService extends VeridaService {
      * @param mergeOverlaps If there is an overlap of messages within the same chat group, they will be merged into a single thread.
      * @returns 
      */
-    public async chatThreadsByKeywords(keywordsList: string[], threadSize: number = 10, limit: number = 20, mergeOverlaps: boolean = true): Promise {
+    public async chatThreadsByKeywords(keywordsList: string[], timeframe: KeywordSearchTimeframe, threadSize: number = 10, limit: number = 20, mergeOverlaps: boolean = true): Promise {
         const query = keywordsList.join(' ')
         const messageSchemaUri = "https://common.schemas.verida.io/social/chat/message/v0.1.0/schema.json"
         const groupSchemaUri = "https://common.schemas.verida.io/social/chat/group/v0.1.0/schema.json"
         const dataService = new DataService(this.did, this.context)
         const miniSearchIndex = await dataService.getIndex(messageSchemaUri)
 
-        console.log('Chat threads: searching for', query)
+        const maxDatetime = Helpers.keywordTimeframeToDate(timeframe)
+        console.log('Chat threads: searching for', query, timeframe, maxDatetime)
         
-        const searchResults = await miniSearchIndex.search(query)
+        const searchResults = await miniSearchIndex.search(query, {
+            filter: (result: any) => maxDatetime ? result.sentAt > maxDatetime.toISOString() : true
+        })
         const chatMessageDs = await this.context.openDatastore(messageSchemaUri)
         const chatGroupDs = await this.context.openDatastore(groupSchemaUri)
 
@@ -263,9 +273,10 @@ export class SearchService extends VeridaService {
         
     }
 
-    public async multiByKeywords(searchTypes: SearchType[], keywordsList: string[], limit: number = 20, minResultsPerType: number = 10) {
+    public async multiByKeywords(searchTypes: SearchType[], keywordsList: string[], timeframe: KeywordSearchTimeframe, limit: number = 20, minResultsPerType: number = 10) {
         const query = keywordsList.join(' ')
         const dataService = new DataService(this.did, this.context)
+        const maxTimeframe = 
 
         console.log('Multi: searching for', query)
 
diff --git a/src/services/tools/promptSearch.ts b/src/services/tools/promptSearch.ts
index 6f555dab..77a95b98 100644
--- a/src/services/tools/promptSearch.ts
+++ b/src/services/tools/promptSearch.ts
@@ -1,3 +1,4 @@
+import { KeywordSearchTimeframe } from "../../helpers/interfaces";
 import { LLM } from "../llm"
 
 const systemPrompt = `You are an expert data analyst. When I give you a prompt, you must generate search metadata that will be used to extract relevant information to help answer the prompt.
@@ -7,18 +8,30 @@ You must generate a JSON response containing the following information:
 - timeframe: one of; day, week, month, quarter, half-year, full-year, all
 - databases: an array of databases to search; emails, chat_messages, files, favourites, web_history, calendar
 - sort: keyword_rank, recent, oldest
-- output_type: The amount of detail in the output of each search result to provide meaningful context. full_content, summary, name
+- output_type: The amount of detail in the output of each search result to provide meaningful context. full_content, summary, headline
 - profile_information; Array of these options only; name, contactInfo, demographics, lifestyle, preferences, habits, financial, health, personality, employment, education, skills, language, interests
 
 JSON only, no explanation.`
 
+export enum PromptSearchSort {
+  KEYWORD_RANK = "keyword_rank",
+  RECENT = "recent",
+  OLDEST = "oldest"
+}
+
+export enum PromptSearchOutputType {
+  FULL = "full_content",
+  SUMMARY = "summary",
+  HEADLINE = "headline"
+}
+
 export interface PromptSearchLLMResponse {
     search_type: "keywords" | "all";
     keywords?: string[]; // Array of single word terms, required if search_type is "keywords"
-    timeframe: "day" | "week" | "month" | "quarter" | "half-year" | "full-year" | "all";
+    timeframe: KeywordSearchTimeframe;
     databases: Array<"emails" | "chat_messages" | "files" | "favourites" | "web_history" | "calendar">;
-    sort: "keyword_rank" | "recent" | "oldest";
-    output_type: "full_content" | "summary" | "name";
+    sort: PromptSearchSort;
+    output_type: PromptSearchOutputType;
     profile_information: Array<
       "name" | "contactInfo" | "demographics" | "lifestyle" | "preferences" | "habits" |
       "financial" | "health" | "personality" | "employment" | "education" | "skills" |

From 6f307fb92a5f0f90dcacf99a916c92146ba1778d Mon Sep 17 00:00:00 2001
From: chime3 
Date: Mon, 26 Aug 2024 15:13:36 -0700
Subject: [PATCH 047/182] fix: updated schema and unit test

---
 src/providers/google/gdrive-document.ts         | 1 +
 src/schemas.ts                                  | 2 +-
 tests/providers/google/gdrive-document.tests.ts | 2 +-
 3 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/providers/google/gdrive-document.ts b/src/providers/google/gdrive-document.ts
index f9c3e496..cdf421e1 100644
--- a/src/providers/google/gdrive-document.ts
+++ b/src/providers/google/gdrive-document.ts
@@ -220,6 +220,7 @@ export default class GoogleDriveDocument extends GoogleHandler {
     
             results.push({
                 _id: this.buildItemId(fileId),
+                schema: CONFIG.verida.schemas.FILE,
                 name: title,
                 mimeType: mimeType,
                 extension: extension,
diff --git a/src/schemas.ts b/src/schemas.ts
index 1e81bf96..3d3fd3d4 100644
--- a/src/schemas.ts
+++ b/src/schemas.ts
@@ -5,7 +5,7 @@ export interface SchemaRecord {
     _rev?: string
     schema?: string
     name: string
-    description?: string
+    summary?: string
     insertedAt?: string
     modifiedAt?: string
     icon?: string
diff --git a/tests/providers/google/gdrive-document.tests.ts b/tests/providers/google/gdrive-document.tests.ts
index cdb507fb..f53e7b2a 100644
--- a/tests/providers/google/gdrive-document.tests.ts
+++ b/tests/providers/google/gdrive-document.tests.ts
@@ -32,7 +32,7 @@ describe(`${providerName} GDrive Document Tests`, function () {
   
     testConfig = {
       idPrefix: `${provider.getProviderName()}-${connection.profile.id}`,
-      timeOrderAttribute: "insertedAt",
+      timeOrderAttribute: "modifiedAt",
       batchSizeLimitAttribute: "batchSize",
     };
   });

From 746d9e6de680a71f602a91c3b85f7618e11772ed Mon Sep 17 00:00:00 2001
From: chime3 
Date: Mon, 26 Aug 2024 15:46:07 -0700
Subject: [PATCH 048/182] feat: added comments for MAX_BATCH_SIZE

---
 src/providers/google/youtube-favourite.ts | 2 ++
 src/providers/google/youtube-following.ts | 2 ++
 src/providers/google/youtube-post.ts      | 2 ++
 3 files changed, 6 insertions(+)

diff --git a/src/providers/google/youtube-favourite.ts b/src/providers/google/youtube-favourite.ts
index 2d71de45..7cfdc785 100644
--- a/src/providers/google/youtube-favourite.ts
+++ b/src/providers/google/youtube-favourite.ts
@@ -14,6 +14,8 @@ import { ItemsRangeTracker } from "../../helpers/itemsRangeTracker";
 
 const _ = require("lodash");
 
+// Set MAX_BATCH_SIZE to 50 because the YouTube Data API v3 'maxResults' parameter is capped at 50.
+// For more details, see: https://developers.google.com/youtube/v3/docs/search/list
 const MAX_BATCH_SIZE = 50;
 
 export interface SyncFavouriteItemsResult extends SyncItemsResult {
diff --git a/src/providers/google/youtube-following.ts b/src/providers/google/youtube-following.ts
index 0315a089..85a3272a 100644
--- a/src/providers/google/youtube-following.ts
+++ b/src/providers/google/youtube-following.ts
@@ -8,6 +8,8 @@ import { ItemsRangeTracker } from "../../helpers/itemsRangeTracker";
 
 const _ = require("lodash");
 
+// Set MAX_BATCH_SIZE to 50 because the YouTube Data API v3 'maxResults' parameter is capped at 50.
+// For more details, see: https://developers.google.com/youtube/v3/docs/search/list
 const MAX_BATCH_SIZE = 50;
 
 export default class YouTubeFollowing extends GoogleHandler {
diff --git a/src/providers/google/youtube-post.ts b/src/providers/google/youtube-post.ts
index d88b35d8..5e63e64f 100644
--- a/src/providers/google/youtube-post.ts
+++ b/src/providers/google/youtube-post.ts
@@ -14,6 +14,8 @@ import { ItemsRangeTracker } from "../../helpers/itemsRangeTracker";
 
 const _ = require("lodash");
 
+// Set MAX_BATCH_SIZE to 50 because the YouTube Data API v3 'maxResults' parameter is capped at 50.
+// For more details, see: https://developers.google.com/youtube/v3/docs/search/list
 const MAX_BATCH_SIZE = 50;
 
 export interface SyncPostItemsResult extends SyncItemsResult {

From 2614faf0bae79943041fe0479626ddffcd7224fe Mon Sep 17 00:00:00 2001
From: Chris 
Date: Tue, 27 Aug 2024 13:42:46 +0930
Subject: [PATCH 049/182] Fix provider connect URL

---
 src/cli/commands/connect.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/cli/commands/connect.ts b/src/cli/commands/connect.ts
index 420111ff..ac09bdd0 100644
--- a/src/cli/commands/connect.ts
+++ b/src/cli/commands/connect.ts
@@ -68,7 +68,7 @@ export const Connect: Command = {
       const rows = await ds.getMany()
       console.log(rows)*/
 
-      const openUrl = `${serverconfig.serverUrl}/api/${serverconfig.apiVersion}/connect/${options.provider}?key=${signature}`
+      const openUrl = `${serverconfig.serverUrl}/api/v1/provider/connect/${options.provider}?key=${signature}`
       open(openUrl)
     }
   };
\ No newline at end of file

From 790103075422be63e35ec9ba030c4c06ee5e74b5 Mon Sep 17 00:00:00 2001
From: Chris 
Date: Tue, 27 Aug 2024 14:17:44 +0930
Subject: [PATCH 050/182] Set default google batchsize to 50

---
 src/serverconfig.example.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/serverconfig.example.json b/src/serverconfig.example.json
index 4034ca14..16600379 100644
--- a/src/serverconfig.example.json
+++ b/src/serverconfig.example.json
@@ -65,7 +65,7 @@
         "google": {
             "clientId": "",
             "clientSecret": "",
-            "batchSize": 100,
+            "batchSize": 50,
             "maxSyncLoops": 1,
             "metadata": {
                 "breakTimestamp": "2000-07-21T12:07:11.000Z"

From 7320924605529ab447fcbd64dad9e197555e7b79 Mon Sep 17 00:00:00 2001
From: Chris 
Date: Tue, 27 Aug 2024 14:17:59 +0930
Subject: [PATCH 051/182] Increase gdrive timeout to handle local downloads

---
 tests/providers/google/gdrive-document.tests.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/providers/google/gdrive-document.tests.ts b/tests/providers/google/gdrive-document.tests.ts
index f53e7b2a..f297c216 100644
--- a/tests/providers/google/gdrive-document.tests.ts
+++ b/tests/providers/google/gdrive-document.tests.ts
@@ -23,7 +23,7 @@ let providerConfig: Omit = {};
 
 
 describe(`${providerName} GDrive Document Tests`, function () {
-  this.timeout(100000);
+  this.timeout(400000);
 
   this.beforeAll(async function () {
     network = await CommonUtils.getNetwork();

From a1e2de3baed98bb0f44c6ba6a6e8747701520710 Mon Sep 17 00:00:00 2001
From: Chris 
Date: Tue, 27 Aug 2024 14:58:33 +0930
Subject: [PATCH 052/182] Support network cache garbage collection

---
 src/utils.ts | 23 ++++++++++++++++++++++-
 1 file changed, 22 insertions(+), 1 deletion(-)

diff --git a/src/utils.ts b/src/utils.ts
index 30f74512..1b1f9f0f 100644
--- a/src/utils.ts
+++ b/src/utils.ts
@@ -10,6 +10,7 @@ import { Request } from 'express'
 const VAULT_CONTEXT_NAME = 'Verida: Vault'
 const DID_CLIENT_CONFIG = serverconfig.verida.didClientConfig
 const SBT_CREDENTIAL_SCHEMA = 'https://common.schemas.verida.io/token/sbt/credential/v0.1.0/schema.json'
+const NETWORK_CONNECTION_CACHE_EXPIRY = 60*3 // 3 mins
 
 export {
     DID_CLIENT_CONFIG,
@@ -20,6 +21,7 @@ export interface NetworkConnectionCache {
     requestIds: string[],
     currentPromise?: Promise,
     networkConnection?: NetworkConnection
+    lastTouch: Date
 }
 
 export interface NetworkConnection {
@@ -71,6 +73,9 @@ export class Utils {
 
         if (Utils.networkCache[did]) {
             Utils.networkCache[did].requestIds.push(requestId)
+            Utils.networkCache[did].lastTouch = new Date()
+
+            Utils.gcNetworkCache()
             return Utils.networkCache[did].networkConnection
         }
 
@@ -80,7 +85,8 @@ export class Utils {
         //     await Utils.networkCache[did].shuttingPromise
         // }
         Utils.networkCache[did] = {
-            requestIds: [requestId]
+            requestIds: [requestId],
+            lastTouch: new Date()
         }
 
         Utils.networkCache[did].currentPromise = new Promise(async (resolve, reject) => {
@@ -96,6 +102,7 @@ export class Utils {
     
             Utils.networkCache[did] = {
                 requestIds: [requestId],
+                lastTouch: new Date(),
                 networkConnection
             }
 
@@ -106,6 +113,20 @@ export class Utils {
         return Utils.networkCache[did].networkConnection
     }
 
+    public static async gcNetworkCache() {
+        // console.log("gcNetworkCache()")
+        for (const did in Utils.networkCache) {
+            const cache = Utils.networkCache[did]
+            const duration = ((new Date()).getTime() - cache.lastTouch.getTime())/1000
+            // console.log("gcNetworkCache()", duration)
+            if (duration > NETWORK_CONNECTION_CACHE_EXPIRY) {
+                // console.log("gcNetworkCache() -- expired!", did)
+                await Utils.networkCache[did].networkConnection.context.close()
+                delete Utils.networkCache[did]
+            }
+        }
+    }
+
     public static async closeConnection(did: string, requestId: string = 'none'): Promise {
         Utils.networkCache[did].requestIds = Utils.networkCache[did].requestIds.filter(id => id !== requestId)
 

From cdcf4038bd52177329c61fc977da1a22c1957efe Mon Sep 17 00:00:00 2001
From: Chris 
Date: Tue, 27 Aug 2024 14:59:21 +0930
Subject: [PATCH 053/182] Support generics on keyword and schema searches.
 Support searching schemas based on AI prompt requests.

---
 src/services/assistants/search.ts  | 42 +++++++++++++++++++++++-------
 src/services/search.ts             | 17 ++++++------
 src/services/tools/promptSearch.ts |  7 ++++-
 3 files changed, 48 insertions(+), 18 deletions(-)

diff --git a/src/services/assistants/search.ts b/src/services/assistants/search.ts
index 6ee7f0ab..efda491e 100644
--- a/src/services/assistants/search.ts
+++ b/src/services/assistants/search.ts
@@ -1,9 +1,9 @@
 const _ = require('lodash')
 import { defaultModel } from "../llm"
-import { PromptSearch, PromptSearchLLMResponse, PromptSearchSort } from "../tools/promptSearch"
+import { PromptSearch, PromptSearchLLMResponse, PromptSearchSort, PromptSearchType } from "../tools/promptSearch"
 import { ChatThreadResult, SearchService, SearchSortType, SearchType } from "../search"
 import { VeridaService } from '../veridaService'
-import { SchemaEmail, SchemaSocialChatMessage } from '../../schemas'
+import { SchemaEmail, SchemaFavourite, SchemaSocialChatMessage } from '../../schemas'
 import { Helpers } from "../helpers"
 
 const llm = defaultModel
@@ -14,6 +14,8 @@ const MAX_CONTEXT_LENGTH = 20000 // (~5000 tokens)
 
 const MAX_DATERANGE_EMAILS = 40
 const MAX_DATERANGE_CHAT_MESSAGES = 100
+const MAX_DATERANGE_FAVOURITES = 20
+const MAX_DATERANGE_FILES = 20
 
 // "You are a personal assistant with the ability to search the following categories; emails, chat_history and documents. You receive a prompt and generate a JSON response (with no other text) that provides search queries that will source useful information to help answer the prompt. Search queries for each category should contain three properties; \"terms\" (an array of 10 individual words), \"beforeDate\" (results must be before this date), \"afterDate\" (results must be after this date), \"resultType\" (either \"count\" to count results or \"results\" to return the search results), \"filter\" (an array of key, value pairs of fields to filter the results). Categories can be empty if not relevant to the prompt. The current date is 2024-08-12.\n\nHere is an example JSON response:\n{\"email\": {\"terms\": [\"golf\", \"tennis\", \"soccer\"], \"beforeDate\": \"2024-06-01\", \"afterDate\": \"2024-01-10\" \"filter\": {\"from\": \"dave\"}, \"resultType\": \"results}}\n\nHere is the prompt:\nWhat subscriptions do I currently pay for?"
 
@@ -32,18 +34,40 @@ export class PromptSearchService extends VeridaService {
 
         let chatThreads: ChatThreadResult[] = []
         let emails: SchemaEmail[] = []
+        let favourites: SchemaFavourite[] = []
+        // let files: SchemaFile[] = []
         let chatMessages: SchemaSocialChatMessage[] = []
 
         const searchService = new SearchService(this.did, this.context)
 
-        if (promptSearchResult.sort == PromptSearchSort.KEYWORD_RANK) {
-            emails = await searchService.emailsByKeywords(promptSearchResult.keywords, promptSearchResult.timeframe, 20)
-            chatThreads = await searchService.chatThreadsByKeywords(promptSearchResult.keywords, promptSearchResult.timeframe, 10, 10)
+        if (promptSearchResult.search_type == PromptSearchType.KEYWORDS) {
+            if (promptSearchResult.databases.indexOf("emails")) {
+                emails = await searchService.schemaByKeywords(SearchType.EMAILS, promptSearchResult.keywords!, promptSearchResult.timeframe, 20)
+            }
+            // if (promptSearchResult.databases.indexOf("files")) {
+            //     files = await searchService.schemaByKeywords(SearchType.FILES, promptSearchResult.keywords!, promptSearchResult.timeframe, 20)
+            // }
+            if (promptSearchResult.databases.indexOf("favourites")) {
+                favourites = await searchService.schemaByKeywords(SearchType.FAVORITES, promptSearchResult.keywords!, promptSearchResult.timeframe, 20)
+            }
+            if (promptSearchResult.databases.indexOf("chat_messages")) {
+                chatThreads = await searchService.chatThreadsByKeywords(promptSearchResult.keywords!, promptSearchResult.timeframe, 10, 10)
+            }
         } else {
             const maxDatetime = Helpers.keywordTimeframeToDate(promptSearchResult.timeframe)
             const sort = promptSearchResult.sort == PromptSearchSort.RECENT ? SearchSortType.RECENT : SearchSortType.OLDEST
-            emails =  await searchService.schemaByDateRange(SearchType.EMAILS, maxDatetime, sort, MAX_DATERANGE_EMAILS)
-            chatMessages =  await searchService.schemaByDateRange(SearchType.CHAT_MESSAGES, maxDatetime, sort, MAX_DATERANGE_CHAT_MESSAGES)
+            if (promptSearchResult.databases.indexOf("emails")) {
+                emails = await searchService.schemaByDateRange(SearchType.EMAILS, maxDatetime, sort, MAX_DATERANGE_EMAILS)
+            }
+            // if (promptSearchResult.databases.indexOf("files")) {
+            //     files = await searchService.schemaByDateRange(SearchType.FILES, maxDatetime, sort, MAX_DATERANGE_FILES)
+            // }
+            if (promptSearchResult.databases.indexOf("favourites")) {
+                favourites = await searchService.schemaByDateRange(SearchType.FAVORITES, maxDatetime, sort, MAX_DATERANGE_FAVOURITES)
+            }
+            if (promptSearchResult.databases.indexOf("chat_messages")) {
+                chatMessages =  await searchService.schemaByDateRange(SearchType.CHAT_MESSAGES, maxDatetime, sort, MAX_DATERANGE_CHAT_MESSAGES)
+            }
         }
 
         let finalPrompt = `Answer this prompt:\n${prompt}\nHere are some recent messages that may help you provide a relevant answer.\n`
@@ -72,7 +96,7 @@ export class PromptSearchService extends VeridaService {
             let body = email.messageText.substring(0, MAX_EMAIL_LENGTH)
             if (email.attachments) {
                 for (const attachment of email.attachments) {
-                    body += attachment.textContent.substring(0, MAX_ATTACHMENT_LENGTH)
+                    body += attachment.textContent!.substring(0, MAX_ATTACHMENT_LENGTH)
                 }
             }
 
@@ -98,7 +122,7 @@ export class PromptSearchService extends VeridaService {
         // console.log(contextString)
 
         return {
-            result: finalResponse.choices[0].message.content,
+            result: finalResponse.choices[0].message.content!,
             duration,
             process: promptSearchResult
         }
diff --git a/src/services/search.ts b/src/services/search.ts
index 336a697b..ee193585 100644
--- a/src/services/search.ts
+++ b/src/services/search.ts
@@ -28,6 +28,7 @@ export enum SearchSortType {
 
 export enum SearchType {
     // CHAT_THREADS = "chat-threads",
+    FILES = "files",
     CHAT_MESSAGES = "chat-messages",
     EMAILS = "emails",
     FAVORITES = "favorites",
@@ -37,6 +38,7 @@ export enum SearchType {
 
 export const SearchTypeSchemas: Record = {
     // [SearchType.CHAT_THREADS]: "https://common.schemas.verida.io/social/chat/message/v0.1.0/schema.json",
+    [SearchType.FILES]: "https://common.schemas.verida.io/file/v0.1.0/schema.json",
     [SearchType.CHAT_MESSAGES]: "https://common.schemas.verida.io/social/chat/message/v0.1.0/schema.json",
     [SearchType.EMAILS]: "https://common.schemas.verida.io/social/email/v0.1.0/schema.json",
     [SearchType.FAVORITES]: "https://common.schemas.verida.io/favourite/v0.1.0/schema.json",
@@ -108,26 +110,25 @@ export class SearchService extends VeridaService {
         return results
     }
 
-    public async emailsByKeywords(keywordsList: string[], timeframe: KeywordSearchTimeframe, limit: number = 20): Promise {
+    public async schemaByKeywords(searchType: SearchType, keywordsList: string[], timeframe: KeywordSearchTimeframe, limit: number = 20): Promise {
         const query = keywordsList.join(' ')
-        const searchType = SearchType.EMAILS
         const schemaUri = SearchTypeSchemas[searchType]
         const dataService = new DataService(this.did, this.context)
         const miniSearchIndex = await dataService.getIndex(schemaUri)
 
         const maxDatetime = Helpers.keywordTimeframeToDate(timeframe)
-        console.log('Emails: searching for', query, timeframe)
+        console.log(`${searchType}: Searching for ${query} (${timeframe})`)
         
         const searchResults = await miniSearchIndex.search(query, {
             filter: (result: any) => maxDatetime ? result.sentAt > maxDatetime.toISOString() : true
         })
-        return  await this.rankAndMergeResults([{
+        return await this.rankAndMergeResults([{
             searchType,
             rows: searchResults
-        }], limit)
+        }], limit) as T[]
     }
 
-    public async schemaByDateRange(searchType: SearchType, maxDatetime: Date, sortType: SearchSortType, limit: number = 20): Promise {
+    public async schemaByDateRange(searchType: SearchType, maxDatetime: Date, sortType: SearchSortType, limit: number = 20): Promise {
         const schemaUri = SearchTypeSchemas[searchType]
         const dataService = new DataService(this.did, this.context)
         const datastore = await dataService.getDatastore(schemaUri)
@@ -146,10 +147,10 @@ export class SearchService extends VeridaService {
         }
 
         console.log(searchType, ': searching for', filter, options)
-        return  await datastore.getMany(filter, options)
+        return await datastore.getMany(filter, options) as T[]
     }
 
-    public async chatHistoryByKeywords(keywordsList: string[], timeframe: KeywordSearchTimeframe, limit: number = 20): Promise {
+    public async chatHistoryByKeywords(keywordsList: string[], timeframe: KeywordSearchTimeframe, limit: number = 20): Promise {
         const searchType = SearchType.CHAT_MESSAGES
         const schemaUri = SearchTypeSchemas[searchType]
         const query = keywordsList.join(' ')
diff --git a/src/services/tools/promptSearch.ts b/src/services/tools/promptSearch.ts
index 77a95b98..63aa7e76 100644
--- a/src/services/tools/promptSearch.ts
+++ b/src/services/tools/promptSearch.ts
@@ -13,6 +13,11 @@ You must generate a JSON response containing the following information:
 
 JSON only, no explanation.`
 
+export enum PromptSearchType {
+  KEYWORDS = "keywords",
+  ALL = "all"
+}
+
 export enum PromptSearchSort {
   KEYWORD_RANK = "keyword_rank",
   RECENT = "recent",
@@ -26,7 +31,7 @@ export enum PromptSearchOutputType {
 }
 
 export interface PromptSearchLLMResponse {
-    search_type: "keywords" | "all";
+    search_type: PromptSearchType;
     keywords?: string[]; // Array of single word terms, required if search_type is "keywords"
     timeframe: KeywordSearchTimeframe;
     databases: Array<"emails" | "chat_messages" | "files" | "favourites" | "web_history" | "calendar">;

From 828b704093d0dcadc246edef710bb372279e3004 Mon Sep 17 00:00:00 2001
From: Chris 
Date: Tue, 27 Aug 2024 15:16:25 +0930
Subject: [PATCH 054/182] Fix issues with fetching data for AI prompts.

---
 src/services/assistants/search.ts  | 12 ++++++------
 src/services/data.ts               |  2 +-
 src/services/search.ts             | 14 ++++++++++++--
 src/services/tools/promptSearch.ts |  5 +++--
 4 files changed, 22 insertions(+), 11 deletions(-)

diff --git a/src/services/assistants/search.ts b/src/services/assistants/search.ts
index efda491e..12414d56 100644
--- a/src/services/assistants/search.ts
+++ b/src/services/assistants/search.ts
@@ -41,31 +41,31 @@ export class PromptSearchService extends VeridaService {
         const searchService = new SearchService(this.did, this.context)
 
         if (promptSearchResult.search_type == PromptSearchType.KEYWORDS) {
-            if (promptSearchResult.databases.indexOf("emails")) {
+            if (promptSearchResult.databases.indexOf(SearchType.EMAILS) !== -1) {
                 emails = await searchService.schemaByKeywords(SearchType.EMAILS, promptSearchResult.keywords!, promptSearchResult.timeframe, 20)
             }
             // if (promptSearchResult.databases.indexOf("files")) {
             //     files = await searchService.schemaByKeywords(SearchType.FILES, promptSearchResult.keywords!, promptSearchResult.timeframe, 20)
             // }
-            if (promptSearchResult.databases.indexOf("favourites")) {
+            if (promptSearchResult.databases.indexOf(SearchType.FAVORITES) !== -1) {
                 favourites = await searchService.schemaByKeywords(SearchType.FAVORITES, promptSearchResult.keywords!, promptSearchResult.timeframe, 20)
             }
-            if (promptSearchResult.databases.indexOf("chat_messages")) {
+            if (promptSearchResult.databases.indexOf(SearchType.CHAT_MESSAGES) !== -1) {
                 chatThreads = await searchService.chatThreadsByKeywords(promptSearchResult.keywords!, promptSearchResult.timeframe, 10, 10)
             }
         } else {
             const maxDatetime = Helpers.keywordTimeframeToDate(promptSearchResult.timeframe)
             const sort = promptSearchResult.sort == PromptSearchSort.RECENT ? SearchSortType.RECENT : SearchSortType.OLDEST
-            if (promptSearchResult.databases.indexOf("emails")) {
+            if (promptSearchResult.databases.indexOf(SearchType.EMAILS) !== -1) {
                 emails = await searchService.schemaByDateRange(SearchType.EMAILS, maxDatetime, sort, MAX_DATERANGE_EMAILS)
             }
             // if (promptSearchResult.databases.indexOf("files")) {
             //     files = await searchService.schemaByDateRange(SearchType.FILES, maxDatetime, sort, MAX_DATERANGE_FILES)
             // }
-            if (promptSearchResult.databases.indexOf("favourites")) {
+            if (promptSearchResult.databases.indexOf(SearchType.FAVORITES) !== -1) {
                 favourites = await searchService.schemaByDateRange(SearchType.FAVORITES, maxDatetime, sort, MAX_DATERANGE_FAVOURITES)
             }
-            if (promptSearchResult.databases.indexOf("chat_messages")) {
+            if (promptSearchResult.databases.indexOf(SearchType.CHAT_MESSAGES) !== -1) {
                 chatMessages =  await searchService.schemaByDateRange(SearchType.CHAT_MESSAGES, maxDatetime, sort, MAX_DATERANGE_CHAT_MESSAGES)
             }
         }
diff --git a/src/services/data.ts b/src/services/data.ts
index 5f98741a..d363552a 100644
--- a/src/services/data.ts
+++ b/src/services/data.ts
@@ -48,7 +48,7 @@ const schemas: Record = {
     "https://common.schemas.verida.io/favourite/v0.1.0/schema.json": {
         label: "Favourite",
         storeFields: ['_id', 'insertedAt'],
-        indexFields: ['name', 'favouriteType', 'contentTYpe', 'summary']
+        indexFields: ['name', 'favouriteType', 'contentType', 'summary']
     }
 }
 
diff --git a/src/services/search.ts b/src/services/search.ts
index ee193585..aaf02ffa 100644
--- a/src/services/search.ts
+++ b/src/services/search.ts
@@ -29,7 +29,7 @@ export enum SearchSortType {
 export enum SearchType {
     // CHAT_THREADS = "chat-threads",
     FILES = "files",
-    CHAT_MESSAGES = "chat-messages",
+    CHAT_MESSAGES = "messages",
     EMAILS = "emails",
     FAVORITES = "favorites",
     FOLLOWING = "following",
@@ -46,6 +46,16 @@ export const SearchTypeSchemas: Record = {
     [SearchType.FOLLOWING]: "https://common.schemas.verida.io/favourite/v0.1.0/schema.json",
 }
 
+export const SearchTypeTimeProperty: Record = {
+    // [SearchType.CHAT_THREADS]: "sentAt",
+    [SearchType.FILES]: "insertedAt",
+    [SearchType.CHAT_MESSAGES]: "sentAt",
+    [SearchType.EMAILS]: "sentAt",
+    [SearchType.FAVORITES]: "insertedAt",
+    [SearchType.POSTS]: "insertedAt",
+    [SearchType.FOLLOWING]: "followedAt",
+}
+
 export interface ChatThreadResult {
     group: SchemaSocialChatGroup,
     messages: SchemaSocialChatMessage[]
@@ -141,7 +151,7 @@ export class SearchService extends VeridaService {
             limit,
             sort: [
                 {
-                    sentAt: sortType == SearchSortType.OLDEST ? "asc" : "desc"
+                    [SearchTypeTimeProperty[searchType]]: sortType == SearchSortType.OLDEST ? "asc" : "desc"
                 }
             ]
         }
diff --git a/src/services/tools/promptSearch.ts b/src/services/tools/promptSearch.ts
index 63aa7e76..57c9567e 100644
--- a/src/services/tools/promptSearch.ts
+++ b/src/services/tools/promptSearch.ts
@@ -1,12 +1,13 @@
 import { KeywordSearchTimeframe } from "../../helpers/interfaces";
 import { LLM } from "../llm"
+import { SearchType } from "../search";
 
 const systemPrompt = `You are an expert data analyst. When I give you a prompt, you must generate search metadata that will be used to extract relevant information to help answer the prompt.
 You must generate a JSON response containing the following information:
 - search_type: keywords (search for specific keywords), all (search with filters, no keywords required)
 - keywords: array of single word terms to search on that match the underlying objective of the search. extract entity names. aim for 5-10 terms.
 - timeframe: one of; day, week, month, quarter, half-year, full-year, all
-- databases: an array of databases to search; emails, chat_messages, files, favourites, web_history, calendar
+- databases: an array of databases to search; emails, messages, files, favourites, web_history, calendar
 - sort: keyword_rank, recent, oldest
 - output_type: The amount of detail in the output of each search result to provide meaningful context. full_content, summary, headline
 - profile_information; Array of these options only; name, contactInfo, demographics, lifestyle, preferences, habits, financial, health, personality, employment, education, skills, language, interests
@@ -34,7 +35,7 @@ export interface PromptSearchLLMResponse {
     search_type: PromptSearchType;
     keywords?: string[]; // Array of single word terms, required if search_type is "keywords"
     timeframe: KeywordSearchTimeframe;
-    databases: Array<"emails" | "chat_messages" | "files" | "favourites" | "web_history" | "calendar">;
+    databases: Array;
     sort: PromptSearchSort;
     output_type: PromptSearchOutputType;
     profile_information: Array<

From 2ecb046047c7b2e21e013ad36059d33d1c2c2a5a Mon Sep 17 00:00:00 2001
From: chime3 
Date: Mon, 26 Aug 2024 23:54:15 -0700
Subject: [PATCH 055/182] fix: typo error

---
 src/web/developer/data/data.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/web/developer/data/data.js b/src/web/developer/data/data.js
index 5bbdfe45..3abe55ab 100644
--- a/src/web/developer/data/data.js
+++ b/src/web/developer/data/data.js
@@ -209,7 +209,7 @@ $(document).ready(function() {
             "Social Post": "https://common.schemas.verida.io/social/post/v0.1.0/schema.json",
             "Favourites": "https://common.schemas.verida.io/favourite/v0.1.0/schema.json",
             "Email": "https://common.schemas.verida.io/social/email/v0.1.0/schema.json",
-            "FILE": "https://common.schemas.verida.io/file/v0.1.0/schema.json",
+            "File": "https://common.schemas.verida.io/file/v0.1.0/schema.json",
             "Chat Group": "https://common.schemas.verida.io/social/chat/group/v0.1.0/schema.json",
             "Chat Message": "https://common.schemas.verida.io/social/chat/message/v0.1.0/schema.json"
         };

From a45a9d336a8a5b7d16aab6d47c05d4222796a0f3 Mon Sep 17 00:00:00 2001
From: chime3 
Date: Tue, 27 Aug 2024 19:47:33 -0700
Subject: [PATCH 056/182] fix: removed unused variable

---
 src/schemas.ts | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/schemas.ts b/src/schemas.ts
index 3d3fd3d4..9174b279 100644
--- a/src/schemas.ts
+++ b/src/schemas.ts
@@ -1,5 +1,3 @@
-import { String$Input } from "tdlib-native/dist/types"
-
 export interface SchemaRecord {
     _id: string
     _rev?: string

From b5a1559767139fd74f9ca5146e15f99cd706b654 Mon Sep 17 00:00:00 2001
From: chime3 
Date: Wed, 28 Aug 2024 02:29:32 -0700
Subject: [PATCH 057/182] chore: update dependencies

---
 yarn.lock | 75 +++++++++++++++++++++++++++++++------------------------
 1 file changed, 42 insertions(+), 33 deletions(-)

diff --git a/yarn.lock b/yarn.lock
index 2d735cb2..49453f64 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -628,6 +628,16 @@
   resolved "https://registry.yarnpkg.com/@oauth-everything/profile/-/profile-1.0.0.tgz#0b5e78749415519fa312dc83347a677903f456ba"
   integrity sha512-OmCuBPhjaLHh9MST9P5jRuVBZaP0z7hBk8nH4Yt7Id5kNM1AXGd5uud6CP7W2zuhKl2nk0KsYmeMT7SkzN6VWg==
 
+"@one-ini/wasm@0.1.1":
+  version "0.1.1"
+  resolved "https://registry.yarnpkg.com/@one-ini/wasm/-/wasm-0.1.1.tgz#6013659736c9dbfccc96e8a9c2b3de317df39323"
+  integrity sha512-XuySG1E38YScSJoMlqovLru4KTUNSjgVTIjyh7qMX6aNN5HY5Ct5LhRJdxO79JtTzKfzV/bnWpz+zquYrISsvw==
+
+"@pkgjs/parseargs@^0.11.0":
+  version "0.11.0"
+  resolved "https://registry.yarnpkg.com/@pkgjs/parseargs/-/parseargs-0.11.0.tgz#a77ea742fab25775145434eb1d2328cf5013ac33"
+  integrity sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==
+
 "@prebuilt-tdlib/darwin@0.1008034.0":
   version "0.1008034.0"
   resolved "https://registry.yarnpkg.com/@prebuilt-tdlib/darwin/-/darwin-0.1008034.0.tgz#a5983898df91b3017fc432874f258a17299dcf3c"
@@ -648,16 +658,6 @@
   resolved "https://registry.yarnpkg.com/@prebuilt-tdlib/win32-x64/-/win32-x64-0.1008034.0.tgz#4cfd1463e7a21aa052afd174380b9773ffdb4744"
   integrity sha512-GdkQVgeyXm5Fzj9Rh9UL/30WwuTTbqS+SAs3MKhwu5IxZXtmtv0kHerCfctW6jI926dBCbysSO1+d6wgEMOhbg==
 
-"@one-ini/wasm@0.1.1":
-  version "0.1.1"
-  resolved "https://registry.yarnpkg.com/@one-ini/wasm/-/wasm-0.1.1.tgz#6013659736c9dbfccc96e8a9c2b3de317df39323"
-  integrity sha512-XuySG1E38YScSJoMlqovLru4KTUNSjgVTIjyh7qMX6aNN5HY5Ct5LhRJdxO79JtTzKfzV/bnWpz+zquYrISsvw==
-
-"@pkgjs/parseargs@^0.11.0":
-  version "0.11.0"
-  resolved "https://registry.yarnpkg.com/@pkgjs/parseargs/-/parseargs-0.11.0.tgz#a77ea742fab25775145434eb1d2328cf5013ac33"
-  integrity sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==
-
 "@sapphire/async-queue@^1.5.0":
   version "1.5.0"
   resolved "https://registry.yarnpkg.com/@sapphire/async-queue/-/async-queue-1.5.0.tgz#2f255a3f186635c4fb5a2381e375d3dfbc5312d8"
@@ -1418,7 +1418,7 @@ abbrev@^2.0.0:
   resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-2.0.0.tgz#cf59829b8b4f03f89dda2771cb7f3653828c89bf"
   integrity sha512-6/mh1E2u2YgEsCHdY0Yx5oW+61gZU+1vXaoiHHrpKeuRNNgFvS+/jrwHiQhB5apAf5oB7UB7E19ol2R2LKH8hQ==
 
-abort-controller@3.0.0:
+abort-controller@3.0.0, abort-controller@^3.0.0:
   version "3.0.0"
   resolved "https://registry.yarnpkg.com/abort-controller/-/abort-controller-3.0.0.tgz#eaf54d53b62bae4138e809ca225c8439a6efb392"
   integrity sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==
@@ -2442,11 +2442,6 @@ create-hmac@^1.1.4:
     safe-buffer "^5.0.1"
     sha.js "^2.4.8"
 
-crypto-js@^4.2.0:
-  version "4.2.0"
-  resolved "https://registry.yarnpkg.com/crypto-js/-/crypto-js-4.2.0.tgz#4d931639ecdfd12ff80e8186dba6af2c2e856631"
-  integrity sha512-KALDyEYgpY+Rlob/iriUtjV6d5Eq+Y191A5g4UqLAi8CyGP9N1+FdVbkc1SxKc2r4YAYqG8JzO2KGL+AizD70Q==
-
 cross-spawn@^7.0.0:
   version "7.0.3"
   resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.3.tgz#f73a85b9d5d41d045551c177e2882d4ac85728a6"
@@ -2456,6 +2451,11 @@ cross-spawn@^7.0.0:
     shebang-command "^2.0.0"
     which "^2.0.1"
 
+crypto-js@^4.2.0:
+  version "4.2.0"
+  resolved "https://registry.yarnpkg.com/crypto-js/-/crypto-js-4.2.0.tgz#4d931639ecdfd12ff80e8186dba6af2c2e856631"
+  integrity sha512-KALDyEYgpY+Rlob/iriUtjV6d5Eq+Y191A5g4UqLAi8CyGP9N1+FdVbkc1SxKc2r4YAYqG8JzO2KGL+AizD70Q==
+
 "crypto-pouch@git+https://github.com/tahpot/crypto-pouch.git#feature/support-key-import":
   version "4.0.1"
   resolved "git+https://github.com/tahpot/crypto-pouch.git#7a712691b4404cfefb34ad3f58db1d83b55ed3bf"
@@ -2542,7 +2542,7 @@ debug@^3.1.0, debug@^3.2.7:
   dependencies:
     ms "^2.1.1"
 
-debug@^4.3.5:
+debug@^4.1.1, debug@^4.3.1, debug@^4.3.5:
   version "4.3.6"
   resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.6.tgz#2ab2c38fbaffebf8aa95fdfe6d88438c7a13c52b"
   integrity sha512-O/09Bd4Z1fBrU4VzkhFqVgpPzaGbw6Sm9FEkBT1A/YBXQFGuuSxa1dN2nxgxS34JmKXqYx8CZAwEVoJFImUXIg==
@@ -4649,16 +4649,16 @@ minimist@^1.2.0, minimist@^1.2.5, minimist@^1.2.6:
   resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.6.tgz#8637a5b759ea0d6e98702cfb3a9283323c93af44"
   integrity sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==
 
-minisearch@^7.1.0:
-  version "7.1.0"
-  resolved "https://registry.yarnpkg.com/minisearch/-/minisearch-7.1.0.tgz#f5830e9109b5919ee7b291c29a304f381aa68770"
-  integrity sha512-tv7c/uefWdEhcu6hvrfTihflgeEi2tN6VV7HJnCjK6VxM75QQJh4t9FwJCsA2EsRS8LCnu3W87CuGPWMocOLCA==
-
 "minipass@^5.0.0 || ^6.0.2 || ^7.0.0", minipass@^7.1.2:
   version "7.1.2"
   resolved "https://registry.yarnpkg.com/minipass/-/minipass-7.1.2.tgz#93a9626ce5e5e66bd4db86849e7515e92340a707"
   integrity sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==
 
+minisearch@^7.1.0:
+  version "7.1.0"
+  resolved "https://registry.yarnpkg.com/minisearch/-/minisearch-7.1.0.tgz#f5830e9109b5919ee7b291c29a304f381aa68770"
+  integrity sha512-tv7c/uefWdEhcu6hvrfTihflgeEi2tN6VV7HJnCjK6VxM75QQJh4t9FwJCsA2EsRS8LCnu3W87CuGPWMocOLCA==
+
 mkdirp-classic@^0.5.2:
   version "0.5.3"
   resolved "https://registry.yarnpkg.com/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz#fa10c9115cc6d8865be221ba47ee9bed78601113"
@@ -5342,16 +5342,6 @@ pouchdb@^7.2.2:
     uuid "8.3.2"
     vuvuzela "1.0.3"
 
-prebuilt-tdlib@^0.1008034.0:
-  version "0.1008034.0"
-  resolved "https://registry.yarnpkg.com/prebuilt-tdlib/-/prebuilt-tdlib-0.1008034.0.tgz#f80d8f2b21a242e43739df54624ae20418ee251d"
-  integrity sha512-8BuDali/P5wz2kQ4prnFbjeA5wieHmNgDNOm+C6Wu4W2mZ7L/4vDy15EdknKhqMfB8OrRbCKZknp6rqJ16FcTw==
-  optionalDependencies:
-    "@prebuilt-tdlib/darwin" "0.1008034.0"
-    "@prebuilt-tdlib/linux-arm64-glibc" "0.1008034.0"
-    "@prebuilt-tdlib/linux-x64-glibc" "0.1008034.0"
-    "@prebuilt-tdlib/win32-x64" "0.1008034.0"
-
 pptx-parser@^1.1.7-beta.9:
   version "1.1.7-beta.9"
   resolved "https://registry.yarnpkg.com/pptx-parser/-/pptx-parser-1.1.7-beta.9.tgz#26648ba5c45c4c5c548b1de7ab03f04764399d37"
@@ -5376,6 +5366,16 @@ pptx-parser@^1.1.7-beta.9:
     txml "3.1.3"
     url-loader "^4.1.0"
 
+prebuilt-tdlib@^0.1008034.0:
+  version "0.1008034.0"
+  resolved "https://registry.yarnpkg.com/prebuilt-tdlib/-/prebuilt-tdlib-0.1008034.0.tgz#f80d8f2b21a242e43739df54624ae20418ee251d"
+  integrity sha512-8BuDali/P5wz2kQ4prnFbjeA5wieHmNgDNOm+C6Wu4W2mZ7L/4vDy15EdknKhqMfB8OrRbCKZknp6rqJ16FcTw==
+  optionalDependencies:
+    "@prebuilt-tdlib/darwin" "0.1008034.0"
+    "@prebuilt-tdlib/linux-arm64-glibc" "0.1008034.0"
+    "@prebuilt-tdlib/linux-x64-glibc" "0.1008034.0"
+    "@prebuilt-tdlib/win32-x64" "0.1008034.0"
+
 prepend-http@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/prepend-http/-/prepend-http-2.0.0.tgz#e92434bfa5ea8c19f41cdfd401d741a3c819d897"
@@ -6018,6 +6018,15 @@ string-strip-html@8.5.0:
   resolved "https://registry.yarnpkg.com/string-strip-html/-/string-strip-html-8.5.0.tgz#5e239fe84016fad7b33ca02d23c591f1ccb6af75"
   integrity sha512-5ICsK1B1j0A3AF1d45m0sqQCcmi1Q+t1QpF+b794LO5FTHV+ITkGR5C+UCDJQZgs5LMuRruqr6j48PxQVIurJQ==
 
+"string-width-cjs@npm:string-width@^4.2.0":
+  version "4.2.3"
+  resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010"
+  integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==
+  dependencies:
+    emoji-regex "^8.0.0"
+    is-fullwidth-code-point "^3.0.0"
+    strip-ansi "^6.0.1"
+
 string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.2:
   version "4.2.3"
   resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010"
@@ -6230,7 +6239,7 @@ tdlib-native@^2.6.0:
     "@tdlib-native/tdjson-linux-x64-glibc" "1.8.33-commit.cb164927417f22811c74cd8678ed4a5ab7cb80ba"
     "@tdlib-native/tdjson-win32-x64" "1.8.33-commit.cb164927417f22811c74cd8678ed4a5ab7cb80ba"
 
-through2@3.0.2:
+through2@3.0.2, through2@^3.0.1:
   version "3.0.2"
   resolved "https://registry.yarnpkg.com/through2/-/through2-3.0.2.tgz#99f88931cfc761ec7678b41d5d7336b5b6a07bf4"
   integrity sha512-enaDQ4MUyP2W6ZyT6EsMzqBPZaM/avg8iuo+l2d3QCs0J+6RaqkHV/2/lOwDTueBHeJ/2LG9lrLW3d5rWPucuQ==

From 837b13af3771600d5d1f8b05cecc0c25ada527d2 Mon Sep 17 00:00:00 2001
From: Chris 
Date: Thu, 29 Aug 2024 10:29:01 +0930
Subject: [PATCH 058/182] Support toName in email

---
 src/providers/google/gmail.ts | 3 ++-
 src/schemas.ts                | 1 +
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/providers/google/gmail.ts b/src/providers/google/gmail.ts
index 3482b4d2..174ed2be 100644
--- a/src/providers/google/gmail.ts
+++ b/src/providers/google/gmail.ts
@@ -231,7 +231,8 @@ export default class Gmail extends GoogleHandler {
         sourceId: message.id,
         fromName: from.name,
         fromEmail: from.email,
-        toEmail: to.name,
+        toEmail: to.email,
+        toName: to.name,
         messageText: text ? text : 'No email body',
         messageHTML: html ? html : 'No email body',
         sentAt: internalDate,
diff --git a/src/schemas.ts b/src/schemas.ts
index 32cfe68f..36ed0120 100644
--- a/src/schemas.ts
+++ b/src/schemas.ts
@@ -60,6 +60,7 @@ export interface SchemaEmail extends SchemaRecord {
     fromName: string
     fromEmail: string
     toEmail: string
+    toName: string
     messageText: string
     messageHTML: string
     sentAt: string

From 8fda382b0f5078bde1e03871a23f881fcd2fc108 Mon Sep 17 00:00:00 2001
From: Chris 
Date: Thu, 29 Aug 2024 10:29:30 +0930
Subject: [PATCH 059/182] Update shcmea indexable fields

---
 src/services/data.ts | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/src/services/data.ts b/src/services/data.ts
index d363552a..484923c7 100644
--- a/src/services/data.ts
+++ b/src/services/data.ts
@@ -28,27 +28,27 @@ const schemas: Record = {
     "https://common.schemas.verida.io/social/following/v0.1.0/schema.json": {
         label: "Social Following",
         storeFields: ['_id', 'name','uri','description','insertedAt','followedTimestamp'],
-        indexFields: ['name','description']
+        indexFields: ['name','description','sourceApplication']
     },
     "https://common.schemas.verida.io/social/post/v0.1.0/schema.json": {
         label: "Social Posts",
         storeFields: ['_id', 'name','content','type','uri','insertedAt'],
-        indexFields: ['name', 'content', 'indexableText']
+        indexFields: ['name', 'content', 'indexableText','sourceApplication']
     },
     "https://common.schemas.verida.io/social/email/v0.1.0/schema.json": {
         label: "Email",
         storeFields: ['_id', 'sentAt'],
-        indexFields: ['name','fromName','fromEmail','messageText','attachments_0.textContent','attachments_1.textContent','attachments_2.textContent', 'indexableText', 'sentAt']
+        indexFields: ['name','fromName','fromEmail','messageText','attachments_0.textContent','attachments_1.textContent','attachments_2.textContent', 'indexableText', 'sentAt','sourceApplication']
     },
     "https://common.schemas.verida.io/social/chat/message/v0.1.0/schema.json": {
         label: "Chat Message",
         storeFields: ['_id', 'groupId', 'sentAt'],
-        indexFields: ['messageText', 'fromHandle', 'fromName', 'groupName', 'indexableText', 'sentAt']
+        indexFields: ['messageText', 'fromHandle', 'fromName', 'groupName', 'indexableText', 'sentAt','sourceApplication']
     },
     "https://common.schemas.verida.io/favourite/v0.1.0/schema.json": {
-        label: "Favourite",
+        label: "Favorite",
         storeFields: ['_id', 'insertedAt'],
-        indexFields: ['name', 'favouriteType', 'contentType', 'summary']
+        indexFields: ['name', 'favouriteType', 'contentType', 'summary','sourceApplication']
     }
 }
 
@@ -96,7 +96,7 @@ export class DataService extends EventEmitter {
             this.emitProgress(schemaConfig.label, HotLoadStatus.StartData, 10)
             const datastore = await this.context.openDatastore(schemaUri)
 
-            console.log('Fetching data')
+            console.log('Fetching data from index ', schemaUri)
             const database = await datastore.getDb()
             const db = await database.getDb()
             const result = await db.allDocs({

From f2f64b6368029032df21ef01e3dcabfdf2e066ae Mon Sep 17 00:00:00 2001
From: Chris 
Date: Thu, 29 Aug 2024 10:30:06 +0930
Subject: [PATCH 060/182] Add timing logs. Support email shortlist. Tweak the
 prompt to better specify databases to query.

---
 src/services/assistants/search.ts    | 57 +++++++++++++++++++++-----
 src/services/search.ts               | 12 ++++--
 src/services/tools/emailShortlist.ts | 60 ++++++++++++++++++++++++++++
 src/services/tools/promptSearch.ts   |  4 +-
 4 files changed, 116 insertions(+), 17 deletions(-)
 create mode 100644 src/services/tools/emailShortlist.ts

diff --git a/src/services/assistants/search.ts b/src/services/assistants/search.ts
index 12414d56..05be9d55 100644
--- a/src/services/assistants/search.ts
+++ b/src/services/assistants/search.ts
@@ -3,18 +3,20 @@ import { defaultModel } from "../llm"
 import { PromptSearch, PromptSearchLLMResponse, PromptSearchSort, PromptSearchType } from "../tools/promptSearch"
 import { ChatThreadResult, SearchService, SearchSortType, SearchType } from "../search"
 import { VeridaService } from '../veridaService'
-import { SchemaEmail, SchemaFavourite, SchemaSocialChatMessage } from '../../schemas'
+import { SchemaEmail, SchemaFavourite, SchemaFollowing, SchemaSocialChatMessage } from '../../schemas'
 import { Helpers } from "../helpers"
+import { EmailShortlist } from "../tools/emailShortlist"
 
 const llm = defaultModel
 
 const MAX_EMAIL_LENGTH = 500
-const MAX_ATTACHMENT_LENGTH = 1000
+const MAX_ATTACHMENT_LENGTH = 500
 const MAX_CONTEXT_LENGTH = 20000 // (~5000 tokens)
 
-const MAX_DATERANGE_EMAILS = 40
+const MAX_DATERANGE_EMAILS = 30
 const MAX_DATERANGE_CHAT_MESSAGES = 100
-const MAX_DATERANGE_FAVOURITES = 20
+const MAX_DATERANGE_FAVORITES = 30
+const MAX_DATERANGE_FOLLOWING = 30
 const MAX_DATERANGE_FILES = 20
 
 // "You are a personal assistant with the ability to search the following categories; emails, chat_history and documents. You receive a prompt and generate a JSON response (with no other text) that provides search queries that will source useful information to help answer the prompt. Search queries for each category should contain three properties; \"terms\" (an array of 10 individual words), \"beforeDate\" (results must be before this date), \"afterDate\" (results must be after this date), \"resultType\" (either \"count\" to count results or \"results\" to return the search results), \"filter\" (an array of key, value pairs of fields to filter the results). Categories can be empty if not relevant to the prompt. The current date is 2024-08-12.\n\nHere is an example JSON response:\n{\"email\": {\"terms\": [\"golf\", \"tennis\", \"soccer\"], \"beforeDate\": \"2024-06-01\", \"afterDate\": \"2024-01-10\" \"filter\": {\"from\": \"dave\"}, \"resultType\": \"results}}\n\nHere is the prompt:\nWhat subscriptions do I currently pay for?"
@@ -26,49 +28,69 @@ export class PromptSearchService extends VeridaService {
         duration: number,
         process: PromptSearchLLMResponse
     }> {
+        console.time("PersonalPromptStart")
         const start = Date.now()
         const promptSearch = new PromptSearch(llm)
+        console.time("KeywordPrompt")
         const promptSearchResult = await promptSearch.search(prompt)
+        console.timeEnd("KeywordPrompt")
 
         console.log(promptSearchResult)
 
         let chatThreads: ChatThreadResult[] = []
         let emails: SchemaEmail[] = []
         let favourites: SchemaFavourite[] = []
+        let following: SchemaFollowing[] = []
         // let files: SchemaFile[] = []
         let chatMessages: SchemaSocialChatMessage[] = []
 
         const searchService = new SearchService(this.did, this.context)
 
+        console.time("DataFetch")
         if (promptSearchResult.search_type == PromptSearchType.KEYWORDS) {
+            console.time("DataFetchKeywords")
             if (promptSearchResult.databases.indexOf(SearchType.EMAILS) !== -1) {
-                emails = await searchService.schemaByKeywords(SearchType.EMAILS, promptSearchResult.keywords!, promptSearchResult.timeframe, 20)
+                emails = await searchService.schemaByKeywords(SearchType.EMAILS, promptSearchResult.keywords!, promptSearchResult.timeframe, 40)
             }
             // if (promptSearchResult.databases.indexOf("files")) {
             //     files = await searchService.schemaByKeywords(SearchType.FILES, promptSearchResult.keywords!, promptSearchResult.timeframe, 20)
             // }
             if (promptSearchResult.databases.indexOf(SearchType.FAVORITES) !== -1) {
-                favourites = await searchService.schemaByKeywords(SearchType.FAVORITES, promptSearchResult.keywords!, promptSearchResult.timeframe, 20)
+                favourites = await searchService.schemaByKeywords(SearchType.FAVORITES, promptSearchResult.keywords!, promptSearchResult.timeframe, 40)
+            }
+            if (promptSearchResult.databases.indexOf(SearchType.FOLLOWING) !== -1) {
+                following = await searchService.schemaByKeywords(SearchType.FOLLOWING, promptSearchResult.keywords!, promptSearchResult.timeframe, 40)
             }
             if (promptSearchResult.databases.indexOf(SearchType.CHAT_MESSAGES) !== -1) {
-                chatThreads = await searchService.chatThreadsByKeywords(promptSearchResult.keywords!, promptSearchResult.timeframe, 10, 10)
+                chatThreads = await searchService.chatThreadsByKeywords(promptSearchResult.keywords!, promptSearchResult.timeframe, 10, 20)
             }
+            console.timeEnd("DataFetchKeywords")
         } else {
+            console.time("DataFetchDaterange")
             const maxDatetime = Helpers.keywordTimeframeToDate(promptSearchResult.timeframe)
             const sort = promptSearchResult.sort == PromptSearchSort.RECENT ? SearchSortType.RECENT : SearchSortType.OLDEST
             if (promptSearchResult.databases.indexOf(SearchType.EMAILS) !== -1) {
-                emails = await searchService.schemaByDateRange(SearchType.EMAILS, maxDatetime, sort, MAX_DATERANGE_EMAILS)
+                emails = await searchService.schemaByDateRange(SearchType.EMAILS, maxDatetime, sort, MAX_DATERANGE_EMAILS*3)
+                const emailShortlist = new EmailShortlist(llm)
+                console.time("EmailShortlist")
+                emails = await emailShortlist.shortlist(prompt, emails, MAX_DATERANGE_EMAILS)
+                console.timeEnd("EmailShortlist")
             }
             // if (promptSearchResult.databases.indexOf("files")) {
             //     files = await searchService.schemaByDateRange(SearchType.FILES, maxDatetime, sort, MAX_DATERANGE_FILES)
             // }
             if (promptSearchResult.databases.indexOf(SearchType.FAVORITES) !== -1) {
-                favourites = await searchService.schemaByDateRange(SearchType.FAVORITES, maxDatetime, sort, MAX_DATERANGE_FAVOURITES)
+                favourites = await searchService.schemaByDateRange(SearchType.FAVORITES, maxDatetime, sort, MAX_DATERANGE_FAVORITES)
+            }
+            if (promptSearchResult.databases.indexOf(SearchType.FOLLOWING) !== -1) {
+                following = await searchService.schemaByDateRange(SearchType.FOLLOWING, maxDatetime, sort, MAX_DATERANGE_FOLLOWING)
             }
             if (promptSearchResult.databases.indexOf(SearchType.CHAT_MESSAGES) !== -1) {
                 chatMessages =  await searchService.schemaByDateRange(SearchType.CHAT_MESSAGES, maxDatetime, sort, MAX_DATERANGE_CHAT_MESSAGES)
             }
+            console.timeEnd("DataFetchDaterange")
         }
+        console.timeEnd("DataFetch")
 
         let finalPrompt = `Answer this prompt:\n${prompt}\nHere are some recent messages that may help you provide a relevant answer.\n`
         let contextString = ''
@@ -88,6 +110,16 @@ export class PromptSearchService extends VeridaService {
             contextString += `From: ${chatMessage.fromName} <${chatMessage.fromHandle}> (${chatMessage.groupName})\nBody: ${chatMessage.messageText}\n\n`
         }
 
+        console.log('favourites: ', favourites.length)
+        for (const favourite of favourites) {
+            contextString += `Favorite: ${favourite.name} ${favourite.description?.substring(0,100)} (via ${favourite.sourceApplication})\n\n`
+        }
+
+        console.log('following: ', following.length)
+        for (const follow of following) {
+            contextString += `Following: ${follow.name} ${follow.description?.substring(0,100)} (via ${follow.sourceApplication})\n\n`
+        }
+
         // console.log('pre-email context string: ', contextString.length)
 
         let emailCount = 0
@@ -100,7 +132,7 @@ export class PromptSearchService extends VeridaService {
                 }
             }
 
-            extraContext = `From: ${email.fromName} <${email.fromEmail}> (${email.name})\nBody: ${body}\n\n`
+            extraContext = `To: ${email.toName} <${email.toEmail}>\nFrom: ${email.fromName} <${email.fromEmail}> (${email.name})\nBody: ${body}\n\n`
             // console.log(email.fromName, email.fromEmail, email.name, body.length, email.messageText.length)
             if ((extraContext.length + contextString.length + finalPrompt.length) > MAX_CONTEXT_LENGTH) {
                 break
@@ -115,12 +147,15 @@ export class PromptSearchService extends VeridaService {
         const now = (new Date()).toISOString()
         finalPrompt += `${contextString}\nThe current time is: ${now}`
 
-        // console.log('Running final prompt', finalPrompt.length)
+        console.log('Running final prompt', finalPrompt.length)
+        console.time("FinalPrompt")
         const finalResponse = await llm.prompt(finalPrompt, undefined, false)
+        console.timeEnd("FinalPrompt")
         const duration = Date.now() - start
 
         // console.log(contextString)
 
+        console.timeEnd("PersonalPromptStart")
         return {
             result: finalResponse.choices[0].message.content!,
             duration,
diff --git a/src/services/search.ts b/src/services/search.ts
index aaf02ffa..6db5e409 100644
--- a/src/services/search.ts
+++ b/src/services/search.ts
@@ -32,7 +32,7 @@ export enum SearchType {
     CHAT_MESSAGES = "messages",
     EMAILS = "emails",
     FAVORITES = "favorites",
-    FOLLOWING = "following",
+    FOLLOWING = "followed_pages",
     POSTS = "posts"
 }
 
@@ -43,7 +43,7 @@ export const SearchTypeSchemas: Record = {
     [SearchType.EMAILS]: "https://common.schemas.verida.io/social/email/v0.1.0/schema.json",
     [SearchType.FAVORITES]: "https://common.schemas.verida.io/favourite/v0.1.0/schema.json",
     [SearchType.POSTS]: "https://common.schemas.verida.io/social/post/v0.1.0/schema.json",
-    [SearchType.FOLLOWING]: "https://common.schemas.verida.io/favourite/v0.1.0/schema.json",
+    [SearchType.FOLLOWING]: "https://common.schemas.verida.io/social/following/v0.1.0/schema.json",
 }
 
 export const SearchTypeTimeProperty: Record = {
@@ -53,7 +53,7 @@ export const SearchTypeTimeProperty: Record = {
     [SearchType.EMAILS]: "sentAt",
     [SearchType.FAVORITES]: "insertedAt",
     [SearchType.POSTS]: "insertedAt",
-    [SearchType.FOLLOWING]: "followedAt",
+    [SearchType.FOLLOWING]: "followedTimestamp",
 }
 
 export interface ChatThreadResult {
@@ -64,6 +64,7 @@ export interface ChatThreadResult {
 export class SearchService extends VeridaService {
 
     protected async rankAndMergeResults(schemaResults: SearchServiceSchemaResult[], limit: number, minResultsPerType: number = 10): Promise {
+        console.time("RankAndMerge")
         const unsortedResults: Record = {}
         const guaranteedResults: Record = {}
 
@@ -117,10 +118,12 @@ export class SearchService extends VeridaService {
             })
         }
 
+        console.timeEnd("RankAndMerge")
         return results
     }
 
     public async schemaByKeywords(searchType: SearchType, keywordsList: string[], timeframe: KeywordSearchTimeframe, limit: number = 20): Promise {
+        console.time("SchemaByKeywords" + searchType, )
         const query = keywordsList.join(' ')
         const schemaUri = SearchTypeSchemas[searchType]
         const dataService = new DataService(this.did, this.context)
@@ -132,6 +135,7 @@ export class SearchService extends VeridaService {
         const searchResults = await miniSearchIndex.search(query, {
             filter: (result: any) => maxDatetime ? result.sentAt > maxDatetime.toISOString() : true
         })
+        console.timeEnd("SchemaByKeywords" + searchType)
         return await this.rankAndMergeResults([{
             searchType,
             rows: searchResults
@@ -143,7 +147,7 @@ export class SearchService extends VeridaService {
         const dataService = new DataService(this.did, this.context)
         const datastore = await dataService.getDatastore(schemaUri)
         const filter = {
-            sentAt: {
+            [SearchTypeTimeProperty[searchType]]: {
                 "$gte": maxDatetime.toISOString()
             }
         }
diff --git a/src/services/tools/emailShortlist.ts b/src/services/tools/emailShortlist.ts
new file mode 100644
index 00000000..7460a962
--- /dev/null
+++ b/src/services/tools/emailShortlist.ts
@@ -0,0 +1,60 @@
+import { KeywordSearchTimeframe } from "../../helpers/interfaces";
+import { SchemaEmail } from "../../schemas";
+import { LLM } from "../llm"
+import { SearchType } from "../search";
+
+const systemPrompt = `You are an expert data analyst. I will provide you with a prompt and a list of email IDs and subjects.
+You must generate a JSON object containing a single key "emailIds" that contains an array of emailIds that are the most relevant to helping answer the prompt.
+Data is in the format:
+[emailId] 
+
+JSON only, no explanation.`
+
+const MAX_EMAILS = 200
+
+
+export class EmailShortlist {
+
+    private llm: LLM
+
+    constructor(llm: LLM) {
+        this.llm = llm
+    }
+
+    public async shortlist(originalPrompt: string, emails: SchemaEmail[], limit=20): Promise {
+        console.log("shortlist")
+        let userPrompt = `${originalPrompt}\n\n`
+        const emailDict: Record = {}
+        let emailLimit = MAX_EMAILS
+        for (const email of emails) {
+            emailDict[email._id] = email
+            userPrompt += `[${email._id}] <${email.name}>\n`
+            emailLimit--
+            if (emailLimit <= 0) {
+                break
+            }
+        }
+
+        userPrompt += `\nMaximum of ${limit} email IDs`
+        const response = await this.llm.prompt(userPrompt, systemPrompt)
+        console.log(response.choices[0])
+        const jsonResponse: any = JSON.parse(response.choices[0].message.content!)
+        console.log(jsonResponse)
+        const emailIds = jsonResponse.emailIds
+        console.log(emailIds)
+        const result: SchemaEmail[] = []
+
+        for (const emailId of emailIds) {
+            result.push(emailDict[emailId])
+            console.log(emailDict[emailId].name)
+            limit--
+            if (limit <= 0) {
+                break
+            }
+        }
+
+        return result
+        
+    }
+
+}
\ No newline at end of file
diff --git a/src/services/tools/promptSearch.ts b/src/services/tools/promptSearch.ts
index 57c9567e..aafbd587 100644
--- a/src/services/tools/promptSearch.ts
+++ b/src/services/tools/promptSearch.ts
@@ -7,7 +7,7 @@ You must generate a JSON response containing the following information:
 - search_type: keywords (search for specific keywords), all (search with filters, no keywords required)
 - keywords: array of single word terms to search on that match the underlying objective of the search. extract entity names. aim for 5-10 terms.
 - timeframe: one of; day, week, month, quarter, half-year, full-year, all
-- databases: an array of databases to search; emails, messages, files, favourites, web_history, calendar
+- databases: an array of databases to search; emails, messages, files, favorites, followed_pages, web_history, calendar
 - sort: keyword_rank, recent, oldest
 - output_type: The amount of detail in the output of each search result to provide meaningful context. full_content, summary, headline
 - profile_information; Array of these options only; name, contactInfo, demographics, lifestyle, preferences, habits, financial, health, personality, employment, education, skills, language, interests
@@ -56,7 +56,7 @@ export class PromptSearch {
     public async search(userPrompt: string): Promise {
         const response = await this.llm.prompt(userPrompt, systemPrompt)
         console.log(response.choices[0])
-        return  JSON.parse(response.choices[0].message.content)
+        return  JSON.parse(response.choices[0].message.content!)
         
     }
 

From dd64f0886d4e4f5fa4e08f98823cb111f521e7e9 Mon Sep 17 00:00:00 2001
From: Chris 
Date: Thu, 29 Aug 2024 11:04:04 +0930
Subject: [PATCH 061/182] Fix google auth handler to stop deleting refresh
 token when a new access token received

---
 src/providers/google/GoogleHandler.ts | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/src/providers/google/GoogleHandler.ts b/src/providers/google/GoogleHandler.ts
index 5b48796c..830c9844 100644
--- a/src/providers/google/GoogleHandler.ts
+++ b/src/providers/google/GoogleHandler.ts
@@ -22,11 +22,16 @@ export default class BaseGoogleHandler extends BaseSyncHandler {
         // Handle update to access or refresh token
         const handler = this
         oAuth2Client.on('tokens', (tokens) => {
-            handler.updateConnection({
-                accessToken: tokens.access_token,
-                refreshToken: tokens.refresh_token
-            })
-          });
+          const updatedConnection: Record = {
+            accessToken: tokens.access_token,
+          }
+
+          if (tokens.refresh_token) {
+            updatedConnection.refreshToken = tokens.refresh_token
+          }
+
+          handler.updateConnection(updatedConnection)
+        })
     
         oAuth2Client.setCredentials(TOKEN);
         return oAuth2Client

From 1149f5a49e4fa608c95d4d07464685ce0b22acdd Mon Sep 17 00:00:00 2001
From: Chris 
Date: Thu, 29 Aug 2024 11:25:58 +0930
Subject: [PATCH 062/182] Fix Browse Data page

---
 src/api/v1/ds/controller.ts    |  4 ++-
 src/web/developer/data/data.js | 59 +++++++++++++++++++++-------------
 2 files changed, 39 insertions(+), 24 deletions(-)

diff --git a/src/api/v1/ds/controller.ts b/src/api/v1/ds/controller.ts
index 89922a7c..89c43fd6 100644
--- a/src/api/v1/ds/controller.ts
+++ b/src/api/v1/ds/controller.ts
@@ -68,7 +68,9 @@ export class DsController {
                 items
             })
         } catch (error) {
-            res.status(500).send(error.message);
+            res.status(500).send({
+                error: error.message
+            });
         }
     }
 }
diff --git a/src/web/developer/data/data.js b/src/web/developer/data/data.js
index 51f58f27..e8a0f33e 100644
--- a/src/web/developer/data/data.js
+++ b/src/web/developer/data/data.js
@@ -4,10 +4,21 @@ $(document).ready(function() {
     let currentSortField = '';
     let currentSortDirection = 'asc';
     let currentFilters = {};
+    const schemas = {
+        "Sync Position": "https://vault.schemas.verida.io/data-connections/sync-position/v0.1.0/schema.json",
+        "Sync Activity Log": "https://vault.schemas.verida.io/data-connections/activity-log/v0.1.0/schema.json",
+        "Connections": "https://vault.schemas.verida.io/data-connections/connection/v0.2.0/schema.json",
+        "Social Following": "https://common.schemas.verida.io/social/following/v0.1.0/schema.json",
+        "Social Post": "https://common.schemas.verida.io/social/post/v0.1.0/schema.json",
+        "Favourites": "https://common.schemas.verida.io/favourite/v0.1.0/schema.json",
+        "Email": "https://common.schemas.verida.io/social/email/v0.1.0/schema.json",
+        "Chat Group": "https://common.schemas.verida.io/social/chat/group/v0.1.0/schema.json",
+        "Chat Message": "https://common.schemas.verida.io/social/chat/message/v0.1.0/schema.json"
+    };
 
     // Load Verida Key and Schema from local storage
     $('#veridaKey').val(localStorage.getItem('veridaKey') || '');
-    $('#schema').val(localStorage.getItem('schema') || '');
+    $('#schema').val(localStorage.getItem('schema') || schemas["Connections"]);
 
     // Function to get query parameters
     function getQueryParams() {
@@ -44,10 +55,23 @@ $(document).ready(function() {
     function fetchData() {
         const veridaKey = $('#veridaKey').val();
         const schema = $('#schema').val();
-        const limit = $('#limit').val();
+        const limit = parseInt($('#limit').val());
+        const offset = parseInt($('#offset').val());
         const sortField = currentSortField;
         const sortDirection = currentSortDirection;
-        const filterParam = Object.keys(currentFilters).map(key => `${key}:${currentFilters[key]}`).join(',');
+
+        let sort = undefined
+        if (sortField) {
+            sort = [{[sortField]: sortDirection}]
+        }
+
+        const options = {
+            limit,
+            skip: offset,
+            sort
+        }
+
+        console.log(currentFilters, options)
 
         // Show loading message
         $('#tableBody').html('Loading...');
@@ -55,16 +79,17 @@ $(document).ready(function() {
 
         $.ajax({
             url: `${apiUrl}/${btoa(schema)}`,
-            data: {
-                key: veridaKey,
-                schema: schema,
-                limit: limit,
-                offset: offset,
-                sort: sortField ? `${sortField}:${sortDirection}` : '',
-                filter: filterParam
+            method: 'POST',
+            headers: {
+                key: veridaKey
             },
+            data: JSON.stringify({
+                options,
+                query: currentFilters
+            }),
+            contentType: 'application/json',
             success: function(response) {
-                const results = response.results;
+                const results = response.items;
                 const options = response.options;
                 const headers = Object.keys(results[0] || {});
 
@@ -201,18 +226,6 @@ $(document).ready(function() {
 
     // Example of listing schemas
     $('#schemaModal').on('show.bs.modal', function() {
-        const schemas = {
-            "Sync Position": "https://vault.schemas.verida.io/data-connections/sync-position/v0.1.0/schema.json",
-            "Sync Activity Log": "https://vault.schemas.verida.io/data-connections/activity-log/v0.1.0/schema.json",
-            "Connections": "https://vault.schemas.verida.io/data-connections/connection/v0.2.0/schema.json",
-            "Social Following": "https://common.schemas.verida.io/social/following/v0.1.0/schema.json",
-            "Social Post": "https://common.schemas.verida.io/social/post/v0.1.0/schema.json",
-            "Favourites": "https://common.schemas.verida.io/favourite/v0.1.0/schema.json",
-            "Email": "https://common.schemas.verida.io/social/email/v0.1.0/schema.json",
-            "Chat Group": "https://common.schemas.verida.io/social/chat/group/v0.1.0/schema.json",
-            "Chat Message": "https://common.schemas.verida.io/social/chat/message/v0.1.0/schema.json"
-        };
-
         // Clear previous list
         $('#schemaList').empty();
 

From 5ab1215db22017641a472bade017118fcf03bc16 Mon Sep 17 00:00:00 2001
From: Chris 
Date: Thu, 29 Aug 2024 11:52:21 +0930
Subject: [PATCH 063/182] Support delete row and destroy database from the
 Browse Data page

---
 src/api/v1/ds/controller.ts       | 35 +++++++++++++++++
 src/api/v1/ds/routes.ts           |  2 +
 src/web/developer/data/data.js    | 65 ++++++++++++++++++++++++++++---
 src/web/developer/data/index.html | 22 +++++++++++
 4 files changed, 119 insertions(+), 5 deletions(-)

diff --git a/src/api/v1/ds/controller.ts b/src/api/v1/ds/controller.ts
index 89c43fd6..0e73a57e 100644
--- a/src/api/v1/ds/controller.ts
+++ b/src/api/v1/ds/controller.ts
@@ -73,6 +73,41 @@ export class DsController {
             });
         }
     }
+
+    public async delete(req: Request, res: Response) {
+        try {
+            const { context } = await Utils.getNetworkFromRequest(req)
+            const permissions = Utils.buildPermissions(req)
+            const schemaName = Utils.getSchemaFromParams(req.params[0])
+
+            const ds = await context.openDatastore(schemaName, {
+                // @ts-ignore
+                permissions
+            })
+
+            const deleteId = req.query.id ? req.query.id.toString() : undefined
+            const destroy = req.query.destroy && req.query.destroy.toString() == "true"
+
+            let action
+            if (destroy) {
+                action = "destroy"
+                const db = await ds.getDb()
+                await db.destroy()
+            } else if (deleteId) {
+                action = "delete"
+                await ds.delete(deleteId)
+            }
+
+            return res.json({
+                success: true,
+                action
+            })
+        } catch (error) {
+            res.status(500).send({
+                error: error.message
+            });
+        }
+    }
 }
 
 export const controller = new DsController()
\ No newline at end of file
diff --git a/src/api/v1/ds/routes.ts b/src/api/v1/ds/routes.ts
index ec642ab2..0f610868 100644
--- a/src/api/v1/ds/routes.ts
+++ b/src/api/v1/ds/routes.ts
@@ -7,5 +7,7 @@ router.get(/get\/(.*)\/(.*)$/, controller.getById)
 router.get(/get\/(.*)$/, controller.get)
 router.post(/query\/(.*)$/, controller.query)
 
+router.delete(/([^\/]*)$/, controller.delete)
+
 
 export default router
\ No newline at end of file
diff --git a/src/web/developer/data/data.js b/src/web/developer/data/data.js
index e8a0f33e..41d4d186 100644
--- a/src/web/developer/data/data.js
+++ b/src/web/developer/data/data.js
@@ -13,7 +13,8 @@ $(document).ready(function() {
         "Favourites": "https://common.schemas.verida.io/favourite/v0.1.0/schema.json",
         "Email": "https://common.schemas.verida.io/social/email/v0.1.0/schema.json",
         "Chat Group": "https://common.schemas.verida.io/social/chat/group/v0.1.0/schema.json",
-        "Chat Message": "https://common.schemas.verida.io/social/chat/message/v0.1.0/schema.json"
+        "Chat Message": "https://common.schemas.verida.io/social/chat/message/v0.1.0/schema.json",
+        "Files": "https://common.schemas.verida.io/file/v0.1.0/schema.json"
     };
 
     // Load Verida Key and Schema from local storage
@@ -71,8 +72,6 @@ $(document).ready(function() {
             sort
         }
 
-        console.log(currentFilters, options)
-
         // Show loading message
         $('#tableBody').html('Loading...');
         $('.alert').hide(); // Hide previous error messages
@@ -99,6 +98,7 @@ $(document).ready(function() {
                     const sortIcon = (header === currentSortField) ? (currentSortDirection === 'asc' ? 'â–²' : 'â–¼') : '';
                     $('#tableHeaders').append(`${header} ${sortIcon}`);
                 });
+                $('#tableHeaders').append('Action'); // Add Action column header
 
                 // Populate table rows
                 $('#tableBody').empty();
@@ -107,6 +107,7 @@ $(document).ready(function() {
                     headers.forEach(header => {
                         rowHtml += `${row[header] || ''}`;
                     });
+                    rowHtml += ``; // Add Delete button
                     rowHtml += '';
                     $('#tableBody').append(rowHtml);
                 });
@@ -142,7 +143,7 @@ $(document).ready(function() {
                 $('#openFilters').show();
 
                 // Handle pagination
-                $('#prevButton').toggleClass('disabled', options.skip === 0);
+                $('#prevButton').toggleClass('disabled', !options || !options.skip ? true : options.skip === 0);
                 $('#nextButton').toggleClass('disabled', results.length < limit);
             },
             error: function(jqXHR) {
@@ -251,7 +252,61 @@ $(document).ready(function() {
     // Hide the filter button initially
     $('#openFilters').hide();
 
+    // Handle delete row button click
+    $('#tableBody').on('click', '.delete-row', function() {
+        const id = $(this).data('id');
+        const schemaUrl = $('#schema').val();
+        const veridaKey = $('#veridaKey').val();
+
+        if (confirm('Are you sure you want to delete this row?')) {
+            $.ajax({
+                url: `/api/v1/ds/${btoa(schemaUrl)}?id=${id}`,
+                method: 'DELETE',
+                headers: {
+                    key: veridaKey
+                },
+                success: function() {
+                    alert('Row deleted successfully');
+                    fetchData();
+                },
+                error: function(jqXHR) {
+                    const error = jqXHR.responseJSON ? jqXHR.responseJSON.error : 'An error occurred while deleting the row';
+                    alert(error);
+                }
+            });
+        }
+    });
+
+    // Handle Destroy button click
+    $('#destroyButton').click(function() {
+        $('#destroyModal').modal('show');
+    });
+
+    // Handle Destroy confirmation
+    $('#confirmDestroy').click(function() {
+        const schemaUrl = $('#schema').val();
+        const veridaKey = $('#veridaKey').val();
+
+        $.ajax({
+            url: `/api/v1/ds/${btoa(schemaUrl)}?destroy=true`,
+            method: 'DELETE',
+            headers: {
+                key: veridaKey
+            },
+            success: function() {
+                alert('Database destroyed successfully');
+                $('#destroyModal').modal('hide');
+                fetchData();
+            },
+            error: function(jqXHR) {
+                const error = jqXHR.responseJSON ? jqXHR.responseJSON.error : 'An error occurred while destroying the database';
+                alert(error);
+                $('#destroyModal').modal('hide');
+            }
+        });
+    });
+
     // Set initial values from query parameters
     setInitialValues();
     fetchData();
-});
+});
\ No newline at end of file
diff --git a/src/web/developer/data/index.html b/src/web/developer/data/index.html
index ab5e0cf6..b121c9e9 100644
--- a/src/web/developer/data/index.html
+++ b/src/web/developer/data/index.html
@@ -76,6 +76,7 @@ 
Database Results

             

             
             
+            
         
 
         
@@ -134,6 +135,27 @@ 
         

     
 
+    
+    
+
     
     
     

From f1fbcfa3ed6d3ed61d61b30c02694405b7f27841 Mon Sep 17 00:00:00 2001
From: chime3 
Date: Wed, 28 Aug 2024 19:27:07 -0700
Subject: [PATCH 064/182] fix: typo error

---
 src/providers/google/helpers.ts | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/src/providers/google/helpers.ts b/src/providers/google/helpers.ts
index 92a46417..d48fa279 100644
--- a/src/providers/google/helpers.ts
+++ b/src/providers/google/helpers.ts
@@ -326,7 +326,7 @@ export class GoogleDriveHelpers {
       if (!textContent) {
         console.warn("No indexable text found, using fallback method.");
 
-        // 10MB limit (5 * 1024 * 1024)
+        // 10MB limit (10 * 1024 * 1024)
         const sizeLimit = 10 * 1024 * 1024;
         const fileSize = response.data.size
           ? parseInt(response.data.size)
@@ -336,6 +336,7 @@ export class GoogleDriveHelpers {
           if (mimeType === "application/pdf") {
             const fileBuffer = await this.downloadFile(drive, fileId);
             textContent = await this.parsePdf(fileBuffer);
+            
           } else if (mimeType === "application/vnd.google-apps.document") {
             textContent = await this.extractGoogleDocsText(drive, fileId);
           } else if (mimeType === "application/vnd.google-apps.spreadsheet") {
@@ -349,26 +350,29 @@ export class GoogleDriveHelpers {
             mimeType === "application/msword" ||
             mimeType ===
               "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
-          ) {
+          ) {            
             const fileBuffer = await this.downloadFile(drive, fileId);
             textContent = await this.parseDocx(fileBuffer);
+            
           } else if (
             mimeType === "application/vnd.ms-excel" ||
             mimeType ===
               "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
-          ) {
+          ) {            
             const fileBuffer = await this.downloadFile(drive, fileId);
             textContent = await this.parseXlsx(fileBuffer);
+            
           } else if (
             mimeType === "application/vnd.ms-powerpoint" ||
             mimeType ===
               "application/vnd.openxmlformats-officedocument.presentationml.presentation"
-          ) {
+          ) {            
             const fileBuffer = await this.downloadFile(drive, fileId);
             textContent = await this.parsePptx(fileBuffer);
+            
           } else {
             console.warn(
-              "Unsupported MIME type or file size exceeds the limit."
+              "Unsupported MIME type."
             );
           }
         } else {

From 6514509fc9e4d2ace411625747a7a59e7695c280 Mon Sep 17 00:00:00 2001
From: Chris 
Date: Thu, 29 Aug 2024 13:03:48 +0930
Subject: [PATCH 065/182] Touch network cache when syncing. Support devmode.
 Delete any cached pouchdb folders on start when not in dev mode.

---
 src/providers/BaseProvider.ts |  4 +++
 src/server-app.ts             |  6 +++++
 src/server.ts                 |  2 +-
 src/serverconfig.example.json |  1 +
 src/utils.ts                  | 47 ++++++++++++++++++++++++++++++++++-
 5 files changed, 58 insertions(+), 2 deletions(-)

diff --git a/src/providers/BaseProvider.ts b/src/providers/BaseProvider.ts
index e323b8cb..27734211 100644
--- a/src/providers/BaseProvider.ts
+++ b/src/providers/BaseProvider.ts
@@ -195,6 +195,10 @@ export default class BaseProvider extends EventEmitter {
     public async sync(accessToken?: string, refreshToken?: string, force: boolean = false): Promise {
         await this.logMessage(SyncProviderLogLevel.INFO, `Starting sync`)
 
+        // Touch network, to ensure cache remains active
+        const account = await this.vault.getAccount()
+        await Utils.touchNetworkCache(await account.did())
+
         if (!accessToken) {
             const connection = this.getConnection()
             accessToken = connection.accessToken
diff --git a/src/server-app.ts b/src/server-app.ts
index b8365b7e..624927cc 100644
--- a/src/server-app.ts
+++ b/src/server-app.ts
@@ -1,5 +1,6 @@
 import { UniqueRequest } from './interfaces'
 import express, { Response, NextFunction } from 'express'
+import { Utils } from './utils'
 const cors = require('cors')
 import bodyParser from 'body-parser'
 import router from './routes'
@@ -41,5 +42,10 @@ app.use(bodyParser.json())
 app.use(bodyParser.urlencoded({ extended: false }))
 app.use(router)
 
+if (CONFIG.verida.devMode) {
+  console.log("Server is in development mode")
+} else {
+  Utils.deleteCachedData()
+}
 
 module.exports=app
\ No newline at end of file
diff --git a/src/server.ts b/src/server.ts
index 293c24d8..3d428ec0 100644
--- a/src/server.ts
+++ b/src/server.ts
@@ -24,5 +24,5 @@ https.createServer(
     },
     app
   ).listen(PORT, () => {
-  console.log(`server running on port ${PORT}`)
+  console.log(`Server running on port ${PORT}`)
 });
\ No newline at end of file
diff --git a/src/serverconfig.example.json b/src/serverconfig.example.json
index d1643fb6..08effd4d 100644
--- a/src/serverconfig.example.json
+++ b/src/serverconfig.example.json
@@ -4,6 +4,7 @@
     "assetsUrl": "https://127.0.0.1:5021/assets",
     "logLevel": "debug", 
     "verida": {
+        "devMode": true,
         "environment": "banksia",
         "testVeridaKey": "0x...",
         "testVeridaNetwork": "banksia",
diff --git a/src/utils.ts b/src/utils.ts
index 1b1f9f0f..1e5d6c54 100644
--- a/src/utils.ts
+++ b/src/utils.ts
@@ -3,6 +3,7 @@ import { Client } from "@verida/client-ts"
 import { Credentials } from '@verida/verifiable-credentials'
 import Providers from "./providers"
 import fs from 'fs'
+import path from 'path'
 import serverconfig from './config'
 import { AutoAccount } from '@verida/account-node'
 import { Request } from 'express'
@@ -73,7 +74,7 @@ export class Utils {
 
         if (Utils.networkCache[did]) {
             Utils.networkCache[did].requestIds.push(requestId)
-            Utils.networkCache[did].lastTouch = new Date()
+            Utils.touchNetworkCache(did)
 
             Utils.gcNetworkCache()
             return Utils.networkCache[did].networkConnection
@@ -113,6 +114,12 @@ export class Utils {
         return Utils.networkCache[did].networkConnection
     }
 
+    public static async touchNetworkCache(did: string) {
+        if (Utils.networkCache[did]) {
+            Utils.networkCache[did].lastTouch = new Date()
+        }
+    }
+
     public static async gcNetworkCache() {
         // console.log("gcNetworkCache()")
         for (const did in Utils.networkCache) {
@@ -239,6 +246,44 @@ export class Utils {
         const buffer = Buffer.from(base64Schema, 'base64')
         return buffer.toString('utf-8')
     }
+
+    public static deleteCachedData() {
+        // Read all files and folders in the directory
+        const directory = "./"
+        fs.readdir(directory, (err, files) => {
+            if (err) {
+                console.error(`Error reading directory: ${err}`);
+                return;
+            }
+
+            // Loop through each file/folder
+            files.forEach(file => {
+                const filePath = path.join(directory, file);
+
+                // Check if the name starts with 'v' and it's a directory
+                if (file.startsWith('v')) {
+                    fs.stat(filePath, (err, stats) => {
+                    if (err) {
+                        console.error(`Error stating file: ${err}`);
+                        return;
+                    }
+
+                    if (stats.isDirectory()) {
+                        // Recursively delete the directory
+                        console.log('rm ', filePath)
+                        fs.rm(filePath, { recursive: true, force: true }, (err) => {
+                            if (err) {
+                                console.error(`Error deleting folder: ${err}`);
+                            } else {
+                                console.log(`Deleted folder: ${filePath}`);
+                            }
+                        });
+                    }
+                    });
+                }
+            });
+        });
+    }
 }
 
 const VERIDA_ENVIRONMENT =  serverconfig.verida.environment

From bad6cdbc5d1a631129e9f74bf068be7581312598 Mon Sep 17 00:00:00 2001
From: chime3 
Date: Thu, 29 Aug 2024 00:10:40 -0700
Subject: [PATCH 066/182] feat: added size limit

---
 package.json                            |    2 +-
 src/providers/google/gdrive-document.ts |   13 +-
 src/providers/google/helpers.ts         |   24 +-
 src/providers/google/index.ts           |    8 +-
 src/providers/google/interfaces.ts      |    1 +
 src/serverconfig.example.json           |    1 +
 yarn.lock                               | 1362 +++++------------------
 7 files changed, 322 insertions(+), 1089 deletions(-)

diff --git a/package.json b/package.json
index 64169d40..9fb3e652 100644
--- a/package.json
+++ b/package.json
@@ -70,6 +70,7 @@
     "mocha": "^9.2.1",
     "nano": "^9.0.5",
     "node-imap": "^0.9.6",
+    "officeparser": "^4.1.1",
     "passport": "^0.5.2",
     "passport-facebook": "^3.0.0",
     "passport-google-oauth20": "^2.0.0",
@@ -78,7 +79,6 @@
     "string-strip-html": "8.5.0",
     "tdl": "^8.0.1",
     "tdlib-native": "^2.6.0",
-    "pptx-parser": "^1.1.7-beta.9",
     "ts-mocha": "^9.0.2",
     "twitter-api-v2": "^1.14.0",
     "uuid": "^10.0.0",
diff --git a/src/providers/google/gdrive-document.ts b/src/providers/google/gdrive-document.ts
index cdf421e1..3b34c6be 100644
--- a/src/providers/google/gdrive-document.ts
+++ b/src/providers/google/gdrive-document.ts
@@ -158,7 +158,15 @@ export default class GoogleDriveDocument extends GoogleHandler {
     
         for (const file of serverResponse.data.files ?? []) {
             
-            const fileId = file.id ?? '';
+            const fileId = file.id;
+            if (!fileId) {
+                const logEvent: SyncProviderLogEvent = {
+                    level: SyncProviderLogLevel.DEBUG,
+                    message: `Invalid ID for file ${fileId}. Ignoring this file.`,
+                };
+                this.emit('log', logEvent);
+                continue;
+            }
             
             if (fileId === breakId) {
                 const logEvent: SyncProviderLogEvent = {
@@ -216,7 +224,8 @@ export default class GoogleDriveDocument extends GoogleHandler {
                 continue;
             }
 
-            const textContent = await GoogleDriveHelpers.extractTextContent(drive, fileId, mimeType, this.getGoogleAuth());
+            const sizeLimit = this.config.sizeLimit * 1024 * 1024;
+            const textContent = await GoogleDriveHelpers.extractTextContent(drive, fileId, mimeType, sizeLimit, this.getGoogleAuth());
     
             results.push({
                 _id: this.buildItemId(fileId),
diff --git a/src/providers/google/helpers.ts b/src/providers/google/helpers.ts
index d48fa279..37d2dc40 100644
--- a/src/providers/google/helpers.ts
+++ b/src/providers/google/helpers.ts
@@ -2,7 +2,11 @@ import { drive_v3, gmail_v1, google } from 'googleapis';
 import { OAuth2Client } from 'google-auth-library';
 import pdf from "pdf-parse";
 import { stripHtml } from "string-strip-html";
-import { DocumentType } from "../../schemas";
+import mammoth from "mammoth";
+import * as XLSX from "xlsx";
+import * as officeParser from "officeparser";
+
+
 
 export const mimeExtensions: {[key: string]: string} = {
   // Google Drive MIME types
@@ -308,6 +312,7 @@ export class GoogleDriveHelpers {
     drive: drive_v3.Drive,
     fileId: string,
     mimeType: string,
+    sizeLimit: number,
     auth: OAuth2Client
   ): Promise {
     let textContent = "";
@@ -326,8 +331,6 @@ export class GoogleDriveHelpers {
       if (!textContent) {
         console.warn("No indexable text found, using fallback method.");
 
-        // 10MB limit (10 * 1024 * 1024)
-        const sizeLimit = 10 * 1024 * 1024;
         const fileSize = response.data.size
           ? parseInt(response.data.size)
           : undefined;
@@ -347,7 +350,6 @@ export class GoogleDriveHelpers {
             const fileBuffer = await this.downloadFile(drive, fileId);
             textContent = fileBuffer.toString("utf8");
           } else if (
-            mimeType === "application/msword" ||
             mimeType ===
               "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
           ) {            
@@ -363,11 +365,11 @@ export class GoogleDriveHelpers {
             textContent = await this.parseXlsx(fileBuffer);
             
           } else if (
-            mimeType === "application/vnd.ms-powerpoint" ||
             mimeType ===
               "application/vnd.openxmlformats-officedocument.presentationml.presentation"
           ) {            
             const fileBuffer = await this.downloadFile(drive, fileId);
+            //const pdfBuffer = await this.convertToPDF(fileBuffer);
             textContent = await this.parsePptx(fileBuffer);
             
           } else {
@@ -388,9 +390,9 @@ export class GoogleDriveHelpers {
 
     return textContent;
   }
-  
+
   static async parseDocx(docxBuffer: Buffer): Promise {
-    const mammoth = require('mammoth');
+    
     try {
       const result = await mammoth.extractRawText({ buffer: docxBuffer });
       return result.value;
@@ -401,7 +403,6 @@ export class GoogleDriveHelpers {
   }
   
   static async parseXlsx(xlsxBuffer: Buffer): Promise {
-    const XLSX = require('xlsx');
     try {
       const workbook = XLSX.read(xlsxBuffer, { type: 'buffer' });
       const text = workbook.SheetNames.map((sheetName: string) => {
@@ -416,12 +417,11 @@ export class GoogleDriveHelpers {
   }
   
   static async parsePptx(pptxBuffer: Buffer): Promise {
-    const PptxParser = require('pptx-parser');
     try {
-      const result = await PptxParser(pptxBuffer);
-      return result.text;
+      const data = await officeParser.parseOfficeAsync(pptxBuffer);
+      return data;
     } catch (error) {
-      console.error("Error parsing PPTX file:", error);
+      console.error("Error parsing Slides file:", error);
       return "";
     }
   }
diff --git a/src/providers/google/index.ts b/src/providers/google/index.ts
index cad84cad..cb087e8b 100644
--- a/src/providers/google/index.ts
+++ b/src/providers/google/index.ts
@@ -28,10 +28,10 @@ export default class GoogleProvider extends Base {
 
   public syncHandlers(): any[] {
     return [
-      Gmail,
-      YouTubeFollowing,
-      YouTubePost,
-      YouTubeFavourite,
+      //Gmail,
+      //YouTubeFollowing,
+      //YouTubePost,
+      //YouTubeFavourite,
       GoogleDriveDocument
     ];
   }
diff --git a/src/providers/google/interfaces.ts b/src/providers/google/interfaces.ts
index ca90302c..3fa81c25 100644
--- a/src/providers/google/interfaces.ts
+++ b/src/providers/google/interfaces.ts
@@ -4,6 +4,7 @@ export interface GoogleProviderConfig extends BaseProviderConfig {
   clientId: string;
   clientSecret: string;
   callbackUrl: string;
+  sizeLimit: number; //Mega bytes
 }
 
 export interface GmailHandlerConfig extends Record<"backdate", string> {}
diff --git a/src/serverconfig.example.json b/src/serverconfig.example.json
index 16600379..c577bdea 100644
--- a/src/serverconfig.example.json
+++ b/src/serverconfig.example.json
@@ -66,6 +66,7 @@
             "clientId": "",
             "clientSecret": "",
             "batchSize": 50,
+            "sizeLimit": 10,
             "maxSyncLoops": 1,
             "metadata": {
                 "breakTimestamp": "2000-07-21T12:07:11.000Z"
diff --git a/yarn.lock b/yarn.lock
index 49453f64..d3a80f6f 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -17,147 +17,6 @@
     call-me-maybe "^1.0.1"
     js-yaml "^4.1.0"
 
-"@babel/code-frame@^7.24.7":
-  version "7.24.7"
-  resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.24.7.tgz#882fd9e09e8ee324e496bd040401c6f046ef4465"
-  integrity sha512-BcYH1CVJBO9tvyIZ2jVeXgSIMvGZ2FDRvDdOIVQyuklNKSsx+eppDEBq/g47Ayw+RqNFE+URvOShmf+f/qwAlA==
-  dependencies:
-    "@babel/highlight" "^7.24.7"
-    picocolors "^1.0.0"
-
-"@babel/compat-data@^7.22.6", "@babel/compat-data@^7.25.2":
-  version "7.25.2"
-  resolved "https://registry.yarnpkg.com/@babel/compat-data/-/compat-data-7.25.2.tgz#e41928bd33475305c586f6acbbb7e3ade7a6f7f5"
-  integrity sha512-bYcppcpKBvX4znYaPEeFau03bp89ShqNMLs+rmdptMw+heSZh9+z84d2YG+K7cYLbWwzdjtDoW/uqZmPjulClQ==
-
-"@babel/generator@^7.25.0":
-  version "7.25.0"
-  resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.25.0.tgz#f858ddfa984350bc3d3b7f125073c9af6988f18e"
-  integrity sha512-3LEEcj3PVW8pW2R1SR1M89g/qrYk/m/mB/tLqn7dn4sbBUQyTqnlod+II2U4dqiGtUmkcnAmkMDralTFZttRiw==
-  dependencies:
-    "@babel/types" "^7.25.0"
-    "@jridgewell/gen-mapping" "^0.3.5"
-    "@jridgewell/trace-mapping" "^0.3.25"
-    jsesc "^2.5.1"
-
-"@babel/helper-compilation-targets@^7.22.6":
-  version "7.25.2"
-  resolved "https://registry.yarnpkg.com/@babel/helper-compilation-targets/-/helper-compilation-targets-7.25.2.tgz#e1d9410a90974a3a5a66e84ff55ef62e3c02d06c"
-  integrity sha512-U2U5LsSaZ7TAt3cfaymQ8WHh0pxvdHoEk6HVpaexxixjyEquMh0L0YNJNM6CTGKMXV1iksi0iZkGw4AcFkPaaw==
-  dependencies:
-    "@babel/compat-data" "^7.25.2"
-    "@babel/helper-validator-option" "^7.24.8"
-    browserslist "^4.23.1"
-    lru-cache "^5.1.1"
-    semver "^6.3.1"
-
-"@babel/helper-define-polyfill-provider@^0.6.2":
-  version "0.6.2"
-  resolved "https://registry.yarnpkg.com/@babel/helper-define-polyfill-provider/-/helper-define-polyfill-provider-0.6.2.tgz#18594f789c3594acb24cfdb4a7f7b7d2e8bd912d"
-  integrity sha512-LV76g+C502biUK6AyZ3LK10vDpDyCzZnhZFXkH1L75zHPj68+qc8Zfpx2th+gzwA2MzyK+1g/3EPl62yFnVttQ==
-  dependencies:
-    "@babel/helper-compilation-targets" "^7.22.6"
-    "@babel/helper-plugin-utils" "^7.22.5"
-    debug "^4.1.1"
-    lodash.debounce "^4.0.8"
-    resolve "^1.14.2"
-
-"@babel/helper-module-imports@^7.24.7":
-  version "7.24.7"
-  resolved "https://registry.yarnpkg.com/@babel/helper-module-imports/-/helper-module-imports-7.24.7.tgz#f2f980392de5b84c3328fc71d38bd81bbb83042b"
-  integrity sha512-8AyH3C+74cgCVVXow/myrynrAGv+nTVg5vKu2nZph9x7RcRwzmh0VFallJuFTZ9mx6u4eSdXZfcOzSqTUm0HCA==
-  dependencies:
-    "@babel/traverse" "^7.24.7"
-    "@babel/types" "^7.24.7"
-
-"@babel/helper-plugin-utils@^7.22.5", "@babel/helper-plugin-utils@^7.24.7":
-  version "7.24.8"
-  resolved "https://registry.yarnpkg.com/@babel/helper-plugin-utils/-/helper-plugin-utils-7.24.8.tgz#94ee67e8ec0e5d44ea7baeb51e571bd26af07878"
-  integrity sha512-FFWx5142D8h2Mgr/iPVGH5G7w6jDn4jUSpZTyDnQO0Yn7Ks2Kuz6Pci8H6MPCoUJegd/UZQ3tAvfLCxQSnWWwg==
-
-"@babel/helper-string-parser@^7.24.8":
-  version "7.24.8"
-  resolved "https://registry.yarnpkg.com/@babel/helper-string-parser/-/helper-string-parser-7.24.8.tgz#5b3329c9a58803d5df425e5785865881a81ca48d"
-  integrity sha512-pO9KhhRcuUyGnJWwyEgnRJTSIZHiT+vMD0kPeD+so0l7mxkMT19g3pjY9GTnHySck/hDzq+dtW/4VgnMkippsQ==
-
-"@babel/helper-validator-identifier@^7.24.7":
-  version "7.24.7"
-  resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.24.7.tgz#75b889cfaf9e35c2aaf42cf0d72c8e91719251db"
-  integrity sha512-rR+PBcQ1SMQDDyF6X0wxtG8QyLCgUB0eRAGguqRLfkCA87l7yAP7ehq8SNj96OOGTO8OBV70KhuFYcIkHXOg0w==
-
-"@babel/helper-validator-option@^7.24.8":
-  version "7.24.8"
-  resolved "https://registry.yarnpkg.com/@babel/helper-validator-option/-/helper-validator-option-7.24.8.tgz#3725cdeea8b480e86d34df15304806a06975e33d"
-  integrity sha512-xb8t9tD1MHLungh/AIoWYN+gVHaB9kwlu8gffXGSt3FFEIT7RjS+xWbc2vUD1UTZdIpKj/ab3rdqJ7ufngyi2Q==
-
-"@babel/highlight@^7.24.7":
-  version "7.24.7"
-  resolved "https://registry.yarnpkg.com/@babel/highlight/-/highlight-7.24.7.tgz#a05ab1df134b286558aae0ed41e6c5f731bf409d"
-  integrity sha512-EStJpq4OuY8xYfhGVXngigBJRWxftKX9ksiGDnmlY3o7B/V7KIAc9X4oiK87uPJSc/vs5L869bem5fhZa8caZw==
-  dependencies:
-    "@babel/helper-validator-identifier" "^7.24.7"
-    chalk "^2.4.2"
-    js-tokens "^4.0.0"
-    picocolors "^1.0.0"
-
-"@babel/parser@^7.25.0", "@babel/parser@^7.25.3":
-  version "7.25.3"
-  resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.25.3.tgz#91fb126768d944966263f0657ab222a642b82065"
-  integrity sha512-iLTJKDbJ4hMvFPgQwwsVoxtHyWpKKPBrxkANrSYewDPaPpT5py5yeVkgPIJ7XYXhndxJpaA3PyALSXQ7u8e/Dw==
-  dependencies:
-    "@babel/types" "^7.25.2"
-
-"@babel/plugin-transform-runtime@^7.7.6":
-  version "7.24.7"
-  resolved "https://registry.yarnpkg.com/@babel/plugin-transform-runtime/-/plugin-transform-runtime-7.24.7.tgz#00a5bfaf8c43cf5c8703a8a6e82b59d9c58f38ca"
-  integrity sha512-YqXjrk4C+a1kZjewqt+Mmu2UuV1s07y8kqcUf4qYLnoqemhR4gRQikhdAhSVJioMjVTu6Mo6pAbaypEA3jY6fw==
-  dependencies:
-    "@babel/helper-module-imports" "^7.24.7"
-    "@babel/helper-plugin-utils" "^7.24.7"
-    babel-plugin-polyfill-corejs2 "^0.4.10"
-    babel-plugin-polyfill-corejs3 "^0.10.1"
-    babel-plugin-polyfill-regenerator "^0.6.1"
-    semver "^6.3.1"
-
-"@babel/runtime-corejs3@^7.10.4", "@babel/runtime-corejs3@^7.7.7":
-  version "7.25.0"
-  resolved "https://registry.yarnpkg.com/@babel/runtime-corejs3/-/runtime-corejs3-7.25.0.tgz#0a318b66dfc765ad10562d829fea372ed7e1eb7d"
-  integrity sha512-BOehWE7MgQ8W8Qn0CQnMtg2tHPHPulcS/5AVpFvs2KCK1ET+0WqZqPvnpRpFN81gYoFopdIEJX9Sgjw3ZBccPg==
-  dependencies:
-    core-js-pure "^3.30.2"
-    regenerator-runtime "^0.14.0"
-
-"@babel/template@^7.25.0":
-  version "7.25.0"
-  resolved "https://registry.yarnpkg.com/@babel/template/-/template-7.25.0.tgz#e733dc3134b4fede528c15bc95e89cb98c52592a"
-  integrity sha512-aOOgh1/5XzKvg1jvVz7AVrx2piJ2XBi227DHmbY6y+bM9H2FlN+IfecYu4Xl0cNiiVejlsCri89LUsbj8vJD9Q==
-  dependencies:
-    "@babel/code-frame" "^7.24.7"
-    "@babel/parser" "^7.25.0"
-    "@babel/types" "^7.25.0"
-
-"@babel/traverse@^7.24.7":
-  version "7.25.3"
-  resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.25.3.tgz#f1b901951c83eda2f3e29450ce92743783373490"
-  integrity sha512-HefgyP1x754oGCsKmV5reSmtV7IXj/kpaE1XYY+D9G5PvKKoFfSbiS4M77MdjuwlZKDIKFCffq9rPU+H/s3ZdQ==
-  dependencies:
-    "@babel/code-frame" "^7.24.7"
-    "@babel/generator" "^7.25.0"
-    "@babel/parser" "^7.25.3"
-    "@babel/template" "^7.25.0"
-    "@babel/types" "^7.25.2"
-    debug "^4.3.1"
-    globals "^11.1.0"
-
-"@babel/types@^7.24.7", "@babel/types@^7.25.0", "@babel/types@^7.25.2":
-  version "7.25.2"
-  resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.25.2.tgz#55fb231f7dc958cd69ea141a4c2997e819646125"
-  integrity sha512-YTnYtra7W9e6/oAZEHj0bJehPRUlLH9/fbpT5LfB0NhQXyALCRkRs3zH9v07IYhkgpqX6Z78FnuccZr/l4Fs4Q==
-  dependencies:
-    "@babel/helper-string-parser" "^7.24.8"
-    "@babel/helper-validator-identifier" "^7.24.7"
-    to-fast-properties "^2.0.0"
-
 "@discordjs/builders@^1.6.0":
   version "1.6.1"
   resolved "https://registry.yarnpkg.com/@discordjs/builders/-/builders-1.6.1.tgz#5b1447cfa493bc1306671ef18ce3aae13c0af0ba"
@@ -544,50 +403,6 @@
     "@ethersproject/properties" "^5.7.0"
     "@ethersproject/strings" "^5.7.0"
 
-"@isaacs/cliui@^8.0.2":
-  version "8.0.2"
-  resolved "https://registry.yarnpkg.com/@isaacs/cliui/-/cliui-8.0.2.tgz#b37667b7bc181c168782259bab42474fbf52b550"
-  integrity sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==
-  dependencies:
-    string-width "^5.1.2"
-    string-width-cjs "npm:string-width@^4.2.0"
-    strip-ansi "^7.0.1"
-    strip-ansi-cjs "npm:strip-ansi@^6.0.1"
-    wrap-ansi "^8.1.0"
-    wrap-ansi-cjs "npm:wrap-ansi@^7.0.0"
-
-"@jridgewell/gen-mapping@^0.3.5":
-  version "0.3.5"
-  resolved "https://registry.yarnpkg.com/@jridgewell/gen-mapping/-/gen-mapping-0.3.5.tgz#dcce6aff74bdf6dad1a95802b69b04a2fcb1fb36"
-  integrity sha512-IzL8ZoEDIBRWEzlCcRhOaCupYyN5gdIK+Q6fbFdPDg6HqX6jpkItn7DFIpW9LQzXG6Df9sA7+OKnq0qlz/GaQg==
-  dependencies:
-    "@jridgewell/set-array" "^1.2.1"
-    "@jridgewell/sourcemap-codec" "^1.4.10"
-    "@jridgewell/trace-mapping" "^0.3.24"
-
-"@jridgewell/resolve-uri@^3.1.0":
-  version "3.1.2"
-  resolved "https://registry.yarnpkg.com/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz#7a0ee601f60f99a20c7c7c5ff0c80388c1189bd6"
-  integrity sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==
-
-"@jridgewell/set-array@^1.2.1":
-  version "1.2.1"
-  resolved "https://registry.yarnpkg.com/@jridgewell/set-array/-/set-array-1.2.1.tgz#558fb6472ed16a4c850b889530e6b36438c49280"
-  integrity sha512-R8gLRTZeyp03ymzP/6Lil/28tGeGEzhx1q2k703KGWRAI1VdvPIXdG70VJc2pAMw3NA6JKL5hhFu1sJX0Mnn/A==
-
-"@jridgewell/sourcemap-codec@^1.4.10", "@jridgewell/sourcemap-codec@^1.4.14":
-  version "1.5.0"
-  resolved "https://registry.yarnpkg.com/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.0.tgz#3188bcb273a414b0d215fd22a58540b989b9409a"
-  integrity sha512-gv3ZRaISU3fjPAgNsriBRqGWQL6quFx04YMPW/zD8XMLsU32mhCCbfbO6KZFLjvYpCZ8zyDEgqsgf+PwPaM7GQ==
-
-"@jridgewell/trace-mapping@^0.3.24", "@jridgewell/trace-mapping@^0.3.25":
-  version "0.3.25"
-  resolved "https://registry.yarnpkg.com/@jridgewell/trace-mapping/-/trace-mapping-0.3.25.tgz#15f190e98895f3fc23276ee14bc76b675c2e50f0"
-  integrity sha512-vNk6aEwybGtawWmy/PzwnGDOjCkLWSD2wqvjGGAgOAwCGWySYXfYoxt00IJkTF+8Lb57DwOb3Aa0o9CApepiYQ==
-  dependencies:
-    "@jridgewell/resolve-uri" "^3.1.0"
-    "@jridgewell/sourcemap-codec" "^1.4.14"
-
 "@jsdevtools/ono@^7.1.3":
   version "7.1.3"
   resolved "https://registry.yarnpkg.com/@jsdevtools/ono/-/ono-7.1.3.tgz#9df03bbd7c696a5c58885c34aa06da41c8543796"
@@ -628,16 +443,6 @@
   resolved "https://registry.yarnpkg.com/@oauth-everything/profile/-/profile-1.0.0.tgz#0b5e78749415519fa312dc83347a677903f456ba"
   integrity sha512-OmCuBPhjaLHh9MST9P5jRuVBZaP0z7hBk8nH4Yt7Id5kNM1AXGd5uud6CP7W2zuhKl2nk0KsYmeMT7SkzN6VWg==
 
-"@one-ini/wasm@0.1.1":
-  version "0.1.1"
-  resolved "https://registry.yarnpkg.com/@one-ini/wasm/-/wasm-0.1.1.tgz#6013659736c9dbfccc96e8a9c2b3de317df39323"
-  integrity sha512-XuySG1E38YScSJoMlqovLru4KTUNSjgVTIjyh7qMX6aNN5HY5Ct5LhRJdxO79JtTzKfzV/bnWpz+zquYrISsvw==
-
-"@pkgjs/parseargs@^0.11.0":
-  version "0.11.0"
-  resolved "https://registry.yarnpkg.com/@pkgjs/parseargs/-/parseargs-0.11.0.tgz#a77ea742fab25775145434eb1d2328cf5013ac33"
-  integrity sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==
-
 "@prebuilt-tdlib/darwin@0.1008034.0":
   version "0.1008034.0"
   resolved "https://registry.yarnpkg.com/@prebuilt-tdlib/darwin/-/darwin-0.1008034.0.tgz#a5983898df91b3017fc432874f258a17299dcf3c"
@@ -960,7 +765,7 @@
     "@types/qs" "*"
     "@types/serve-static" "*"
 
-"@types/json-schema@^7.0.5", "@types/json-schema@^7.0.6", "@types/json-schema@^7.0.8":
+"@types/json-schema@^7.0.6":
   version "7.0.15"
   resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.15.tgz#596a1747233694d50f6ad8a7869fcb6f56cf5841"
   integrity sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==
@@ -1379,31 +1184,7 @@
     axios "^1.2.3"
     ethers "^5.7.2"
 
-"@vf.js/gui@^3.0.2":
-  version "3.0.3"
-  resolved "https://registry.yarnpkg.com/@vf.js/gui/-/gui-3.0.3.tgz#095b509676d5473e8af55ba221a59b5478954cab"
-  integrity sha512-ej5GKK7pe8+U1TmHov8L1MUoarDGPq8s8ThpBd/CMkwg8IrVyq11n/vEUpKZi71mqtFKLkDvIWb2frJUYyMY9A==
-
-"@vf.js/launcher@2.0.12":
-  version "2.0.12"
-  resolved "https://registry.yarnpkg.com/@vf.js/launcher/-/launcher-2.0.12.tgz#32cac047091faeb1dccb989dbde05c3aecfd69e5"
-  integrity sha512-5zXvkLfIM+VydHZy3qInhsdBY4Ydp6TueVZQNBZyk9rb4Go/nAh8ouKdjfdpPQ2iZU8xT106qpS28VChQqRoHQ==
-  dependencies:
-    "@vf.js/gui" "^3.0.2"
-    "@vf.js/player" "^2.0.4"
-    "@vf.js/vf" "^6.0.2-v63"
-
-"@vf.js/player@^2.0.4":
-  version "2.0.4"
-  resolved "https://registry.yarnpkg.com/@vf.js/player/-/player-2.0.4.tgz#f6daf3f125b343d0886f4bd85de4e7afa93b16ff"
-  integrity sha512-exIUcWt/G/E91ckK0LAiavG84+uxsqWngSO3FRNedGHhkU6uPj+sTI0MLLX78lkHRfcC2zpvRQKF5Q7e2qTsZg==
-
-"@vf.js/vf@^6.0.2-v63":
-  version "6.0.2-v65"
-  resolved "https://registry.yarnpkg.com/@vf.js/vf/-/vf-6.0.2-v65.tgz#1db0bb92cf5a7dfd8b97aad2e662aa7c57f63742"
-  integrity sha512-W1GtAFKeLIfM/GDO0gKBzteaGJc1+5ND5rpxdOsBLWNkKdqVrMn+pAbf908nkgxGtoNvDbsoFzpmb2/TNRQB9Q==
-
-"@xmldom/xmldom@^0.8.6":
+"@xmldom/xmldom@^0.8.10", "@xmldom/xmldom@^0.8.6":
   version "0.8.10"
   resolved "https://registry.yarnpkg.com/@xmldom/xmldom/-/xmldom-0.8.10.tgz#a1337ca426aa61cef9fe15b5b28e340a72f6fa99"
   integrity sha512-2WALfTl4xo2SkGCYRt6rDTFfk9R1czmBvUQy12gK2KuRKIpWEhcbbzy8EZXtz/jkRqHX8bFEc6FC1HjX4TUWYw==
@@ -1413,11 +1194,6 @@ abbrev@1:
   resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-1.1.1.tgz#f8f2c887ad10bf67f634f005b6987fed3179aac8"
   integrity sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==
 
-abbrev@^2.0.0:
-  version "2.0.0"
-  resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-2.0.0.tgz#cf59829b8b4f03f89dda2771cb7f3653828c89bf"
-  integrity sha512-6/mh1E2u2YgEsCHdY0Yx5oW+61gZU+1vXaoiHHrpKeuRNNgFvS+/jrwHiQhB5apAf5oB7UB7E19ol2R2LKH8hQ==
-
 abort-controller@3.0.0, abort-controller@^3.0.0:
   version "3.0.0"
   resolved "https://registry.yarnpkg.com/abort-controller/-/abort-controller-3.0.0.tgz#eaf54d53b62bae4138e809ca225c8439a6efb392"
@@ -1425,11 +1201,6 @@ abort-controller@3.0.0, abort-controller@^3.0.0:
   dependencies:
     event-target-shim "^5.0.0"
 
-abs-svg-path@^0.1.1:
-  version "0.1.1"
-  resolved "https://registry.yarnpkg.com/abs-svg-path/-/abs-svg-path-0.1.1.tgz#df601c8e8d2ba10d4a76d625e236a9a39c2723bf"
-  integrity sha512-d8XPSGjfyzlXC3Xx891DJRyZfqk5JU0BJrDQcsWomFIV1/BIzPW5HDH5iDdWpqWaav0YVIEzT1RHTwWr0FFshA==
-
 abstract-leveldown@^6.2.1:
   version "6.3.0"
   resolved "https://registry.yarnpkg.com/abstract-leveldown/-/abstract-leveldown-6.3.0.tgz#d25221d1e6612f820c35963ba4bd739928f6026a"
@@ -1503,12 +1274,7 @@ ajv-formats@^2.1.1:
   dependencies:
     ajv "^8.0.0"
 
-ajv-keywords@^3.5.2:
-  version "3.5.2"
-  resolved "https://registry.yarnpkg.com/ajv-keywords/-/ajv-keywords-3.5.2.tgz#31f29da5ab6e00d1c2d329acf7b5929614d5014d"
-  integrity sha512-5p6WTN0DdTGVQk6VjcEju19IgaHudalcfabD7yhDGeA6bcQnmL+CpveLJq/3hvfwd1aof6L386Ougkx6RfyMIQ==
-
-ajv@^6.12.3, ajv@^6.12.4, ajv@^6.12.5:
+ajv@^6.12.3:
   version "6.12.6"
   resolved "https://registry.yarnpkg.com/ajv/-/ajv-6.12.6.tgz#baf5a62e802b07d977034586f8c3baf5adf26df4"
   integrity sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==
@@ -1545,11 +1311,6 @@ ansi-regex@^5.0.1:
   resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-5.0.1.tgz#082cb2c89c9fe8659a311a53bd6a4dc5301db304"
   integrity sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==
 
-ansi-regex@^6.0.1:
-  version "6.0.1"
-  resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-6.0.1.tgz#3183e38fae9a65d7cb5e53945cd5897d0260a06a"
-  integrity sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA==
-
 ansi-styles@^3.2.1:
   version "3.2.1"
   resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-3.2.1.tgz#41fbb20243e50b12be0f04b8dedbf07520ce841d"
@@ -1564,11 +1325,6 @@ ansi-styles@^4.0.0, ansi-styles@^4.1.0:
   dependencies:
     color-convert "^2.0.1"
 
-ansi-styles@^6.1.0:
-  version "6.2.1"
-  resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-6.2.1.tgz#0e62320cf99c21afff3b3012192546aacbfb05c5"
-  integrity sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==
-
 any-promise@^1.3.0:
   version "1.3.0"
   resolved "https://registry.yarnpkg.com/any-promise/-/any-promise-1.3.0.tgz#abc6afeedcea52e809cdc0376aed3ce39635d17f"
@@ -1754,30 +1510,6 @@ axios@^1.2.3, axios@^1.3.3, axios@^1.6.2, axios@^1.7.2:
     form-data "^4.0.0"
     proxy-from-env "^1.1.0"
 
-babel-plugin-polyfill-corejs2@^0.4.10:
-  version "0.4.11"
-  resolved "https://registry.yarnpkg.com/babel-plugin-polyfill-corejs2/-/babel-plugin-polyfill-corejs2-0.4.11.tgz#30320dfe3ffe1a336c15afdcdafd6fd615b25e33"
-  integrity sha512-sMEJ27L0gRHShOh5G54uAAPaiCOygY/5ratXuiyb2G46FmlSpc9eFCzYVyDiPxfNbwzA7mYahmjQc5q+CZQ09Q==
-  dependencies:
-    "@babel/compat-data" "^7.22.6"
-    "@babel/helper-define-polyfill-provider" "^0.6.2"
-    semver "^6.3.1"
-
-babel-plugin-polyfill-corejs3@^0.10.1:
-  version "0.10.6"
-  resolved "https://registry.yarnpkg.com/babel-plugin-polyfill-corejs3/-/babel-plugin-polyfill-corejs3-0.10.6.tgz#2deda57caef50f59c525aeb4964d3b2f867710c7"
-  integrity sha512-b37+KR2i/khY5sKmWNVQAnitvquQbNdWy6lJdsr0kmquCKEEUgMKK4SboVM3HtfnZilfjr4MMQ7vY58FVWDtIA==
-  dependencies:
-    "@babel/helper-define-polyfill-provider" "^0.6.2"
-    core-js-compat "^3.38.0"
-
-babel-plugin-polyfill-regenerator@^0.6.1:
-  version "0.6.2"
-  resolved "https://registry.yarnpkg.com/babel-plugin-polyfill-regenerator/-/babel-plugin-polyfill-regenerator-0.6.2.tgz#addc47e240edd1da1058ebda03021f382bba785e"
-  integrity sha512-2R25rQZWP63nGwaAswvDazbPXfrM3HwVoBXK6HcqeKrSrL/JqcC/rDcf95l4r7LXLyxDXc8uQDa064GubtCABg==
-  dependencies:
-    "@babel/helper-define-polyfill-provider" "^0.6.2"
-
 babel-runtime@^6.23.0:
   version "6.26.0"
   resolved "https://registry.yarnpkg.com/babel-runtime/-/babel-runtime-6.26.0.tgz#965c7058668e82b55d7bfe04ff2337bc8b5647fe"
@@ -1837,11 +1569,6 @@ bech32@^2.0.0:
   resolved "https://registry.yarnpkg.com/bech32/-/bech32-2.0.0.tgz#078d3686535075c8c79709f054b1b226a133b355"
   integrity sha512-LcknSilhIGatDAsY1ak2I8VtGaHNhgMSYVxFrGLXv+xLHytaKZKcaUJJUE7qmBr7h33o5YQwP55pMI0xmkpJwg==
 
-big.js@^5.2.2:
-  version "5.2.2"
-  resolved "https://registry.yarnpkg.com/big.js/-/big.js-5.2.2.tgz#65f0af382f578bcdc742bd9c281e9cb2d7768328"
-  integrity sha512-vyL2OymJxmarO8gxMr0mhChsO9QGwhynfuu4+MHTAW6czfq9humCB7rKpUjDd9YUiDPU4mzpyupFSvOClAwbmQ==
-
 bignumber.js@^9.0.0:
   version "9.1.2"
   resolved "https://registry.yarnpkg.com/bignumber.js/-/bignumber.js-9.1.2.tgz#b7c4242259c008903b13707983b5f4bbd31eda0c"
@@ -1857,6 +1584,14 @@ binary-extensions@^2.0.0:
   resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-2.2.0.tgz#75f502eeaf9ffde42fc98829645be4ea76bd9e2d"
   integrity sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==
 
+bl@^1.0.0:
+  version "1.2.3"
+  resolved "https://registry.yarnpkg.com/bl/-/bl-1.2.3.tgz#1e8dd80142eac80d7158c9dccc047fb620e035e7"
+  integrity sha512-pvcNpa0UU69UT341rO6AYy4FVAIkUHuZXRIWbq+zHnsVcRzDDjIAhGuuYoi0d//cwIwtt4pkpKycWEfjdV+vww==
+  dependencies:
+    readable-stream "^2.3.5"
+    safe-buffer "^5.1.1"
+
 bl@^4.0.3:
   version "4.1.0"
   resolved "https://registry.yarnpkg.com/bl/-/bl-4.1.0.tgz#451535264182bec2fbbc83a62ab98cf11d9f7b3a"
@@ -1921,13 +1656,6 @@ brace-expansion@^1.1.7:
     balanced-match "^1.0.0"
     concat-map "0.0.1"
 
-brace-expansion@^2.0.1:
-  version "2.0.1"
-  resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-2.0.1.tgz#1edc459e0f0c548486ecf9fc99f2221364b9a0ae"
-  integrity sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==
-  dependencies:
-    balanced-match "^1.0.0"
-
 braces@~3.0.2:
   version "3.0.2"
   resolved "https://registry.yarnpkg.com/braces/-/braces-3.0.2.tgz#3454e1a462ee8d599e236df336cd9ea4f8afe107"
@@ -1952,16 +1680,6 @@ browserify-zlib@^0.1.4:
   dependencies:
     pako "~0.2.0"
 
-browserslist@^4.23.1, browserslist@^4.23.3:
-  version "4.23.3"
-  resolved "https://registry.yarnpkg.com/browserslist/-/browserslist-4.23.3.tgz#debb029d3c93ebc97ffbc8d9cbb03403e227c800"
-  integrity sha512-btwCFJVjI4YWDNfau8RhZ+B1Q/VLoUITrm3RlP6y1tYGWIOa+InuYiRGXUBXo8nA1qKmHMyLB/iVQg5TT4eFoA==
-  dependencies:
-    caniuse-lite "^1.0.30001646"
-    electron-to-chromium "^1.5.4"
-    node-releases "^2.0.18"
-    update-browserslist-db "^1.1.0"
-
 bs58@^4.0.1:
   version "4.0.1"
   resolved "https://registry.yarnpkg.com/bs58/-/bs58-4.0.1.tgz#be161e76c354f6f788ae4071f63f34e8c4f0a42a"
@@ -1976,7 +1694,20 @@ bs58@^5.0.0:
   dependencies:
     base-x "^4.0.0"
 
-buffer-crc32@^0.2.1, buffer-crc32@^0.2.13:
+buffer-alloc-unsafe@^1.1.0:
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/buffer-alloc-unsafe/-/buffer-alloc-unsafe-1.1.0.tgz#bd7dc26ae2972d0eda253be061dba992349c19f0"
+  integrity sha512-TEM2iMIEQdJ2yjPJoSIsldnleVaAk1oW3DBVUykyOLsEsFmEc9kn+SFFPz+gl54KQNxlDnAwCXosOS9Okx2xAg==
+
+buffer-alloc@^1.2.0:
+  version "1.2.0"
+  resolved "https://registry.yarnpkg.com/buffer-alloc/-/buffer-alloc-1.2.0.tgz#890dd90d923a873e08e10e5fd51a57e5b7cce0ec"
+  integrity sha512-CFsHQgjtW1UChdXgbyJGtnm+O/uLQeZdtbDo8mfUgYXCHSM1wgrVxXm6bSyrUuErEb+4sYVGCzASBRot7zyrow==
+  dependencies:
+    buffer-alloc-unsafe "^1.1.0"
+    buffer-fill "^1.0.0"
+
+buffer-crc32@^0.2.1, buffer-crc32@^0.2.13, buffer-crc32@~0.2.3:
   version "0.2.13"
   resolved "https://registry.yarnpkg.com/buffer-crc32/-/buffer-crc32-0.2.13.tgz#0d333e3f00eac50aa1454abd30ef8c2a5d9a7242"
   integrity sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==
@@ -1986,6 +1717,11 @@ buffer-equal-constant-time@1.0.1:
   resolved "https://registry.yarnpkg.com/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz#f8e71132f7ffe6e01a5c9697a4c6f3e48d5cc819"
   integrity sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==
 
+buffer-fill@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/buffer-fill/-/buffer-fill-1.0.0.tgz#f8f78b76789888ef39f205cd637f68e702122b2c"
+  integrity sha512-T7zexNBwiiaCOGDg9xNX9PBmjrubblRkENuptryuI64URkXDFum9il/JGL8Lm8wYfAXpredVXXZz7eMHilimiQ==
+
 buffer-from@1.1.2, buffer-from@^1.0.0, buffer-from@^1.1.0:
   version "1.1.2"
   resolved "https://registry.yarnpkg.com/buffer-from/-/buffer-from-1.1.2.tgz#2b146a6fd72e80b4f55d255f35ed59a3a9a41bd5"
@@ -2000,7 +1736,7 @@ buffer@4.9.2:
     ieee754 "^1.1.4"
     isarray "^1.0.0"
 
-buffer@^5.1.0, buffer@^5.5.0, buffer@^5.6.0:
+buffer@^5.1.0, buffer@^5.2.1, buffer@^5.5.0, buffer@^5.6.0:
   version "5.7.1"
   resolved "https://registry.yarnpkg.com/buffer/-/buffer-5.7.1.tgz#ba62e7c13133053582197160851a8f648e99eed0"
   integrity sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==
@@ -2057,21 +1793,11 @@ call-me-maybe@^1.0.1:
   resolved "https://registry.yarnpkg.com/call-me-maybe/-/call-me-maybe-1.0.2.tgz#03f964f19522ba643b1b0693acb9152fe2074baa"
   integrity sha512-HpX65o1Hnr9HH25ojC1YGs7HCQLq0GCOibSaWER0eNpgJ/Z1MZv2mTc7+xh6WOPxbRVcmgbv4hGU+uSQ/2xFZQ==
 
-camelcase@^5.3.1:
-  version "5.3.1"
-  resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-5.3.1.tgz#e3c9b31569e106811df242f715725a1f4c494320"
-  integrity sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==
-
 camelcase@^6.0.0, camelcase@^6.2.0:
   version "6.3.0"
   resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-6.3.0.tgz#5685b95eb209ac9c0c177467778c9c84df58ba9a"
   integrity sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==
 
-caniuse-lite@^1.0.30001646:
-  version "1.0.30001651"
-  resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001651.tgz#52de59529e8b02b1aedcaaf5c05d9e23c0c28138"
-  integrity sha512-9Cf+Xv1jJNe1xPZLGuUXLNkE1BoDkqRqYyFJ9TDYSqhduqA4hu4oR9HluGoWYQC/aj8WHjsGVV+bwkh0+tegRg==
-
 canonicalize@^1.0.5:
   version "1.0.8"
   resolved "https://registry.yarnpkg.com/canonicalize/-/canonicalize-1.0.8.tgz#24d1f1a00ed202faafd9bf8e63352cd4450c6df1"
@@ -2219,11 +1945,6 @@ color-name@~1.1.4:
   resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.4.tgz#c2a09a87acbde69543de6f63fa3995c826c536a2"
   integrity sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==
 
-colors@^1.4.0:
-  version "1.4.0"
-  resolved "https://registry.yarnpkg.com/colors/-/colors-1.4.0.tgz#c50491479d4c1bdaed2c9ced32cf7c7dc2360f78"
-  integrity sha512-a+UqTh4kgZg/SlGvfbzDHpgRu7AAQOmmqRHJnxhRZICKFUT91brVhNNt58CMWU9PsBbv3PDCZUHbVxuDiH2mtA==
-
 combined-stream@^1.0.6, combined-stream@^1.0.8, combined-stream@~1.0.6:
   version "1.0.8"
   resolved "https://registry.yarnpkg.com/combined-stream/-/combined-stream-1.0.8.tgz#c3d45a8b34fd730631a110a8a2520682b31d5a7f"
@@ -2274,10 +1995,10 @@ command-line-usage@6.1.3:
     table-layout "^1.0.2"
     typical "^5.2.0"
 
-commander@^10.0.0:
-  version "10.0.1"
-  resolved "https://registry.yarnpkg.com/commander/-/commander-10.0.1.tgz#881ee46b4f77d1c1dccc5823433aa39b022cbe06"
-  integrity sha512-y4Mg2tXshplEbSGzx7amzPwKKOCGuoSRP/CjEdwwk0FOGlUbq6lKuoyDZTNZkmxHdJtp54hdfY/JUrdL7Xfdug==
+commander@^2.8.1:
+  version "2.20.3"
+  resolved "https://registry.yarnpkg.com/commander/-/commander-2.20.3.tgz#fd485e84c03eb4881c20722ba48035e8531aeb33"
+  integrity sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==
 
 compress-commons@^2.1.1:
   version "2.1.1"
@@ -2294,14 +2015,6 @@ concat-map@0.0.1:
   resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b"
   integrity sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=
 
-config-chain@^1.1.13:
-  version "1.1.13"
-  resolved "https://registry.yarnpkg.com/config-chain/-/config-chain-1.1.13.tgz#fad0795aa6a6cdaff9ed1b68e9dff94372c232f4"
-  integrity sha512-qj+f8APARXHrM0hraqXYb2/bOVSV4PvJQlNZ/DVj0QrmNM2q2euizkeuVckQ57J+W0mRH6Hvi+k50M4Jul2VRQ==
-  dependencies:
-    ini "^1.3.4"
-    proto-list "~1.2.1"
-
 configstore@^5.0.1:
   version "5.0.1"
   resolved "https://registry.yarnpkg.com/configstore/-/configstore-5.0.1.tgz#d365021b5df4b98cdd187d6a3b0e3f6a7cc5ed96"
@@ -2359,28 +2072,11 @@ core-decorators@^0.17.0:
   resolved "https://registry.yarnpkg.com/core-decorators/-/core-decorators-0.17.0.tgz#3f43180a86d2ab0cc51069f46a1ec3e49e7cebd6"
   integrity sha1-P0MYCobSqwzFEGn0ah7D5J5869Y=
 
-core-js-compat@^3.38.0:
-  version "3.38.0"
-  resolved "https://registry.yarnpkg.com/core-js-compat/-/core-js-compat-3.38.0.tgz#d93393b1aa346b6ee683377b0c31172ccfe607aa"
-  integrity sha512-75LAicdLa4OJVwFxFbQR3NdnZjNgX6ILpVcVzcC4T2smerB5lELMrJQQQoWV6TiuC/vlaFqgU2tKQx9w5s0e0A==
-  dependencies:
-    browserslist "^4.23.3"
-
-core-js-pure@^3.30.2:
-  version "3.38.0"
-  resolved "https://registry.yarnpkg.com/core-js-pure/-/core-js-pure-3.38.0.tgz#bc802cd152e33d5b0ec733b656c71cb847cac701"
-  integrity sha512-8balb/HAXo06aHP58mZMtXgD8vcnXz9tUDePgqBgJgKdmTlMt+jw3ujqniuBDQXMvTzxnMpxHFeuSM3g1jWQuQ==
-
 core-js@^2.4.0:
   version "2.6.12"
   resolved "https://registry.yarnpkg.com/core-js/-/core-js-2.6.12.tgz#d9333dfa7b065e347cc5682219d6f690859cc2ec"
   integrity sha512-Kb2wC0fvsWfQrgk8HU5lW6U/Lcs8+9aaYcy4ZFc6DDlo4nZ7n70dEgE5rtR0oG6ufKDUnrwfWL1mXR5ljDatrQ==
 
-core-js@^3.6.0:
-  version "3.38.0"
-  resolved "https://registry.yarnpkg.com/core-js/-/core-js-3.38.0.tgz#8acb7c050bf2ccbb35f938c0d040132f6110f636"
-  integrity sha512-XPpwqEodRljce9KswjZShh95qJ1URisBeKCjUdq27YdenkslVe7OO0ZJhlYXAChW7OhXaRLl8AAba7IBfoIHug==
-
 core-util-is@1.0.2:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.2.tgz#b5fd54220aa2bc5ab57aab7140c940754503c1a7"
@@ -2442,15 +2138,6 @@ create-hmac@^1.1.4:
     safe-buffer "^5.0.1"
     sha.js "^2.4.8"
 
-cross-spawn@^7.0.0:
-  version "7.0.3"
-  resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.3.tgz#f73a85b9d5d41d045551c177e2882d4ac85728a6"
-  integrity sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==
-  dependencies:
-    path-key "^3.1.0"
-    shebang-command "^2.0.0"
-    which "^2.0.1"
-
 crypto-js@^4.2.0:
   version "4.2.0"
   resolved "https://registry.yarnpkg.com/crypto-js/-/crypto-js-4.2.0.tgz#4d931639ecdfd12ff80e8186dba6af2c2e856631"
@@ -2468,30 +2155,6 @@ crypto-random-string@^2.0.0:
   resolved "https://registry.yarnpkg.com/crypto-random-string/-/crypto-random-string-2.0.0.tgz#ef2a7a966ec11083388369baa02ebead229b30d5"
   integrity sha512-v1plID3y9r/lPhviJ1wrXpLeyUIGAZ2SHNYTEapm7/8A9nLPoyvVp3RK/EPFqn5kEznyWgYZNsRtYYIWbuG8KA==
 
-css-loader@^3.6.0:
-  version "3.6.0"
-  resolved "https://registry.yarnpkg.com/css-loader/-/css-loader-3.6.0.tgz#2e4b2c7e6e2d27f8c8f28f61bffcd2e6c91ef645"
-  integrity sha512-M5lSukoWi1If8dhQAUCvj4H8vUt3vOnwbQBH9DdTm/s4Ym2B/3dPMtYZeJmq7Q3S3Pa+I94DcZ7pc9bP14cWIQ==
-  dependencies:
-    camelcase "^5.3.1"
-    cssesc "^3.0.0"
-    icss-utils "^4.1.1"
-    loader-utils "^1.2.3"
-    normalize-path "^3.0.0"
-    postcss "^7.0.32"
-    postcss-modules-extract-imports "^2.0.0"
-    postcss-modules-local-by-default "^3.0.2"
-    postcss-modules-scope "^2.2.0"
-    postcss-modules-values "^3.0.0"
-    postcss-value-parser "^4.1.0"
-    schema-utils "^2.7.0"
-    semver "^6.3.0"
-
-cssesc@^3.0.0:
-  version "3.0.0"
-  resolved "https://registry.yarnpkg.com/cssesc/-/cssesc-3.0.0.tgz#37741919903b868565e1c09ea747445cd18983ee"
-  integrity sha512-/Tb/JcjK111nNScGob5MNtsntNM1aCNUDipB/TkwZFhyDrrE47SOx/18wF2bbjgc3ZzCSKW1T5nt5EbFoAz/Vg==
-
 dashdash@^1.12.0:
   version "1.14.1"
   resolved "https://registry.yarnpkg.com/dashdash/-/dashdash-1.14.1.tgz#853cfa0f7cbe2fed5de20326b8dd581035f6e2f0"
@@ -2542,7 +2205,7 @@ debug@^3.1.0, debug@^3.2.7:
   dependencies:
     ms "^2.1.1"
 
-debug@^4.1.1, debug@^4.3.1, debug@^4.3.5:
+debug@^4.3.5:
   version "4.3.6"
   resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.6.tgz#2ab2c38fbaffebf8aa95fdfe6d88438c7a13c52b"
   integrity sha512-O/09Bd4Z1fBrU4VzkhFqVgpPzaGbw6Sm9FEkBT1A/YBXQFGuuSxa1dN2nxgxS34JmKXqYx8CZAwEVoJFImUXIg==
@@ -2561,19 +2224,64 @@ decompress-response@^3.3.0:
   dependencies:
     mimic-response "^1.0.0"
 
+decompress-tar@^4.0.0, decompress-tar@^4.1.0, decompress-tar@^4.1.1:
+  version "4.1.1"
+  resolved "https://registry.yarnpkg.com/decompress-tar/-/decompress-tar-4.1.1.tgz#718cbd3fcb16209716e70a26b84e7ba4592e5af1"
+  integrity sha512-JdJMaCrGpB5fESVyxwpCx4Jdj2AagLmv3y58Qy4GE6HMVjWz1FeVQk1Ct4Kye7PftcdOo/7U7UKzYBJgqnGeUQ==
+  dependencies:
+    file-type "^5.2.0"
+    is-stream "^1.1.0"
+    tar-stream "^1.5.2"
+
+decompress-tarbz2@^4.0.0:
+  version "4.1.1"
+  resolved "https://registry.yarnpkg.com/decompress-tarbz2/-/decompress-tarbz2-4.1.1.tgz#3082a5b880ea4043816349f378b56c516be1a39b"
+  integrity sha512-s88xLzf1r81ICXLAVQVzaN6ZmX4A6U4z2nMbOwobxkLoIIfjVMBg7TeguTUXkKeXni795B6y5rnvDw7rxhAq9A==
+  dependencies:
+    decompress-tar "^4.1.0"
+    file-type "^6.1.0"
+    is-stream "^1.1.0"
+    seek-bzip "^1.0.5"
+    unbzip2-stream "^1.0.9"
+
+decompress-targz@^4.0.0:
+  version "4.1.1"
+  resolved "https://registry.yarnpkg.com/decompress-targz/-/decompress-targz-4.1.1.tgz#c09bc35c4d11f3de09f2d2da53e9de23e7ce1eee"
+  integrity sha512-4z81Znfr6chWnRDNfFNqLwPvm4db3WuZkqV+UgXQzSngG3CEKdBkw5jrv3axjjL96glyiiKjsxJG3X6WBZwX3w==
+  dependencies:
+    decompress-tar "^4.1.1"
+    file-type "^5.2.0"
+    is-stream "^1.1.0"
+
+decompress-unzip@^4.0.1:
+  version "4.0.1"
+  resolved "https://registry.yarnpkg.com/decompress-unzip/-/decompress-unzip-4.0.1.tgz#deaaccdfd14aeaf85578f733ae8210f9b4848f69"
+  integrity sha512-1fqeluvxgnn86MOh66u8FjbtJpAFv5wgCT9Iw8rcBqQcCo5tO8eiJw7NNTrvt9n4CRBVq7CstiS922oPgyGLrw==
+  dependencies:
+    file-type "^3.8.0"
+    get-stream "^2.2.0"
+    pify "^2.3.0"
+    yauzl "^2.4.2"
+
+decompress@^4.2.0:
+  version "4.2.1"
+  resolved "https://registry.yarnpkg.com/decompress/-/decompress-4.2.1.tgz#007f55cc6a62c055afa37c07eb6a4ee1b773f118"
+  integrity sha512-e48kc2IjU+2Zw8cTb6VZcJQ3lgVbS4uuB1TfCHbiZIP/haNXm+SVyhu+87jts5/3ROpd82GSVCoNs/z8l4ZOaQ==
+  dependencies:
+    decompress-tar "^4.0.0"
+    decompress-tarbz2 "^4.0.0"
+    decompress-targz "^4.0.0"
+    decompress-unzip "^4.0.1"
+    graceful-fs "^4.1.10"
+    make-dir "^1.0.0"
+    pify "^2.3.0"
+    strip-dirs "^2.0.0"
+
 deep-extend@^0.6.0, deep-extend@~0.6.0:
   version "0.6.0"
   resolved "https://registry.yarnpkg.com/deep-extend/-/deep-extend-0.6.0.tgz#c4fa7c95404a17a9c3e8ca7e1537312b736330ac"
   integrity sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==
 
-deep-rename-keys@^0.2.1:
-  version "0.2.1"
-  resolved "https://registry.yarnpkg.com/deep-rename-keys/-/deep-rename-keys-0.2.1.tgz#ede78537d7a66a2be61517e2af956d7f58a3f1d8"
-  integrity sha512-RHd9ABw4Fvk+gYDWqwOftG849x0bYOySl/RgX0tLI9i27ZIeSO91mLZJEp7oPHOMFqHvpgu21YptmDt0FYD/0A==
-  dependencies:
-    kind-of "^3.0.2"
-    rename-keys "^1.1.2"
-
 deepcopy@^2.1.0:
   version "2.1.0"
   resolved "https://registry.yarnpkg.com/deepcopy/-/deepcopy-2.1.0.tgz#2deb0dd52d079c2ecb7924b640a7c3abd4db1d6d"
@@ -2581,7 +2289,7 @@ deepcopy@^2.1.0:
   dependencies:
     type-detect "^4.0.8"
 
-deepmerge@^4.2.2, deepmerge@^4.3.1:
+deepmerge@^4.3.1:
   version "4.3.1"
   resolved "https://registry.yarnpkg.com/deepmerge/-/deepmerge-4.3.1.tgz#44b5f2147cd3b00d4b56137685966f26fd25dd4a"
   integrity sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A==
@@ -2798,11 +2506,6 @@ duplexify@^3.5.0, duplexify@^3.6.0:
     readable-stream "^2.0.0"
     stream-shift "^1.0.0"
 
-eastasianwidth@^0.2.0:
-  version "0.2.0"
-  resolved "https://registry.yarnpkg.com/eastasianwidth/-/eastasianwidth-0.2.0.tgz#696ce2ec0aa0e6ea93a397ffcf24aa7840c827cb"
-  integrity sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==
-
 ecc-jsbn@~0.1.1:
   version "0.1.2"
   resolved "https://registry.yarnpkg.com/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz#3a83a904e54353287874c564b7549386849a98c9"
@@ -2818,31 +2521,11 @@ ecdsa-sig-formatter@1.0.11, ecdsa-sig-formatter@^1.0.11:
   dependencies:
     safe-buffer "^5.0.1"
 
-editorconfig@^1.0.4:
-  version "1.0.4"
-  resolved "https://registry.yarnpkg.com/editorconfig/-/editorconfig-1.0.4.tgz#040c9a8e9a6c5288388b87c2db07028aa89f53a3"
-  integrity sha512-L9Qe08KWTlqYMVvMcTIvMAdl1cDUubzRNYL+WfA4bLDMHe4nemKkpmYzkznE1FwLKu0EEmy6obgQKzMJrg4x9Q==
-  dependencies:
-    "@one-ini/wasm" "0.1.1"
-    commander "^10.0.0"
-    minimatch "9.0.1"
-    semver "^7.5.3"
-
 ee-first@1.1.1:
   version "1.1.1"
   resolved "https://registry.yarnpkg.com/ee-first/-/ee-first-1.1.1.tgz#590c61156b0ae2f4f0255732a158b266bc56b21d"
   integrity sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=
 
-electron-to-chromium@^1.5.4:
-  version "1.5.11"
-  resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.5.11.tgz#258077f1077a1c72f2925cd5b326c470a7f5adef"
-  integrity sha512-R1CccCDYqndR25CaXFd6hp/u9RaaMcftMkphmvuepXr5b1vfLkRml6aWVeBhXJ7rbevHkKEMJtz8XqPf7ffmew==
-
-element-to-path@^1.2.0:
-  version "1.2.1"
-  resolved "https://registry.yarnpkg.com/element-to-path/-/element-to-path-1.2.1.tgz#06afb439a50fa2870d11f846a9e876fe60c7e72c"
-  integrity sha512-JNFZS0yI3Myywn/ltFj/yTihHNzMTYk0ycHcgcjlvA/dYMUjMIGqvbezPZeXN3U1Klp/aiigr2mpmhVRfudtbg==
-
 elliptic@6.5.4:
   version "6.5.4"
   resolved "https://registry.yarnpkg.com/elliptic/-/elliptic-6.5.4.tgz#da37cebd31e79a1367e941b592ed1fbebd58abbb"
@@ -2874,16 +2557,6 @@ emoji-regex@^8.0.0:
   resolved "https://registry.yarnpkg.com/emoji-regex/-/emoji-regex-8.0.0.tgz#e818fd69ce5ccfcb404594f842963bf53164cc37"
   integrity sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==
 
-emoji-regex@^9.2.2:
-  version "9.2.2"
-  resolved "https://registry.yarnpkg.com/emoji-regex/-/emoji-regex-9.2.2.tgz#840c8803b0d8047f4ff0cf963176b32d4ef3ed72"
-  integrity sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==
-
-emojis-list@^3.0.0:
-  version "3.0.0"
-  resolved "https://registry.yarnpkg.com/emojis-list/-/emojis-list-3.0.0.tgz#5570662046ad29e2e916e71aae260abdff4f6a78"
-  integrity sha512-/kyM18EfinwXZbno9FyUGeFh87KC8HRQBQGildHZbEuRyWFOmv1U10o9BBp8XVZDVNNuQKyIGIu5ZYAAXJ0V2Q==
-
 encodeurl@~1.0.2:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/encodeurl/-/encodeurl-1.0.2.tgz#ad3ff4c86ec2d029322f5a02c3a9a606c95b3f59"
@@ -2952,11 +2625,6 @@ escalade@^3.1.1:
   resolved "https://registry.yarnpkg.com/escalade/-/escalade-3.1.1.tgz#d8cfdc7000965c5a0174b4a82eaa5c0552742e40"
   integrity sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==
 
-escalade@^3.1.2:
-  version "3.1.2"
-  resolved "https://registry.yarnpkg.com/escalade/-/escalade-3.1.2.tgz#54076e9ab29ea5bf3d8f1ed62acffbb88272df27"
-  integrity sha512-ErCHMCae19vR8vQGe50xIsVomy19rg6gFu3+r3jkEO46suLMWBksvVyoGgQV+jOfl84ZSOSlmv6Gxa89PmTGmA==
-
 escape-goat@^2.0.0:
   version "2.1.1"
   resolved "https://registry.yarnpkg.com/escape-goat/-/escape-goat-2.1.1.tgz#1b2dc77003676c457ec760b2dc68edb648188675"
@@ -3036,11 +2704,6 @@ event-target-shim@^5.0.0:
   resolved "https://registry.yarnpkg.com/event-target-shim/-/event-target-shim-5.0.1.tgz#5d4d3ebdf9583d63a5333ce2deb7480ab2b05789"
   integrity sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==
 
-eventemitter3@^2.0.0:
-  version "2.0.3"
-  resolved "https://registry.yarnpkg.com/eventemitter3/-/eventemitter3-2.0.3.tgz#b5e1079b59fb5e1ba2771c0a993be060a58c99ba"
-  integrity sha512-jLN68Dx5kyFHaePoXWPsCGW5qdyZQtLYHkxkg02/Mz6g0kYpDx4FyP6XfArhQdlOC4b8Mv+EMxPo/8La7Tzghg==
-
 events@1.1.1:
   version "1.1.1"
   resolved "https://registry.yarnpkg.com/events/-/events-1.1.1.tgz#9ebdb7635ad099c70dcc4c2a1f5004288e8bd924"
@@ -3144,6 +2807,13 @@ fb@^2.0.0:
     debug "^2.6.3"
     request "^2.81.0"
 
+fd-slicer@~1.1.0:
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/fd-slicer/-/fd-slicer-1.1.0.tgz#25c7c89cb1f9077f8891bbe61d8f390eae256f1e"
+  integrity sha512-cE1qsB/VwyQozZ+q1dGxR8LBYNZeofhEdUNGSMbQD3Gw2lAzX9Zb3uIU6Ebc/Fmyjo9AWWfnn0AUCHqtevs/8g==
+  dependencies:
+    pend "~1.2.0"
+
 fetch-cookie@0.11.0:
   version "0.11.0"
   resolved "https://registry.yarnpkg.com/fetch-cookie/-/fetch-cookie-0.11.0.tgz#e046d2abadd0ded5804ce7e2cae06d4331c15407"
@@ -3151,6 +2821,15 @@ fetch-cookie@0.11.0:
   dependencies:
     tough-cookie "^2.3.3 || ^3.0.1 || ^4.0.0"
 
+file-type@^16.5.4:
+  version "16.5.4"
+  resolved "https://registry.yarnpkg.com/file-type/-/file-type-16.5.4.tgz#474fb4f704bee427681f98dd390058a172a6c2fd"
+  integrity sha512-/yFHK0aGjFEgDJjEKP0pWCplsPFPhwyfwevf/pVxiN0tmE4L9LmwWxWukdJSHdoCli4VgQLehjJtwQBnqmsKcw==
+  dependencies:
+    readable-web-to-node-stream "^3.0.0"
+    strtok3 "^6.2.4"
+    token-types "^4.1.1"
+
 file-type@^18.2.1:
   version "18.2.1"
   resolved "https://registry.yarnpkg.com/file-type/-/file-type-18.2.1.tgz#6d8f1fa3b079606f6ecf89483346f55fcd2c671b"
@@ -3160,6 +2839,21 @@ file-type@^18.2.1:
     strtok3 "^7.0.0"
     token-types "^5.0.1"
 
+file-type@^3.8.0:
+  version "3.9.0"
+  resolved "https://registry.yarnpkg.com/file-type/-/file-type-3.9.0.tgz#257a078384d1db8087bc449d107d52a52672b9e9"
+  integrity sha512-RLoqTXE8/vPmMuTI88DAzhMYC99I8BWv7zYP4A1puo5HIjEJ5EX48ighy4ZyKMG9EDXxBgW6e++cn7d1xuFghA==
+
+file-type@^5.2.0:
+  version "5.2.0"
+  resolved "https://registry.yarnpkg.com/file-type/-/file-type-5.2.0.tgz#2ddbea7c73ffe36368dfae49dc338c058c2b8ad6"
+  integrity sha512-Iq1nJ6D2+yIO4c8HHg4fyVb8mAJieo1Oloy1mLLaB2PvezNedhBVm+QU7g0qM42aiMbRXTxKKwGD17rjKNJYVQ==
+
+file-type@^6.1.0:
+  version "6.2.0"
+  resolved "https://registry.yarnpkg.com/file-type/-/file-type-6.2.0.tgz#e50cd75d356ffed4e306dc4f5bcf52a79903a919"
+  integrity sha512-YPcTBDV+2Tm0VqjybVd32MHdlEGAtuxS3VAYsumFokDSMG+ROT5wawGlnHDoz7bfMcMDt9hxuXvXwoKUx2fkOg==
+
 fill-range@^7.0.1:
   version "7.0.1"
   resolved "https://registry.yarnpkg.com/fill-range/-/fill-range-7.0.1.tgz#1919a6a7c75fe38b2c7c77e5198535da9acdda40"
@@ -3215,14 +2909,6 @@ follow-redirects@^1.14.4, follow-redirects@^1.14.9, follow-redirects@^1.15.0, fo
   resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.6.tgz#7f815c0cda4249c74ff09e95ef97c23b5fd0399b"
   integrity sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA==
 
-foreground-child@^3.1.0:
-  version "3.3.0"
-  resolved "https://registry.yarnpkg.com/foreground-child/-/foreground-child-3.3.0.tgz#0ac8644c06e431439f8561db8ecf29a7b5519c77"
-  integrity sha512-Ld2g8rrAyMYFXBhEqMz8ZAHBi4J4uS1i/CxGMDnjyFWddMXLVcDp051DZfu+t7+ab7Wv6SMqpWmyFIj5UbfFvg==
-  dependencies:
-    cross-spawn "^7.0.0"
-    signal-exit "^4.0.1"
-
 forever-agent@~0.6.1:
   version "0.6.1"
   resolved "https://registry.yarnpkg.com/forever-agent/-/forever-agent-0.6.1.tgz#fbc71f0c41adeb37f96c577ad1ed42d8fdacca91"
@@ -3297,15 +2983,6 @@ fs-extra@^6.0.1:
     jsonfile "^4.0.0"
     universalify "^0.1.0"
 
-fs-extra@^8.1.0:
-  version "8.1.0"
-  resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-8.1.0.tgz#49d43c45a88cd9677668cb7be1b46efdb8d2e1c0"
-  integrity sha512-yhlQgA6mnOJUKOsRUFsgJdQCvkKhcz8tlZG5HBQfReYZy46OwLcY+Zia0mtdHsOo9y/hP+CxMN0TU9QxoOtG4g==
-  dependencies:
-    graceful-fs "^4.2.0"
-    jsonfile "^4.0.0"
-    universalify "^0.1.0"
-
 fs.realpath@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f"
@@ -3388,6 +3065,14 @@ get-stdin@^5.0.1:
   resolved "https://registry.yarnpkg.com/get-stdin/-/get-stdin-5.0.1.tgz#122e161591e21ff4c52530305693f20e6393a398"
   integrity sha512-jZV7n6jGE3Gt7fgSTJoz91Ak5MuTLwMwkoYdjxuJ/AmjIsE1UC03y/IWkZCQGEvVNS9qoRNwy5BCqxImv0FVeA==
 
+get-stream@^2.2.0:
+  version "2.3.1"
+  resolved "https://registry.yarnpkg.com/get-stream/-/get-stream-2.3.1.tgz#5f38f93f346009666ee0150a054167f91bdd95de"
+  integrity sha512-AUGhbbemXxrZJRD5cDvKtQxLuYaIbNtDTK8YqupCI393Q2KSTreEsLUN3ZxAWFGiKTzL6nKuzfcIvieflUX9qA==
+  dependencies:
+    object-assign "^4.0.1"
+    pinkie-promise "^2.0.0"
+
 get-stream@^4.1.0:
   version "4.1.0"
   resolved "https://registry.yarnpkg.com/get-stream/-/get-stream-4.1.0.tgz#c1b255575f3dc21d59bfc79cd3d2b46b1c3a54b5"
@@ -3402,11 +3087,6 @@ get-stream@^5.1.0:
   dependencies:
     pump "^3.0.0"
 
-get-value@^2.0.3:
-  version "2.0.6"
-  resolved "https://registry.yarnpkg.com/get-value/-/get-value-2.0.6.tgz#dc15ca1c672387ca76bd37ac0a395ba2042a2c28"
-  integrity sha512-Ln0UQDlxH1BapMu3GPtf7CuYNwRZf2gwCuPqbyG6pB8WfmFpzqcy4xtAaAMUhnNqjMKTiCPZG2oMT3YSx8U2NA==
-
 getpass@^0.1.1:
   version "0.1.7"
   resolved "https://registry.yarnpkg.com/getpass/-/getpass-0.1.7.tgz#5eff8e3e684d569ae4cb2b1282604e8ba62149fa"
@@ -3433,19 +3113,7 @@ glob@7.2.0:
     once "^1.3.0"
     path-is-absolute "^1.0.0"
 
-glob@^10.3.3:
-  version "10.4.5"
-  resolved "https://registry.yarnpkg.com/glob/-/glob-10.4.5.tgz#f4d9f0b90ffdbab09c9d77f5f29b4262517b0956"
-  integrity sha512-7Bv8RF0k6xjo7d4A/PxYLbUCfb6c+Vpd2/mB2yRDlew7Jb5hEXiCD9ibfO7wpk8i4sevK6DFny9h7EYbM3/sHg==
-  dependencies:
-    foreground-child "^3.1.0"
-    jackspeak "^3.1.2"
-    minimatch "^9.0.4"
-    minipass "^7.1.2"
-    package-json-from-dist "^1.0.0"
-    path-scurry "^1.11.1"
-
-glob@^7.0.5, glob@^7.1.2, glob@^7.1.4, glob@^7.1.6:
+glob@^7.0.5, glob@^7.1.2, glob@^7.1.3, glob@^7.1.4:
   version "7.2.3"
   resolved "https://registry.yarnpkg.com/glob/-/glob-7.2.3.tgz#b8df0fb802bbfa8e89bd1d938b4e16578ed44f2b"
   integrity sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==
@@ -3464,11 +3132,6 @@ global-dirs@^3.0.0:
   dependencies:
     ini "2.0.0"
 
-globals@^11.1.0:
-  version "11.12.0"
-  resolved "https://registry.yarnpkg.com/globals/-/globals-11.12.0.tgz#ab8795338868a0babd8525758018c2a7eb95c42e"
-  integrity sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA==
-
 google-auth-library@^9.0.0, google-auth-library@^9.7.0:
   version "9.11.0"
   resolved "https://registry.yarnpkg.com/google-auth-library/-/google-auth-library-9.11.0.tgz#bd6da364bcde4e0cc4ed70a0e0df5112b6a671dd"
@@ -3525,6 +3188,11 @@ got@^9.6.0:
     to-readable-stream "^1.0.0"
     url-parse-lax "^3.0.0"
 
+graceful-fs@^4.1.10:
+  version "4.2.11"
+  resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.11.tgz#4183e4e8bf08bb6e05bbb2f7d2e0c8f712ca40e3"
+  integrity sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==
+
 graceful-fs@^4.1.2, graceful-fs@^4.1.6, graceful-fs@^4.2.0:
   version "4.2.10"
   resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.10.tgz#147d3a006da4ca3ce14728c7aefc287c367d7a6c"
@@ -3609,20 +3277,6 @@ has-symbols@^1.0.1, has-symbols@^1.0.3:
   resolved "https://registry.yarnpkg.com/has-symbols/-/has-symbols-1.0.3.tgz#bb7b2c4349251dce87b125f7bdf874aa7c8b39f8"
   integrity sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==
 
-has-value@^0.3.1:
-  version "0.3.1"
-  resolved "https://registry.yarnpkg.com/has-value/-/has-value-0.3.1.tgz#7b1f58bada62ca827ec0a2078025654845995e1f"
-  integrity sha512-gpG936j8/MzaeID5Yif+577c17TxaDmhuyVgSwtnL/q8UUTySg8Mecb+8Cf1otgLoD7DDH75axp86ER7LFsf3Q==
-  dependencies:
-    get-value "^2.0.3"
-    has-values "^0.1.4"
-    isobject "^2.0.0"
-
-has-values@^0.1.4:
-  version "0.1.4"
-  resolved "https://registry.yarnpkg.com/has-values/-/has-values-0.1.4.tgz#6d61de95d91dfca9b9a02089ad384bff8f62b771"
-  integrity sha512-J8S0cEdWuQbqD9//tlZxiMuMNmxB8PlEwvYwuxsTmR1G5RXUePEX/SJn7aD0GMLieuZYSwNH0cQuJGwnYunXRQ==
-
 has-yarn@^2.1.0:
   version "2.1.0"
   resolved "https://registry.yarnpkg.com/has-yarn/-/has-yarn-2.1.0.tgz#137e11354a7b5bf11aa5cb649cf0c6f3ff2b2e77"
@@ -3652,7 +3306,7 @@ hash.js@1.1.7, hash.js@^1.0.0, hash.js@^1.0.3:
     inherits "^2.0.3"
     minimalistic-assert "^1.0.1"
 
-hasown@^2.0.0, hasown@^2.0.2:
+hasown@^2.0.0:
   version "2.0.2"
   resolved "https://registry.yarnpkg.com/hasown/-/hasown-2.0.2.tgz#003eaf91be7adc372e84ec59dc37252cedb80003"
   integrity sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==
@@ -3756,13 +3410,6 @@ iconv-lite@0.6.3:
   dependencies:
     safer-buffer ">= 2.1.2 < 3.0.0"
 
-icss-utils@^4.0.0, icss-utils@^4.1.1:
-  version "4.1.1"
-  resolved "https://registry.yarnpkg.com/icss-utils/-/icss-utils-4.1.1.tgz#21170b53789ee27447c2f47dd683081403f9a467"
-  integrity sha512-4aFq7wvWyMHKgxsH8QQtGpvbASCf+eM3wPRLI6R+MgAnTCZ6STYsRvttLvRWK0Nfif5piF394St3HeJDaljGPA==
-  dependencies:
-    postcss "^7.0.14"
-
 ieee754@1.1.13:
   version "1.1.13"
   resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.1.13.tgz#ec168558e95aa181fd87d37f55c32bbcb6708b84"
@@ -3816,7 +3463,7 @@ ini@2.0.0:
   resolved "https://registry.yarnpkg.com/ini/-/ini-2.0.0.tgz#e5fd556ecdd5726be978fa1001862eacb0a94bc5"
   integrity sha512-7PnF4oN3CvZF23ADhA5wRaYEQpJ8qygSkbtTXWBeXWXmEVRXK+1ITciHWwHhsjv1TmW0MgacIv6hEi5pX5NQdA==
 
-ini@^1.3.4, ini@~1.3.0:
+ini@~1.3.0:
   version "1.3.8"
   resolved "https://registry.yarnpkg.com/ini/-/ini-1.3.8.tgz#a29da425b48806f34767a4efce397269af28432c"
   integrity sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==
@@ -3833,11 +3480,6 @@ is-binary-path@~2.1.0:
   dependencies:
     binary-extensions "^2.0.0"
 
-is-buffer@^1.1.5:
-  version "1.1.6"
-  resolved "https://registry.yarnpkg.com/is-buffer/-/is-buffer-1.1.6.tgz#efaa2ea9daa0d7ab2ea13a97b2b8ad51fefbe8be"
-  integrity sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==
-
 is-ci@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/is-ci/-/is-ci-2.0.0.tgz#6bc6334181810e04b5c22b3d589fdca55026404c"
@@ -3845,13 +3487,6 @@ is-ci@^2.0.0:
   dependencies:
     ci-info "^2.0.0"
 
-is-core-module@^2.13.0:
-  version "2.15.0"
-  resolved "https://registry.yarnpkg.com/is-core-module/-/is-core-module-2.15.0.tgz#71c72ec5442ace7e76b306e9d48db361f22699ea"
-  integrity sha512-Dd+Lb2/zvk9SKy1TGCt1wFJFo/MWBPMX5x7KcvLajWTGuomczdQX61PvY5yK6SVACwpoexWo81IfFyoKY2QnTA==
-  dependencies:
-    hasown "^2.0.2"
-
 is-deflate@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/is-deflate/-/is-deflate-1.0.0.tgz#c862901c3c161fb09dac7cdc7e784f80e98f2f14"
@@ -3892,6 +3527,11 @@ is-installed-globally@^0.4.0:
     global-dirs "^3.0.0"
     is-path-inside "^3.0.2"
 
+is-natural-number@^4.0.1:
+  version "4.0.1"
+  resolved "https://registry.yarnpkg.com/is-natural-number/-/is-natural-number-4.0.1.tgz#ab9d76e1db4ced51e35de0c72ebecf09f734cde8"
+  integrity sha512-Y4LTamMe0DDQIIAlaer9eKebAlDSV6huy+TWhJVPlzZh2o4tRP5SQWFlLn5N0To4mDD22/qdOq+veo1cSISLgQ==
+
 is-npm@^5.0.0:
   version "5.0.0"
   resolved "https://registry.yarnpkg.com/is-npm/-/is-npm-5.0.0.tgz#43e8d65cc56e1b67f8d47262cf667099193f45a8"
@@ -3917,28 +3557,21 @@ is-plain-obj@^2.1.0:
   resolved "https://registry.yarnpkg.com/is-plain-obj/-/is-plain-obj-2.1.0.tgz#45e42e37fccf1f40da8e5f76ee21515840c09287"
   integrity sha512-YWnfyRwxL/+SsrWYfOpUtz5b3YD+nyfkHvjbcanzk8zgyO4ASD67uVMRt8k5bM4lLMDnXfriRhOpemw+NfT1eA==
 
-is-plain-object@^2.0.1:
-  version "2.0.4"
-  resolved "https://registry.yarnpkg.com/is-plain-object/-/is-plain-object-2.0.4.tgz#2c163b3fafb1b606d9d17928f05c2a1c38e07677"
-  integrity sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og==
-  dependencies:
-    isobject "^3.0.1"
-
 is-redirect@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/is-redirect/-/is-redirect-1.0.0.tgz#1d03dded53bd8db0f30c26e4f95d36fc7c87dc24"
   integrity sha1-HQPd7VO9jbDzDCbk+V02/HyH3CQ=
 
+is-stream@^1.1.0:
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/is-stream/-/is-stream-1.1.0.tgz#12d4a3dd4e68e0b79ceb8dbc84173ae80d91ca44"
+  integrity sha512-uQPm8kcs47jx38atAcWTVxyltQYoPT68y9aWYdV6yWXSyW8mzSat0TL6CiWdZeCdF3KrAvpVtnHbTv4RN+rqdQ==
+
 is-stream@^2.0.0:
   version "2.0.1"
   resolved "https://registry.yarnpkg.com/is-stream/-/is-stream-2.0.1.tgz#fac1e3d53b97ad5a9d0ae9cef2389f5810a5c077"
   integrity sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==
 
-is-svg-path@^1.0.1:
-  version "1.0.2"
-  resolved "https://registry.yarnpkg.com/is-svg-path/-/is-svg-path-1.0.2.tgz#77ab590c12b3d20348e5c7a13d0040c87784dda0"
-  integrity sha512-Lj4vePmqpPR1ZnRctHv8ltSh1OrSxHkhUkd7wi+VQdcdP15/KvQFyk7LhNuM7ZW0EVbJz8kZLVmL9quLrfq4Kg==
-
 is-typedarray@^1.0.0, is-typedarray@~1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/is-typedarray/-/is-typedarray-1.0.0.tgz#e479c80858df0c1b11ddda6940f96011fcda4a9a"
@@ -3966,7 +3599,7 @@ isarray@0.0.1:
   resolved "https://registry.yarnpkg.com/isarray/-/isarray-0.0.1.tgz#8a18acfca9a8f4177e09abfc6038939b05d1eedf"
   integrity sha512-D2S+3GLxWH+uhrNEcoh/fnmYeP8E8/zHl644d/jdA0g2uyXvy3sb0qxotE+ne0LtccHknQzWwZEzhak7oJ0COQ==
 
-isarray@1.0.0, isarray@^1.0.0, isarray@~1.0.0:
+isarray@^1.0.0, isarray@~1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11"
   integrity sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=
@@ -3976,68 +3609,21 @@ isexe@^2.0.0:
   resolved "https://registry.yarnpkg.com/isexe/-/isexe-2.0.0.tgz#e8fbf374dc556ff8947a10dcb0572d633f2cfa10"
   integrity sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=
 
-isobject@^2.0.0:
-  version "2.1.0"
-  resolved "https://registry.yarnpkg.com/isobject/-/isobject-2.1.0.tgz#f065561096a3f1da2ef46272f815c840d87e0c89"
-  integrity sha512-+OUdGJlgjOBZDfxnDjYYG6zp487z0JGNQq3cYQYg5f5hKR+syHMsaztzGeml/4kGG55CSpKSpWTY+jYGgsHLgA==
-  dependencies:
-    isarray "1.0.0"
-
-isobject@^3.0.0, isobject@^3.0.1:
-  version "3.0.1"
-  resolved "https://registry.yarnpkg.com/isobject/-/isobject-3.0.1.tgz#4e431e92b11a9731636aa1f9c8d1ccbcfdab78df"
-  integrity sha512-WhB9zCku7EGTj/HQQRz5aUQEUeoQZH2bWcltRErOpymJ4boYE6wL9Tbr23krRPSZ+C5zqNSrSw+Cc7sZZ4b7vg==
-
 isstream@~0.1.2:
   version "0.1.2"
   resolved "https://registry.yarnpkg.com/isstream/-/isstream-0.1.2.tgz#47e63f7af55afa6f92e1500e690eb8b8529c099a"
   integrity sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=
 
-jackspeak@^3.1.2:
-  version "3.4.3"
-  resolved "https://registry.yarnpkg.com/jackspeak/-/jackspeak-3.4.3.tgz#8833a9d89ab4acde6188942bd1c53b6390ed5a8a"
-  integrity sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw==
-  dependencies:
-    "@isaacs/cliui" "^8.0.2"
-  optionalDependencies:
-    "@pkgjs/parseargs" "^0.11.0"
-
 jmespath@0.16.0:
   version "0.16.0"
   resolved "https://registry.yarnpkg.com/jmespath/-/jmespath-0.16.0.tgz#b15b0a85dfd4d930d43e69ed605943c802785076"
   integrity sha512-9FzQjJ7MATs1tSpnco1K6ayiYE3figslrXA72G2HQ/n76RzvYlofyi5QM+iX4YRs/pu3yzxlVQSST23+dMDknw==
 
-jquery@^3.5.1:
-  version "3.7.1"
-  resolved "https://registry.yarnpkg.com/jquery/-/jquery-3.7.1.tgz#083ef98927c9a6a74d05a6af02806566d16274de"
-  integrity sha512-m4avr8yL8kmFN8psrbFFFmB/If14iN5o9nw/NgnnM+kybDJpRsAynV2BsfpTYrTRysYUdADVD7CkUUizgkpLfg==
-
-js-beautify@^1.13.0:
-  version "1.15.1"
-  resolved "https://registry.yarnpkg.com/js-beautify/-/js-beautify-1.15.1.tgz#4695afb508c324e1084ee0b952a102023fc65b64"
-  integrity sha512-ESjNzSlt/sWE8sciZH8kBF8BPlwXPwhR6pWKAw8bw4Bwj+iZcnKW6ONWUutJ7eObuBZQpiIb8S7OYspWrKt7rA==
-  dependencies:
-    config-chain "^1.1.13"
-    editorconfig "^1.0.4"
-    glob "^10.3.3"
-    js-cookie "^3.0.5"
-    nopt "^7.2.0"
-
-js-cookie@^3.0.5:
-  version "3.0.5"
-  resolved "https://registry.yarnpkg.com/js-cookie/-/js-cookie-3.0.5.tgz#0b7e2fd0c01552c58ba86e0841f94dc2557dcdbc"
-  integrity sha512-cEiJEAEoIbWfCZYKWhVwFuvPX1gETRYPw6LlaTKoxD3s2AkXzkCjnp6h0V77ozyqj0jakteJ4YqDJT830+lVGw==
-
 js-sha3@0.8.0, js-sha3@^0.8.0:
   version "0.8.0"
   resolved "https://registry.yarnpkg.com/js-sha3/-/js-sha3-0.8.0.tgz#b9b7a5da73afad7dedd0f8c463954cbde6818840"
   integrity sha512-gF1cRrHhIzNfToc802P800N8PpXS+evLLXfsVpowqmAFR9uwbi89WvXg2QspOmXL8QL86J4T1EpFu+yUkwJY3Q==
 
-js-tokens@^4.0.0:
-  version "4.0.0"
-  resolved "https://registry.yarnpkg.com/js-tokens/-/js-tokens-4.0.0.tgz#19203fb59991df98e3a287050d4647cdeaf32499"
-  integrity sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==
-
 js-yaml@4.1.0, js-yaml@^4.1.0:
   version "4.1.0"
   resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-4.1.0.tgz#c1fb65f8f5017901cdd2c951864ba18458a10602"
@@ -4050,11 +3636,6 @@ jsbn@~0.1.0:
   resolved "https://registry.yarnpkg.com/jsbn/-/jsbn-0.1.1.tgz#a5e654c2e5a2deb5f201d96cefbca80c0ef2f513"
   integrity sha1-peZUwuWi3rXyAdls77yoDA7y9RM=
 
-jsesc@^2.5.1:
-  version "2.5.2"
-  resolved "https://registry.yarnpkg.com/jsesc/-/jsesc-2.5.2.tgz#80564d2e483dacf6e8ef209650a67df3f0c283a4"
-  integrity sha512-OYu7XEzjkCQ3C5Ps3QIZsQfNpqoJyZZA99wd9aWd05NCtC5pWOkShK2mkL6HXQR6/Cy2lbNdPlZBpuQHXE63gA==
-
 json-bigint@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/json-bigint/-/json-bigint-1.0.0.tgz#ae547823ac0cad8398667f8cd9ef4730f5b01ff1"
@@ -4114,11 +3695,6 @@ json5@^1.0.1:
   dependencies:
     minimist "^1.2.0"
 
-json5@^2.1.2:
-  version "2.2.3"
-  resolved "https://registry.yarnpkg.com/json5/-/json5-2.2.3.tgz#78cd6f1a19bdc12b73db5ad0c61efd66c1e29283"
-  integrity sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==
-
 jsonfile@^4.0.0:
   version "4.0.0"
   resolved "https://registry.yarnpkg.com/jsonfile/-/jsonfile-4.0.0.tgz#8771aae0799b64076b76640fca058f9c10e33ecb"
@@ -4145,18 +3721,6 @@ jsprim@^1.2.2:
     json-schema "0.4.0"
     verror "1.10.0"
 
-jszip-utils@^0.1.0:
-  version "0.1.0"
-  resolved "https://registry.yarnpkg.com/jszip-utils/-/jszip-utils-0.1.0.tgz#8c04cdedcdb291e83f055f5b261b3a3188ceca0b"
-  integrity sha512-tBNe0o3HAf8vo0BrOYnLPnXNo5A3KsRMnkBFYjh20Y3GPYGfgyoclEMgvVchx0nnL+mherPi74yLPIusHUQpZg==
-
-jszip@2.6.1:
-  version "2.6.1"
-  resolved "https://registry.yarnpkg.com/jszip/-/jszip-2.6.1.tgz#b88f3a7b2e67a2a048152982c7a3756d9c4828f0"
-  integrity sha512-C4Z++nYQv+CudUkCWUdz+yKVhQiFJjuWSmRJ5Sg3d3/OzcJ6U4ooUYlmE3+rJXrVk89KWQaiJ9mPp/VLQ4D66g==
-  dependencies:
-    pako "~1.0.2"
-
 jszip@^3.7.1:
   version "3.10.1"
   resolved "https://registry.yarnpkg.com/jszip/-/jszip-3.10.1.tgz#34aee70eb18ea1faec2f589208a157d1feb091c2"
@@ -4191,13 +3755,6 @@ keyv@^3.0.0:
   dependencies:
     json-buffer "3.0.0"
 
-kind-of@^3.0.2:
-  version "3.2.2"
-  resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-3.2.2.tgz#31ea21a734bab9bbb0f32466d893aea51e4a3c64"
-  integrity sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==
-  dependencies:
-    is-buffer "^1.1.5"
-
 latest-version@^5.1.0:
   version "5.1.0"
   resolved "https://registry.yarnpkg.com/latest-version/-/latest-version-5.1.0.tgz#119dfe908fe38d15dfa43ecd13fa12ec8832face"
@@ -4360,24 +3917,6 @@ linkify-it@5.0.0:
   dependencies:
     uc.micro "^2.0.0"
 
-loader-utils@^1.2.3:
-  version "1.4.2"
-  resolved "https://registry.yarnpkg.com/loader-utils/-/loader-utils-1.4.2.tgz#29a957f3a63973883eb684f10ffd3d151fec01a3"
-  integrity sha512-I5d00Pd/jwMD2QCduo657+YM/6L3KZu++pmX9VFncxaxvHcru9jx1lBaFft+r4Mt2jK0Yhp41XlRAihzPxHNCg==
-  dependencies:
-    big.js "^5.2.2"
-    emojis-list "^3.0.0"
-    json5 "^1.0.1"
-
-loader-utils@^2.0.0:
-  version "2.0.4"
-  resolved "https://registry.yarnpkg.com/loader-utils/-/loader-utils-2.0.4.tgz#8b5cb38b5c34a9a018ee1fc0e6a066d1dfcc528c"
-  integrity sha512-xXqpXoINfFhgua9xiqD8fPFHgkoq1mmmpE92WlDbm9rNRd/EbRb+Gqf908T2DMfuHjjJlksiK2RbHVOdD/MqSw==
-  dependencies:
-    big.js "^5.2.2"
-    emojis-list "^3.0.0"
-    json5 "^2.1.2"
-
 locate-path@^6.0.0:
   version "6.0.0"
   resolved "https://registry.yarnpkg.com/locate-path/-/locate-path-6.0.0.tgz#55321eb309febbc59c4801d931a72452a681d286"
@@ -4390,11 +3929,6 @@ lodash.camelcase@^4.3.0:
   resolved "https://registry.yarnpkg.com/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz#b28aa6288a2b9fc651035c7711f65ab6190331a6"
   integrity sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==
 
-lodash.debounce@^4.0.8:
-  version "4.0.8"
-  resolved "https://registry.yarnpkg.com/lodash.debounce/-/lodash.debounce-4.0.8.tgz#82d79bff30a67c4005ffd5e2515300ad9ca4d7af"
-  integrity sha512-FT1yDzDYEoYWhnSGnpE/4Kj1fLZkDFyqRb7fNt6FdYOSxlUWAtp42Eh6Wb0rGIv/m9Bgo7x4GhQbm5Ys4SG5ow==
-
 lodash.defaults@^4.2.0:
   version "4.2.0"
   resolved "https://registry.yarnpkg.com/lodash.defaults/-/lodash.defaults-4.2.0.tgz#d09178716ffea4dde9e5fb7b37f6f0802274580c"
@@ -4425,7 +3959,7 @@ lodash.union@^4.6.0:
   resolved "https://registry.yarnpkg.com/lodash.union/-/lodash.union-4.6.0.tgz#48bb5088409f16f1821666641c44dd1aaae3cd88"
   integrity sha1-SLtQiECfFvGCFmZkHETdGqrjzYg=
 
-lodash@^4.14.0, lodash@^4.17.14, lodash@^4.17.19, lodash@^4.17.20, lodash@^4.17.21:
+lodash@^4.14.0, lodash@^4.17.14, lodash@^4.17.21:
   version "4.17.21"
   resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c"
   integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==
@@ -4468,18 +4002,6 @@ lowercase-keys@^2.0.0:
   resolved "https://registry.yarnpkg.com/lowercase-keys/-/lowercase-keys-2.0.0.tgz#2603e78b7b4b0006cbca2fbcc8a3202558ac9479"
   integrity sha512-tqNXrS78oMOE73NMxK4EMLQsQowWf8jKooH9g7xPavRT706R6bkQJ6DY2Te7QukaZsulxa30wQ7bk0pm4XiHmA==
 
-lru-cache@^10.2.0:
-  version "10.4.3"
-  resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-10.4.3.tgz#410fc8a17b70e598013df257c2446b7f3383f119"
-  integrity sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==
-
-lru-cache@^5.1.1:
-  version "5.1.1"
-  resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-5.1.1.tgz#1da27e6710271947695daf6848e847f01d84b920"
-  integrity sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==
-  dependencies:
-    yallist "^3.0.2"
-
 lru-cache@^6.0.0:
   version "6.0.0"
   resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-6.0.0.tgz#6d6fe6570ebd96aaf90fcad1dafa3b2566db3a94"
@@ -4517,6 +4039,13 @@ mailsplit@5.4.0:
     libmime "5.2.0"
     libqp "2.0.1"
 
+make-dir@^1.0.0:
+  version "1.3.0"
+  resolved "https://registry.yarnpkg.com/make-dir/-/make-dir-1.3.0.tgz#79c1033b80515bd6d24ec9933e860ca75ee27f0c"
+  integrity sha512-2w31R7SJtieJJnQtGc7RVL2StM2vGYVfqUOvUDxH6bC6aJTxPxTF0GnIgCyu7tjockiUWAYQRbxa7vKn34s5sQ==
+  dependencies:
+    pify "^3.0.0"
+
 make-dir@^3.0.0:
   version "3.1.0"
   resolved "https://registry.yarnpkg.com/make-dir/-/make-dir-3.1.0.tgz#415e967046b3a7f1d185277d84aa58203726a13f"
@@ -4579,7 +4108,7 @@ mime-db@1.52.0:
   resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.52.0.tgz#bbabcdc02859f4987301c856e3387ce5ec43bf70"
   integrity sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==
 
-mime-types@^2.1.12, mime-types@^2.1.27, mime-types@~2.1.19, mime-types@~2.1.24, mime-types@~2.1.34:
+mime-types@^2.1.12, mime-types@~2.1.19, mime-types@~2.1.24, mime-types@~2.1.34:
   version "2.1.35"
   resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.35.tgz#381a871b62a734450660ae3deee44813f70d959a"
   integrity sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==
@@ -4623,13 +4152,6 @@ minimatch@4.2.1:
   dependencies:
     brace-expansion "^1.1.7"
 
-minimatch@9.0.1:
-  version "9.0.1"
-  resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-9.0.1.tgz#8a555f541cf976c622daf078bb28f29fb927c253"
-  integrity sha512-0jWhJpD/MdhPXwPuiRkCbfYfSKp2qnn2eOc279qI7f+osl/l+prKSrvhg157zSYvx/1nmgn2NqdT6k2Z7zSH9w==
-  dependencies:
-    brace-expansion "^2.0.1"
-
 minimatch@^3.0.3, minimatch@^3.0.4, minimatch@^3.1.1:
   version "3.1.2"
   resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.1.2.tgz#19cd194bfd3e428f049a70817c038d89ab4be35b"
@@ -4637,23 +4159,11 @@ minimatch@^3.0.3, minimatch@^3.0.4, minimatch@^3.1.1:
   dependencies:
     brace-expansion "^1.1.7"
 
-minimatch@^9.0.4:
-  version "9.0.5"
-  resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-9.0.5.tgz#d74f9dd6b57d83d8e98cfb82133b03978bc929e5"
-  integrity sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==
-  dependencies:
-    brace-expansion "^2.0.1"
-
 minimist@^1.2.0, minimist@^1.2.5, minimist@^1.2.6:
   version "1.2.6"
   resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.6.tgz#8637a5b759ea0d6e98702cfb3a9283323c93af44"
   integrity sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==
 
-"minipass@^5.0.0 || ^6.0.2 || ^7.0.0", minipass@^7.1.2:
-  version "7.1.2"
-  resolved "https://registry.yarnpkg.com/minipass/-/minipass-7.1.2.tgz#93a9626ce5e5e66bd4db86849e7515e92340a707"
-  integrity sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==
-
 minisearch@^7.1.0:
   version "7.1.0"
   resolved "https://registry.yarnpkg.com/minisearch/-/minisearch-7.1.0.tgz#f5830e9109b5919ee7b291c29a304f381aa68770"
@@ -4804,11 +4314,6 @@ node-imap@^0.9.6:
     readable-stream "^3.6.0"
     utf7 "^1.0.2"
 
-node-releases@^2.0.18:
-  version "2.0.18"
-  resolved "https://registry.yarnpkg.com/node-releases/-/node-releases-2.0.18.tgz#f010e8d35e2fe8d6b2944f03f70213ecedc4ca3f"
-  integrity sha512-d9VeXT4SJ7ZeOqGX6R5EM022wpL+eWPooLI+5UpWn2jCT1aosUQEhQP214x33Wkwx3JQMvIm+tIoVOdodFS40g==
-
 nodemailer@6.9.13:
   version "6.9.13"
   resolved "https://registry.yarnpkg.com/nodemailer/-/nodemailer-6.9.13.tgz#5b292bf1e92645f4852ca872c56a6ba6c4a3d3d6"
@@ -4838,13 +4343,6 @@ noms@0.0.0:
     inherits "^2.0.1"
     readable-stream "~1.0.31"
 
-nopt@^7.2.0:
-  version "7.2.1"
-  resolved "https://registry.yarnpkg.com/nopt/-/nopt-7.2.1.tgz#1cac0eab9b8e97c9093338446eddd40b2c8ca1e7"
-  integrity sha512-taM24ViiimT/XntxbPyJQzCG+p4EKOpgD3mxFwW38mGjVUrfERQOeY4EDHjdnptttfHuHQXFx+lTP08Q+mLa/w==
-  dependencies:
-    abbrev "^2.0.0"
-
 nopt@~1.0.10:
   version "1.0.10"
   resolved "https://registry.yarnpkg.com/nopt/-/nopt-1.0.10.tgz#6ddd21bd2a31417b92727dd585f8a6f37608ebee"
@@ -4857,13 +4355,6 @@ normalize-path@^3.0.0, normalize-path@~3.0.0:
   resolved "https://registry.yarnpkg.com/normalize-path/-/normalize-path-3.0.0.tgz#0dcd69ff23a1c9b11fd0978316644a0388216a65"
   integrity sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==
 
-normalize-svg-path@^1.0.0:
-  version "1.1.0"
-  resolved "https://registry.yarnpkg.com/normalize-svg-path/-/normalize-svg-path-1.1.0.tgz#0e614eca23c39f0cffe821d6be6cd17e569a766c"
-  integrity sha512-r9KHKG2UUeB5LoTouwDzBy2VxXlHsiM6fyLQvnJa0S5hrhzqElH/CH7TUGhT1fVvIYBIKf3OpY4YJ4CK+iaqHg==
-  dependencies:
-    svg-arc-to-cubic-bezier "^3.0.0"
-
 normalize-url@^4.1.0:
   version "4.5.1"
   resolved "https://registry.yarnpkg.com/normalize-url/-/normalize-url-4.5.1.tgz#0dd90cf1288ee1d1313b87081c9a5932ee48518a"
@@ -4879,7 +4370,7 @@ oauth@0.9.x:
   resolved "https://registry.yarnpkg.com/oauth/-/oauth-0.9.15.tgz#bd1fefaf686c96b75475aed5196412ff60cfb9c1"
   integrity sha1-vR/vr2hslrdUda7VGWQS/2DPucE=
 
-object-assign@^4:
+object-assign@^4, object-assign@^4.0.1:
   version "4.1.1"
   resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.1.tgz#2109adc7965887cfc05cbbd442cac8bfbb360863"
   integrity sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=
@@ -4894,19 +4385,22 @@ object-inspect@^1.9.0:
   resolved "https://registry.yarnpkg.com/object-inspect/-/object-inspect-1.12.1.tgz#28a661153bad7e470e4b01479ef1cb91ce511191"
   integrity sha512-Y/jF6vnvEtOPGiKD1+q+X0CiUYRQtEHp89MLLUJ7TUivtH8Ugn2+3A7Rynqk7BRsAoqeOQWnFnjpDrKSxDgIGA==
 
+officeparser@^4.1.1:
+  version "4.1.1"
+  resolved "https://registry.yarnpkg.com/officeparser/-/officeparser-4.1.1.tgz#5c08eb3158c0f8d6d85f12e1e6a95b54f590a677"
+  integrity sha512-bOh7l6Bt/caeyU9t+9yGdQF2N30j8puR7PhXmSI/NqssHNnfnTLp1ehpBo4KuIMeOvzhr8mvkXHFpR2qhH1uhg==
+  dependencies:
+    "@xmldom/xmldom" "^0.8.10"
+    decompress "^4.2.0"
+    file-type "^16.5.4"
+    node-ensure "^0.0.0"
+    rimraf "^2.6.3"
+
 oh-no-i-insist@^1.1.1:
   version "1.1.1"
   resolved "https://registry.yarnpkg.com/oh-no-i-insist/-/oh-no-i-insist-1.1.1.tgz#af6f12e2d43366839bae45f8c870b976a11eee35"
   integrity sha1-r28S4tQzZoObrkX4yHC5dqEe7jU=
 
-omit-deep@0.3.0:
-  version "0.3.0"
-  resolved "https://registry.yarnpkg.com/omit-deep/-/omit-deep-0.3.0.tgz#21c8af3499bcadd29651a232cbcacbc52445ebec"
-  integrity sha512-Lbl/Ma59sss2b15DpnWnGmECBRL8cRl/PjPbPMVW+Y8zIQzRrwMaI65Oy6HvxyhYeILVKBJb2LWeG81bj5zbMg==
-  dependencies:
-    is-plain-object "^2.0.1"
-    unset-value "^0.1.1"
-
 on-finished@2.4.1:
   version "2.4.1"
   resolved "https://registry.yarnpkg.com/on-finished/-/on-finished-2.4.1.tgz#58c8c44116e54845ad57f14ab10b03533184ac3f"
@@ -4959,11 +4453,6 @@ p-locate@^5.0.0:
   dependencies:
     p-limit "^3.0.2"
 
-package-json-from-dist@^1.0.0:
-  version "1.0.0"
-  resolved "https://registry.yarnpkg.com/package-json-from-dist/-/package-json-from-dist-1.0.0.tgz#e501cd3094b278495eb4258d4c9f6d5ac3019f00"
-  integrity sha512-dATvCeZN/8wQsGywez1mzHtTlP22H8OEfPrVMLNr4/eGa+ijtLn/6M5f0dY8UKNrC2O9UCU6SSoG3qRKnt7STw==
-
 package-json@^6.3.0:
   version "6.5.0"
   resolved "https://registry.yarnpkg.com/package-json/-/package-json-6.5.0.tgz#6feedaca35e75725876d0b0e64974697fed145b0"
@@ -4984,11 +4473,6 @@ pako@~1.0.2:
   resolved "https://registry.yarnpkg.com/pako/-/pako-1.0.11.tgz#6c9599d340d54dfd3946380252a35705a6b992bf"
   integrity sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==
 
-parse-svg-path@^0.1.2:
-  version "0.1.2"
-  resolved "https://registry.yarnpkg.com/parse-svg-path/-/parse-svg-path-0.1.2.tgz#7a7ec0d1eb06fa5325c7d3e009b859a09b5d49eb"
-  integrity sha512-JyPSBnkTJ0AI8GGJLfMXvKq42cj5c006fnLz6fXy6zfoVjJizi8BNTpu8on8ziI1cKy9d9DGNuY17Ce7wuejpQ==
-
 parseley@^0.12.0:
   version "0.12.1"
   resolved "https://registry.yarnpkg.com/parseley/-/parseley-0.12.1.tgz#4afd561d50215ebe259e3e7a853e62f600683aef"
@@ -5061,24 +4545,6 @@ path-is-absolute@^1.0.0:
   resolved "https://registry.yarnpkg.com/path-is-absolute/-/path-is-absolute-1.0.1.tgz#174b9268735534ffbc7ace6bf53a5a9e1b5c5f5f"
   integrity sha1-F0uSaHNVNP+8es5r9TpanhtcX18=
 
-path-key@^3.1.0:
-  version "3.1.1"
-  resolved "https://registry.yarnpkg.com/path-key/-/path-key-3.1.1.tgz#581f6ade658cbba65a0d3380de7753295054f375"
-  integrity sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==
-
-path-parse@^1.0.7:
-  version "1.0.7"
-  resolved "https://registry.yarnpkg.com/path-parse/-/path-parse-1.0.7.tgz#fbc114b60ca42b30d9daf5858e4bd68bbedb6735"
-  integrity sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==
-
-path-scurry@^1.11.1:
-  version "1.11.1"
-  resolved "https://registry.yarnpkg.com/path-scurry/-/path-scurry-1.11.1.tgz#7960a668888594a0720b12a911d1a742ab9f11d2"
-  integrity sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA==
-  dependencies:
-    lru-cache "^10.2.0"
-    minipass "^5.0.0 || ^6.0.2 || ^7.0.0"
-
 path-to-regexp@0.1.7:
   version "0.1.7"
   resolved "https://registry.yarnpkg.com/path-to-regexp/-/path-to-regexp-0.1.7.tgz#df604178005f522f15eb4490e7247a1bfaa67f8c"
@@ -5113,6 +4579,11 @@ peberminta@^0.9.0:
   resolved "https://registry.yarnpkg.com/peberminta/-/peberminta-0.9.0.tgz#8ec9bc0eb84b7d368126e71ce9033501dca2a352"
   integrity sha512-XIxfHpEuSJbITd1H3EeQwpcZbTLHc+VVr8ANI9t5sit565tsI4/xK3KWTUFE2e6QiangUkh3B0jihzmGnNrRsQ==
 
+peek-readable@^4.1.0:
+  version "4.1.0"
+  resolved "https://registry.yarnpkg.com/peek-readable/-/peek-readable-4.1.0.tgz#4ece1111bf5c2ad8867c314c81356847e8a62e72"
+  integrity sha512-ZI3LnwUv5nOGbQzD9c2iDG6toheuXSZP5esSHBjopsXH4dg19soufvpUGA3uohi5anFtGb2lhAVdHzH6R/Evvg==
+
 peek-readable@^5.0.0:
   version "5.0.0"
   resolved "https://registry.yarnpkg.com/peek-readable/-/peek-readable-5.0.0.tgz#7ead2aff25dc40458c60347ea76cfdfd63efdfec"
@@ -5127,89 +4598,47 @@ peek-stream@^1.1.0:
     duplexify "^3.5.0"
     through2 "^2.0.3"
 
+pend@~1.2.0:
+  version "1.2.0"
+  resolved "https://registry.yarnpkg.com/pend/-/pend-1.2.0.tgz#7a57eb550a6783f9115331fcf4663d5c8e007a50"
+  integrity sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==
+
 performance-now@^2.1.0:
   version "2.1.0"
   resolved "https://registry.yarnpkg.com/performance-now/-/performance-now-2.1.0.tgz#6309f4e0e5fa913ec1c69307ae364b4b377c9e7b"
   integrity sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=
 
-picocolors@^0.2.1:
-  version "0.2.1"
-  resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-0.2.1.tgz#570670f793646851d1ba135996962abad587859f"
-  integrity sha512-cMlDqaLEqfSaW8Z7N5Jw+lyIW869EzT73/F5lhtY9cLGoVxSXznfgfXMO0Z5K0o0Q2TkTXq+0KFsdnSe3jDViA==
-
-picocolors@^1.0.0, picocolors@^1.0.1:
-  version "1.0.1"
-  resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-1.0.1.tgz#a8ad579b571952f0e5d25892de5445bcfe25aaa1"
-  integrity sha512-anP1Z8qwhkbmu7MFP5iTt+wQKXgwzf7zTyGlcdzabySa9vd0Xt392U0rVmz9poOaBj0uHJKyyo9/upk0HrEQew==
-
 picomatch@^2.0.4, picomatch@^2.2.1:
   version "2.3.1"
   resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.3.1.tgz#3ba3833733646d9d3e4995946c1365a67fb07a42"
   integrity sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==
 
+pify@^2.3.0:
+  version "2.3.0"
+  resolved "https://registry.yarnpkg.com/pify/-/pify-2.3.0.tgz#ed141a6ac043a849ea588498e7dca8b15330e90c"
+  integrity sha512-udgsAY+fTnvv7kI7aaxbqwWNb0AHiB0qBO89PZKPkoTmGOgdbrHDKD+0B2X4uTfJ/FT1R09r9gTsjUjNJotuog==
+
+pify@^3.0.0:
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/pify/-/pify-3.0.0.tgz#e5a4acd2c101fdf3d9a4d07f0dbc4db49dd28176"
+  integrity sha512-C3FsVNH1udSEX48gGX1xfvwTWfsYWj5U+8/uK15BGzIGrKoUpghX8hWZwa/OFnakBiiVNmBvemTJR5mcy7iPcg==
+
 pify@^5.0.0:
   version "5.0.0"
   resolved "https://registry.yarnpkg.com/pify/-/pify-5.0.0.tgz#1f5eca3f5e87ebec28cc6d54a0e4aaf00acc127f"
   integrity sha512-eW/gHNMlxdSP6dmG6uJip6FXN0EQBwm2clYYd8Wul42Cwu/DK8HEftzsapcNdYe2MfLiIwZqsDk2RDEsTE79hA==
 
-polf@^0.0.3:
-  version "0.0.3"
-  resolved "https://registry.yarnpkg.com/polf/-/polf-0.0.3.tgz#340f97dccadefeb2d90264d7ae010a8864775a30"
-  integrity sha512-K8zUZu9VGKL+ldcvuU78EkMnzA0Oq5JglIKZNG0rA7aqo7kfW87nXoAK88u3aIjXkTa20i4sAZv0NvP6Qs80qw==
-
-postcss-modules-extract-imports@^2.0.0:
-  version "2.0.0"
-  resolved "https://registry.yarnpkg.com/postcss-modules-extract-imports/-/postcss-modules-extract-imports-2.0.0.tgz#818719a1ae1da325f9832446b01136eeb493cd7e"
-  integrity sha512-LaYLDNS4SG8Q5WAWqIJgdHPJrDDr/Lv775rMBFUbgjTz6j34lUznACHcdRWroPvXANP2Vj7yNK57vp9eFqzLWQ==
-  dependencies:
-    postcss "^7.0.5"
-
-postcss-modules-local-by-default@^3.0.2:
-  version "3.0.3"
-  resolved "https://registry.yarnpkg.com/postcss-modules-local-by-default/-/postcss-modules-local-by-default-3.0.3.tgz#bb14e0cc78279d504dbdcbfd7e0ca28993ffbbb0"
-  integrity sha512-e3xDq+LotiGesympRlKNgaJ0PCzoUIdpH0dj47iWAui/kyTgh3CiAr1qP54uodmJhl6p9rN6BoNcdEDVJx9RDw==
-  dependencies:
-    icss-utils "^4.1.1"
-    postcss "^7.0.32"
-    postcss-selector-parser "^6.0.2"
-    postcss-value-parser "^4.1.0"
-
-postcss-modules-scope@^2.2.0:
-  version "2.2.0"
-  resolved "https://registry.yarnpkg.com/postcss-modules-scope/-/postcss-modules-scope-2.2.0.tgz#385cae013cc7743f5a7d7602d1073a89eaae62ee"
-  integrity sha512-YyEgsTMRpNd+HmyC7H/mh3y+MeFWevy7V1evVhJWewmMbjDHIbZbOXICC2y+m1xI1UVfIT1HMW/O04Hxyu9oXQ==
-  dependencies:
-    postcss "^7.0.6"
-    postcss-selector-parser "^6.0.0"
-
-postcss-modules-values@^3.0.0:
-  version "3.0.0"
-  resolved "https://registry.yarnpkg.com/postcss-modules-values/-/postcss-modules-values-3.0.0.tgz#5b5000d6ebae29b4255301b4a3a54574423e7f10"
-  integrity sha512-1//E5jCBrZ9DmRX+zCtmQtRSV6PV42Ix7Bzj9GbwJceduuf7IqP8MgeTXuRDHOWj2m0VzZD5+roFWDuU8RQjcg==
-  dependencies:
-    icss-utils "^4.0.0"
-    postcss "^7.0.6"
-
-postcss-selector-parser@^6.0.0, postcss-selector-parser@^6.0.2:
-  version "6.1.2"
-  resolved "https://registry.yarnpkg.com/postcss-selector-parser/-/postcss-selector-parser-6.1.2.tgz#27ecb41fb0e3b6ba7a1ec84fff347f734c7929de"
-  integrity sha512-Q8qQfPiZ+THO/3ZrOrO0cJJKfpYCagtMUkXbnEfmgUjwXg6z/WBeOyS9APBBPCTSiDV+s4SwQGu8yFsiMRIudg==
+pinkie-promise@^2.0.0:
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/pinkie-promise/-/pinkie-promise-2.0.1.tgz#2135d6dfa7a358c069ac9b178776288228450ffa"
+  integrity sha512-0Gni6D4UcLTbv9c57DfxDGdr41XfgUjqWZu492f0cIGr16zDU06BWP/RAEvOuo7CQ0CNjHaLlM59YJJFm3NWlw==
   dependencies:
-    cssesc "^3.0.0"
-    util-deprecate "^1.0.2"
-
-postcss-value-parser@^4.1.0:
-  version "4.2.0"
-  resolved "https://registry.yarnpkg.com/postcss-value-parser/-/postcss-value-parser-4.2.0.tgz#723c09920836ba6d3e5af019f92bc0971c02e514"
-  integrity sha512-1NNCs6uurfkVbeXG4S8JFT9t19m45ICnif8zWLd5oPSZ50QnwMfK+H3jv408d4jw/7Bttv5axS5IiHoLaVNHeQ==
+    pinkie "^2.0.0"
 
-postcss@^7.0.14, postcss@^7.0.32, postcss@^7.0.5, postcss@^7.0.6:
-  version "7.0.39"
-  resolved "https://registry.yarnpkg.com/postcss/-/postcss-7.0.39.tgz#9624375d965630e2e1f2c02a935c82a59cb48309"
-  integrity sha512-yioayjNbHn6z1/Bywyb2Y4s3yvDAeXGOyxqD+LnVOinq6Mdmd++SW2wUNVzavyyHxd6+DxzWGIuosg6P1Rj8uA==
-  dependencies:
-    picocolors "^0.2.1"
-    source-map "^0.6.1"
+pinkie@^2.0.0:
+  version "2.0.4"
+  resolved "https://registry.yarnpkg.com/pinkie/-/pinkie-2.0.4.tgz#72556b80cfa0d48a974e80e77248e80ed4f7f870"
+  integrity sha512-MnUuEycAemtSaeFSjXKW/aroV7akBbY+Sv+RkyqFjgAe73F+MR0TBWKBRDkmfWq/HiFmdavfZ1G7h4SPZXaCSg==
 
 pouchdb-abstract-mapreduce@7.3.1:
   version "7.3.1"
@@ -5342,30 +4771,6 @@ pouchdb@^7.2.2:
     uuid "8.3.2"
     vuvuzela "1.0.3"
 
-pptx-parser@^1.1.7-beta.9:
-  version "1.1.7-beta.9"
-  resolved "https://registry.yarnpkg.com/pptx-parser/-/pptx-parser-1.1.7-beta.9.tgz#26648ba5c45c4c5c548b1de7ab03f04764399d37"
-  integrity sha512-xIjw65wRMVGNCmRLF91F3+f84gQ0uITDJEJp21bprsLkznurkSCG5sWGpvmnVZr81/hHqPy+kKsAU3/wgZQYLw==
-  dependencies:
-    "@babel/runtime-corejs3" "^7.10.4"
-    "@vf.js/launcher" "2.0.12"
-    css-loader "^3.6.0"
-    deepmerge "^4.2.2"
-    element-to-path "^1.2.0"
-    jquery "^3.5.1"
-    jszip "2.6.1"
-    jszip-utils "^0.1.0"
-    lodash "^4.17.19"
-    prst-shape-transform "^1.0.5-beta.0"
-    style-loader "^1.2.1"
-    svg-path-bbox "0.0.49"
-    svg-path-bounds "^1.0.1"
-    svgson "^4.1.0"
-    tinycolor2 "^1.4.1"
-    transformation-matrix "^2.5.0"
-    txml "3.1.3"
-    url-loader "^4.1.0"
-
 prebuilt-tdlib@^0.1008034.0:
   version "0.1008034.0"
   resolved "https://registry.yarnpkg.com/prebuilt-tdlib/-/prebuilt-tdlib-0.1008034.0.tgz#f80d8f2b21a242e43739df54624ae20418ee251d"
@@ -5386,11 +4791,6 @@ process-nextick-args@~2.0.0:
   resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-2.0.1.tgz#7820d9b16120cc55ca9ae7792680ae7dba6d7fe2"
   integrity sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==
 
-proto-list@~1.2.1:
-  version "1.2.4"
-  resolved "https://registry.yarnpkg.com/proto-list/-/proto-list-1.2.4.tgz#212d5bfe1318306a420f6402b8e26ff39647a849"
-  integrity sha512-vtK/94akxsTMhe0/cbfpR+syPuszcuwhqVjJq26CuNDgFGj682oRBXOP5MJpv2r7JtE8MsiepGIqvvOTBwn2vA==
-
 proxy-addr@~2.0.7:
   version "2.0.7"
   resolved "https://registry.yarnpkg.com/proxy-addr/-/proxy-addr-2.0.7.tgz#f19fe69ceab311eeb94b42e70e8c2070f9ba1025"
@@ -5409,23 +4809,6 @@ prr@~1.0.1:
   resolved "https://registry.yarnpkg.com/prr/-/prr-1.0.1.tgz#d3fc114ba06995a45ec6893f484ceb1d78f5f476"
   integrity sha512-yPw4Sng1gWghHQWj0B3ZggWUm4qVbPwPFcRG8KyxiU7J2OHFSoEHKS+EZ3fv5l1t9CyCiop6l/ZYeWbrgoQejw==
 
-prst-shape-transform@^1.0.5-beta.0:
-  version "1.0.5-beta.0"
-  resolved "https://registry.yarnpkg.com/prst-shape-transform/-/prst-shape-transform-1.0.5-beta.0.tgz#0e33043df63948d06729745a3fa14873866a9627"
-  integrity sha512-AsFdub+qDdqwEnF6CVOkbrVab4un/Ag1uc5uLTTBGlVCjan8wrQN1oNTtQC0+8PBs8DHGY11hiUNO2E9mC2k0w==
-  dependencies:
-    "@babel/plugin-transform-runtime" "^7.7.6"
-    "@babel/runtime-corejs3" "^7.7.7"
-    colors "^1.4.0"
-    core-js "^3.6.0"
-    fs-extra "^8.1.0"
-    glob "^7.1.6"
-    js-beautify "^1.13.0"
-    lodash "^4.17.20"
-    transformation-matrix "^2.4.0"
-    txml "^3.1.3"
-    xml-js "^1.6.11"
-
 psl@^1.1.28, psl@^1.1.33:
   version "1.8.0"
   resolved "https://registry.yarnpkg.com/psl/-/psl-1.8.0.tgz#9326f8bcfb013adcc005fdff056acce020e51c24"
@@ -5586,6 +4969,19 @@ readable-stream@^2.0.0, readable-stream@^2.0.5, readable-stream@^2.3.6, readable
     string_decoder "~1.1.1"
     util-deprecate "~1.0.1"
 
+readable-stream@^2.3.0, readable-stream@^2.3.5:
+  version "2.3.8"
+  resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.8.tgz#91125e8042bba1b9887f49345f6277027ce8be9b"
+  integrity sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==
+  dependencies:
+    core-util-is "~1.0.0"
+    inherits "~2.0.3"
+    isarray "~1.0.0"
+    process-nextick-args "~2.0.0"
+    safe-buffer "~5.1.1"
+    string_decoder "~1.1.1"
+    util-deprecate "~1.0.1"
+
 readable-stream@^3.1.1, readable-stream@^3.4.0:
   version "3.6.0"
   resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-3.6.0.tgz#337bbda3adc0706bd3e024426a286d4b4b2c9198"
@@ -5610,7 +5006,7 @@ readable-stream@~1.0.31:
     isarray "0.0.1"
     string_decoder "~0.10.x"
 
-readable-web-to-node-stream@^3.0.2:
+readable-web-to-node-stream@^3.0.0, readable-web-to-node-stream@^3.0.2:
   version "3.0.2"
   resolved "https://registry.yarnpkg.com/readable-web-to-node-stream/-/readable-web-to-node-stream-3.0.2.tgz#5d52bb5df7b54861fd48d015e93a2cb87b3ee0bb"
   integrity sha512-ePeK6cc1EcKLEhJFt/AebMCLL+GgSKhuygrZ/GLaKZYEecIgIECf4UaUuaByiGtzckwR4ain9VzUh95T1exYGw==
@@ -5634,11 +5030,6 @@ regenerator-runtime@^0.11.0:
   resolved "https://registry.yarnpkg.com/regenerator-runtime/-/regenerator-runtime-0.11.1.tgz#be05ad7f9bf7d22e056f9726cee5017fbf19e2e9"
   integrity sha512-MguG95oij0fC3QV3URf4V2SDYGJhJnJGqvIIgdECeODCT98wSWDAJ94SSuVpYQUoTcGUIL6L4yNB7j1DFFHSBg==
 
-regenerator-runtime@^0.14.0:
-  version "0.14.1"
-  resolved "https://registry.yarnpkg.com/regenerator-runtime/-/regenerator-runtime-0.14.1.tgz#356ade10263f685dda125100cd862c1db895327f"
-  integrity sha512-dYnhHh0nJoMfnkZs6GmmhFknAGRrLznOu5nc9ML+EJxGvrx6H7teuevqVqCuPcPK//3eDrrjQhehXVx9cnkGdw==
-
 registry-auth-token@^4.0.0:
   version "4.2.1"
   resolved "https://registry.yarnpkg.com/registry-auth-token/-/registry-auth-token-4.2.1.tgz#6d7b4006441918972ccd5fedcd41dc322c79b250"
@@ -5653,11 +5044,6 @@ registry-url@^5.0.0:
   dependencies:
     rc "^1.2.8"
 
-rename-keys@^1.1.2:
-  version "1.2.0"
-  resolved "https://registry.yarnpkg.com/rename-keys/-/rename-keys-1.2.0.tgz#be602fb0b750476b513ebe85ba4465d03254f0a3"
-  integrity sha512-U7XpAktpbSgHTRSNRrjKSrjYkZKuhUukfoBlXWXUExCAqhzh1TU3BDRAfJmarcl5voKS+pbKU9MvyLWKZ4UEEg==
-
 request@^2.81.0:
   version "2.88.2"
   resolved "https://registry.yarnpkg.com/request/-/request-2.88.2.tgz#d73c918731cb5a87da047e207234146f664d12b3"
@@ -5699,15 +5085,6 @@ requires-port@^1.0.0:
   resolved "https://registry.yarnpkg.com/requires-port/-/requires-port-1.0.0.tgz#925d2601d39ac485e091cf0da5c6e694dc3dcaff"
   integrity sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==
 
-resolve@^1.14.2:
-  version "1.22.8"
-  resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.22.8.tgz#b6c87a9f2aa06dfab52e3d70ac8cde321fa5a48d"
-  integrity sha512-oKWePCxqpd6FlLvGV1VU0x7bkPmmCNolxzjMf4NczoDnQcIWrAF+cPtZn5i6n+RfD2d9i0tzpKnG6Yk168yIyw==
-  dependencies:
-    is-core-module "^2.13.0"
-    path-parse "^1.0.7"
-    supports-preserve-symlinks-flag "^1.0.0"
-
 responselike@^1.0.2:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/responselike/-/responselike-1.0.2.tgz#918720ef3b631c5642be068f15ade5a46f4ba1e7"
@@ -5720,6 +5097,13 @@ rfdc@^1.3.0:
   resolved "https://registry.yarnpkg.com/rfdc/-/rfdc-1.3.0.tgz#d0b7c441ab2720d05dc4cf26e01c89631d9da08b"
   integrity sha512-V2hovdzFbOi77/WajaSMXk2OLm+xNIeQdMMuB7icj7bk6zi2F8GGAxigcnDFpJHbNyNcgyJDiP+8nOrY5cZGrA==
 
+rimraf@^2.6.3:
+  version "2.7.1"
+  resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.7.1.tgz#35797f13a7fdadc566142c29d4f07ccad483e3ec"
+  integrity sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==
+  dependencies:
+    glob "^7.1.3"
+
 ripemd160@^2.0.0, ripemd160@^2.0.1:
   version "2.0.2"
   resolved "https://registry.yarnpkg.com/ripemd160/-/ripemd160-2.0.2.tgz#a1c1a6f624751577ba5d07914cbc92850585890c"
@@ -5733,7 +5117,7 @@ safe-buffer@5.1.2, safe-buffer@~5.1.0, safe-buffer@~5.1.1:
   resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d"
   integrity sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==
 
-safe-buffer@5.2.1, safe-buffer@^5.0.1, safe-buffer@^5.1.0, safe-buffer@^5.1.2, safe-buffer@^5.2.0, safe-buffer@~5.2.0:
+safe-buffer@5.2.1, safe-buffer@^5.0.1, safe-buffer@^5.1.0, safe-buffer@^5.1.1, safe-buffer@^5.1.2, safe-buffer@^5.2.0, safe-buffer@~5.2.0:
   version "5.2.1"
   resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.2.1.tgz#1eaf9fa9bdb1fdd4ec75f58f9cdb4e6b7827eec6"
   integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==
@@ -5753,34 +5137,18 @@ sax@>=0.6.0:
   resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9"
   integrity sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==
 
-sax@^1.2.4:
-  version "1.4.1"
-  resolved "https://registry.yarnpkg.com/sax/-/sax-1.4.1.tgz#44cc8988377f126304d3b3fc1010c733b929ef0f"
-  integrity sha512-+aWOz7yVScEGoKNd4PA10LZ8sk0A/z5+nXQG5giUO5rprX9jgYsTdov9qCchZiPIZezbZH+jRut8nPodFAX4Jg==
-
-schema-utils@^2.7.0:
-  version "2.7.1"
-  resolved "https://registry.yarnpkg.com/schema-utils/-/schema-utils-2.7.1.tgz#1ca4f32d1b24c590c203b8e7a50bf0ea4cd394d7"
-  integrity sha512-SHiNtMOUGWBQJwzISiVYKu82GiV4QYGePp3odlY1tuKO7gPtphAT5R/py0fA6xtbgLL/RvtJZnU9b8s0F1q0Xg==
-  dependencies:
-    "@types/json-schema" "^7.0.5"
-    ajv "^6.12.4"
-    ajv-keywords "^3.5.2"
-
-schema-utils@^3.0.0:
-  version "3.3.0"
-  resolved "https://registry.yarnpkg.com/schema-utils/-/schema-utils-3.3.0.tgz#f50a88877c3c01652a15b622ae9e9795df7a60fe"
-  integrity sha512-pN/yOAvcC+5rQ5nERGuwrjLlYvLTbCibnZ1I7B1LaiAz9BRBlE9GMgE/eqV30P7aJQUf7Ddimy/RsbYO/GrVGg==
-  dependencies:
-    "@types/json-schema" "^7.0.8"
-    ajv "^6.12.5"
-    ajv-keywords "^3.5.2"
-
 scrypt-js@3.0.1:
   version "3.0.1"
   resolved "https://registry.yarnpkg.com/scrypt-js/-/scrypt-js-3.0.1.tgz#d314a57c2aef69d1ad98a138a21fe9eafa9ee312"
   integrity sha512-cdwTTnqPu0Hyvf5in5asVdZocVDTNRmR7XEcJuIzMjJeSHybHl7vpB66AzwTaIg6CLSbtjcxc8fqcySfnTkccA==
 
+seek-bzip@^1.0.5:
+  version "1.0.6"
+  resolved "https://registry.yarnpkg.com/seek-bzip/-/seek-bzip-1.0.6.tgz#35c4171f55a680916b52a07859ecf3b5857f21c4"
+  integrity sha512-e1QtP3YL5tWww8uKaOCQ18UxIT2laNBXHjV/S2WYCiK4udiv8lkG89KRIoCjUagnAmCBurjF4zEVX2ByBbnCjQ==
+  dependencies:
+    commander "^2.8.1"
+
 selderee@^0.11.0:
   version "0.11.0"
   resolved "https://registry.yarnpkg.com/selderee/-/selderee-0.11.0.tgz#6af0c7983e073ad3e35787ffe20cefd9daf0ec8a"
@@ -5805,11 +5173,6 @@ semver@^6.0.0, semver@^6.2.0, semver@^6.3.0:
   resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.0.tgz#ee0a64c8af5e8ceea67687b133761e1becbd1d3d"
   integrity sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==
 
-semver@^6.3.1:
-  version "6.3.1"
-  resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.1.tgz#556d2ef8689146e46dcea4bfdd095f3434dffcb4"
-  integrity sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==
-
 semver@^7.3.4:
   version "7.3.7"
   resolved "https://registry.yarnpkg.com/semver/-/semver-7.3.7.tgz#12c5b649afdbf9049707796e22a4028814ce523f"
@@ -5817,11 +5180,6 @@ semver@^7.3.4:
   dependencies:
     lru-cache "^6.0.0"
 
-semver@^7.5.3:
-  version "7.6.3"
-  resolved "https://registry.yarnpkg.com/semver/-/semver-7.6.3.tgz#980f7b5550bc175fb4dc09403085627f9eb33143"
-  integrity sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==
-
 semver@~5.3.0:
   version "5.3.0"
   resolved "https://registry.yarnpkg.com/semver/-/semver-5.3.0.tgz#9b2ce5d3de02d17c6012ad326aa6b4d0cf54f94f"
@@ -5898,18 +5256,6 @@ sha.js@^2.4.0, sha.js@^2.4.8:
     inherits "^2.0.1"
     safe-buffer "^5.0.1"
 
-shebang-command@^2.0.0:
-  version "2.0.0"
-  resolved "https://registry.yarnpkg.com/shebang-command/-/shebang-command-2.0.0.tgz#ccd0af4f8835fbdc265b82461aaf0c36663f34ea"
-  integrity sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==
-  dependencies:
-    shebang-regex "^3.0.0"
-
-shebang-regex@^3.0.0:
-  version "3.0.0"
-  resolved "https://registry.yarnpkg.com/shebang-regex/-/shebang-regex-3.0.0.tgz#ae16f1644d873ecad843b0307b143362d4c42172"
-  integrity sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==
-
 side-channel@^1.0.4:
   version "1.0.4"
   resolved "https://registry.yarnpkg.com/side-channel/-/side-channel-1.0.4.tgz#efce5c8fdc104ee751b25c58d4290011fa5ea2cf"
@@ -5934,11 +5280,6 @@ signal-exit@^3.0.2:
   resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.7.tgz#a9a1767f8af84155114eaabd73f99273c8f59ad9"
   integrity sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==
 
-signal-exit@^4.0.1:
-  version "4.1.0"
-  resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-4.1.0.tgz#952188c1cbd546070e2dd20d0f41c0ae0530cb04"
-  integrity sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==
-
 source-map-support@^0.5.6:
   version "0.5.21"
   resolved "https://registry.yarnpkg.com/source-map-support/-/source-map-support-0.5.21.tgz#04fe7c7f9e1ed2d662233c28cb2b35b9f63f6e4f"
@@ -5947,7 +5288,7 @@ source-map-support@^0.5.6:
     buffer-from "^1.0.0"
     source-map "^0.6.0"
 
-source-map@^0.6.0, source-map@^0.6.1:
+source-map@^0.6.0:
   version "0.6.1"
   resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.6.1.tgz#74722af32e9614e9c287a8d0bbde48b5e2f1a263"
   integrity sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==
@@ -6018,15 +5359,6 @@ string-strip-html@8.5.0:
   resolved "https://registry.yarnpkg.com/string-strip-html/-/string-strip-html-8.5.0.tgz#5e239fe84016fad7b33ca02d23c591f1ccb6af75"
   integrity sha512-5ICsK1B1j0A3AF1d45m0sqQCcmi1Q+t1QpF+b794LO5FTHV+ITkGR5C+UCDJQZgs5LMuRruqr6j48PxQVIurJQ==
 
-"string-width-cjs@npm:string-width@^4.2.0":
-  version "4.2.3"
-  resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010"
-  integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==
-  dependencies:
-    emoji-regex "^8.0.0"
-    is-fullwidth-code-point "^3.0.0"
-    strip-ansi "^6.0.1"
-
 string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.2:
   version "4.2.3"
   resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010"
@@ -6036,15 +5368,6 @@ string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2
     is-fullwidth-code-point "^3.0.0"
     strip-ansi "^6.0.1"
 
-string-width@^5.0.1, string-width@^5.1.2:
-  version "5.1.2"
-  resolved "https://registry.yarnpkg.com/string-width/-/string-width-5.1.2.tgz#14f8daec6d81e7221d2a357e668cab73bdbca794"
-  integrity sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==
-  dependencies:
-    eastasianwidth "^0.2.0"
-    emoji-regex "^9.2.2"
-    strip-ansi "^7.0.1"
-
 string_decoder@^1.1.1:
   version "1.3.0"
   resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.3.0.tgz#42f114594a46cf1a8e30b0a84f56c78c3edac21e"
@@ -6064,13 +5387,6 @@ string_decoder@~1.1.1:
   dependencies:
     safe-buffer "~5.1.0"
 
-"strip-ansi-cjs@npm:strip-ansi@^6.0.1":
-  version "6.0.1"
-  resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9"
-  integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==
-  dependencies:
-    ansi-regex "^5.0.1"
-
 strip-ansi@^6.0.0, strip-ansi@^6.0.1:
   version "6.0.1"
   resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9"
@@ -6078,18 +5394,18 @@ strip-ansi@^6.0.0, strip-ansi@^6.0.1:
   dependencies:
     ansi-regex "^5.0.1"
 
-strip-ansi@^7.0.1:
-  version "7.1.0"
-  resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-7.1.0.tgz#d5b6568ca689d8561370b0707685d22434faff45"
-  integrity sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==
-  dependencies:
-    ansi-regex "^6.0.1"
-
 strip-bom@^3.0.0:
   version "3.0.0"
   resolved "https://registry.yarnpkg.com/strip-bom/-/strip-bom-3.0.0.tgz#2334c18e9c759f7bdd56fdef7e9ae3d588e68ed3"
   integrity sha1-IzTBjpx1n3vdVv3vfprj1YjmjtM=
 
+strip-dirs@^2.0.0:
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/strip-dirs/-/strip-dirs-2.1.0.tgz#4987736264fc344cf20f6c34aca9d13d1d4ed6c5"
+  integrity sha512-JOCxOeKLm2CAS73y/U4ZeZPTkE+gNVCzKt7Eox84Iej1LT/2pTWYpZKJuxwQpvX1LiZb1xokNR7RLfuBAa7T3g==
+  dependencies:
+    is-natural-number "^4.0.1"
+
 strip-indent@3.0.0:
   version "3.0.0"
   resolved "https://registry.yarnpkg.com/strip-indent/-/strip-indent-3.0.0.tgz#c32e1cee940b6b3432c771bc2c54bcce73cd3001"
@@ -6107,6 +5423,14 @@ strip-json-comments@~2.0.1:
   resolved "https://registry.yarnpkg.com/strip-json-comments/-/strip-json-comments-2.0.1.tgz#3c531942e908c2697c0ec344858c286c7ca0a60a"
   integrity sha1-PFMZQukIwml8DsNEhYwobHygpgo=
 
+strtok3@^6.2.4:
+  version "6.3.0"
+  resolved "https://registry.yarnpkg.com/strtok3/-/strtok3-6.3.0.tgz#358b80ffe6d5d5620e19a073aa78ce947a90f9a0"
+  integrity sha512-fZtbhtvI9I48xDSywd/somNqgUHl2L2cstmXCCif0itOf96jeW18MBSyrLuNicYQVkvpOxkZtkzujiTJ9LW5Jw==
+  dependencies:
+    "@tokenizer/token" "^0.3.0"
+    peek-readable "^4.1.0"
+
 strtok3@^7.0.0:
   version "7.0.0"
   resolved "https://registry.yarnpkg.com/strtok3/-/strtok3-7.0.0.tgz#868c428b4ade64a8fd8fee7364256001c1a4cbe5"
@@ -6115,14 +5439,6 @@ strtok3@^7.0.0:
     "@tokenizer/token" "^0.3.0"
     peek-readable "^5.0.0"
 
-style-loader@^1.2.1:
-  version "1.3.0"
-  resolved "https://registry.yarnpkg.com/style-loader/-/style-loader-1.3.0.tgz#828b4a3b3b7e7aa5847ce7bae9e874512114249e"
-  integrity sha512-V7TCORko8rs9rIqkSrlMfkqA63DfoGBBJmK1kKGCcSi+BWb4cqz0SRsnp4l6rU5iwOEd0/2ePv68SV22VXon4Q==
-  dependencies:
-    loader-utils "^2.0.0"
-    schema-utils "^2.7.0"
-
 supports-color@8.1.1:
   version "8.1.1"
   resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-8.1.1.tgz#cd6fc17e28500cff56c1b86c0a7fd4a54a73005c"
@@ -6144,48 +5460,6 @@ supports-color@^7.1.0:
   dependencies:
     has-flag "^4.0.0"
 
-supports-preserve-symlinks-flag@^1.0.0:
-  version "1.0.0"
-  resolved "https://registry.yarnpkg.com/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz#6eda4bd344a3c94aea376d4cc31bc77311039e09"
-  integrity sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==
-
-svg-arc-to-cubic-bezier@^3.0.0:
-  version "3.2.0"
-  resolved "https://registry.yarnpkg.com/svg-arc-to-cubic-bezier/-/svg-arc-to-cubic-bezier-3.2.0.tgz#390c450035ae1c4a0104d90650304c3bc814abe6"
-  integrity sha512-djbJ/vZKZO+gPoSDThGNpKDO+o+bAeA4XQKovvkNCqnIS2t+S4qnLAGQhyyrulhCFRl1WWzAp0wUDV8PpTVU3g==
-
-svg-path-bbox@0.0.49:
-  version "0.0.49"
-  resolved "https://registry.yarnpkg.com/svg-path-bbox/-/svg-path-bbox-0.0.49.tgz#0d1c84b9b84341fe085a244f0d286ea2ecf95d15"
-  integrity sha512-QXKc4LhdZTlk3thjxR1jrWqoRGCMSjNjqqdgDTS1vqOSdYFTnTNQvGrKv3dvSCqB3VC1pyRBQGJKJdtXE3mp5Q==
-  dependencies:
-    polf "^0.0.3"
-    svgpath "^2.3.0"
-
-svg-path-bounds@^1.0.1:
-  version "1.0.2"
-  resolved "https://registry.yarnpkg.com/svg-path-bounds/-/svg-path-bounds-1.0.2.tgz#00312f672b08afc432a66ddfbd06db40cec8d0d0"
-  integrity sha512-H4/uAgLWrppIC0kHsb2/dWUYSmb4GE5UqH06uqWBcg6LBjX2fu0A8+JrO2/FJPZiSsNOKZAhyFFgsLTdYUvSqQ==
-  dependencies:
-    abs-svg-path "^0.1.1"
-    is-svg-path "^1.0.1"
-    normalize-svg-path "^1.0.0"
-    parse-svg-path "^0.1.2"
-
-svgpath@^2.3.0:
-  version "2.6.0"
-  resolved "https://registry.yarnpkg.com/svgpath/-/svgpath-2.6.0.tgz#5b160ef3d742b7dfd2d721bf90588d3450d7a90d"
-  integrity sha512-OIWR6bKzXvdXYyO4DK/UWa1VA1JeKq8E+0ug2DG98Y/vOmMpfZNj+TIG988HjfYSqtcy/hFOtZq/n/j5GSESNg==
-
-svgson@^4.1.0:
-  version "4.1.0"
-  resolved "https://registry.yarnpkg.com/svgson/-/svgson-4.1.0.tgz#eb70dac8d0075c61e5bfd45411a56014e2d3610e"
-  integrity sha512-DodISxHtdLKUghDYA+PGK4Qq350+CbBAkdvGLkBFSmWd9WKSg4dijgjB1IiRPTmsUCd+a7KYe+ILHtklYgQyzQ==
-  dependencies:
-    deep-rename-keys "^0.2.1"
-    omit-deep "0.3.0"
-    xml-reader "2.4.3"
-
 table-layout@^1.0.2:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/table-layout/-/table-layout-1.0.2.tgz#c4038a1853b0136d63365a734b6931cf4fad4a04"
@@ -6206,6 +5480,19 @@ tar-fs@^2.1.1:
     pump "^3.0.0"
     tar-stream "^2.1.4"
 
+tar-stream@^1.5.2:
+  version "1.6.2"
+  resolved "https://registry.yarnpkg.com/tar-stream/-/tar-stream-1.6.2.tgz#8ea55dab37972253d9a9af90fdcd559ae435c555"
+  integrity sha512-rzS0heiNf8Xn7/mpdSVVSMAWAoy9bfb1WOTYC78Z0UQKeKa/CWS8FOq0lKGNa8DWKAn9gxjCvMLYc5PGXYlK2A==
+  dependencies:
+    bl "^1.0.0"
+    buffer-alloc "^1.2.0"
+    end-of-stream "^1.0.0"
+    fs-constants "^1.0.0"
+    readable-stream "^2.3.0"
+    to-buffer "^1.1.1"
+    xtend "^4.0.0"
+
 tar-stream@^2.1.0, tar-stream@^2.1.4:
   version "2.2.0"
   resolved "https://registry.yarnpkg.com/tar-stream/-/tar-stream-2.2.0.tgz#acad84c284136b060dc3faa64474aa9aebd77287"
@@ -6239,7 +5526,7 @@ tdlib-native@^2.6.0:
     "@tdlib-native/tdjson-linux-x64-glibc" "1.8.33-commit.cb164927417f22811c74cd8678ed4a5ab7cb80ba"
     "@tdlib-native/tdjson-win32-x64" "1.8.33-commit.cb164927417f22811c74cd8678ed4a5ab7cb80ba"
 
-through2@3.0.2, through2@^3.0.1:
+through2@3.0.2:
   version "3.0.2"
   resolved "https://registry.yarnpkg.com/through2/-/through2-3.0.2.tgz#99f88931cfc761ec7678b41d5d7336b5b6a07bf4"
   integrity sha512-enaDQ4MUyP2W6ZyT6EsMzqBPZaM/avg8iuo+l2d3QCs0J+6RaqkHV/2/lOwDTueBHeJ/2LG9lrLW3d5rWPucuQ==
@@ -6255,20 +5542,20 @@ through2@^2.0.1, through2@^2.0.3:
     readable-stream "~2.3.6"
     xtend "~4.0.1"
 
-tinycolor2@^1.4.1:
-  version "1.6.0"
-  resolved "https://registry.yarnpkg.com/tinycolor2/-/tinycolor2-1.6.0.tgz#f98007460169b0263b97072c5ae92484ce02d09e"
-  integrity sha512-XPaBkWQJdsf3pLKJV9p4qN/S+fm2Oj8AIPo1BTUhg5oxkvm9+SVEGFdhyOz7tTdUTfvxMiAs4sp6/eZO2Ew+pw==
+through@^2.3.8:
+  version "2.3.8"
+  resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5"
+  integrity sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==
 
 tlds@1.252.0:
   version "1.252.0"
   resolved "https://registry.yarnpkg.com/tlds/-/tlds-1.252.0.tgz#71d9617f4ef4cc7347843bee72428e71b8b0f419"
   integrity sha512-GA16+8HXvqtfEnw/DTcwB0UU354QE1n3+wh08oFjr6Znl7ZLAeUgYzCcK+/CCrOyE0vnHR8/pu3XXG3vDijXpQ==
 
-to-fast-properties@^2.0.0:
-  version "2.0.0"
-  resolved "https://registry.yarnpkg.com/to-fast-properties/-/to-fast-properties-2.0.0.tgz#dc5e698cbd079265bc73e0377681a4e4e83f616e"
-  integrity sha512-/OaKK0xYrs3DmxRYqL/yDc+FxFUVYhDlXMhRmv3z915w2HF1tnN1omB354j8VUGO/hbRzyD6Y3sA7v7GS/ceog==
+to-buffer@^1.1.1:
+  version "1.1.1"
+  resolved "https://registry.yarnpkg.com/to-buffer/-/to-buffer-1.1.1.tgz#493bd48f62d7c43fcded313a03dcadb2e1213a80"
+  integrity sha512-lx9B5iv7msuFYE3dytT+KE5tap+rNYw+K4jVkb9R/asAb+pbBSM17jtunHplhBe6RRJdZx3Pn2Jph24O32mOVg==
 
 to-readable-stream@^1.0.0:
   version "1.0.0"
@@ -6287,6 +5574,14 @@ toidentifier@1.0.1:
   resolved "https://registry.yarnpkg.com/toidentifier/-/toidentifier-1.0.1.tgz#3be34321a88a820ed1bd80dfaa33e479fbb8dd35"
   integrity sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==
 
+token-types@^4.1.1:
+  version "4.2.1"
+  resolved "https://registry.yarnpkg.com/token-types/-/token-types-4.2.1.tgz#0f897f03665846982806e138977dbe72d44df753"
+  integrity sha512-6udB24Q737UD/SDsKAHI9FCRP7Bqc9D/MQUV02ORQg5iskjtLJlZJNdN4kKtcdtwCeWIwIHDGaUsTsCCAa8sFQ==
+  dependencies:
+    "@tokenizer/token" "^0.3.0"
+    ieee754 "^1.2.1"
+
 token-types@^5.0.1:
   version "5.0.1"
   resolved "https://registry.yarnpkg.com/token-types/-/token-types-5.0.1.tgz#aa9d9e6b23c420a675e55413b180635b86a093b4"
@@ -6341,11 +5636,6 @@ transform-pouch@^2.0.0:
   dependencies:
     pouchdb-wrappers "^5.0.0"
 
-transformation-matrix@^2.4.0, transformation-matrix@^2.5.0:
-  version "2.16.1"
-  resolved "https://registry.yarnpkg.com/transformation-matrix/-/transformation-matrix-2.16.1.tgz#4a2de06331b94ae953193d1b9a5ba002ec5f658a"
-  integrity sha512-tdtC3wxVEuzU7X/ydL131Q3JU5cPMEn37oqVLITjRDSDsnSHVFzW2JiCLfZLIQEgWzZHdSy3J6bZzvKEN24jGA==
-
 ts-mixer@^6.0.3:
   version "6.0.3"
   resolved "https://registry.yarnpkg.com/ts-mixer/-/ts-mixer-6.0.3.tgz#69bd50f406ff39daa369885b16c77a6194c7cae6"
@@ -6421,20 +5711,6 @@ twitter-api-v2@^1.14.0:
   resolved "https://registry.yarnpkg.com/twitter-api-v2/-/twitter-api-v2-1.14.0.tgz#2aa186087aae58083dcbbafef8727b42cb703483"
   integrity sha512-kBc0X6hTl0qWYTSNH9Gp87S6d9wIM2qOxHg7jlWH054HnMPC8XMcttsrKyPwP8SVha28dt5DPQvlqhPxsWRC+Q==
 
-txml@3.1.3:
-  version "3.1.3"
-  resolved "https://registry.yarnpkg.com/txml/-/txml-3.1.3.tgz#c2d890b18f1eb0685845c6e320cfe0e8f7aadeb7"
-  integrity sha512-JOXZxzZtdXqxczL3aYs6ZtJdHKbqrzdb/BOOj9M48kmL095RHmT8Ad+Ax+UVhE3t7XZLaCjaRsUo6qE4RYudIQ==
-  dependencies:
-    through2 "^3.0.1"
-
-txml@^3.1.3:
-  version "3.2.5"
-  resolved "https://registry.yarnpkg.com/txml/-/txml-3.2.5.tgz#607eeef7e021dba8c6dd173b3971b12443deb9ec"
-  integrity sha512-AtN8AgJLiDanttIXJaQlxH8/R0NOCNwto8kcO7BaxdLgsN9b7itM9lnTD7c2O3TadP+hHB9j7ra5XGFRPNnk/g==
-  dependencies:
-    through2 "^3.0.1"
-
 type-detect@^4.0.8:
   version "4.1.0"
   resolved "https://registry.yarnpkg.com/type-detect/-/type-detect-4.1.0.tgz#deb2453e8f08dcae7ae98c626b13dddb0155906c"
@@ -6499,6 +5775,14 @@ uint8arrays@^3.0.0:
   dependencies:
     multiformats "^9.4.2"
 
+unbzip2-stream@^1.0.9:
+  version "1.4.3"
+  resolved "https://registry.yarnpkg.com/unbzip2-stream/-/unbzip2-stream-1.4.3.tgz#b0da04c4371311df771cdc215e87f2130991ace7"
+  integrity sha512-mlExGW4w71ebDJviH16lQLtZS32VKqsSfk80GCfUlwT/4/hNRFsoscrF/c++9xinkMzECL1uL9DDwXqFWkruPg==
+  dependencies:
+    buffer "^5.2.1"
+    through "^2.3.8"
+
 undefsafe@^2.0.5:
   version "2.0.5"
   resolved "https://registry.yarnpkg.com/undefsafe/-/undefsafe-2.0.5.tgz#38733b9327bdcd226db889fb723a6efd162e6e2c"
@@ -6548,27 +5832,11 @@ unpipe@1.0.0, unpipe@~1.0.0:
   resolved "https://registry.yarnpkg.com/unpipe/-/unpipe-1.0.0.tgz#b2bf4ee8514aae6165b4817829d21b2ef49904ec"
   integrity sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw=
 
-unset-value@^0.1.1:
-  version "0.1.2"
-  resolved "https://registry.yarnpkg.com/unset-value/-/unset-value-0.1.2.tgz#506810b867f27c2a5a6e9b04833631f6de58d310"
-  integrity sha512-yhv5I4TsldLdE3UcVQn0hD2T5sNCPv4+qm/CTUpRKIpwthYRIipsAPdsrNpOI79hPQa0rTTeW22Fq6JWRcTgNg==
-  dependencies:
-    has-value "^0.3.1"
-    isobject "^3.0.0"
-
 untildify@^4.0.0:
   version "4.0.0"
   resolved "https://registry.yarnpkg.com/untildify/-/untildify-4.0.0.tgz#2bc947b953652487e4600949fb091e3ae8cd919b"
   integrity sha512-KK8xQ1mkzZeg9inewmFVDNkg3l5LUhoq9kN6iWYB/CC9YMG8HA+c1Q8HwDe6dEX7kErrEVNVBO3fWsVq5iDgtw==
 
-update-browserslist-db@^1.1.0:
-  version "1.1.0"
-  resolved "https://registry.yarnpkg.com/update-browserslist-db/-/update-browserslist-db-1.1.0.tgz#7ca61c0d8650766090728046e416a8cde682859e"
-  integrity sha512-EdRAaAyk2cUE1wOf2DkEhzxqOQvFOoRJFNS6NeyJ01Gp2beMRpBAINjM2iDXE3KCuKhwnvHIQCJm6ThL2Z+HzQ==
-  dependencies:
-    escalade "^3.1.2"
-    picocolors "^1.0.1"
-
 update-notifier@^5.1.0:
   version "5.1.0"
   resolved "https://registry.yarnpkg.com/update-notifier/-/update-notifier-5.1.0.tgz#4ab0d7c7f36a231dd7316cf7729313f0214d9ad9"
@@ -6596,15 +5864,6 @@ uri-js@^4.2.2:
   dependencies:
     punycode "^2.1.0"
 
-url-loader@^4.1.0:
-  version "4.1.1"
-  resolved "https://registry.yarnpkg.com/url-loader/-/url-loader-4.1.1.tgz#28505e905cae158cf07c92ca622d7f237e70a4e2"
-  integrity sha512-3BTV812+AVHHOJQO8O5MkWgZ5aosP7GnROJwvzLS9hWDj00lZ6Z0wNak423Lp9PBZN05N+Jk/N5Si8jRAlGyWA==
-  dependencies:
-    loader-utils "^2.0.0"
-    mime-types "^2.1.27"
-    schema-utils "^3.0.0"
-
 url-parse-lax@^3.0.0:
   version "3.0.0"
   resolved "https://registry.yarnpkg.com/url-parse-lax/-/url-parse-lax-3.0.0.tgz#16b5cafc07dbe3676c1b1999177823d6503acb0c"
@@ -6648,7 +5907,7 @@ utf7@^1.0.2:
   dependencies:
     semver "~5.3.0"
 
-util-deprecate@^1.0.1, util-deprecate@^1.0.2, util-deprecate@~1.0.1:
+util-deprecate@^1.0.1, util-deprecate@~1.0.1:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf"
   integrity sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=
@@ -6725,7 +5984,7 @@ whatwg-url@^5.0.0:
     tr46 "~0.0.3"
     webidl-conversions "^3.0.0"
 
-which@2.0.2, which@^2.0.1, which@^2.0.2:
+which@2.0.2, which@^2.0.2:
   version "2.0.2"
   resolved "https://registry.yarnpkg.com/which/-/which-2.0.2.tgz#7c6a8dd0a636a0327e10b59c9286eee93f3f51b1"
   integrity sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==
@@ -6762,15 +6021,6 @@ workerpool@6.2.0:
   resolved "https://registry.yarnpkg.com/workerpool/-/workerpool-6.2.0.tgz#827d93c9ba23ee2019c3ffaff5c27fccea289e8b"
   integrity sha512-Rsk5qQHJ9eowMH28Jwhe8HEbmdYDX4lwoMWshiCXugjtHqMD9ZbiqSDLxcsfdqsETPzVUtX5s1Z5kStiIM6l4A==
 
-"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0":
-  version "7.0.0"
-  resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43"
-  integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==
-  dependencies:
-    ansi-styles "^4.0.0"
-    string-width "^4.1.0"
-    strip-ansi "^6.0.0"
-
 wrap-ansi@^7.0.0:
   version "7.0.0"
   resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43"
@@ -6780,15 +6030,6 @@ wrap-ansi@^7.0.0:
     string-width "^4.1.0"
     strip-ansi "^6.0.0"
 
-wrap-ansi@^8.1.0:
-  version "8.1.0"
-  resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-8.1.0.tgz#56dc22368ee570face1b49819975d9b9a5ead214"
-  integrity sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==
-  dependencies:
-    ansi-styles "^6.1.0"
-    string-width "^5.0.1"
-    strip-ansi "^7.0.1"
-
 wrappy@1:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f"
@@ -6844,28 +6085,6 @@ xlsx@^0.18.5:
     wmf "~1.0.1"
     word "~0.3.0"
 
-xml-js@^1.6.11:
-  version "1.6.11"
-  resolved "https://registry.yarnpkg.com/xml-js/-/xml-js-1.6.11.tgz#927d2f6947f7f1c19a316dd8eea3614e8b18f8e9"
-  integrity sha512-7rVi2KMfwfWFl+GpPg6m80IVMWXLRjO+PxTq7V2CDhoGak0wzYzFgUY2m4XJ47OGdXd8eLE8EmwfAmdjw7lC1g==
-  dependencies:
-    sax "^1.2.4"
-
-xml-lexer@^0.2.2:
-  version "0.2.2"
-  resolved "https://registry.yarnpkg.com/xml-lexer/-/xml-lexer-0.2.2.tgz#518193a4aa334d58fc7d248b549079b89907e046"
-  integrity sha512-G0i98epIwiUEiKmMcavmVdhtymW+pCAohMRgybyIME9ygfVu8QheIi+YoQh3ngiThsT0SQzJT4R0sKDEv8Ou0w==
-  dependencies:
-    eventemitter3 "^2.0.0"
-
-xml-reader@2.4.3:
-  version "2.4.3"
-  resolved "https://registry.yarnpkg.com/xml-reader/-/xml-reader-2.4.3.tgz#9f810caf7c425a5aafb848b1c45103c9e71d7530"
-  integrity sha512-xWldrIxjeAMAu6+HSf9t50ot1uL5M+BtOidRCWHXIeewvSeIpscWCsp4Zxjk8kHHhdqFBrfK8U0EJeCcnyQ/gA==
-  dependencies:
-    eventemitter3 "^2.0.0"
-    xml-lexer "^0.2.2"
-
 xml2js@0.4.19:
   version "0.4.19"
   resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.19.tgz#686c20f213209e94abf0d1bcf1efaa291c7827a7"
@@ -6889,7 +6108,7 @@ xoauth2@^1.2.0:
   resolved "https://registry.yarnpkg.com/xoauth2/-/xoauth2-1.2.0.tgz#f2eefac11472c971ea3bc46e554eb4b1232146e5"
   integrity sha512-hKuNbkj3q/ifCcfWnW6KURP+6ExSuLdLG007gasNhMEMKlLaejNkIA6eu5Ol1xPP0/kzTuA87XHDaAcUw5k73Q==
 
-xtend@^4.0.2, xtend@~4.0.0, xtend@~4.0.1:
+xtend@^4.0.0, xtend@^4.0.2, xtend@~4.0.0, xtend@~4.0.1:
   version "4.0.2"
   resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.2.tgz#bb72779f5fa465186b1f438f674fa347fdb5db54"
   integrity sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==
@@ -6899,11 +6118,6 @@ y18n@^5.0.5:
   resolved "https://registry.yarnpkg.com/y18n/-/y18n-5.0.8.tgz#7f4934d0f7ca8c56f95314939ddcd2dd91ce1d55"
   integrity sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==
 
-yallist@^3.0.2:
-  version "3.1.1"
-  resolved "https://registry.yarnpkg.com/yallist/-/yallist-3.1.1.tgz#dbb7daf9bfd8bac9ab45ebf602b8cbad0d5d08fd"
-  integrity sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==
-
 yallist@^4.0.0:
   version "4.0.0"
   resolved "https://registry.yarnpkg.com/yallist/-/yallist-4.0.0.tgz#9bb92790d9c0effec63be73519e11a35019a3a72"
@@ -6942,6 +6156,14 @@ yargs@16.2.0, yargs@^16.1.0:
     y18n "^5.0.5"
     yargs-parser "^20.2.2"
 
+yauzl@^2.4.2:
+  version "2.10.0"
+  resolved "https://registry.yarnpkg.com/yauzl/-/yauzl-2.10.0.tgz#c7eb17c93e112cb1086fa6d8e51fb0667b79a5f9"
+  integrity sha512-p4a9I6X6nu6IhoGmBqAcbJy1mlC4j27vEPZX9F4L4/vZT3Lyq1VkFHw/V/PUcB9Buo+DG3iHkT0x3Qya58zc3g==
+  dependencies:
+    buffer-crc32 "~0.2.3"
+    fd-slicer "~1.1.0"
+
 yn@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/yn/-/yn-2.0.0.tgz#e5adabc8acf408f6385fc76495684c88e6af689a"

From de0d9785cf80dcc7ad2114392afe1310b5af42f8 Mon Sep 17 00:00:00 2001
From: chime3 
Date: Thu, 29 Aug 2024 00:41:57 -0700
Subject: [PATCH 067/182] fix: added exception and skipped folders

---
 src/providers/google/gdrive-document.ts |  2 +-
 src/providers/google/helpers.ts         | 19 ++++++++++++-------
 2 files changed, 13 insertions(+), 8 deletions(-)

diff --git a/src/providers/google/gdrive-document.ts b/src/providers/google/gdrive-document.ts
index 3b34c6be..57f25aa3 100644
--- a/src/providers/google/gdrive-document.ts
+++ b/src/providers/google/gdrive-document.ts
@@ -60,7 +60,7 @@ export default class GoogleDriveDocument extends GoogleHandler {
         let query: drive_v3.Params$Resource$Files$List = {
             pageSize: this.config.batchSize,
             fields: 'nextPageToken, files(id, name, mimeType, modifiedTime, webViewLink, thumbnailLink)',
-            q: "", // Fetch all files without restricting mimeType
+            q: "mimeType != 'application/vnd.google-apps.folder'", // Fetch all files without restricting mimeType
             orderBy: "modifiedTime desc", // Fetch files ordered by modifiedTime descending
         };
     
diff --git a/src/providers/google/helpers.ts b/src/providers/google/helpers.ts
index 37d2dc40..f1cee428 100644
--- a/src/providers/google/helpers.ts
+++ b/src/providers/google/helpers.ts
@@ -292,17 +292,22 @@ export class GoogleDriveHelpers {
     fileId: string
   ): Promise {
     const file = await this.getFile(drive, fileId);
-
+  
     if (file.size) {
       // For non-Google docs (like PDF, image)
       return parseInt(file.size);
     } else if (file.mimeType && file.mimeType.startsWith("application/vnd.google-apps.")) {
-      // For Google Docs, export the file as plain text to estimate size
-      const exportedFile = await drive.files.export(
-        { fileId: fileId, mimeType: "text/plain" }, 
-        { responseType: "arraybuffer" }
-      );
-      return Buffer.byteLength(exportedFile.data as ArrayBuffer);
+      try {
+        // For Google Docs, export the file as plain text to estimate size
+        const exportedFile = await drive.files.export(
+          { fileId: fileId, mimeType: "text/plain" }, 
+          { responseType: "arraybuffer" }
+        );
+        return Buffer.byteLength(exportedFile.data as ArrayBuffer);
+      } catch (error) {
+        // Ignore the file if there's an error during export
+        return undefined;
+      }
     } else {
       return undefined;
     }

From 00060210de4db0615b61c49c7fa72e08d5971678 Mon Sep 17 00:00:00 2001
From: chime3 
Date: Thu, 29 Aug 2024 19:08:18 -0700
Subject: [PATCH 068/182] fix: remove comments

---
 src/providers/google/index.ts | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/providers/google/index.ts b/src/providers/google/index.ts
index cb087e8b..cad84cad 100644
--- a/src/providers/google/index.ts
+++ b/src/providers/google/index.ts
@@ -28,10 +28,10 @@ export default class GoogleProvider extends Base {
 
   public syncHandlers(): any[] {
     return [
-      //Gmail,
-      //YouTubeFollowing,
-      //YouTubePost,
-      //YouTubeFavourite,
+      Gmail,
+      YouTubeFollowing,
+      YouTubePost,
+      YouTubeFavourite,
       GoogleDriveDocument
     ];
   }

From 08f19a1a3185e4ed81d1616fbe0cc7b6f0ef7a72 Mon Sep 17 00:00:00 2001
From: Chris 
Date: Fri, 30 Aug 2024 15:39:42 +0930
Subject: [PATCH 069/182] Improve performance and quality of personal prompt
 with telegram message threads. Better handle unexpeted LLM issues.

---
 src/providers/telegram/chat-message.ts |   8 +-
 src/services/assistants/search.ts      |  29 +----
 src/services/data.ts                   |  28 ++++-
 src/services/llm.ts                    |   4 +-
 src/services/search.ts                 | 142 +++++++++++++------------
 src/services/tools/emailShortlist.ts   |  11 +-
 src/services/tools/promptSearch.ts     |   3 +-
 7 files changed, 116 insertions(+), 109 deletions(-)

diff --git a/src/providers/telegram/chat-message.ts b/src/providers/telegram/chat-message.ts
index c3286494..d9a4d2ac 100644
--- a/src/providers/telegram/chat-message.ts
+++ b/src/providers/telegram/chat-message.ts
@@ -59,12 +59,8 @@ export default class TelegramChatMessageHandler extends BaseSyncHandler {
     // Fetch all the latest chat groups, fetches 500 by default
     const latestChatGroupIds = await api.getChatGroupIds()
 
-    // Append the chat group list with any new chat groups so we don't miss any
-    for (const groupId of latestChatGroupIds) {
-      if (chatGroupIds.indexOf(groupId) === -1) {
-        chatGroupIds.push(groupId)
-      }
-    }
+    // Make sure we process new groups first
+    chatGroupIds = latestChatGroupIds
 
     // Build chat group data for each group ID
     const chatGroupResults = await this.buildChatGroupResults(api, chatGroupIds, this.config.groupLimit)
diff --git a/src/services/assistants/search.ts b/src/services/assistants/search.ts
index 05be9d55..e81326be 100644
--- a/src/services/assistants/search.ts
+++ b/src/services/assistants/search.ts
@@ -28,12 +28,9 @@ export class PromptSearchService extends VeridaService {
         duration: number,
         process: PromptSearchLLMResponse
     }> {
-        console.time("PersonalPromptStart")
         const start = Date.now()
         const promptSearch = new PromptSearch(llm)
-        console.time("KeywordPrompt")
         const promptSearchResult = await promptSearch.search(prompt)
-        console.timeEnd("KeywordPrompt")
 
         console.log(promptSearchResult)
 
@@ -46,9 +43,8 @@ export class PromptSearchService extends VeridaService {
 
         const searchService = new SearchService(this.did, this.context)
 
-        console.time("DataFetch")
         if (promptSearchResult.search_type == PromptSearchType.KEYWORDS) {
-            console.time("DataFetchKeywords")
+            console.log(`Searching by keywords: ${promptSearchResult.keywords!}`)
             if (promptSearchResult.databases.indexOf(SearchType.EMAILS) !== -1) {
                 emails = await searchService.schemaByKeywords(SearchType.EMAILS, promptSearchResult.keywords!, promptSearchResult.timeframe, 40)
             }
@@ -64,17 +60,14 @@ export class PromptSearchService extends VeridaService {
             if (promptSearchResult.databases.indexOf(SearchType.CHAT_MESSAGES) !== -1) {
                 chatThreads = await searchService.chatThreadsByKeywords(promptSearchResult.keywords!, promptSearchResult.timeframe, 10, 20)
             }
-            console.timeEnd("DataFetchKeywords")
         } else {
-            console.time("DataFetchDaterange")
             const maxDatetime = Helpers.keywordTimeframeToDate(promptSearchResult.timeframe)
             const sort = promptSearchResult.sort == PromptSearchSort.RECENT ? SearchSortType.RECENT : SearchSortType.OLDEST
+            console.log(`Searching by timeframe: ${maxDatetime} ${sort}`)
             if (promptSearchResult.databases.indexOf(SearchType.EMAILS) !== -1) {
                 emails = await searchService.schemaByDateRange(SearchType.EMAILS, maxDatetime, sort, MAX_DATERANGE_EMAILS*3)
                 const emailShortlist = new EmailShortlist(llm)
-                console.time("EmailShortlist")
                 emails = await emailShortlist.shortlist(prompt, emails, MAX_DATERANGE_EMAILS)
-                console.timeEnd("EmailShortlist")
             }
             // if (promptSearchResult.databases.indexOf("files")) {
             //     files = await searchService.schemaByDateRange(SearchType.FILES, maxDatetime, sort, MAX_DATERANGE_FILES)
@@ -88,9 +81,10 @@ export class PromptSearchService extends VeridaService {
             if (promptSearchResult.databases.indexOf(SearchType.CHAT_MESSAGES) !== -1) {
                 chatMessages =  await searchService.schemaByDateRange(SearchType.CHAT_MESSAGES, maxDatetime, sort, MAX_DATERANGE_CHAT_MESSAGES)
             }
-            console.timeEnd("DataFetchDaterange")
         }
-        console.timeEnd("DataFetch")
+
+        console.log('emails / favourites / following / chatThreads')
+        console.log(emails.length, favourites.length, following.length, chatThreads.length)
 
         let finalPrompt = `Answer this prompt:\n${prompt}\nHere are some recent messages that may help you provide a relevant answer.\n`
         let contextString = ''
@@ -110,18 +104,14 @@ export class PromptSearchService extends VeridaService {
             contextString += `From: ${chatMessage.fromName} <${chatMessage.fromHandle}> (${chatMessage.groupName})\nBody: ${chatMessage.messageText}\n\n`
         }
 
-        console.log('favourites: ', favourites.length)
         for (const favourite of favourites) {
             contextString += `Favorite: ${favourite.name} ${favourite.description?.substring(0,100)} (via ${favourite.sourceApplication})\n\n`
         }
 
-        console.log('following: ', following.length)
         for (const follow of following) {
             contextString += `Following: ${follow.name} ${follow.description?.substring(0,100)} (via ${follow.sourceApplication})\n\n`
         }
 
-        // console.log('pre-email context string: ', contextString.length)
-
         let emailCount = 0
         for (const email of emails) {
             let extraContext = ""
@@ -133,7 +123,6 @@ export class PromptSearchService extends VeridaService {
             }
 
             extraContext = `To: ${email.toName} <${email.toEmail}>\nFrom: ${email.fromName} <${email.fromEmail}> (${email.name})\nBody: ${body}\n\n`
-            // console.log(email.fromName, email.fromEmail, email.name, body.length, email.messageText.length)
             if ((extraContext.length + contextString.length + finalPrompt.length) > MAX_CONTEXT_LENGTH) {
                 break
             }
@@ -142,20 +131,12 @@ export class PromptSearchService extends VeridaService {
             emailCount++
         }
 
-        // console.log('email count', emailCount)
-
         const now = (new Date()).toISOString()
         finalPrompt += `${contextString}\nThe current time is: ${now}`
 
-        console.log('Running final prompt', finalPrompt.length)
-        console.time("FinalPrompt")
         const finalResponse = await llm.prompt(finalPrompt, undefined, false)
-        console.timeEnd("FinalPrompt")
         const duration = Date.now() - start
 
-        // console.log(contextString)
-
-        console.timeEnd("PersonalPromptStart")
         return {
             result: finalResponse.choices[0].message.content!,
             duration,
diff --git a/src/services/data.ts b/src/services/data.ts
index 484923c7..8b59f316 100644
--- a/src/services/data.ts
+++ b/src/services/data.ts
@@ -1,7 +1,7 @@
 import { IContext, IDatastore } from '@verida/types';
 import * as CryptoJS from 'crypto-js';
 import { EventEmitter } from 'events'
-import MiniSearch from 'minisearch';
+import MiniSearch, { SearchOptions, SearchResult } from 'minisearch';
 
 export const indexCache: Record> = {}
 
@@ -85,6 +85,29 @@ export class DataService extends EventEmitter {
         return this.context.openDatastore(schemaUri)
     }
 
+    public async searchIndex(schemaUri: string, query: string, maxResults: number = 50, cutoffPercent: number = 0.5, searchOptions?: SearchOptions): Promise {
+        const miniSearchIndex = await this.getIndex(schemaUri)
+        const searchResults = await miniSearchIndex.search(query, searchOptions)
+        
+        if (!searchResults.length) {
+            return []
+        }
+
+        const results: any[] = []
+        const cutoffScore = searchResults[0].score * cutoffPercent
+        for (const result of searchResults) {
+            if (result.score < cutoffScore || results.length >= maxResults) {
+                break
+            }
+
+            results.push(result)
+        }
+
+        // console.log(schemaUri, results)
+
+        return results
+    }
+
     public async getIndex(schemaUri: string): Promise> {
         const schemaConfig = schemas[schemaUri]
         const indexFields = schemaConfig.indexFields
@@ -96,7 +119,6 @@ export class DataService extends EventEmitter {
             this.emitProgress(schemaConfig.label, HotLoadStatus.StartData, 10)
             const datastore = await this.context.openDatastore(schemaUri)
 
-            console.log('Fetching data from index ', schemaUri)
             const database = await datastore.getDb()
             const db = await database.getDb()
             const result = await db.allDocs({
@@ -177,6 +199,8 @@ export class DataService extends EventEmitter {
             this.emitProgress(schemaConfig.label, HotLoadStatus.Complete, 10)
 
             indexCache[cacheKey] = miniSearch
+
+            console.log(`Index created for ${schemaUri}`)
         } else {
             this.stepCount += 2
             this.emitProgress(schemaConfig.label, HotLoadStatus.Complete, 10)
diff --git a/src/services/llm.ts b/src/services/llm.ts
index 54930616..af9e5865 100644
--- a/src/services/llm.ts
+++ b/src/services/llm.ts
@@ -93,7 +93,7 @@ export class GroqLLM implements LLM {
       temperature: 1,
       top_p: 1
     });
-    console.log(JSON.stringify(response, null, 2))
+
     return response
 }
 }
@@ -141,8 +141,6 @@ export class OpenAILLM implements LLM {
       headers
     });
 
-    // return response.data.choices[0].message.content
-    console.log(JSON.stringify(response.data, null, 2))
     return response.data
   }
 }
diff --git a/src/services/search.ts b/src/services/search.ts
index 6db5e409..6caa5824 100644
--- a/src/services/search.ts
+++ b/src/services/search.ts
@@ -64,7 +64,6 @@ export interface ChatThreadResult {
 export class SearchService extends VeridaService {
 
     protected async rankAndMergeResults(schemaResults: SearchServiceSchemaResult[], limit: number, minResultsPerType: number = 10): Promise {
-        console.time("RankAndMerge")
         const unsortedResults: Record = {}
         const guaranteedResults: Record = {}
 
@@ -72,7 +71,6 @@ export class SearchService extends VeridaService {
         for (const schemaResult of schemaResults) {
             let schemaResultCount = 0
             for (const row of schemaResult.rows) {
-                // console.log(row.id, row.score)
                 const result = {
                     ...row,
                     schemaUrl: SearchTypeSchemas[schemaResult.searchType]
@@ -90,7 +88,6 @@ export class SearchService extends VeridaService {
         }
 
         const unsortedResultCount = Object.values(unsortedResults).length
-        console.log(`Have ${unsortedResultCount} unsorted schema results`)
         if (unsortedResultCount == 0) {
             return []
         }
@@ -118,24 +115,20 @@ export class SearchService extends VeridaService {
             })
         }
 
-        console.timeEnd("RankAndMerge")
         return results
     }
 
     public async schemaByKeywords(searchType: SearchType, keywordsList: string[], timeframe: KeywordSearchTimeframe, limit: number = 20): Promise {
-        console.time("SchemaByKeywords" + searchType, )
         const query = keywordsList.join(' ')
         const schemaUri = SearchTypeSchemas[searchType]
         const dataService = new DataService(this.did, this.context)
-        const miniSearchIndex = await dataService.getIndex(schemaUri)
 
         const maxDatetime = Helpers.keywordTimeframeToDate(timeframe)
-        console.log(`${searchType}: Searching for ${query} (${timeframe})`)
         
-        const searchResults = await miniSearchIndex.search(query, {
+        const searchResults = await dataService.searchIndex(schemaUri, query, limit, undefined, {
             filter: (result: any) => maxDatetime ? result.sentAt > maxDatetime.toISOString() : true
         })
-        console.timeEnd("SchemaByKeywords" + searchType)
+
         return await this.rankAndMergeResults([{
             searchType,
             rows: searchResults
@@ -160,7 +153,6 @@ export class SearchService extends VeridaService {
             ]
         }
 
-        console.log(searchType, ': searching for', filter, options)
         return await datastore.getMany(filter, options) as T[]
     }
 
@@ -172,7 +164,6 @@ export class SearchService extends VeridaService {
         const miniSearchIndex = await dataService.getIndex(schemaUri)
 
         const maxDatetime = Helpers.keywordTimeframeToDate(timeframe)
-        console.log('Chat history: searching for', query, timeframe, maxDatetime)
         
         const searchResults = await miniSearchIndex.search(query, {
             filter: (result: any) => maxDatetime ? result.sentAt > maxDatetime.toISOString() : true
@@ -200,50 +191,16 @@ export class SearchService extends VeridaService {
         const messageSchemaUri = "https://common.schemas.verida.io/social/chat/message/v0.1.0/schema.json"
         const groupSchemaUri = "https://common.schemas.verida.io/social/chat/group/v0.1.0/schema.json"
         const dataService = new DataService(this.did, this.context)
-        const miniSearchIndex = await dataService.getIndex(messageSchemaUri)
 
         const maxDatetime = Helpers.keywordTimeframeToDate(timeframe)
-        console.log('Chat threads: searching for', query, timeframe, maxDatetime)
         
-        const searchResults = await miniSearchIndex.search(query, {
+        const searchResults = await dataService.searchIndex(messageSchemaUri, query, 50, 0.5, {
             filter: (result: any) => maxDatetime ? result.sentAt > maxDatetime.toISOString() : true
         })
+        
         const chatMessageDs = await this.context.openDatastore(messageSchemaUri)
         const chatGroupDs = await this.context.openDatastore(groupSchemaUri)
 
-        // Create a thread for each message
-        async function buildThread(messageId: string, groupId: string): Promise {
-            const maxMessages = Math.round(threadSize / 2)
-            const startMessages =  await chatMessageDs.getMany({
-                _id: {
-                    "$lte": messageId
-                },
-                groupId
-            }, {
-                limit: maxMessages,
-                sort: [{_id: "desc"}]
-            })
-            const lastMessages =  await chatMessageDs.getMany({
-                _id: {
-                    "$gt": messageId
-                },
-                groupId
-            }, {
-                limit: maxMessages,
-                sort: [{_id: "asc"}]
-            })
-
-            const messages: SchemaSocialChatMessage[] = []
-            for (const message of startMessages.concat(lastMessages)) {
-                delete message['sourceData']
-                messages.push( message)
-            }
-
-            // @todo: if not enough messages, fetch more to match threadSize
-
-            return messages
-        }
-
         const groupCache: Record = {}
         async function getGroup(groupId: string) {
             if (groupCache[groupId]) {
@@ -256,44 +213,97 @@ export class SearchService extends VeridaService {
             return groupCache[groupId]
         }
 
+        // Build a list of messages for each chat group
+        const chatThreadMessageIds: Record = {}
+        // Build a chat group of results for each chat group
         const chatThreads: Record = {}
-        let foundThreads = 0
         for (const searchResult of searchResults) {
-            const messages = await buildThread(searchResult.id, searchResult.groupId)
-
             if (!chatThreads[searchResult.groupId]) {
                 chatThreads[searchResult.groupId] = {
-                    group: await getGroup(messages[0].groupId),
+                    group: await getGroup(searchResult.groupId),
                     messages: []
                 }
             }
 
-            chatThreads[searchResult.groupId].messages = chatThreads[searchResult.groupId].messages.concat(messages)
-
-            if (foundThreads++ >= limit) {
-                break
+            if (!chatThreadMessageIds[searchResult.groupId]) {
+                chatThreadMessageIds[searchResult.groupId] = []
             }
+
+            chatThreadMessageIds[searchResult.groupId].push(searchResult.id)
         }
 
-        const results: ChatThreadResult[] = []
-        for (const chatThread of Object.values(chatThreads)) {
-            // Remove duplicate messages
-            chatThread.messages = _.uniqBy(chatThread.messages, '_id')
-            // Sort chat thread messages by _id
-            chatThread.messages = _.sortBy(chatThread.messages, '_id')
-            results.push(chatThread)
+        async function fetchMessagesWithContext(db: any, groupId: string, messageIds: string[], windowSize = 10) {
+            const results: SchemaSocialChatMessage[] = []
+            const fetchedMessages = new Set(); // Track fetched messages
+        
+            // Sort the message IDs lexicographically
+            const sortedMessageIds = messageIds.sort();
+        
+            // Determine the range for initial query
+            const startKey = sortedMessageIds[0];
+            const endKey = sortedMessageIds[sortedMessageIds.length - 1];
+        
+            let lastFetchedKey = startKey; // Track the last fetched message ID
+        
+            try {
+            // Use a sliding window to fetch messages in chunks
+            for (const messageId of sortedMessageIds) {
+                if (lastFetchedKey > messageId) {
+                    continue
+                }
+
+                const response = await db.find({
+                    selector: {
+                        groupId: groupId,
+                        _id: { $gte: lastFetchedKey, $lte: endKey }
+                    },
+                    sort: [{ _id: 'asc' }],
+                    limit: windowSize
+                });
+        
+                // Break the loop if no more messages are found
+                if (response.docs.length === 0) break;
+        
+                // Step 4: Process and collect fetched messages
+                response.docs.forEach((doc: any) => {
+                if (!fetchedMessages.has(doc._id)) {
+                    results.push(doc);
+                    fetchedMessages.add(doc._id); // Mark this message ID as fetched
+                }
+                });
+        
+                // Update the last fetched key to the next message ID for the sliding window
+                lastFetchedKey = response.docs[response.docs.length - 1]._id;
+        
+                // Slide the window to the next set of messages
+                if (lastFetchedKey === endKey) break; // Stop if the last fetched key reaches the end
+            }
+        
+            } catch (err) {
+                console.error(`Error fetching messages for group ${groupId}:`, err);
+            }
+        
+            // console.log(`Finished processing group: ${groupId} (${fetchedMessages.values.length})`);
+
+            return results
+          }
+
+        // Process each chat group
+        const veridaDb = await chatMessageDs.getDb()
+        const pouchDb = await veridaDb.getDb()
+
+        for (const groupId in chatThreadMessageIds) {
+            chatThreads[groupId].messages = await fetchMessagesWithContext(pouchDb, groupId, chatThreadMessageIds[groupId])
+            break
         }
 
+        const results = Object.values(chatThreads)
         return results
-        
     }
 
     public async multiByKeywords(searchTypes: SearchType[], keywordsList: string[], timeframe: KeywordSearchTimeframe, limit: number = 20, minResultsPerType: number = 10) {
         const query = keywordsList.join(' ')
         const dataService = new DataService(this.did, this.context)
-        const maxTimeframe = 
-
-        console.log('Multi: searching for', query)
 
         const searchResults = []
         for (const searchType of searchTypes) {
diff --git a/src/services/tools/emailShortlist.ts b/src/services/tools/emailShortlist.ts
index 7460a962..85f3620e 100644
--- a/src/services/tools/emailShortlist.ts
+++ b/src/services/tools/emailShortlist.ts
@@ -8,7 +8,7 @@ You must generate a JSON object containing a single key "emailIds" that contains
 Data is in the format:
 [emailId] 
 
-JSON only, no explanation.`
+JSON only, no explanation, no formatting.`
 
 const MAX_EMAILS = 200
 
@@ -22,7 +22,6 @@ export class EmailShortlist {
     }
 
     public async shortlist(originalPrompt: string, emails: SchemaEmail[], limit=20): Promise {
-        console.log("shortlist")
         let userPrompt = `${originalPrompt}\n\n`
         const emailDict: Record = {}
         let emailLimit = MAX_EMAILS
@@ -37,16 +36,16 @@ export class EmailShortlist {
 
         userPrompt += `\nMaximum of ${limit} email IDs`
         const response = await this.llm.prompt(userPrompt, systemPrompt)
-        console.log(response.choices[0])
         const jsonResponse: any = JSON.parse(response.choices[0].message.content!)
-        console.log(jsonResponse)
         const emailIds = jsonResponse.emailIds
-        console.log(emailIds)
         const result: SchemaEmail[] = []
 
         for (const emailId of emailIds) {
+            if (!emailDict[emailId]) {
+                continue
+            }
+
             result.push(emailDict[emailId])
-            console.log(emailDict[emailId].name)
             limit--
             if (limit <= 0) {
                 break
diff --git a/src/services/tools/promptSearch.ts b/src/services/tools/promptSearch.ts
index aafbd587..06a7120d 100644
--- a/src/services/tools/promptSearch.ts
+++ b/src/services/tools/promptSearch.ts
@@ -12,7 +12,7 @@ You must generate a JSON response containing the following information:
 - output_type: The amount of detail in the output of each search result to provide meaningful context. full_content, summary, headline
 - profile_information; Array of these options only; name, contactInfo, demographics, lifestyle, preferences, habits, financial, health, personality, employment, education, skills, language, interests
 
-JSON only, no explanation.`
+JSON only, no explanation or formatting.`
 
 export enum PromptSearchType {
   KEYWORDS = "keywords",
@@ -55,7 +55,6 @@ export class PromptSearch {
 
     public async search(userPrompt: string): Promise {
         const response = await this.llm.prompt(userPrompt, systemPrompt)
-        console.log(response.choices[0])
         return  JSON.parse(response.choices[0].message.content!)
         
     }

From 94118775a1bcf58a872f75f55a58a13236d8a5b0 Mon Sep 17 00:00:00 2001
From: Chris 
Date: Fri, 30 Aug 2024 17:11:25 +0930
Subject: [PATCH 070/182] Fix issue with most importnt search results not being
 returned

---
 src/services/search.ts | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/services/search.ts b/src/services/search.ts
index 6caa5824..ae232bd9 100644
--- a/src/services/search.ts
+++ b/src/services/search.ts
@@ -87,11 +87,6 @@ export class SearchService extends VeridaService {
             datastores[schemaUri] = await this.context.openDatastore(schemaUri)
         }
 
-        const unsortedResultCount = Object.values(unsortedResults).length
-        if (unsortedResultCount == 0) {
-            return []
-        }
-
         // Sort results by score
         const sortedResults = Object.values(unsortedResults)
         sortedResults.sort((a: any, b: any) => b.score - a.score)
@@ -115,6 +110,7 @@ export class SearchService extends VeridaService {
             })
         }
 
+        // console.log('returning ', results.length, 'items')
         return results
     }
 
@@ -124,11 +120,15 @@ export class SearchService extends VeridaService {
         const dataService = new DataService(this.did, this.context)
 
         const maxDatetime = Helpers.keywordTimeframeToDate(timeframe)
+
+        console.log(query, maxDatetime)
         
         const searchResults = await dataService.searchIndex(schemaUri, query, limit, undefined, {
             filter: (result: any) => maxDatetime ? result.sentAt > maxDatetime.toISOString() : true
         })
 
+        console.log(searchResults)
+
         return await this.rankAndMergeResults([{
             searchType,
             rows: searchResults

From b87bd8349f6552d3e129e769a3234cc791b70125 Mon Sep 17 00:00:00 2001
From: Chris 
Date: Mon, 2 Sep 2024 10:09:01 +0930
Subject: [PATCH 071/182] Resolve issue with google handlers where the last
 page of results would be processed over and over again for each sync

---
 src/providers/google/gmail.ts             | 10 +++-------
 src/providers/google/youtube-favourite.ts |  9 +++------
 src/providers/google/youtube-following.ts |  9 +++------
 src/providers/google/youtube-post.ts      |  9 +++------
 4 files changed, 12 insertions(+), 25 deletions(-)

diff --git a/src/providers/google/gmail.ts b/src/providers/google/gmail.ts
index 174ed2be..347f5f8d 100644
--- a/src/providers/google/gmail.ts
+++ b/src/providers/google/gmail.ts
@@ -111,18 +111,14 @@ export default class Gmail extends GoogleHandler {
       }, false) // No results and first batch, so break ID couldn't have been hit
     }
 
-    if (items.length != this.config.batchSize) {
+    currentRange = rangeTracker.nextRange();
+    if (items.length != this.config.batchSize && currentRange.startId) {
       // Not enough items, fetch more from the next page of results
-      currentRange = rangeTracker.nextRange()
-
       query = {
         userId: "me",
         maxResults: this.config.batchSize - items.length, // only fetch enough items needed to complete the batch size
+        pageToken: currentRange.startId
       };
-  
-      if (currentRange.startId) {
-        query.pageToken = currentRange.startId
-      }
 
       const backfillResponse = await gmail.users.messages.list(query);
       const backfillResult = await this.buildResults(
diff --git a/src/providers/google/youtube-favourite.ts b/src/providers/google/youtube-favourite.ts
index 7cfdc785..1bda2c6a 100644
--- a/src/providers/google/youtube-favourite.ts
+++ b/src/providers/google/youtube-favourite.ts
@@ -99,18 +99,15 @@ export default class YouTubeFavourite extends GoogleHandler {
             }, false);
         }
 
-        if (items.length != this.config.batchSize) {
-            currentRange = rangeTracker.nextRange();
+        currentRange = rangeTracker.nextRange();
+        if (items.length != this.config.batchSize && currentRange.startId) {
             query = {
                 part: ["snippet", "contentDetails"],
                 myRating: "like",
                 maxResults: this.config.batchSize - items.length,
+                pageToken: currentRange.startId
             };
 
-            if (currentRange.startId) {
-                query.pageToken = currentRange.startId;
-            }
-
             const backfillResponse = await youtube.videos.list(query);
             const backfillResult = await this.buildResults(
                 backfillResponse,
diff --git a/src/providers/google/youtube-following.ts b/src/providers/google/youtube-following.ts
index 85a3272a..24628d82 100644
--- a/src/providers/google/youtube-following.ts
+++ b/src/providers/google/youtube-following.ts
@@ -88,18 +88,15 @@ export default class YouTubeFollowing extends GoogleHandler {
             }, false);
         }
 
-        if (items.length != this.config.batchSize) {
-            currentRange = rangeTracker.nextRange();
+        currentRange = rangeTracker.nextRange();
+        if (items.length != this.config.batchSize && currentRange.startId) {
             query = {
                 part: ["snippet"],
                 mine: true,
                 maxResults: this.config.batchSize - items.length,
+                pageToken: currentRange.startId
             };
 
-            if (currentRange.startId) {
-                query.pageToken = currentRange.startId;
-            }
-
             const backfillResponse = await youtube.subscriptions.list(query);
             const backfillResult = await this.buildResults(
                 backfillResponse,
diff --git a/src/providers/google/youtube-post.ts b/src/providers/google/youtube-post.ts
index 5e63e64f..a08e05b9 100644
--- a/src/providers/google/youtube-post.ts
+++ b/src/providers/google/youtube-post.ts
@@ -99,18 +99,15 @@ export default class YouTubePost extends GoogleHandler {
             }, false);
         }
 
-        if (items.length != this.config.batchSize) {
-            currentRange = rangeTracker.nextRange();
+        currentRange = rangeTracker.nextRange();
+        if (items.length != this.config.batchSize && currentRange.startId) {
             query = {
                 part: ["snippet", "contentDetails"],
                 mine: true,
                 maxResults: this.config.batchSize - items.length,
+                pageToken: currentRange.startId
             };
 
-            if (currentRange.startId) {
-                query.pageToken = currentRange.startId;
-            }
-
             const backfillResponse = await youtube.activities.list(query);
             const backfillResult = await this.buildResults(
                 backfillResponse,

From 658d577d30599f1ceec59003aa5bc040c3314036 Mon Sep 17 00:00:00 2001
From: Chris 
Date: Mon, 2 Sep 2024 10:10:21 +0930
Subject: [PATCH 072/182] Fix handler log messages always being logged as error

---
 src/providers/BaseProvider.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/providers/BaseProvider.ts b/src/providers/BaseProvider.ts
index 27734211..d755f9ef 100644
--- a/src/providers/BaseProvider.ts
+++ b/src/providers/BaseProvider.ts
@@ -259,7 +259,7 @@ export default class BaseProvider extends EventEmitter {
         syncPosition.status = SyncHandlerStatus.SYNCING
         
         handler.on('log', async (syncLog: SyncProviderLogEvent) => {
-            await providerInstance.logMessage(SyncProviderLogLevel.ERROR, syncLog.message, handler.getName(), schemaUri)
+            await providerInstance.logMessage(syncLog.level, syncLog.message, handler.getName(), schemaUri)
         })
 
         await this.logMessage(SyncProviderLogLevel.DEBUG, `Syncing ${handler.getName()}`, handler.getName(),schemaUri)

From 580011d97d17369226bbc415f10142ddb9eb2893 Mon Sep 17 00:00:00 2001
From: Chris 
Date: Mon, 2 Sep 2024 10:58:12 +0930
Subject: [PATCH 073/182] Enable File RAG for private AI

---
 src/services/assistants/search.ts | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/src/services/assistants/search.ts b/src/services/assistants/search.ts
index 05be9d55..7ec0ff48 100644
--- a/src/services/assistants/search.ts
+++ b/src/services/assistants/search.ts
@@ -3,7 +3,7 @@ import { defaultModel } from "../llm"
 import { PromptSearch, PromptSearchLLMResponse, PromptSearchSort, PromptSearchType } from "../tools/promptSearch"
 import { ChatThreadResult, SearchService, SearchSortType, SearchType } from "../search"
 import { VeridaService } from '../veridaService'
-import { SchemaEmail, SchemaFavourite, SchemaFollowing, SchemaSocialChatMessage } from '../../schemas'
+import { SchemaEmail, SchemaFavourite, SchemaFile, SchemaFollowing, SchemaSocialChatMessage } from '../../schemas'
 import { Helpers } from "../helpers"
 import { EmailShortlist } from "../tools/emailShortlist"
 
@@ -41,7 +41,7 @@ export class PromptSearchService extends VeridaService {
         let emails: SchemaEmail[] = []
         let favourites: SchemaFavourite[] = []
         let following: SchemaFollowing[] = []
-        // let files: SchemaFile[] = []
+        let files: SchemaFile[] = []
         let chatMessages: SchemaSocialChatMessage[] = []
 
         const searchService = new SearchService(this.did, this.context)
@@ -52,9 +52,9 @@ export class PromptSearchService extends VeridaService {
             if (promptSearchResult.databases.indexOf(SearchType.EMAILS) !== -1) {
                 emails = await searchService.schemaByKeywords(SearchType.EMAILS, promptSearchResult.keywords!, promptSearchResult.timeframe, 40)
             }
-            // if (promptSearchResult.databases.indexOf("files")) {
-            //     files = await searchService.schemaByKeywords(SearchType.FILES, promptSearchResult.keywords!, promptSearchResult.timeframe, 20)
-            // }
+            if (promptSearchResult.databases.indexOf(SearchType.FILES)) {
+                files = await searchService.schemaByKeywords(SearchType.FILES, promptSearchResult.keywords!, promptSearchResult.timeframe, 20)
+            }
             if (promptSearchResult.databases.indexOf(SearchType.FAVORITES) !== -1) {
                 favourites = await searchService.schemaByKeywords(SearchType.FAVORITES, promptSearchResult.keywords!, promptSearchResult.timeframe, 40)
             }
@@ -76,9 +76,9 @@ export class PromptSearchService extends VeridaService {
                 emails = await emailShortlist.shortlist(prompt, emails, MAX_DATERANGE_EMAILS)
                 console.timeEnd("EmailShortlist")
             }
-            // if (promptSearchResult.databases.indexOf("files")) {
-            //     files = await searchService.schemaByDateRange(SearchType.FILES, maxDatetime, sort, MAX_DATERANGE_FILES)
-            // }
+            if (promptSearchResult.databases.indexOf(SearchType.FILES)) {
+                files = await searchService.schemaByDateRange(SearchType.FILES, maxDatetime, sort, MAX_DATERANGE_FILES)
+            }
             if (promptSearchResult.databases.indexOf(SearchType.FAVORITES) !== -1) {
                 favourites = await searchService.schemaByDateRange(SearchType.FAVORITES, maxDatetime, sort, MAX_DATERANGE_FAVORITES)
             }
@@ -147,8 +147,8 @@ export class PromptSearchService extends VeridaService {
         const now = (new Date()).toISOString()
         finalPrompt += `${contextString}\nThe current time is: ${now}`
 
-        console.log('Running final prompt', finalPrompt.length)
-        console.time("FinalPrompt")
+        console.log(finalPrompt)
+
         const finalResponse = await llm.prompt(finalPrompt, undefined, false)
         console.timeEnd("FinalPrompt")
         const duration = Date.now() - start

From 1ad0183dfac53c44ec071bfe1c3374717a45ef11 Mon Sep 17 00:00:00 2001
From: Chris 
Date: Mon, 2 Sep 2024 11:01:26 +0930
Subject: [PATCH 074/182] Fix files not processing in private AI

---
 src/services/assistants/search.ts | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/services/assistants/search.ts b/src/services/assistants/search.ts
index 7ec0ff48..84772851 100644
--- a/src/services/assistants/search.ts
+++ b/src/services/assistants/search.ts
@@ -10,6 +10,7 @@ import { EmailShortlist } from "../tools/emailShortlist"
 const llm = defaultModel
 
 const MAX_EMAIL_LENGTH = 500
+const MAX_DOC_LENGTH = 2000
 const MAX_ATTACHMENT_LENGTH = 500
 const MAX_CONTEXT_LENGTH = 20000 // (~5000 tokens)
 
@@ -110,6 +111,11 @@ export class PromptSearchService extends VeridaService {
             contextString += `From: ${chatMessage.fromName} <${chatMessage.fromHandle}> (${chatMessage.groupName})\nBody: ${chatMessage.messageText}\n\n`
         }
 
+        console.log('files: ', files.length)
+        for (const file of files) {
+            contextString += `File: ${file.name} ${file.indexableText?.substring(0,MAX_DOC_LENGTH)} (via ${file.sourceApplication})\n\n`
+        }
+
         console.log('favourites: ', favourites.length)
         for (const favourite of favourites) {
             contextString += `Favorite: ${favourite.name} ${favourite.description?.substring(0,100)} (via ${favourite.sourceApplication})\n\n`

From 2d80fb63862d5a2882d445ed2886d332cdc2ce89 Mon Sep 17 00:00:00 2001
From: Chris 
Date: Mon, 2 Sep 2024 11:03:41 +0930
Subject: [PATCH 075/182] Add more detail to console output

---
 src/providers/google/helpers.ts | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/providers/google/helpers.ts b/src/providers/google/helpers.ts
index f1cee428..f2d3c64b 100644
--- a/src/providers/google/helpers.ts
+++ b/src/providers/google/helpers.ts
@@ -379,11 +379,11 @@ export class GoogleDriveHelpers {
             
           } else {
             console.warn(
-              "Unsupported MIME type."
+              `Unsupported MIME type (${mimeType})`
             );
           }
         } else {
-          console.warn("File size exceeds the limit or unsupported file type.");
+          console.warn(`File size ${fileSize} exceeds the limit (${sizeLimit}) or unsupported file type.`);
         }
       } else {
         console.log("Indexable text extracted successfully from contentHints.");

From 15ea07a48b36535ea2107da3565b6d5311e2a6f8 Mon Sep 17 00:00:00 2001
From: Chris 
Date: Mon, 2 Sep 2024 11:20:14 +0930
Subject: [PATCH 076/182] Add missing File schema in data service.

---
 src/services/data.ts | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/services/data.ts b/src/services/data.ts
index 484923c7..c9ca5e31 100644
--- a/src/services/data.ts
+++ b/src/services/data.ts
@@ -49,6 +49,11 @@ const schemas: Record = {
         label: "Favorite",
         storeFields: ['_id', 'insertedAt'],
         indexFields: ['name', 'favouriteType', 'contentType', 'summary','sourceApplication']
+    },
+    "https://common.schemas.verida.io/file/v0.1.0/schema.json": {
+        label: "File",
+        storeFields: ['_id', 'insertedAt'],
+        indexFields: ['name', 'contentText', 'indexableText', 'sourceApplication', "modifiedAt", "insertedAt"]
     }
 }
 

From 2251600fc766e40fee7e1451792be7babff16123 Mon Sep 17 00:00:00 2001
From: Chris 
Date: Mon, 2 Sep 2024 11:20:33 +0930
Subject: [PATCH 077/182] Fix bug with files not being included in private AI
 processing

---
 src/services/assistants/search.ts | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/services/assistants/search.ts b/src/services/assistants/search.ts
index 84772851..6fd06f7d 100644
--- a/src/services/assistants/search.ts
+++ b/src/services/assistants/search.ts
@@ -53,7 +53,7 @@ export class PromptSearchService extends VeridaService {
             if (promptSearchResult.databases.indexOf(SearchType.EMAILS) !== -1) {
                 emails = await searchService.schemaByKeywords(SearchType.EMAILS, promptSearchResult.keywords!, promptSearchResult.timeframe, 40)
             }
-            if (promptSearchResult.databases.indexOf(SearchType.FILES)) {
+            if (promptSearchResult.databases.indexOf(SearchType.FILES) !== -1) {
                 files = await searchService.schemaByKeywords(SearchType.FILES, promptSearchResult.keywords!, promptSearchResult.timeframe, 20)
             }
             if (promptSearchResult.databases.indexOf(SearchType.FAVORITES) !== -1) {
@@ -77,7 +77,7 @@ export class PromptSearchService extends VeridaService {
                 emails = await emailShortlist.shortlist(prompt, emails, MAX_DATERANGE_EMAILS)
                 console.timeEnd("EmailShortlist")
             }
-            if (promptSearchResult.databases.indexOf(SearchType.FILES)) {
+            if (promptSearchResult.databases.indexOf(SearchType.FILES) !== -1) {
                 files = await searchService.schemaByDateRange(SearchType.FILES, maxDatetime, sort, MAX_DATERANGE_FILES)
             }
             if (promptSearchResult.databases.indexOf(SearchType.FAVORITES) !== -1) {

From aaa28eb9b906b49d66ac0a61695e1449639bec4f Mon Sep 17 00:00:00 2001
From: chime3 
Date: Sun, 1 Sep 2024 19:28:34 -0700
Subject: [PATCH 078/182] Fix: refactored Gdrive document handler

---
 src/providers/google/gdrive-document.ts | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/src/providers/google/gdrive-document.ts b/src/providers/google/gdrive-document.ts
index 57f25aa3..622614d1 100644
--- a/src/providers/google/gdrive-document.ts
+++ b/src/providers/google/gdrive-document.ts
@@ -93,17 +93,16 @@ export default class GoogleDriveDocument extends GoogleHandler {
             }, false);
         }
     
-        if (items.length != this.config.batchSize) {
+        currentRange = rangeTracker.nextRange();
+        if (items.length != this.config.batchSize && currentRange.startId) {
+        
             currentRange = rangeTracker.nextRange();
             query = {
                 ...query,
                 pageSize: this.config.batchSize - items.length,
+                pageToken: currentRange.startId
             };
     
-            if (currentRange.startId) {
-                query.pageToken = currentRange.startId;
-            }
-    
             const backfillResponse = await drive.files.list(query);
             const backfillResult = await this.buildResults(
                 drive,

From 3a5f80b88bd5a092b9e362a16c905e68699f500d Mon Sep 17 00:00:00 2001
From: chime3 
Date: Sun, 1 Sep 2024 19:30:14 -0700
Subject: [PATCH 079/182] fix: refactored Gdrive document handler

---
 src/providers/google/gdrive-document.ts | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/providers/google/gdrive-document.ts b/src/providers/google/gdrive-document.ts
index 622614d1..a87d9a92 100644
--- a/src/providers/google/gdrive-document.ts
+++ b/src/providers/google/gdrive-document.ts
@@ -95,8 +95,6 @@ export default class GoogleDriveDocument extends GoogleHandler {
     
         currentRange = rangeTracker.nextRange();
         if (items.length != this.config.batchSize && currentRange.startId) {
-        
-            currentRange = rangeTracker.nextRange();
             query = {
                 ...query,
                 pageSize: this.config.batchSize - items.length,

From 13e42740cbb14cf44fe52aba99246067d6edfff2 Mon Sep 17 00:00:00 2001
From: Chris 
Date: Mon, 2 Sep 2024 15:11:05 +0930
Subject: [PATCH 080/182] Fix issues with indexing files for private AI

---
 src/services/assistants/search.ts | 9 ++++-----
 src/services/data.ts              | 8 ++++----
 src/services/search.ts            | 8 +++-----
 3 files changed, 11 insertions(+), 14 deletions(-)

diff --git a/src/services/assistants/search.ts b/src/services/assistants/search.ts
index 35fe358f..58f6ee83 100644
--- a/src/services/assistants/search.ts
+++ b/src/services/assistants/search.ts
@@ -84,8 +84,8 @@ export class PromptSearchService extends VeridaService {
             }
         }
 
-        console.log('emails / favourites / following / chatThreads')
-        console.log(emails.length, favourites.length, following.length, chatThreads.length)
+        console.log('files / emails / favourites / following / chatThreads')
+        console.log(files.length, emails.length, favourites.length, following.length, chatThreads.length)
 
         let finalPrompt = `Answer this prompt:\n${prompt}\nHere are some recent messages that may help you provide a relevant answer.\n`
         let contextString = ''
@@ -105,9 +105,8 @@ export class PromptSearchService extends VeridaService {
             contextString += `From: ${chatMessage.fromName} <${chatMessage.fromHandle}> (${chatMessage.groupName})\nBody: ${chatMessage.messageText}\n\n`
         }
 
-        console.log('files: ', files.length)
         for (const file of files) {
-            contextString += `File: ${file.name} ${file.indexableText?.substring(0,MAX_DOC_LENGTH)} (via ${file.sourceApplication})\n\n`
+            contextString += `File: ${file.name} ${file.contentText.substring(0,MAX_DOC_LENGTH)} (via ${file.sourceApplication})\n\n`
         }
 
         for (const favourite of favourites) {
@@ -140,7 +139,7 @@ export class PromptSearchService extends VeridaService {
         const now = (new Date()).toISOString()
         finalPrompt += `${contextString}\nThe current time is: ${now}`
 
-        console.log(finalPrompt)
+        // console.log(finalPrompt)
 
         const finalResponse = await llm.prompt(finalPrompt, undefined, false)
         const duration = Date.now() - start
diff --git a/src/services/data.ts b/src/services/data.ts
index daac2ccd..88ad4b9d 100644
--- a/src/services/data.ts
+++ b/src/services/data.ts
@@ -165,13 +165,13 @@ export class DataService extends EventEmitter {
                 // Flatten array fields for indexing
                 for (const arrayProperty of arrayProperties) {
                     if (row[arrayProperty] && row[arrayProperty].length) {
-                        let i = 0
+                        let j = 0
                         for (const arrayItem of row[arrayProperty]) {
                             if (!arrayItem.filename.match('pdf')) {
                                 continue
                             }
 
-                            const arrayItemProperty = `${arrayProperty}_${i}`
+                            const arrayItemProperty = `${arrayProperty}_${j}`
                             row[arrayItemProperty] = arrayItem
 
                             // Make sure this field is stored
@@ -180,7 +180,7 @@ export class DataService extends EventEmitter {
                             }
 
                             // @todo: Make sure the original field isn't stored (`arrayProperty`)
-                            i++
+                            j++
                         }
                     }
                 }
@@ -195,7 +195,7 @@ export class DataService extends EventEmitter {
                 // Add support for nested fields (`ie: attachments_0.textContent)
                 extractField: (document, fieldName) => {
                     return fieldName.split('.').reduce((doc, key) => doc && doc[key], document)
-                    }
+                }
             })
 
             // Index all documents
diff --git a/src/services/search.ts b/src/services/search.ts
index ae232bd9..53978124 100644
--- a/src/services/search.ts
+++ b/src/services/search.ts
@@ -124,11 +124,9 @@ export class SearchService extends VeridaService {
         console.log(query, maxDatetime)
         
         const searchResults = await dataService.searchIndex(schemaUri, query, limit, undefined, {
-            filter: (result: any) => maxDatetime ? result.sentAt > maxDatetime.toISOString() : true
+            filter: (result: any) => maxDatetime ? result[SearchTypeTimeProperty[searchType]] > maxDatetime.toISOString() : true
         })
 
-        console.log(searchResults)
-
         return await this.rankAndMergeResults([{
             searchType,
             rows: searchResults
@@ -166,7 +164,7 @@ export class SearchService extends VeridaService {
         const maxDatetime = Helpers.keywordTimeframeToDate(timeframe)
         
         const searchResults = await miniSearchIndex.search(query, {
-            filter: (result: any) => maxDatetime ? result.sentAt > maxDatetime.toISOString() : true
+            filter: (result: any) => maxDatetime ? result[SearchTypeTimeProperty[searchType]] > maxDatetime.toISOString() : true
         })
 
         return this.rankAndMergeResults([{
@@ -195,7 +193,7 @@ export class SearchService extends VeridaService {
         const maxDatetime = Helpers.keywordTimeframeToDate(timeframe)
         
         const searchResults = await dataService.searchIndex(messageSchemaUri, query, 50, 0.5, {
-            filter: (result: any) => maxDatetime ? result.sentAt > maxDatetime.toISOString() : true
+            filter: (result: any) => maxDatetime ? result[SearchTypeTimeProperty[SearchType.CHAT_MESSAGES]] > maxDatetime.toISOString() : true
         })
         
         const chatMessageDs = await this.context.openDatastore(messageSchemaUri)

From f12d417c10c36fba0e24f41f8abebd215b1d96d2 Mon Sep 17 00:00:00 2001
From: Chris 
Date: Mon, 2 Sep 2024 15:39:12 +0930
Subject: [PATCH 081/182] Move breakTimestamp config to root of config. Support
 custom config per handler.

---
 src/interfaces.ts                       |  5 +++--
 src/providers/BaseProvider.ts           |  3 ++-
 src/providers/BaseSyncHandler.ts        | 10 ++++++++++
 src/providers/google/gdrive-document.ts |  8 ++++----
 src/providers/google/gmail.ts           |  8 ++++----
 src/providers/index.ts                  |  2 +-
 src/serverconfig.example.json           |  7 +++++--
 7 files changed, 29 insertions(+), 14 deletions(-)

diff --git a/src/interfaces.ts b/src/interfaces.ts
index 03377f75..1bcebca0 100644
--- a/src/interfaces.ts
+++ b/src/interfaces.ts
@@ -115,8 +115,9 @@ export interface BaseProviderConfig {
     sbtImage: string
     batchSize?: number
     maxSyncLoops?: number
-    // Other metadata useful to configure for the handler
-    metadata?: object
+    breakTimestamp?: object
+    // Custom config for each handler
+    handlers?: Record
 }
 
 export interface DatastoreSaveResponse {
diff --git a/src/providers/BaseProvider.ts b/src/providers/BaseProvider.ts
index d755f9ef..2fbb966a 100644
--- a/src/providers/BaseProvider.ts
+++ b/src/providers/BaseProvider.ts
@@ -6,6 +6,7 @@ import { IContext, IDatastore } from '@verida/types'
 import BaseSyncHandler from './BaseSyncHandler'
 import { SchemaRecord } from '../schemas'
 import EventEmitter from 'events'
+const _ = require("lodash")
 
 const SCHEMA_SYNC_POSITIONS = serverconfig.verida.schemas.SYNC_POSITION
 const SCHEMA_SYNC_LOG = serverconfig.verida.schemas.SYNC_LOG
@@ -337,7 +338,7 @@ export default class BaseProvider extends EventEmitter {
         const syncHandlers = []
         for (let h in handlers) {
             const handler = handlers[h]
-            
+
             const handlerInstance = new handler(this.config, this.connection, this)
             syncHandlers.push(handlerInstance)
         }
diff --git a/src/providers/BaseSyncHandler.ts b/src/providers/BaseSyncHandler.ts
index 4977f37d..caf5107d 100644
--- a/src/providers/BaseSyncHandler.ts
+++ b/src/providers/BaseSyncHandler.ts
@@ -4,6 +4,7 @@ import { EventEmitter } from "events"
 import { Utils } from "../utils"
 import { SchemaRecord } from "../schemas"
 import BaseProvider from "./BaseProvider"
+const _ = require("lodash")
 
 export default class BaseSyncHandler extends EventEmitter {
 
@@ -15,6 +16,15 @@ export default class BaseSyncHandler extends EventEmitter {
 
     constructor(config: any, connection: Connection, provider: BaseProvider) {
         super()
+        // Handle any custom config for this handler
+        if (config.handlers) {
+            if (config.handlers[this.getName()]) {
+                config = _.merge({}, config, config.handlers[this.getName()])
+            }
+
+            delete config["handlers"]
+        }
+
         this.config = config
         this.connection = connection
         this.provider = provider
diff --git a/src/providers/google/gdrive-document.ts b/src/providers/google/gdrive-document.ts
index a87d9a92..ac6eff21 100644
--- a/src/providers/google/gdrive-document.ts
+++ b/src/providers/google/gdrive-document.ts
@@ -73,8 +73,8 @@ export default class GoogleDriveDocument extends GoogleHandler {
             drive,
             latestResponse,
             currentRange.endId,
-            _.has(this.config, "metadata.breakTimestamp")
-                ? this.config.metadata.breakTimestamp
+            _.has(this.config, "breakTimestamp")
+                ? this.config.breakTimestamp
                 : undefined
         );
     
@@ -106,8 +106,8 @@ export default class GoogleDriveDocument extends GoogleHandler {
                 drive,
                 backfillResponse,
                 currentRange.endId,
-                _.has(this.config, "metadata.breakTimestamp")
-                    ? this.config.metadata.breakTimestamp
+                _.has(this.config, "breakTimestamp")
+                    ? this.config.breakTimestamp
                     : undefined
             );
     
diff --git a/src/providers/google/gmail.ts b/src/providers/google/gmail.ts
index 347f5f8d..ae9c80fb 100644
--- a/src/providers/google/gmail.ts
+++ b/src/providers/google/gmail.ts
@@ -90,8 +90,8 @@ export default class Gmail extends GoogleHandler {
       latestResponse,
       currentRange.endId,
       SchemaEmailType.RECEIVE,
-      _.has(this.config, "metadata.breakTimestamp")
-        ? this.config.metadata.breakTimestamp
+      _.has(this.config, "breakTimestamp")
+        ? this.config.breakTimestamp
         : undefined
     );
 
@@ -126,8 +126,8 @@ export default class Gmail extends GoogleHandler {
         backfillResponse,
         currentRange.endId,
         SchemaEmailType.RECEIVE,
-        _.has(this.config, "metadata.breakTimestamp")
-          ? this.config.metadata.breakTimestamp
+        _.has(this.config, "breakTimestamp")
+          ? this.config.breakTimestamp
           : undefined
       );
 
diff --git a/src/providers/index.ts b/src/providers/index.ts
index 473bd6b4..35e13791 100644
--- a/src/providers/index.ts
+++ b/src/providers/index.ts
@@ -8,7 +8,7 @@ export default function (providerName: string, vault?: IContext, connection?: Co
     const provider = require(`./${providerName}`)
 
     // @ts-ignore
-    const providerConfig = _.merge(CONFIG.providerDefaults, CONFIG.providers[providerName])
+    const providerConfig = _.merge({}, CONFIG.providerDefaults, CONFIG.providers[providerName])
     providerConfig.callbackUrl = `${CONFIG.serverUrl}/callback/${providerName}`
 
     return new provider.default(providerConfig, vault, connection)
diff --git a/src/serverconfig.example.json b/src/serverconfig.example.json
index f69dc6f0..5faf2412 100644
--- a/src/serverconfig.example.json
+++ b/src/serverconfig.example.json
@@ -69,8 +69,11 @@
             "batchSize": 50,
             "sizeLimit": 10,
             "maxSyncLoops": 1,
-            "metadata": {
-                "breakTimestamp": "2000-07-21T12:07:11.000Z"
+            "breakTimestamp": "2000-07-21T12:07:11.000Z",
+            "handlers": {
+                "gmail": {
+                    "batchSize": 200
+                }
             }
         },
         "telegram": {

From 80846301d9c36e6680017edfb1938bf64c1725d7 Mon Sep 17 00:00:00 2001
From: Chris 
Date: Mon, 2 Sep 2024 15:46:09 +0930
Subject: [PATCH 082/182] Fix avatar profile display on connection screen

---
 src/web/user/connections/connections.js | 4 ++--
 src/web/user/connections/index.html     | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/web/user/connections/connections.js b/src/web/user/connections/connections.js
index 132c94de..a5543333 100644
--- a/src/web/user/connections/connections.js
+++ b/src/web/user/connections/connections.js
@@ -63,7 +63,7 @@ $(document).ready(function() {
                 const formattedSyncTimes = `Start: ${new Date(connection.syncStart).toLocaleString()}
End: ${new Date(connection.syncEnd).toLocaleString()}`;
 
                 const providerDetails = getProviderDetails(connection.provider);
-                const avatar = connection.profile.avatarUrl ? `${connection.profile.name}` : ''
+                const avatar = connection.profile.avatar.uri ? `${connection.profile.name}` : ''
 
                 const row = $(`
                     
@@ -72,7 +72,7 @@ $(document).ready(function() {
                             ${providerDetails.label}
                         
                             ${avatar}
-                            ${connection.profile.name}
${connection.profile.email ? '('+ connection.profile.email +')' : ''} (${connection.providerId})
+                            ${connection.profile.name}
${connection.profile.email ? '('+ connection.profile.email +')' : ''} (${connection.providerId})
                         ${connection.syncStatus}
${formattedSyncTimes}
                         ${handlers.map(handler => `[${handler.handlerName}] ${handler.syncMessage ? handler.syncMessage : ""} (${handler.status})
`).join('')}
                         
diff --git a/src/web/user/connections/index.html b/src/web/user/connections/index.html
index dd0274d8..4d44706a 100644
--- a/src/web/user/connections/index.html
+++ b/src/web/user/connections/index.html
@@ -69,7 +69,7 @@ 
Connections

         
-                    
+                    

From d6d55b666d10bc5c94fadbe3eb963be845d732ea Mon Sep 17 00:00:00 2001
From: tahpot 
Date: Tue, 3 Sep 2024 16:17:40 +0930
Subject: [PATCH 083/182] Feature/100 reorganize providers endpoint (#101)

---
 src/api/v1/base/controller.ts             | 16 +++++++-----
 src/interfaces.ts                         | 17 ++++++++++---
 src/providers/BaseSyncHandler.ts          | 16 ++++++++++--
 src/providers/discord/following.ts        |  4 +++
 src/providers/facebook/following.ts       |  4 +++
 src/providers/facebook/post.ts            |  4 +++
 src/providers/google/gdrive-document.ts   | 30 +++++++++++++++++------
 src/providers/google/gmail.ts             | 26 ++++++++++++++++----
 src/providers/google/youtube-favourite.ts | 28 ++++++++++++++++-----
 src/providers/google/youtube-following.ts | 28 ++++++++++++++++-----
 src/providers/google/youtube-post.ts      | 28 ++++++++++++++++-----
 src/providers/telegram/chat-message.ts    | 30 +++++++++++++++--------
 src/providers/twitter/posts.ts            |  2 +-
 src/sync-manager.ts                       |  4 +--
 14 files changed, 183 insertions(+), 54 deletions(-)

diff --git a/src/api/v1/base/controller.ts b/src/api/v1/base/controller.ts
index 28f17dac..105fb82f 100644
--- a/src/api/v1/base/controller.ts
+++ b/src/api/v1/base/controller.ts
@@ -1,7 +1,7 @@
 import { Request, Response } from 'express'
 import Providers from "../../../providers"
 import SyncManager from '../../../sync-manager'
-import { HandlerOption, SyncHandlerPosition, UniqueRequest } from '../../../interfaces'
+import { ProviderHandler, SyncHandlerPosition, UniqueRequest } from '../../../interfaces'
 import { Utils } from '../../../utils'
 import CONFIG from '../../../config'
 import { SchemaRecord } from '../../../schemas'
@@ -201,26 +201,30 @@ export default class Controller {
     public static async providers(req: Request, res: Response) {
         const providers = Object.keys(CONFIG.providers)
 
-        const results: any = {}
+        const results: any = []
         for (let p in providers) {
             const providerName = providers[p]
 
             try {
                 const provider = Providers(providerName)
                 const syncHandlers = await provider.getSyncHandlers()
-                const handlers: Record = {}
+                const handlers: ProviderHandler[] = []
                 for (const handler of syncHandlers) {
-                    handlers[handler.getName()] = handler.getOptions()
+                    handlers.push({
+                        id: handler.getName(),
+                        label: handler.getLabel(),
+                        options: handler.getOptions()
+                    })
                 }
 
-                results[providerName] = {
+                results.push({
                     name: providerName,
                     label: provider.getProviderLabel(),
                     icon: provider.getProviderImageUrl(),
                     description: provider.getDescription(),
                     options: provider.getOptions(),
                     handlers
-                }
+                })
             } catch (err) {
                 // skip broken providers
             }
diff --git a/src/interfaces.ts b/src/interfaces.ts
index 1bcebca0..c33582b5 100644
--- a/src/interfaces.ts
+++ b/src/interfaces.ts
@@ -16,11 +16,16 @@ export enum ConnectionOptionType {
     BOOLEAN = "boolean"
 }
 
+export interface ConnectionOptionEnumOption {
+    value: string
+    label: string
+}
+
 export interface ConnectionOption {
-    name: string
+    id: string
     label: string
     type: ConnectionOptionType
-    enumOptions?: string[]
+    enumOptions?: ConnectionOptionEnumOption[]
     defaultValue: string
 }
 
@@ -49,7 +54,13 @@ export interface PassportProfile {
     connectionProfile?: Partial
 }
 
-export interface HandlerOption extends ConnectionOption {}
+export interface ProviderHandler {
+    id: string
+    label: string
+    options: ProviderHandlerOption[]
+}
+
+export interface ProviderHandlerOption extends ConnectionOption {}
 
 export interface AvatarObject extends Object {
     uri: string
diff --git a/src/providers/BaseSyncHandler.ts b/src/providers/BaseSyncHandler.ts
index caf5107d..99e64230 100644
--- a/src/providers/BaseSyncHandler.ts
+++ b/src/providers/BaseSyncHandler.ts
@@ -1,4 +1,4 @@
-import { Connection, HandlerOption, SyncHandlerResponse, SyncHandlerStatus, SyncProviderLogLevel, SyncResponse, SyncHandlerPosition } from "../interfaces"
+import { Connection, ProviderHandlerOption, SyncHandlerResponse, SyncHandlerStatus, SyncProviderLogLevel, SyncResponse, SyncHandlerPosition } from "../interfaces"
 import { IDatastore } from '@verida/types'
 import { EventEmitter } from "events"
 import { Utils } from "../utils"
@@ -34,11 +34,23 @@ export default class BaseSyncHandler extends EventEmitter {
         throw new Error('Not implemented')
     }
 
+    /**
+     * Set a default label
+     */
+    public getLabel(): string {
+        let label = this.getName()
+        // Replace all instances of "-" with a space
+        label = label.replace(/-/g, ' ');
+
+        // Uppercase the first letter
+        return label.charAt(0).toUpperCase() + label.slice(1);
+    }
+
     public getConfig(): any {
         return this.config
     }
 
-    public getOptions(): HandlerOption[] {
+    public getOptions(): ProviderHandlerOption[] {
         return []
     }
 
diff --git a/src/providers/discord/following.ts b/src/providers/discord/following.ts
index c723c916..890e45f3 100644
--- a/src/providers/discord/following.ts
+++ b/src/providers/discord/following.ts
@@ -11,6 +11,10 @@ export default class Following extends BaseSyncHandler {
 
     protected static schemaUri: string = 'https://common.schemas.verida.io/social/following/v0.1.0/schema.json'
 
+    public getLabel(): string {
+        return "Joined Servers"
+    }
+
     public getName(): string {
         return 'following'
     }
diff --git a/src/providers/facebook/following.ts b/src/providers/facebook/following.ts
index 3b0ced32..51073591 100644
--- a/src/providers/facebook/following.ts
+++ b/src/providers/facebook/following.ts
@@ -12,6 +12,10 @@ export default class Following extends BaseSyncHandler {
 
     protected apiEndpoint = '/me/likes'
 
+    public getLabel(): string {
+        return "Liked Pages"
+    }
+
     public getName(): string {
         return 'following'
     }
diff --git a/src/providers/facebook/post.ts b/src/providers/facebook/post.ts
index 8bbbab8f..f19df870 100644
--- a/src/providers/facebook/post.ts
+++ b/src/providers/facebook/post.ts
@@ -19,6 +19,10 @@ export const enum PostSyncRefTypes {
 
 export default class Posts extends BaseSyncHandler {
 
+    public getLabel(): string {
+        return "Posts"
+    }
+
     public getName(): string {
         return 'post'
     }
diff --git a/src/providers/google/gdrive-document.ts b/src/providers/google/gdrive-document.ts
index ac6eff21..b63143b3 100644
--- a/src/providers/google/gdrive-document.ts
+++ b/src/providers/google/gdrive-document.ts
@@ -1,5 +1,5 @@
 import CONFIG from "../../config";
-import { SyncProviderLogEvent, SyncProviderLogLevel, SyncHandlerPosition, SyncItemsBreak, SyncResponse, SyncHandlerStatus, SyncItemsResult, HandlerOption, ConnectionOptionType } from '../../interfaces';
+import { SyncProviderLogEvent, SyncProviderLogLevel, SyncHandlerPosition, SyncItemsBreak, SyncResponse, SyncHandlerStatus, SyncItemsResult, ConnectionOptionType, ProviderHandlerOption } from '../../interfaces';
 import { SchemaFile } from "../../schemas";
 import { google, drive_v3 } from "googleapis";
 import { GaxiosResponse } from "gaxios";
@@ -17,8 +17,12 @@ export interface SyncDocumentItemsResult extends SyncItemsResult {
 
 export default class GoogleDriveDocument extends GoogleHandler {
 
+    public getLabel(): string {
+        return "Google Drive Documents"
+    }
+
     public getName(): string {
-        return "google-drive-documents";
+        return "google-drive-document";
     }
 
     public getSchemaUri(): string {
@@ -34,14 +38,26 @@ export default class GoogleDriveDocument extends GoogleHandler {
         return google.drive({ version: "v3", auth });
     }
 
-    public getOptions(): HandlerOption[] {
+    public getOptions(): ProviderHandlerOption[] {
         return [{
-            name: 'backdate',
+            id: 'backdate',
             label: 'Backdate history',
             type: ConnectionOptionType.ENUM,
-            enumOptions: ['1 month', '3 months', '6 months', '12 months'],
-            defaultValue: '3 months'
-        }];
+            enumOptions: [{
+              value: '1-month',
+              label: '1 month'
+            }, {
+              value: '3-months',
+              label: '3 months'
+            }, {
+              value: '6-months',
+              label: '6 months'
+            }, {
+              value: '12-months',
+              label: '12 months'
+            }],
+            defaultValue: '3-months'
+          }]
     }
 
     public async _sync(
diff --git a/src/providers/google/gmail.ts b/src/providers/google/gmail.ts
index ae9c80fb..f0137d12 100644
--- a/src/providers/google/gmail.ts
+++ b/src/providers/google/gmail.ts
@@ -8,7 +8,7 @@ import { ItemsRangeTracker } from "../../helpers/itemsRangeTracker"
 import {
   SyncResponse,
   SyncHandlerStatus,
-  HandlerOption,
+  ProviderHandlerOption,
   ConnectionOptionType,
 } from "../../interfaces";
 import { SchemaEmail, SchemaEmailType, SchemaRecord } from "../../schemas";
@@ -25,6 +25,10 @@ export interface SyncEmailItemsResult extends SyncItemsResult {
 
 export default class Gmail extends GoogleHandler {
 
+  public getLabel(): string {
+    return "Gmail"
+  }
+
   public getName(): string {
     return 'gmail'
 }
@@ -44,13 +48,25 @@ export default class Gmail extends GoogleHandler {
     return gmail;
   }
   
-  public getOptions(): HandlerOption[] {
+  public getOptions(): ProviderHandlerOption[] {
     return [{
-      name: 'backdate',
+      id: 'backdate',
       label: 'Backdate history',
       type: ConnectionOptionType.ENUM,
-      enumOptions: ['1 month', '3 months', '6 months', '12 months'],
-      defaultValue: '3 months'
+      enumOptions: [{
+        value: '1-month',
+        label: '1 month'
+      }, {
+        value: '3-months',
+        label: '3 months'
+      }, {
+        value: '6-months',
+        label: '6 months'
+      }, {
+        value: '12-months',
+        label: '12 months'
+      }],
+      defaultValue: '3-months'
     }]
   }
 
diff --git a/src/providers/google/youtube-favourite.ts b/src/providers/google/youtube-favourite.ts
index 1bda2c6a..903dec54 100644
--- a/src/providers/google/youtube-favourite.ts
+++ b/src/providers/google/youtube-favourite.ts
@@ -4,7 +4,7 @@ import { SyncHandlerPosition, SyncItemsBreak, SyncItemsResult, SyncProviderLogEv
 import {
     SyncResponse,
     SyncHandlerStatus,
-    HandlerOption,
+    ProviderHandlerOption,
     ConnectionOptionType,
 } from "../../interfaces";
 import { SchemaFavouriteContentType, SchemaFavouriteType, SchemaFavourite } from "../../schemas";
@@ -24,6 +24,10 @@ export interface SyncFavouriteItemsResult extends SyncItemsResult {
 
 export default class YouTubeFavourite extends GoogleHandler {
 
+    public getLabel(): string {
+        return "Youtube Favourites"
+    }
+
     public getName(): string {
         return "youtube-favourite";
     }
@@ -42,14 +46,26 @@ export default class YouTubeFavourite extends GoogleHandler {
         return youtube;
     }
 
-    public getOptions(): HandlerOption[] {
+    public getOptions(): ProviderHandlerOption[] {
         return [{
-            name: 'backdate',
+            id: 'backdate',
             label: 'Backdate history',
             type: ConnectionOptionType.ENUM,
-            enumOptions: ['1 month', '3 months', '6 months', '12 months'],
-            defaultValue: '3 months'
-        }];
+            enumOptions: [{
+              value: '1-month',
+              label: '1 month'
+            }, {
+              value: '3-months',
+              label: '3 months'
+            }, {
+              value: '6-months',
+              label: '6 months'
+            }, {
+              value: '12-months',
+              label: '12 months'
+            }],
+            defaultValue: '3-months'
+          }]
     }
 
     public async _sync(
diff --git a/src/providers/google/youtube-following.ts b/src/providers/google/youtube-following.ts
index 24628d82..caf7f40b 100644
--- a/src/providers/google/youtube-following.ts
+++ b/src/providers/google/youtube-following.ts
@@ -1,6 +1,6 @@
 import GoogleHandler from "./GoogleHandler";
 import CONFIG from "../../config";
-import { SyncProviderLogEvent, SyncProviderLogLevel, SyncHandlerPosition, SyncResponse, SyncHandlerStatus, SyncItemsBreak, HandlerOption, ConnectionOptionType } from "../../interfaces";
+import { SyncProviderLogEvent, SyncProviderLogLevel, SyncHandlerPosition, SyncResponse, SyncHandlerStatus, SyncItemsBreak, ProviderHandlerOption, ConnectionOptionType } from "../../interfaces";
 import { SchemaFollowing } from "../../schemas";
 import { google, youtube_v3 } from "googleapis";
 import { GaxiosResponse } from "gaxios";
@@ -14,6 +14,10 @@ const MAX_BATCH_SIZE = 50;
 
 export default class YouTubeFollowing extends GoogleHandler {
 
+    public getLabel(): string {
+        return "Youtube Subscriptions"
+    }
+
     public getName(): string {
         return "youtube-following";
     }
@@ -31,14 +35,26 @@ export default class YouTubeFollowing extends GoogleHandler {
         return google.youtube({ version: "v3", auth: oAuth2Client });
     }
 
-    public getOptions(): HandlerOption[] {
+    public getOptions(): ProviderHandlerOption[] {
         return [{
-            name: 'backdate',
+            id: 'backdate',
             label: 'Backdate history',
             type: ConnectionOptionType.ENUM,
-            enumOptions: ['1 month', '3 months', '6 months', '12 months'],
-            defaultValue: '3 months'
-        }];
+            enumOptions: [{
+              value: '1-month',
+              label: '1 month'
+            }, {
+              value: '3-months',
+              label: '3 months'
+            }, {
+              value: '6-months',
+              label: '6 months'
+            }, {
+              value: '12-months',
+              label: '12 months'
+            }],
+            defaultValue: '3-months'
+          }]
     }
 
     public async _sync(
diff --git a/src/providers/google/youtube-post.ts b/src/providers/google/youtube-post.ts
index a08e05b9..e79852ff 100644
--- a/src/providers/google/youtube-post.ts
+++ b/src/providers/google/youtube-post.ts
@@ -4,7 +4,7 @@ import { ConnectionOptionType, SyncHandlerPosition, SyncItemsBreak, SyncItemsRes
 import {
     SyncResponse,
     SyncHandlerStatus,
-    HandlerOption,
+    ProviderHandlerOption,
 } from "../../interfaces";
 import { SchemaPostType, SchemaPost } from "../../schemas";
 import { google, youtube_v3 } from "googleapis";
@@ -24,6 +24,10 @@ export interface SyncPostItemsResult extends SyncItemsResult {
 
 export default class YouTubePost extends GoogleHandler {
 
+    public getLabel(): string {
+        return "Youtube Posts"
+    }
+
     public getName(): string {
         return "youtube-post";
     }
@@ -42,14 +46,26 @@ export default class YouTubePost extends GoogleHandler {
         return youtube;
     }
 
-    public getOptions(): HandlerOption[] {
+    public getOptions(): ProviderHandlerOption[] {
         return [{
-            name: 'backdate',
+            id: 'backdate',
             label: 'Backdate history',
             type: ConnectionOptionType.ENUM,
-            enumOptions: ['1 month', '3 months', '6 months', '12 months'],
-            defaultValue: '3 months'
-        }];
+            enumOptions: [{
+              value: '1-month',
+              label: '1 month'
+            }, {
+              value: '3-months',
+              label: '3 months'
+            }, {
+              value: '6-months',
+              label: '6 months'
+            }, {
+              value: '12-months',
+              label: '12 months'
+            }],
+            defaultValue: '3-months'
+          }]
     }
 
     public async _sync(
diff --git a/src/providers/telegram/chat-message.ts b/src/providers/telegram/chat-message.ts
index d9a4d2ac..787887f6 100644
--- a/src/providers/telegram/chat-message.ts
+++ b/src/providers/telegram/chat-message.ts
@@ -5,7 +5,7 @@ import {
   SyncResponse,
   SyncHandlerPosition,
   SyncHandlerStatus,
-  HandlerOption,
+  ProviderHandlerOption,
   ConnectionOptionType,
 } from "../../interfaces";
 import {
@@ -28,6 +28,10 @@ export default class TelegramChatMessageHandler extends BaseSyncHandler {
     return "chat-message";
   }
 
+  public getLabel(): string {
+    return "Chat Messages"
+  }
+
   public getSchemaUri(): string {
     return CONFIG.verida.schemas.CHAT_MESSAGE;
   }
@@ -36,12 +40,21 @@ export default class TelegramChatMessageHandler extends BaseSyncHandler {
     return "https://telegram.com";
   }
 
-  public getOptions(): HandlerOption[] {
+  public getOptions(): ProviderHandlerOption[] {
     return [{
-      name: 'groupTypes',
+      id: 'groupTypes',
       label: 'Group types',
       type: ConnectionOptionType.ENUM_MULTI,
-      enumOptions: [TelegramChatGroupType.BASIC, TelegramChatGroupType.PRIVATE, TelegramChatGroupType.SECRET, TelegramChatGroupType.SUPERGROUP],
+      enumOptions: [{
+        label: "Basic",
+        value: TelegramChatGroupType.BASIC
+      }, {
+        label: "Private",
+        value: TelegramChatGroupType.PRIVATE
+      }, {
+        label: "Secret",
+        value: TelegramChatGroupType.SECRET
+      }],
       // Exclude super groups by default
       defaultValue: [TelegramChatGroupType.BASIC, TelegramChatGroupType.PRIVATE, TelegramChatGroupType.SECRET].join(',')
     }]
@@ -200,7 +213,7 @@ export default class TelegramChatMessageHandler extends BaseSyncHandler {
     chatGroup: SchemaSocialChatGroup,
     chatHistory: SchemaSocialChatMessage[]
   }> {
-    console.log(`- Processing group: ${chatGroup.name} (${chatGroup.sourceId}) - ${chatGroup.syncData}`)
+    // console.log(`- Processing group: ${chatGroup.name} (${chatGroup.sourceId}) - ${chatGroup.syncData}`)
     const chatHistory: SchemaSocialChatMessage[] = []
     const rangeTracker = new ItemsRangeTracker(chatGroup.syncData)
     let groupMessageCount = 0
@@ -283,7 +296,6 @@ export default class TelegramChatMessageHandler extends BaseSyncHandler {
       const userCache = new UsersCache(api)
       const chatGroups: SchemaSocialChatGroup[] = []
       const chatGroupsBacklog = await this.buildChatGroupList(api, syncPosition)
-      console.log(`- Fetched ${chatGroupsBacklog.length} chat groups as backlog`)
       let chatHistory: SchemaSocialChatMessage[] = []
 
       // Process each chat group
@@ -337,8 +349,8 @@ export default class TelegramChatMessageHandler extends BaseSyncHandler {
     }
 
     if (content == "") {
-      console.log('empty content')
-      console.log(rawMessage.content)
+      // console.log('empty content')
+      // console.log(rawMessage.content)
       return
     }
 
@@ -377,11 +389,9 @@ export default class TelegramChatMessageHandler extends BaseSyncHandler {
       }
 
       const groupDetails = await api.getChatGroup(parseInt(groupId))
-      console.log(groupDetails.title, groupDetails.type._)
       if (groupDetails.type._ == TelegramChatGroupType.SUPERGROUP) {
         const supergroupDetails = await api.getSupergroup(groupDetails.type.supergroup_id)
         if (supergroupDetails.member_count > this.config.maxGroupSize) {
-          console.log('group too big')
           continue
         }
       }
diff --git a/src/providers/twitter/posts.ts b/src/providers/twitter/posts.ts
index cd610c17..62c6db60 100644
--- a/src/providers/twitter/posts.ts
+++ b/src/providers/twitter/posts.ts
@@ -11,7 +11,7 @@ export default class Posts extends BaseSyncHandler {
     protected static schemaUri: string = 'https://common.schemas.verida.io/social/post/v0.1.0/schema.json'
 
     public getName(): string {
-        return 'posts'
+        return 'post'
     }
 
     /**
diff --git a/src/sync-manager.ts b/src/sync-manager.ts
index ad1bf493..cd1a36fb 100644
--- a/src/sync-manager.ts
+++ b/src/sync-manager.ts
@@ -178,7 +178,7 @@ export default class SyncManager {
             const handlerConfig: Record = {}
 
             for (const handlerOption of handlerOptions) {
-                handlerConfig[handlerOption.name] = handlerOption.defaultValue
+                handlerConfig[handlerOption.id] = handlerOption.defaultValue
             }
 
             connectionHandlers.push({
@@ -190,7 +190,7 @@ export default class SyncManager {
 
         const providerConfig: Record = {}
         for (const providerOption of provider.getOptions()) {
-            providerConfig[providerOption.name] = providerOption.defaultValue
+            providerConfig[providerOption.id] = providerOption.defaultValue
         }
 
         providerConnection = {

From 4f143066735fefec377ba127b5b072cec38b4848 Mon Sep 17 00:00:00 2001
From: Chris 
Date: Tue, 3 Sep 2024 16:18:22 +0930
Subject: [PATCH 084/182] Improve AI page layout

---
 src/web/user/ai/index.html | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/src/web/user/ai/index.html b/src/web/user/ai/index.html
index 56a39eeb..a884fe37 100644
--- a/src/web/user/ai/index.html
+++ b/src/web/user/ai/index.html
@@ -13,7 +13,7 @@
             background-color: #f0f0f5;
         }
         .chat-container {
-            height: 60vh;
+            height: 75vh;
             overflow-y: auto;
             border: 1px solid #ddd;
             border-radius: 10px;
@@ -55,7 +55,6 @@
             font-style: italic;
         }
         .input-group {
-            position: fixed;
             bottom: 20px;
             left: 0;
             right: 0;
@@ -159,7 +158,7 @@
     
     

         

-            

+            

                 

                     
                     
@@ -178,7 +177,7 @@
                 

                 

                 

-                    
+                    
                     

                         
                     


From be9e181012962c8958d6f0553757c576dd27bded Mon Sep 17 00:00:00 2001
From: Chris 
Date: Tue, 3 Sep 2024 16:18:40 +0930
Subject: [PATCH 085/182] Disable databases for telegram

---
 src/providers/telegram/api.ts | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/providers/telegram/api.ts b/src/providers/telegram/api.ts
index 58ac8539..7a7cca13 100644
--- a/src/providers/telegram/api.ts
+++ b/src/providers/telegram/api.ts
@@ -59,6 +59,9 @@ export class TelegramApi {
             application_version: '0.1',
             database_directory: `${this.tdPath}/db`,
             files_directory: `${this.tdPath}/files`,
+            use_chat_info_database: false,
+            use_message_database: false,
+            use_file_database: false
         })
     }
 

From a371bdc7ec7fbba43c23c48b89494e7ecd4a8ca8 Mon Sep 17 00:00:00 2001
From: Chris 
Date: Tue, 3 Sep 2024 16:26:22 +0930
Subject: [PATCH 086/182] Fix connection page to use updated provider response

---
 src/web/user/connections/connections.js | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/web/user/connections/connections.js b/src/web/user/connections/connections.js
index a5543333..495f9278 100644
--- a/src/web/user/connections/connections.js
+++ b/src/web/user/connections/connections.js
@@ -167,7 +167,10 @@ $(document).ready(function() {
 
     function loadProviders(callback) {
         $.getJSON('/api/v1/providers', function(providersResponse) {
-            window.providersData = providersResponse; // Store globally or manage differently as needed
+            window.providersData = {}
+            for (const provider of providersResponse) {
+                window.providersData[provider.name] = provider
+            }
             populateConnectionDropdown(providersResponse);
             if (callback) callback();
         });

From 9a3cc7e2c136d397f6f9e2a91165e9bad470273f Mon Sep 17 00:00:00 2001
From: chime3 
Date: Tue, 3 Sep 2024 22:46:38 -0700
Subject: [PATCH 087/182] feat: added calendar schema and handler

---
 src/providers/google/calendar.ts | 213 +++++++++++++++++++++++++++++++
 src/schemas.ts                   |   6 +
 src/serverconfig.example.json    |   4 +-
 3 files changed, 222 insertions(+), 1 deletion(-)
 create mode 100644 src/providers/google/calendar.ts

diff --git a/src/providers/google/calendar.ts b/src/providers/google/calendar.ts
new file mode 100644
index 00000000..8070cdaf
--- /dev/null
+++ b/src/providers/google/calendar.ts
@@ -0,0 +1,213 @@
+import GoogleHandler from "./GoogleHandler";
+import CONFIG from "../../config";
+import { SyncHandlerPosition, SyncItemsBreak, SyncItemsResult, SyncProviderLogEvent, SyncProviderLogLevel } from '../../interfaces';
+import { google, calendar_v3 } from "googleapis";
+import { GaxiosResponse } from "gaxios";
+import { ItemsRangeTracker } from "../../helpers/itemsRangeTracker";
+
+import {
+  SyncResponse,
+  SyncHandlerStatus,
+  ProviderHandlerOption,
+  ConnectionOptionType,
+} from "../../interfaces";
+import { SchemaCalendar } from "../../schemas";
+
+const _ = require("lodash");
+
+const MAX_BATCH_SIZE = 250;
+
+export interface SyncCalendarItemsResult extends SyncItemsResult {
+  items: SchemaCalendar[];
+}
+
+export default class Calendar extends GoogleHandler {
+
+  public getName(): string {
+    return 'calendar';
+  }
+
+  public getSchemaUri(): string {
+    return CONFIG.verida.schemas.CALENDAR_LIST;
+  }
+
+  public getProviderApplicationUrl() {
+    return 'https://calendar.google.com/';
+  }
+
+  public getCalendar(): calendar_v3.Calendar {
+    const oAuth2Client = this.getGoogleAuth();
+
+    const calendar = google.calendar({ version: "v3", auth: oAuth2Client });
+    return calendar;
+  }
+
+  public getOptions(): ProviderHandlerOption[] {
+    return [{
+      id: 'backdate',
+      label: 'Backdate history',
+      type: ConnectionOptionType.ENUM,
+      enumOptions: [{
+        value: '1-month',
+        label: '1 month'
+      }, {
+        value: '3-months',
+        label: '3 months'
+      }, {
+        value: '6-months',
+        label: '6 months'
+      }, {
+        value: '12-months',
+        label: '12 months'
+      }],
+      defaultValue: '3-months'
+    }]
+  }
+
+  public async _sync(
+    api: any,
+    syncPosition: SyncHandlerPosition
+  ): Promise {
+    if (this.config.batchSize > MAX_BATCH_SIZE) {
+      throw new Error(`Batch size (${this.config.batchSize}) is larger than permitted (${MAX_BATCH_SIZE})`);
+    }
+
+    const calendar = this.getCalendar();
+    const rangeTracker = new ItemsRangeTracker(syncPosition.thisRef);
+
+    let items: SchemaCalendar[] = [];
+
+    // Fetch any new items
+    let currentRange = rangeTracker.nextRange();
+
+    let query: calendar_v3.Params$Resource$Calendarlist$List = {
+      maxResults: this.config.batchSize, // default = 250, max = 250
+    };
+
+    if (currentRange.startId) {
+      query.pageToken = currentRange.startId;
+    }
+
+    const latestResponse = await calendar.calendarList.list(query);
+    const latestResult = await this.buildResults(
+      calendar,
+      latestResponse,
+      currentRange.endId
+    );
+
+    items = latestResult.items;
+
+    let nextPageToken = _.has(latestResponse, "data.nextPageToken") ? latestResponse.data.nextPageToken : undefined;
+
+    if (items.length) {
+      rangeTracker.completedRange({
+        startId: items[0].sourceId,
+        endId: nextPageToken
+      }, latestResult.breakHit == SyncItemsBreak.ID);
+    } else {
+      rangeTracker.completedRange({
+        startId: undefined,
+        endId: undefined
+      }, false);
+    }
+
+    if (items.length != this.config.batchSize) {
+      currentRange = rangeTracker.nextRange();
+
+      query = {
+        maxResults: this.config.batchSize - items.length,
+      };
+
+      if (currentRange.startId) {
+        query.pageToken = currentRange.startId;
+      }
+
+      const backfillResponse = await calendar.calendarList.list(query);
+      const backfillResult = await this.buildResults(
+        calendar,
+        backfillResponse,
+        currentRange.endId
+      );
+
+      items = items.concat(backfillResult.items);
+
+      nextPageToken = _.has(backfillResponse, "data.nextPageToken") ? backfillResponse.data.nextPageToken : undefined;
+
+      if (backfillResult.items.length) {
+        rangeTracker.completedRange({
+          startId: backfillResult.items[0].sourceId,
+          endId: nextPageToken
+        }, backfillResult.breakHit == SyncItemsBreak.ID);
+      } else {
+        rangeTracker.completedRange({
+          startId: undefined,
+          endId: undefined
+        }, backfillResult.breakHit == SyncItemsBreak.ID);
+      }
+    }
+
+    if (!items.length) {
+      syncPosition.syncMessage = `Stopping. No results found.`;
+      syncPosition.status = SyncHandlerStatus.ENABLED;
+    } else {
+      if (items.length != this.config.batchSize && !nextPageToken) {
+        syncPosition.syncMessage = `Processed ${items.length} items. Stopping. No more results.`;
+        syncPosition.status = SyncHandlerStatus.ENABLED;
+      } else {
+        syncPosition.syncMessage = `Batch complete (${this.config.batchSize}). More results pending.`;
+      }
+    }
+
+    syncPosition.thisRef = rangeTracker.export();
+
+    return {
+      results: items,
+      position: syncPosition,
+    };
+  }
+
+  protected async buildResults(
+    calendar: calendar_v3.Calendar,
+    serverResponse: GaxiosResponse,
+    breakId: string
+  ): Promise {
+    const results: SchemaCalendar[] = [];
+    let breakHit: SyncItemsBreak;
+
+    for (const listItem of serverResponse.data.items) {
+      const calendarId = listItem.id;
+
+      if (calendarId == breakId) {
+        const logEvent: SyncProviderLogEvent = {
+          level: SyncProviderLogLevel.DEBUG,
+          message: `Break ID hit (${breakId})`
+        };
+        this.emit('log', logEvent);
+        breakHit = SyncItemsBreak.ID;
+        break;
+      }
+
+      const summary = listItem.summary;
+      const timeZone = listItem.timeZone;
+      const description = listItem.description;
+      const location = listItem.location;
+
+      results.push({
+        _id: this.buildItemId(calendarId),
+        name: summary || 'No calendar title',
+        sourceAccountId: this.provider.getProviderId(),
+        sourceData: listItem,
+        sourceApplication: this.getProviderApplicationUrl(),
+        sourceId: listItem.id,
+        timezone: timeZone || 'No time zone',
+        description: description || 'No description',
+        location: location || 'No location',
+      });
+    }
+
+    return {
+      items: results,
+      breakHit
+    };
+  }
+}
diff --git a/src/schemas.ts b/src/schemas.ts
index b7697114..e61887ea 100644
--- a/src/schemas.ts
+++ b/src/schemas.ts
@@ -131,3 +131,9 @@ export interface SchemaFile extends SchemaRecord {
     uri?: string
 
 }
+
+export interface SchemaCalendar extends SchemaRecord {
+    description?: string
+    timezone: string
+    location?: string
+}
diff --git a/src/serverconfig.example.json b/src/serverconfig.example.json
index 5faf2412..1ddab795 100644
--- a/src/serverconfig.example.json
+++ b/src/serverconfig.example.json
@@ -26,7 +26,9 @@
             "FAVOURITE": "https://common.schemas.verida.io/favourite/v0.1.0/schema.json",
             "FILE": "https://common.schemas.verida.io/file/v0.1.0/schema.json",
             "CHAT_GROUP": "https://common.schemas.verida.io/social/chat/group/v0.1.0/schema.json",
-            "CHAT_MESSAGE": "https://common.schemas.verida.io/social/chat/message/v0.1.0/schema.json"
+            "CHAT_MESSAGE": "https://common.schemas.verida.io/social/chat/message/v0.1.0/schema.json",
+            "CALENDAR": "https://common.schemas.verida.io/social/calendar/v0.1.0/schema.json",
+            "EVENT": "https://common.schemas.verida.io/social/event/v0.1.0/schema.json"
         },
         "llms": {
             "bedrockEndpoint": "",

From 3a769cf2eb3652c6f5eac3d1179a9af34021177b Mon Sep 17 00:00:00 2001
From: chime3 
Date: Tue, 3 Sep 2024 23:01:35 -0700
Subject: [PATCH 088/182] fix: added comment

---
 src/providers/google/calendar.ts | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/providers/google/calendar.ts b/src/providers/google/calendar.ts
index 8070cdaf..1825fb0b 100644
--- a/src/providers/google/calendar.ts
+++ b/src/providers/google/calendar.ts
@@ -15,6 +15,8 @@ import { SchemaCalendar } from "../../schemas";
 
 const _ = require("lodash");
 
+// Set MAX_BATCH_SIZE to 250 because the Google Calendar API v3 'maxResults' parameter is capped at 250.
+// For more details, see: https://developers.google.com/calendar/api/v3/reference/calendarList/list 
 const MAX_BATCH_SIZE = 250;
 
 export interface SyncCalendarItemsResult extends SyncItemsResult {
@@ -28,7 +30,7 @@ export default class Calendar extends GoogleHandler {
   }
 
   public getSchemaUri(): string {
-    return CONFIG.verida.schemas.CALENDAR_LIST;
+    return CONFIG.verida.schemas.CALENDAR;
   }
 
   public getProviderApplicationUrl() {

From c097a9c9eb553e978bed5e93826561b4e739d9de Mon Sep 17 00:00:00 2001
From: chime3 
Date: Tue, 3 Sep 2024 23:13:25 -0700
Subject: [PATCH 089/182] feat: added event schema and handler

---
 src/providers/google/calendar-event.ts | 238 +++++++++++++++++++++++++
 src/schemas.ts                         |  14 ++
 2 files changed, 252 insertions(+)
 create mode 100644 src/providers/google/calendar-event.ts

diff --git a/src/providers/google/calendar-event.ts b/src/providers/google/calendar-event.ts
new file mode 100644
index 00000000..a79152d4
--- /dev/null
+++ b/src/providers/google/calendar-event.ts
@@ -0,0 +1,238 @@
+import GoogleHandler from "./GoogleHandler";
+import CONFIG from "../../config";
+import { SyncHandlerPosition, SyncItemsBreak, SyncItemsResult, SyncProviderLogEvent, SyncProviderLogLevel } from '../../interfaces';
+import { google, calendar_v3 } from "googleapis";
+import { GaxiosResponse } from "gaxios";
+import { ItemsRangeTracker } from "../../helpers/itemsRangeTracker";
+
+import {
+  SyncResponse,
+  SyncHandlerStatus,
+  ProviderHandlerOption,
+  ConnectionOptionType,
+} from "../../interfaces";
+import { SchemaEvent } from "../../schemas";
+
+const _ = require("lodash");
+
+// Set MAX_BATCH_SIZE to 2500 because the Google Calendar API v3 'maxResults' parameter is capped at 2500.
+// For more details, see: https://developers.google.com/calendar/api/v3/reference/events/list 
+const MAX_BATCH_SIZE = 2500;
+
+export interface SyncCalendarItemsResult extends SyncItemsResult {
+  items: SchemaEvent[];
+}
+
+export default class CalendarEvent extends GoogleHandler {
+
+  public getName(): string {
+    return 'calendar-event';
+  }
+
+  public getSchemaUri(): string {
+    return CONFIG.verida.schemas.EVENT;
+  }
+
+  public getProviderApplicationUrl() {
+    return 'https://calendar.google.com/';
+  }
+
+  public getCalendar(): calendar_v3.Calendar {
+    const oAuth2Client = this.getGoogleAuth();
+
+    const calendar = google.calendar({ version: "v3", auth: oAuth2Client });
+    return calendar;
+  }
+
+  public getOptions(): ProviderHandlerOption[] {
+    return [{
+      id: 'backdate',
+      label: 'Backdate history',
+      type: ConnectionOptionType.ENUM,
+      enumOptions: [{
+        value: '1-month',
+        label: '1 month'
+      }, {
+        value: '3-months',
+        label: '3 months'
+      }, {
+        value: '6-months',
+        label: '6 months'
+      }, {
+        value: '12-months',
+        label: '12 months'
+      }],
+      defaultValue: '3-months'
+    }]
+  }
+
+  public async _sync(
+    api: any,
+    syncPosition: SyncHandlerPosition
+  ): Promise {
+    if (this.config.batchSize > MAX_BATCH_SIZE) {
+      throw new Error(`Batch size (${this.config.batchSize}) is larger than permitted (${MAX_BATCH_SIZE})`);
+    }
+
+    const calendar = this.getCalendar();
+    const rangeTracker = new ItemsRangeTracker(syncPosition.thisRef);
+
+    let items: SchemaEvent[] = [];
+
+    // Fetch any new items
+    let currentRange = rangeTracker.nextRange();
+
+    let query: calendar_v3.Params$Resource$Events$List = {
+      calendarId: 'primary',
+      maxResults: this.config.batchSize, // default = 250, max = 2500
+      singleEvents: true,
+      orderBy: "startTime",
+    };
+
+    if (currentRange.startId) {
+      query.pageToken = currentRange.startId;
+    }
+
+    const latestResponse = await calendar.events.list(query);
+    const latestResult = await this.buildResults(
+      calendar,
+      latestResponse,
+      currentRange.endId,
+      _.has(this.config, "breakTimestamp")
+        ? this.config.breakTimestamp
+        : undefined
+    );
+
+    items = latestResult.items;
+
+    let nextPageToken = _.has(latestResponse, "data.nextPageToken") ? latestResponse.data.nextPageToken : undefined;
+
+    if (items.length) {
+      rangeTracker.completedRange({
+        startId: items[0].sourceId,
+        endId: nextPageToken
+      }, latestResult.breakHit == SyncItemsBreak.ID);
+    } else {
+      rangeTracker.completedRange({
+        startId: undefined,
+        endId: undefined
+      }, false);
+    }
+
+    if (items.length != this.config.batchSize) {
+      currentRange = rangeTracker.nextRange();
+
+      query = {
+        calendarId: 'primary',
+        maxResults: this.config.batchSize - items.length,
+        singleEvents: true,
+        orderBy: "startTime",
+      };
+
+      if (currentRange.startId) {
+        query.pageToken = currentRange.startId;
+      }
+
+      const backfillResponse = await calendar.events.list(query);
+      const backfillResult = await this.buildResults(
+        calendar,
+        backfillResponse,
+        currentRange.endId,
+        _.has(this.config, "breakTimestamp")
+          ? this.config.breakTimestamp
+          : undefined
+      );
+
+      items = items.concat(backfillResult.items);
+
+      nextPageToken = _.has(backfillResponse, "data.nextPageToken") ? backfillResponse.data.nextPageToken : undefined;
+
+      if (backfillResult.items.length) {
+        rangeTracker.completedRange({
+          startId: backfillResult.items[0].sourceId,
+          endId: nextPageToken
+        }, backfillResult.breakHit == SyncItemsBreak.ID);
+      } else {
+        rangeTracker.completedRange({
+          startId: undefined,
+          endId: undefined
+        }, backfillResult.breakHit == SyncItemsBreak.ID);
+      }
+    }
+
+    if (!items.length) {
+      syncPosition.syncMessage = `Stopping. No results found.`;
+      syncPosition.status = SyncHandlerStatus.ENABLED;
+    } else {
+      if (items.length != this.config.batchSize && !nextPageToken) {
+        syncPosition.syncMessage = `Processed ${items.length} items. Stopping. No more results.`;
+        syncPosition.status = SyncHandlerStatus.ENABLED;
+      } else {
+        syncPosition.syncMessage = `Batch complete (${this.config.batchSize}). More results pending.`;
+      }
+    }
+
+    syncPosition.thisRef = rangeTracker.export();
+
+    return {
+      results: items,
+      position: syncPosition,
+    };
+  }
+
+  protected async buildResults(
+    calendar: calendar_v3.Calendar,
+    serverResponse: GaxiosResponse,
+    breakId: string,
+    breakTimestamp?: string
+  ): Promise {
+    const results: SchemaEvent[] = [];
+    let breakHit: SyncItemsBreak;
+    
+    for (const event of serverResponse.data.items) {
+      const eventId = event.id;
+
+      if (eventId == breakId) {
+        const logEvent: SyncProviderLogEvent = {
+          level: SyncProviderLogLevel.DEBUG,
+          message: `Break ID hit (${breakId})`
+        };
+        this.emit('log', logEvent);
+        breakHit = SyncItemsBreak.ID;
+        break;
+      }
+
+      const startTime = event.start?.dateTime || event.start?.date;
+      const endTime = event.end?.dateTime || event.end?.date;
+
+      if (breakTimestamp && startTime < breakTimestamp) {
+        const logEvent: SyncProviderLogEvent = {
+          level: SyncProviderLogLevel.DEBUG,
+          message: `Break timestamp hit (${breakTimestamp})`
+        };
+        this.emit('log', logEvent);
+        breakHit = SyncItemsBreak.TIMESTAMP;
+        break;
+      }
+
+      results.push({
+        _id: this.buildItemId(eventId),
+        name: event.summary || 'No event title',
+        sourceAccountId: this.provider.getProviderId(),
+        sourceData: event,
+        sourceApplication: this.getProviderApplicationUrl(),
+        sourceId: event.id,
+        startTime,
+        endTime,
+        location: event.location || 'No location',
+        description: event.description || 'No description',
+        attendees: event.attendees || [],
+      });
+    }
+
+    return {
+      items: results,
+      breakHit
+    };
+  }
+}
diff --git a/src/schemas.ts b/src/schemas.ts
index e61887ea..000ce248 100644
--- a/src/schemas.ts
+++ b/src/schemas.ts
@@ -137,3 +137,17 @@ export interface SchemaCalendar extends SchemaRecord {
     timezone: string
     location?: string
 }
+
+export interface SchemaEvent extends SchemaRecord {
+    status?: string
+    description?: string
+    calendarId: string
+    location?: string
+    creator?: object
+    start: object
+    end: object
+    attendees?: []
+    conferenceData?: object
+    attachments?: []
+    
+}

From f1fb820c6bf471cfaeb8720361d36899ad8d22be Mon Sep 17 00:00:00 2001
From: chime3 
Date: Wed, 4 Sep 2024 06:17:36 -0700
Subject: [PATCH 090/182] feat: define calendar data types

---
 src/providers/google/interfaces.ts | 19 +++++++++++++++++++
 src/schemas.ts                     | 14 ++++++++------
 2 files changed, 27 insertions(+), 6 deletions(-)

diff --git a/src/providers/google/interfaces.ts b/src/providers/google/interfaces.ts
index 3fa81c25..aea96ca2 100644
--- a/src/providers/google/interfaces.ts
+++ b/src/providers/google/interfaces.ts
@@ -33,4 +33,23 @@ export enum YoutubeActivityType {
   COMMENT = "comment", // ingored
   PLAYLIST_ITEM = "playlistItem", //ignored
   RECOMMENDATION = "recommendation", // favourite
+}
+
+export interface Person {
+  email: string
+  displayName?: string
+}
+
+export interface DateTimeInfo {
+  dateTime?: string;  // ISO format: YYYY-MM-DDTHH:mm:ss.sssZ
+  date: string;      // ISO ate format: YYYY-MM-DD
+  timeZone?: string;  // UTC offset format: ±HH:MM
+}
+
+export interface CalendarAttachment {
+  fileUrl?: string;   // URL of the file
+  title?: string;     // Title of the attachment
+  mimeType?: string;  // MIME type of the file
+  iconLink?: string;  // URL of the icon representing the file
+  fileId?: string;    // Unique identifier for the file
 }
\ No newline at end of file
diff --git a/src/schemas.ts b/src/schemas.ts
index 000ce248..b460aa16 100644
--- a/src/schemas.ts
+++ b/src/schemas.ts
@@ -1,3 +1,5 @@
+import { CalendarAttachment, DateTimeInfo, Person } from "./providers/google/interfaces"
+
 export interface SchemaRecord {
     _id: string
     _rev?: string
@@ -143,11 +145,11 @@ export interface SchemaEvent extends SchemaRecord {
     description?: string
     calendarId: string
     location?: string
-    creator?: object
-    start: object
-    end: object
-    attendees?: []
+    creator?: Person
+    start: DateTimeInfo
+    end: DateTimeInfo
+    attendees?: Person[]
     conferenceData?: object
-    attachments?: []
-    
+    attachments?: CalendarAttachment[]
+
 }

From 687e8dc7ee4127f0fe8fc272b9dad47a95e05730 Mon Sep 17 00:00:00 2001
From: chime3 
Date: Wed, 4 Sep 2024 20:20:28 -0700
Subject: [PATCH 091/182] fix: start & end date time info

---
 src/providers/google/calendar-event.ts | 17 +++++++++--------
 src/providers/google/interfaces.ts     |  2 +-
 2 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/src/providers/google/calendar-event.ts b/src/providers/google/calendar-event.ts
index a79152d4..23487a02 100644
--- a/src/providers/google/calendar-event.ts
+++ b/src/providers/google/calendar-event.ts
@@ -12,6 +12,7 @@ import {
   ConnectionOptionType,
 } from "../../interfaces";
 import { SchemaEvent } from "../../schemas";
+import { DateTimeInfo } from "./interfaces";
 
 const _ = require("lodash");
 
@@ -202,10 +203,10 @@ export default class CalendarEvent extends GoogleHandler {
         break;
       }
 
-      const startTime = event.start?.dateTime || event.start?.date;
-      const endTime = event.end?.dateTime || event.end?.date;
+      const start: DateTimeInfo = event.start;
+      const end: DateTimeInfo = event.end;
 
-      if (breakTimestamp && startTime < breakTimestamp) {
+      if (breakTimestamp && start.dateTime < breakTimestamp) {
         const logEvent: SyncProviderLogEvent = {
           level: SyncProviderLogLevel.DEBUG,
           message: `Break timestamp hit (${breakTimestamp})`
@@ -221,12 +222,12 @@ export default class CalendarEvent extends GoogleHandler {
         sourceAccountId: this.provider.getProviderId(),
         sourceData: event,
         sourceApplication: this.getProviderApplicationUrl(),
-        sourceId: event.id,
-        startTime,
-        endTime,
+        sourceId: "primary",
+        calendarId: eventId,
+        start,
+        end,
         location: event.location || 'No location',
-        description: event.description || 'No description',
-        attendees: event.attendees || [],
+        description: event.description || 'No description'       
       });
     }
 
diff --git a/src/providers/google/interfaces.ts b/src/providers/google/interfaces.ts
index aea96ca2..54527b51 100644
--- a/src/providers/google/interfaces.ts
+++ b/src/providers/google/interfaces.ts
@@ -42,7 +42,7 @@ export interface Person {
 
 export interface DateTimeInfo {
   dateTime?: string;  // ISO format: YYYY-MM-DDTHH:mm:ss.sssZ
-  date: string;      // ISO ate format: YYYY-MM-DD
+  date?: string;      // ISO ate format: YYYY-MM-DD
   timeZone?: string;  // UTC offset format: ±HH:MM
 }
 

From 3aa8880cb9b0a3aab17502f28a11b328ca147bb0 Mon Sep 17 00:00:00 2001
From: chime3 
Date: Wed, 4 Sep 2024 20:22:25 -0700
Subject: [PATCH 092/182] feat: added calendar scope and handlers

---
 src/providers/google/index.ts | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/src/providers/google/index.ts b/src/providers/google/index.ts
index cad84cad..9debbf0c 100644
--- a/src/providers/google/index.ts
+++ b/src/providers/google/index.ts
@@ -6,6 +6,8 @@ import YouTubePost from "./youtube-post";
 import { GoogleProviderConfig, GoogleProviderConnection } from "./interfaces";
 import YouTubeFavourite from "./youtube-favourite";
 import GoogleDriveDocument from "./gdrive-document";
+import Calendar from "./calendar";
+import CalendarEvent from "./calendar-event";
 
 const passport = require("passport");
 const GoogleStrategy = require("passport-google-oauth20");
@@ -32,7 +34,9 @@ export default class GoogleProvider extends Base {
       YouTubeFollowing,
       YouTubePost,
       YouTubeFavourite,
-      GoogleDriveDocument
+      GoogleDriveDocument,
+      Calendar,
+      CalendarEvent
     ];
   }
 
@@ -43,7 +47,8 @@ export default class GoogleProvider extends Base {
         "email",
         "https://www.googleapis.com/auth/gmail.readonly",
         "https://www.googleapis.com/auth/youtube.readonly",
-        "https://www.googleapis.com/auth/drive.readonly"
+        "https://www.googleapis.com/auth/drive.readonly",
+        "https://www.googleapis.com/auth/calendar.readonly"
     ];
   }
 

From 765f5410f5a6a661fed17099271b8710718c54a9 Mon Sep 17 00:00:00 2001
From: chime3 
Date: Wed, 4 Sep 2024 22:56:25 -0700
Subject: [PATCH 093/182] feat: added timezone lib

---
 package.json |  1 +
 yarn.lock    | 12 ++++++++++++
 2 files changed, 13 insertions(+)

diff --git a/package.json b/package.json
index 9fb3e652..faf0a553 100644
--- a/package.json
+++ b/package.json
@@ -68,6 +68,7 @@
     "memory-cache": "^0.2.0",
     "minisearch": "^7.1.0",
     "mocha": "^9.2.1",
+    "moment-timezone": "^0.5.45",
     "nano": "^9.0.5",
     "node-imap": "^0.9.6",
     "officeparser": "^4.1.1",
diff --git a/yarn.lock b/yarn.lock
index d3a80f6f..3edc3b33 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -4216,6 +4216,18 @@ mocha@^9.2.1:
     yargs-parser "20.2.4"
     yargs-unparser "2.0.0"
 
+moment-timezone@^0.5.45:
+  version "0.5.45"
+  resolved "https://registry.yarnpkg.com/moment-timezone/-/moment-timezone-0.5.45.tgz#cb685acd56bac10e69d93c536366eb65aa6bcf5c"
+  integrity sha512-HIWmqA86KcmCAhnMAN0wuDOARV/525R2+lOLotuGFzn4HO+FH+/645z2wx0Dt3iDv6/p61SIvKnDstISainhLQ==
+  dependencies:
+    moment "^2.29.4"
+
+moment@^2.29.4:
+  version "2.30.1"
+  resolved "https://registry.yarnpkg.com/moment/-/moment-2.30.1.tgz#f8c91c07b7a786e30c59926df530b4eac96974ae"
+  integrity sha512-uEmtNhbDOrWPFS+hdjFCBfy9f2YoyzRpwcl+DqpC6taX21FzsTLQVbMV/W7PzNSX6x/bhC1zA3c2UQ5NzH6how==
+
 ms@2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8"

From d1980deb58329207b6eabd45c9dcf8a6fd569809 Mon Sep 17 00:00:00 2001
From: chime3 
Date: Wed, 4 Sep 2024 22:58:47 -0700
Subject: [PATCH 094/182] refactor: calendar buildresults function

---
 src/providers/google/calendar.ts | 55 ++++++++++++++++++++++----------
 1 file changed, 38 insertions(+), 17 deletions(-)

diff --git a/src/providers/google/calendar.ts b/src/providers/google/calendar.ts
index 1825fb0b..22ef78bf 100644
--- a/src/providers/google/calendar.ts
+++ b/src/providers/google/calendar.ts
@@ -5,6 +5,8 @@ import { google, calendar_v3 } from "googleapis";
 import { GaxiosResponse } from "gaxios";
 import { ItemsRangeTracker } from "../../helpers/itemsRangeTracker";
 
+import moment from "moment-timezone";
+
 import {
   SyncResponse,
   SyncHandlerStatus,
@@ -39,9 +41,7 @@ export default class Calendar extends GoogleHandler {
 
   public getCalendar(): calendar_v3.Calendar {
     const oAuth2Client = this.getGoogleAuth();
-
-    const calendar = google.calendar({ version: "v3", auth: oAuth2Client });
-    return calendar;
+    return google.calendar({ version: "v3", auth: oAuth2Client });
   }
 
   public getOptions(): ProviderHandlerOption[] {
@@ -63,7 +63,7 @@ export default class Calendar extends GoogleHandler {
         label: '12 months'
       }],
       defaultValue: '3-months'
-    }]
+    }];
   }
 
   public async _sync(
@@ -81,9 +81,8 @@ export default class Calendar extends GoogleHandler {
 
     // Fetch any new items
     let currentRange = rangeTracker.nextRange();
-
     let query: calendar_v3.Params$Resource$Calendarlist$List = {
-      maxResults: this.config.batchSize, // default = 250, max = 250
+      maxResults: this.config.batchSize,
     };
 
     if (currentRange.startId) {
@@ -115,7 +114,6 @@ export default class Calendar extends GoogleHandler {
 
     if (items.length != this.config.batchSize) {
       currentRange = rangeTracker.nextRange();
-
       query = {
         maxResults: this.config.batchSize - items.length,
       };
@@ -132,7 +130,6 @@ export default class Calendar extends GoogleHandler {
       );
 
       items = items.concat(backfillResult.items);
-
       nextPageToken = _.has(backfillResponse, "data.nextPageToken") ? backfillResponse.data.nextPageToken : undefined;
 
       if (backfillResult.items.length) {
@@ -178,7 +175,16 @@ export default class Calendar extends GoogleHandler {
 
     for (const listItem of serverResponse.data.items) {
       const calendarId = listItem.id;
-
+      
+      if (!calendarId) {
+        const logEvent: SyncProviderLogEvent = {
+            level: SyncProviderLogLevel.DEBUG,
+            message: `Invalid calendar ID. Ignoring this calendar.`,
+        };
+        this.emit('log', logEvent);
+        continue;
+      }
+      
       if (calendarId == breakId) {
         const logEvent: SyncProviderLogEvent = {
           level: SyncProviderLogLevel.DEBUG,
@@ -189,21 +195,36 @@ export default class Calendar extends GoogleHandler {
         break;
       }
 
-      const summary = listItem.summary;
+      const summary = listItem.summary ?? 'No calendar title';
       const timeZone = listItem.timeZone;
-      const description = listItem.description;
-      const location = listItem.location;
+
+      if (!timeZone) {
+        const logEvent: SyncProviderLogEvent = {
+            level: SyncProviderLogLevel.DEBUG,
+            message: `Invalid timezone for calendar ${calendarId}. Ignoring this calendar.`,
+        };
+        this.emit('log', logEvent);
+        continue;
+      }
+      
+      const now = moment.tz(timeZone);
+      const utcOffset   = now.format("Z");
+
+      const description = listItem.description ?? 'No description';
+      const location = listItem.location ?? 'No location';
+      const insertedAt = new Date().toISOString(); // Adding insertedAt field
 
       results.push({
         _id: this.buildItemId(calendarId),
-        name: summary || 'No calendar title',
+        name: summary,
         sourceAccountId: this.provider.getProviderId(),
         sourceData: listItem,
         sourceApplication: this.getProviderApplicationUrl(),
-        sourceId: listItem.id,
-        timezone: timeZone || 'No time zone',
-        description: description || 'No description',
-        location: location || 'No location',
+        sourceId: calendarId,
+        timezone: utcOffset,
+        description,
+        location,
+        insertedAt,  // insertedAt field
       });
     }
 

From 2776521334182253f077564ae6e2aedf16a621c9 Mon Sep 17 00:00:00 2001
From: chime3 
Date: Wed, 4 Sep 2024 23:37:01 -0700
Subject: [PATCH 095/182] feat: added calendar unit test

---
 tests/providers/google/calendar.tests.ts | 54 ++++++++++++++++++++++++
 1 file changed, 54 insertions(+)
 create mode 100644 tests/providers/google/calendar.tests.ts

diff --git a/tests/providers/google/calendar.tests.ts b/tests/providers/google/calendar.tests.ts
new file mode 100644
index 00000000..bbf2230c
--- /dev/null
+++ b/tests/providers/google/calendar.tests.ts
@@ -0,0 +1,54 @@
+const assert = require("assert");
+import {
+  BaseProviderConfig,
+  Connection,
+  SyncHandlerStatus,
+  SyncHandlerPosition,
+} from "../../../src/interfaces";
+import Providers from "../../../src/providers";
+import CommonUtils, { NetworkInstance } from "../../common.utils";
+
+import GoogleCalendar from "../../../src/providers/google/calendar";
+import BaseProvider from "../../../src/providers/BaseProvider";
+import { CommonTests, GenericTestConfig } from "../../common.tests";
+
+const providerName = "google";
+let network: NetworkInstance;
+let connection: Connection;
+let provider: BaseProvider;
+let handlerName = "google-calendar";
+let testConfig: GenericTestConfig;
+let providerConfig: Omit = {};
+
+describe(`${providerName} Google Calendar Tests`, function () {
+  this.timeout(100000);
+
+  this.beforeAll(async function () {
+    network = await CommonUtils.getNetwork();
+    connection = await CommonUtils.getConnection(providerName);
+    provider = Providers(providerName, network.context, connection);
+
+    testConfig = {
+      idPrefix: `${provider.getProviderName()}-${connection.profile.id}`,
+      batchSizeLimitAttribute: "batchSize",
+    };
+  });
+
+  describe(`Fetch ${providerName} data`, () => {
+   
+    it(`Can pass basic tests: ${handlerName}`, async () => {
+      await CommonTests.runGenericTests(
+        providerName,
+        GoogleCalendar,
+        testConfig,
+        providerConfig,
+        connection
+      );
+    });
+  });
+
+  this.afterAll(async function () {
+    const { context } = await CommonUtils.getNetwork();
+    await context.close();
+  });
+});

From e2b332dcb5a6cf24643461e42f99552b7d389dfe Mon Sep 17 00:00:00 2001
From: chime3 
Date: Thu, 5 Sep 2024 00:02:08 -0700
Subject: [PATCH 096/182] refactor: calendar event handler

---
 src/providers/google/calendar-event.ts | 42 +++++++++++++++-----------
 1 file changed, 24 insertions(+), 18 deletions(-)

diff --git a/src/providers/google/calendar-event.ts b/src/providers/google/calendar-event.ts
index 23487a02..81cd36cc 100644
--- a/src/providers/google/calendar-event.ts
+++ b/src/providers/google/calendar-event.ts
@@ -40,9 +40,7 @@ export default class CalendarEvent extends GoogleHandler {
 
   public getCalendar(): calendar_v3.Calendar {
     const oAuth2Client = this.getGoogleAuth();
-
-    const calendar = google.calendar({ version: "v3", auth: oAuth2Client });
-    return calendar;
+    return google.calendar({ version: "v3", auth: oAuth2Client });
   }
 
   public getOptions(): ProviderHandlerOption[] {
@@ -64,7 +62,7 @@ export default class CalendarEvent extends GoogleHandler {
         label: '12 months'
       }],
       defaultValue: '3-months'
-    }]
+    }];
   }
 
   public async _sync(
@@ -99,14 +97,12 @@ export default class CalendarEvent extends GoogleHandler {
       calendar,
       latestResponse,
       currentRange.endId,
-      _.has(this.config, "breakTimestamp")
-        ? this.config.breakTimestamp
-        : undefined
+      this.config.breakTimestamp ?? undefined
     );
 
     items = latestResult.items;
 
-    let nextPageToken = _.has(latestResponse, "data.nextPageToken") ? latestResponse.data.nextPageToken : undefined;
+    let nextPageToken = latestResponse.data.nextPageToken ?? undefined;
 
     if (items.length) {
       rangeTracker.completedRange({
@@ -139,14 +135,12 @@ export default class CalendarEvent extends GoogleHandler {
         calendar,
         backfillResponse,
         currentRange.endId,
-        _.has(this.config, "breakTimestamp")
-          ? this.config.breakTimestamp
-          : undefined
+        this.config.breakTimestamp ?? undefined
       );
 
       items = items.concat(backfillResult.items);
 
-      nextPageToken = _.has(backfillResponse, "data.nextPageToken") ? backfillResponse.data.nextPageToken : undefined;
+      nextPageToken = backfillResponse.data.nextPageToken ?? undefined;
 
       if (backfillResult.items.length) {
         rangeTracker.completedRange({
@@ -189,7 +183,7 @@ export default class CalendarEvent extends GoogleHandler {
   ): Promise {
     const results: SchemaEvent[] = [];
     let breakHit: SyncItemsBreak;
-    
+
     for (const event of serverResponse.data.items) {
       const eventId = event.id;
 
@@ -206,6 +200,15 @@ export default class CalendarEvent extends GoogleHandler {
       const start: DateTimeInfo = event.start;
       const end: DateTimeInfo = event.end;
 
+      if (!start.date || !end.date) {
+        const logEvent: SyncProviderLogEvent = {
+            level: SyncProviderLogLevel.DEBUG,
+            message: `Invalid date for the event ${eventId}. Ignoring this event.`,
+        };
+        this.emit('log', logEvent);
+        continue;
+      }
+
       if (breakTimestamp && start.dateTime < breakTimestamp) {
         const logEvent: SyncProviderLogEvent = {
           level: SyncProviderLogLevel.DEBUG,
@@ -216,18 +219,21 @@ export default class CalendarEvent extends GoogleHandler {
         break;
       }
 
+      const insertedAt = new Date().toISOString();
+
       results.push({
         _id: this.buildItemId(eventId),
-        name: event.summary || 'No event title',
+        name: event.summary ?? 'No event title',
         sourceAccountId: this.provider.getProviderId(),
         sourceData: event,
         sourceApplication: this.getProviderApplicationUrl(),
-        sourceId: "primary",
-        calendarId: eventId,
+        sourceId: eventId,
+        calendarId: "primary",
         start,
         end,
-        location: event.location || 'No location',
-        description: event.description || 'No description'       
+        location: event.location ?? 'No location',
+        description: event.description ?? 'No description',
+        insertedAt
       });
     }
 

From 69913a2b719af8d20cf0997c234bd394353f75c1 Mon Sep 17 00:00:00 2001
From: chime3 
Date: Thu, 5 Sep 2024 19:45:56 -0700
Subject: [PATCH 097/182] feat: added calendar helper

---
 src/providers/google/calendar.ts |  8 ++++----
 src/providers/google/helpers.ts  | 13 ++++++++++---
 2 files changed, 14 insertions(+), 7 deletions(-)

diff --git a/src/providers/google/calendar.ts b/src/providers/google/calendar.ts
index 22ef78bf..99b1cbf0 100644
--- a/src/providers/google/calendar.ts
+++ b/src/providers/google/calendar.ts
@@ -14,6 +14,7 @@ import {
   ConnectionOptionType,
 } from "../../interfaces";
 import { SchemaCalendar } from "../../schemas";
+import { CalendarHelpers } from "./helpers";
 
 const _ = require("lodash");
 
@@ -196,7 +197,7 @@ export default class Calendar extends GoogleHandler {
       }
 
       const summary = listItem.summary ?? 'No calendar title';
-      const timeZone = listItem.timeZone;
+      let timeZone = listItem.timeZone;
 
       if (!timeZone) {
         const logEvent: SyncProviderLogEvent = {
@@ -207,8 +208,7 @@ export default class Calendar extends GoogleHandler {
         continue;
       }
       
-      const now = moment.tz(timeZone);
-      const utcOffset   = now.format("Z");
+      timeZone = CalendarHelpers.getUTCOffsetTimezone(timeZone);
 
       const description = listItem.description ?? 'No description';
       const location = listItem.location ?? 'No location';
@@ -221,7 +221,7 @@ export default class Calendar extends GoogleHandler {
         sourceData: listItem,
         sourceApplication: this.getProviderApplicationUrl(),
         sourceId: calendarId,
-        timezone: utcOffset,
+        timezone: timeZone,
         description,
         location,
         insertedAt,  // insertedAt field
diff --git a/src/providers/google/helpers.ts b/src/providers/google/helpers.ts
index f2d3c64b..403a909a 100644
--- a/src/providers/google/helpers.ts
+++ b/src/providers/google/helpers.ts
@@ -5,8 +5,7 @@ import { stripHtml } from "string-strip-html";
 import mammoth from "mammoth";
 import * as XLSX from "xlsx";
 import * as officeParser from "officeparser";
-
-
+import moment from "moment-timezone";
 
 export const mimeExtensions: {[key: string]: string} = {
   // Google Drive MIME types
@@ -565,5 +564,13 @@ export class GoogleDriveHelpers {
         throw error;
     }
   }
-
 }
+
+export class CalendarHelpers {
+  static getUTCOffsetTimezone(timezone: string) {
+    const now = moment.tz(timezone);
+    const utcOffset = now.format("Z");
+
+    return utcOffset;
+  }
+}
\ No newline at end of file

From 38cacdb0438612e030fa96fd0b2f0344b5847816 Mon Sep 17 00:00:00 2001
From: chime3 
Date: Thu, 5 Sep 2024 20:09:28 -0700
Subject: [PATCH 098/182] fix: added date as required field

---
 src/providers/google/calendar-event.ts | 27 ++++++++++++++++++++++----
 src/providers/google/interfaces.ts     |  2 +-
 2 files changed, 24 insertions(+), 5 deletions(-)

diff --git a/src/providers/google/calendar-event.ts b/src/providers/google/calendar-event.ts
index 81cd36cc..542aab27 100644
--- a/src/providers/google/calendar-event.ts
+++ b/src/providers/google/calendar-event.ts
@@ -13,6 +13,7 @@ import {
 } from "../../interfaces";
 import { SchemaEvent } from "../../schemas";
 import { DateTimeInfo } from "./interfaces";
+import { CalendarHelpers } from "./helpers";
 
 const _ = require("lodash");
 
@@ -185,7 +186,7 @@ export default class CalendarEvent extends GoogleHandler {
     let breakHit: SyncItemsBreak;
 
     for (const event of serverResponse.data.items) {
-      const eventId = event.id;
+      const eventId = event.id ?? '';
 
       if (eventId == breakId) {
         const logEvent: SyncProviderLogEvent = {
@@ -197,10 +198,20 @@ export default class CalendarEvent extends GoogleHandler {
         break;
       }
 
-      const start: DateTimeInfo = event.start;
-      const end: DateTimeInfo = event.end;
+      let start: DateTimeInfo = {
+        date: ""
+      };
+      let end: DateTimeInfo = {
+        date: ""
+      };
+
+      start.dateTime = event.start?.dateTime;
+      end.dateTime = event.end?.dateTime;
 
-      if (!start.date || !end.date) {
+      const startDate = event.start?.date ?? start.dateTime?.split('T')[0];
+      const endDate = event.end?.date ?? end.dateTime?.split('T')[0];
+
+      if (!startDate || !endDate) {
         const logEvent: SyncProviderLogEvent = {
             level: SyncProviderLogLevel.DEBUG,
             message: `Invalid date for the event ${eventId}. Ignoring this event.`,
@@ -209,6 +220,14 @@ export default class CalendarEvent extends GoogleHandler {
         continue;
       }
 
+      // `start.date` and `end.date` are required
+      start.date = startDate;
+      end.date = endDate;
+
+      // UTC offset time zone
+      start.timeZone = CalendarHelpers.getUTCOffsetTimezone(event.start?.timeZone)
+      end.timeZone = CalendarHelpers.getUTCOffsetTimezone(event.end?.timeZone)
+
       if (breakTimestamp && start.dateTime < breakTimestamp) {
         const logEvent: SyncProviderLogEvent = {
           level: SyncProviderLogLevel.DEBUG,
diff --git a/src/providers/google/interfaces.ts b/src/providers/google/interfaces.ts
index 54527b51..aea96ca2 100644
--- a/src/providers/google/interfaces.ts
+++ b/src/providers/google/interfaces.ts
@@ -42,7 +42,7 @@ export interface Person {
 
 export interface DateTimeInfo {
   dateTime?: string;  // ISO format: YYYY-MM-DDTHH:mm:ss.sssZ
-  date?: string;      // ISO ate format: YYYY-MM-DD
+  date: string;      // ISO ate format: YYYY-MM-DD
   timeZone?: string;  // UTC offset format: ±HH:MM
 }
 

From b421eab264fb86ad5f0e46e1e493c9216623a7b0 Mon Sep 17 00:00:00 2001
From: chime3 
Date: Sun, 8 Sep 2024 21:40:13 -0700
Subject: [PATCH 099/182] feat: added event and calendar into web interface

---
 src/web/developer/data/data.js | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/web/developer/data/data.js b/src/web/developer/data/data.js
index 734a2919..267d885a 100644
--- a/src/web/developer/data/data.js
+++ b/src/web/developer/data/data.js
@@ -237,7 +237,9 @@ $(document).ready(function() {
             "Email": "https://common.schemas.verida.io/social/email/v0.1.0/schema.json",
             "File": "https://common.schemas.verida.io/file/v0.1.0/schema.json",
             "Chat Group": "https://common.schemas.verida.io/social/chat/group/v0.1.0/schema.json",
-            "Chat Message": "https://common.schemas.verida.io/social/chat/message/v0.1.0/schema.json"
+            "Chat Message": "https://common.schemas.verida.io/social/chat/message/v0.1.0/schema.json",
+            "CALENDAR": "https://common.schemas.verida.io/social/calendar/v0.1.0/schema.json",
+            "EVENT": "https://common.schemas.verida.io/social/event/v0.1.0/schema.json"
         };
 
         // Clear previous list

From 9e347a642f1bd0d61eb72326f6b8f14b62529331 Mon Sep 17 00:00:00 2001
From: chime3 
Date: Sun, 8 Sep 2024 21:41:16 -0700
Subject: [PATCH 100/182] fix: removed date from DateTimeInfo

---
 src/providers/google/calendar-event.ts | 13 +++----------
 src/providers/google/interfaces.ts     |  3 +--
 2 files changed, 4 insertions(+), 12 deletions(-)

diff --git a/src/providers/google/calendar-event.ts b/src/providers/google/calendar-event.ts
index 542aab27..81521a36 100644
--- a/src/providers/google/calendar-event.ts
+++ b/src/providers/google/calendar-event.ts
@@ -199,19 +199,16 @@ export default class CalendarEvent extends GoogleHandler {
       }
 
       let start: DateTimeInfo = {
-        date: ""
+        dateTime: event.start?.dateTime
       };
       let end: DateTimeInfo = {
-        date: ""
+        dateTime: event.end?.dateTime
       };
 
       start.dateTime = event.start?.dateTime;
       end.dateTime = event.end?.dateTime;
 
-      const startDate = event.start?.date ?? start.dateTime?.split('T')[0];
-      const endDate = event.end?.date ?? end.dateTime?.split('T')[0];
-
-      if (!startDate || !endDate) {
+      if (!start.dateTime) {
         const logEvent: SyncProviderLogEvent = {
             level: SyncProviderLogLevel.DEBUG,
             message: `Invalid date for the event ${eventId}. Ignoring this event.`,
@@ -220,10 +217,6 @@ export default class CalendarEvent extends GoogleHandler {
         continue;
       }
 
-      // `start.date` and `end.date` are required
-      start.date = startDate;
-      end.date = endDate;
-
       // UTC offset time zone
       start.timeZone = CalendarHelpers.getUTCOffsetTimezone(event.start?.timeZone)
       end.timeZone = CalendarHelpers.getUTCOffsetTimezone(event.end?.timeZone)
diff --git a/src/providers/google/interfaces.ts b/src/providers/google/interfaces.ts
index aea96ca2..e4ff891b 100644
--- a/src/providers/google/interfaces.ts
+++ b/src/providers/google/interfaces.ts
@@ -41,8 +41,7 @@ export interface Person {
 }
 
 export interface DateTimeInfo {
-  dateTime?: string;  // ISO format: YYYY-MM-DDTHH:mm:ss.sssZ
-  date: string;      // ISO ate format: YYYY-MM-DD
+  dateTime: string;  // ISO format: YYYY-MM-DDTHH:mm:ss.sssZ
   timeZone?: string;  // UTC offset format: ±HH:MM
 }
 

From bbf6b81916b254ce34eb782d21786adc9969f385 Mon Sep 17 00:00:00 2001
From: chime3 
Date: Sun, 8 Sep 2024 22:57:30 -0700
Subject: [PATCH 101/182] feat: added calendar event unit test

---
 src/providers/google/calendar-event.ts        | 22 +++++++-
 src/schemas.ts                                |  1 +
 .../providers/google/calendar-event.tests.ts  | 54 +++++++++++++++++++
 tests/providers/google/calendar.tests.ts      |  2 +-
 4 files changed, 77 insertions(+), 2 deletions(-)
 create mode 100644 tests/providers/google/calendar-event.tests.ts

diff --git a/src/providers/google/calendar-event.ts b/src/providers/google/calendar-event.ts
index 81521a36..e86ee13e 100644
--- a/src/providers/google/calendar-event.ts
+++ b/src/providers/google/calendar-event.ts
@@ -12,7 +12,7 @@ import {
   ConnectionOptionType,
 } from "../../interfaces";
 import { SchemaEvent } from "../../schemas";
-import { DateTimeInfo } from "./interfaces";
+import { CalendarAttachment, DateTimeInfo, Person } from "./interfaces";
 import { CalendarHelpers } from "./helpers";
 
 const _ = require("lodash");
@@ -233,6 +233,20 @@ export default class CalendarEvent extends GoogleHandler {
 
       const insertedAt = new Date().toISOString();
 
+      const creator: Person = {
+        email: event.creator.email ?? 'info@example.com',
+        displayName: event.creator.displayName
+      }
+
+      const organizer: Person = {
+        email: event.organizer.email ?? 'info@example.com',
+        displayName: event.organizer.displayName
+      }
+
+      const attendees: Person[] = event.attendees.filter(attendee => attendee.email) as Person[];
+
+      const attachments: CalendarAttachment[] = event.attachments as CalendarAttachment[];
+
       results.push({
         _id: this.buildItemId(eventId),
         name: event.summary ?? 'No event title',
@@ -243,8 +257,14 @@ export default class CalendarEvent extends GoogleHandler {
         calendarId: "primary",
         start,
         end,
+        creator,
+        organizer,
         location: event.location ?? 'No location',
         description: event.description ?? 'No description',
+        status: event.status ?? 'Unkown',
+        conferenceData: event.conferenceData,
+        attendees,
+        attachments,
         insertedAt
       });
     }
diff --git a/src/schemas.ts b/src/schemas.ts
index b460aa16..f63ac200 100644
--- a/src/schemas.ts
+++ b/src/schemas.ts
@@ -146,6 +146,7 @@ export interface SchemaEvent extends SchemaRecord {
     calendarId: string
     location?: string
     creator?: Person
+    organizer?: Person
     start: DateTimeInfo
     end: DateTimeInfo
     attendees?: Person[]
diff --git a/tests/providers/google/calendar-event.tests.ts b/tests/providers/google/calendar-event.tests.ts
new file mode 100644
index 00000000..17e26ab0
--- /dev/null
+++ b/tests/providers/google/calendar-event.tests.ts
@@ -0,0 +1,54 @@
+const assert = require("assert");
+import {
+  BaseProviderConfig,
+  Connection,
+  SyncHandlerStatus,
+  SyncHandlerPosition,
+} from "../../../src/interfaces";
+import Providers from "../../../src/providers";
+import CommonUtils, { NetworkInstance } from "../../common.utils";
+
+import CalendarEvent from "../../../src/providers/google/calendar-event";
+import BaseProvider from "../../../src/providers/BaseProvider";
+import { CommonTests, GenericTestConfig } from "../../common.tests";
+
+const providerName = "google";
+let network: NetworkInstance;
+let connection: Connection;
+let provider: BaseProvider;
+let handlerName = "calendar-event";
+let testConfig: GenericTestConfig;
+let providerConfig: Omit = {};
+
+describe(`${providerName} Google Calendar Event Tests`, function () {
+  this.timeout(100000);
+
+  this.beforeAll(async function () {
+    network = await CommonUtils.getNetwork();
+    connection = await CommonUtils.getConnection(providerName);
+    provider = Providers(providerName, network.context, connection);
+
+    testConfig = {
+      idPrefix: `${provider.getProviderName()}-${connection.profile.id}`,
+      batchSizeLimitAttribute: "batchSize",
+    };
+  });
+
+  describe(`Fetch ${providerName} data`, () => {
+   
+    it(`Can pass basic tests: ${handlerName}`, async () => {
+      await CommonTests.runGenericTests(
+        providerName,
+        CalendarEvent,
+        testConfig,
+        providerConfig,
+        connection
+      );
+    });
+  });
+
+  this.afterAll(async function () {
+    const { context } = await CommonUtils.getNetwork();
+    await context.close();
+  });
+});
diff --git a/tests/providers/google/calendar.tests.ts b/tests/providers/google/calendar.tests.ts
index bbf2230c..55113155 100644
--- a/tests/providers/google/calendar.tests.ts
+++ b/tests/providers/google/calendar.tests.ts
@@ -16,7 +16,7 @@ const providerName = "google";
 let network: NetworkInstance;
 let connection: Connection;
 let provider: BaseProvider;
-let handlerName = "google-calendar";
+let handlerName = "calendar";
 let testConfig: GenericTestConfig;
 let providerConfig: Omit = {};
 

From c947acc1640700b593db0a744bf34e649a545e6e Mon Sep 17 00:00:00 2001
From: chime3 
Date: Mon, 9 Sep 2024 19:51:29 -0700
Subject: [PATCH 102/182] fix: modified common unit test config

---
 tests/common.tests.ts | 265 ++++++++++++------------------------------
 1 file changed, 76 insertions(+), 189 deletions(-)

diff --git a/tests/common.tests.ts b/tests/common.tests.ts
index 7bee5138..32dbada9 100644
--- a/tests/common.tests.ts
+++ b/tests/common.tests.ts
@@ -15,12 +15,12 @@ import CommonUtils from "./common.utils";
 const assert = require("assert");
 
 export interface GenericTestConfig {
-  // Attribute in the results that is used for time ordering (ie: insertedAt)
-  timeOrderAttribute?: string; // Made optional
-  // Attribute used to limit the batch size (ie: batchLimit)
-  batchSizeLimitAttribute: string;
-  // Prefix used for record ID's (override default which is providerName)
-  idPrefix?: string;
+  timeOrderAttribute?: string; // Optional, used for time ordering
+  batchSizeLimitAttribute: string; // Used for limiting the batch size
+  idPrefix?: string; // Prefix for record ID's
+  resultsPerPage?: number; // Number of items per page
+  pageCount?: number; // Number of pages to fetch
+  allowBackfill?: boolean; // Whether backfill is allowed
 }
 
 // info,debug,error
@@ -37,6 +37,9 @@ export class CommonTests {
     testConfig: GenericTestConfig = {
       timeOrderAttribute: "insertedAt",
       batchSizeLimitAttribute: "batchSize",
+      resultsPerPage: 2, // Default results per page
+      pageCount: 2, // Default number of pages
+      allowBackfill: true, // Allow backfill by default
     },
     syncPositionConfig: Omit,
     providerConfig?: Omit
@@ -106,6 +109,9 @@ export class CommonTests {
     testConfig: GenericTestConfig = {
       timeOrderAttribute: "insertedAt",
       batchSizeLimitAttribute: "batchSize",
+      resultsPerPage: 3, // Default to 3 results per page
+      pageCount: 2, // Default to 2 pages
+      allowBackfill: true, // Backfill allowed by default
     },
     providerConfig: Omit = {},
     connection?: Connection
@@ -114,14 +120,8 @@ export class CommonTests {
     handler: BaseSyncHandler;
     provider: BaseProvider;
   }> {
-    // * - New items are processed
-    //    * - Backfill items are processed
-    //    * - Not enough new items? Process backfill
-    //    * - Backfill twice doesn't process the same items
-    //    * - No more backfill produces empty rangeTracker
-
-    // Set result limit to 3 results so page tests can work correctly
-    providerConfig[testConfig.batchSizeLimitAttribute] = 3;
+    // Set result limit to resultsPerPage from testConfig
+    providerConfig[testConfig.batchSizeLimitAttribute] = testConfig.resultsPerPage!;
 
     const { api, handler, schemaUri, provider } = await this.buildTestObjects(
       providerName,
@@ -134,192 +134,79 @@ export class CommonTests {
       ? testConfig.idPrefix
       : `${provider.getProviderName()}-${connection!.profile.id}`;
 
+    let syncPosition: SyncHandlerPosition = {
+      _id: `${providerName}-${schemaUri}`,
+      providerName,
+      handlerName: handler.getName(),
+      providerId: provider.getProviderId(),
+      status: SyncHandlerStatus.SYNCING,
+    };
+
+    let processedBackfillItems: Set = new Set(); // Track backfill items
+
     try {
-      const syncPosition: SyncHandlerPosition = {
-        _id: `${providerName}-${schemaUri}`,
-        providerName,
-        handlerName: handler.getName(),
-        providerId: provider.getProviderId(),
-        status: SyncHandlerStatus.SYNCING,
-      };
-      
-      // 1. Test new items are processed
-      const response = await handler._sync(api, syncPosition);
-      const results = response.results;
-
-      // console.log(response.position)
-      // console.log(CommonTests.outputItems(results, testConfig.timeOrderAttribute))
-
-      assert.ok(results && results.length, "Have results returned");
-      assert.equal(
-        providerConfig[testConfig.batchSizeLimitAttribute],
-        results.length,
-        "Have correct number of results returned on page 1"
-      );
-
-      if (testConfig.timeOrderAttribute) {
-        assert.ok(
-          results[0][testConfig.timeOrderAttribute] >
-            results[1][testConfig.timeOrderAttribute],
-          "Results are most recent first"
+      // Loop for each page based on the configured pageCount
+      for (let page = 0; page < testConfig.pageCount!; page++) {
+        const response = await handler._sync(api, syncPosition);
+        const results = response.results;
+
+        assert.ok(results && results.length, `Page ${page + 1}: Have results returned`);
+        assert.equal(
+          providerConfig[testConfig.batchSizeLimitAttribute],
+          results.length,
+          `Page ${page + 1}: Have correct number of results returned`
         );
-      }
 
-      CommonTests.checkItem(results[0], handler, provider)
-
-      assert.equal(
-        SyncHandlerStatus.SYNCING,
-        response.position.status,
-        "Sync is active"
-      );
-      assert.ok(response.position.thisRef, "Have a defined processing range");
-
-      const currentRangeParts = response.position.thisRef!.split(':')
-      assert.ok(currentRangeParts.length == 2, "Have correct number of parts for the processing range");
-      assert.ok(currentRangeParts[0] == results[0]._id.replace(`${idPrefix}-`, ''), "Have correct break ID");
-      assert.ok(currentRangeParts[1].length, "Have an end range");
-
-      // 2. Backfill items are processed
-      const syncPosition2 = response.position
-      const response2 = await handler._sync(api, syncPosition2);
-      const results2 = response2.results;
-      
-      // console.log(response2.position)
-      // console.log(CommonTests.outputItems(results2, testConfig.timeOrderAttribute))
-
-      assert.ok(
-        results2 && results2.length,
-        "Have backfill results returned"
-      );
-      assert.ok(
-        results2 &&
-          results2.length == providerConfig[testConfig.batchSizeLimitAttribute],
-        "Have correct number of results returned in second page"
-      );
-
-      if (testConfig.timeOrderAttribute) {
-        assert.ok(
-          results2[0][testConfig.timeOrderAttribute] >
-            results2[1][testConfig.timeOrderAttribute],
-          "Results are most recent first"
-        );
-        assert.ok(
-          results2[0][testConfig.timeOrderAttribute] <
-            results[2][testConfig.timeOrderAttribute],
-          "First item on second page of results have earlier timestamp than last item on first page"
-        );
-      }
+        if (testConfig.timeOrderAttribute) {
+          assert.ok(
+            results[0][testConfig.timeOrderAttribute] >
+              results[1][testConfig.timeOrderAttribute],
+            `Page ${page + 1}: Results are most recent first`
+          );
+        }
 
-      assert.equal(
-        response2.position.status,
-        SyncHandlerStatus.SYNCING,
-        "Sync is active"
-      );
-
-      assert.ok(response2.position.thisRef, "Have a defined processing range");
-
-      const currentRangeParts2 = response2.position.thisRef!.split(':')
-      assert.ok(currentRangeParts2.length == 2, "Have correct number of parts for the processing range");
-      assert.ok(currentRangeParts2[0] == results[0]._id.replace(`${idPrefix}-`, ''), "Have correct break ID matching the very first result");
-      assert.ok(currentRangeParts2[1].length, "Have an end range");
-      assert.ok(results[0]._id != results2[0]._id, "Have different result IDs")
-
-      // 3. Not enough new items? Process backfill
-      const syncPosition3 = response2.position
-      syncPosition3.thisRef = `${results[1].sourceId}:${currentRangeParts2[1]}` // Ensure the first item (only) is fetched
-      const response3 = await handler._sync(api, syncPosition3);
-      const results3 = response3.results;
-      
-      // console.log(response3.position)
-      // console.log(CommonTests.outputItems(results3, testConfig.timeOrderAttribute))
-
-      assert.ok(
-        results3 && results3.length,
-        "Have results returned"
-      );
-      assert.ok(
-        results3 &&
-        results3.length == providerConfig[testConfig.batchSizeLimitAttribute],
-        "Have correct number of results returned"
-      );
-      assert.equal(results3[0]._id, results[0]._id, 'First result item matches the very first item')
-      assert.ok(results3[1]._id != results[1]._id, 'Second result item does not match the very first batch second item')
-
-      if (testConfig.timeOrderAttribute) {
-        assert.ok(
-          results3[0][testConfig.timeOrderAttribute] >
-            results3[1][testConfig.timeOrderAttribute],
-          "Results are most recent first"
-        );
-        // this will break?
-        assert.ok(
-          results3[2][testConfig.timeOrderAttribute] <
-            results[2][testConfig.timeOrderAttribute],
-          "Last item on return results have earlier timestamp than last item on first page"
-        );
-      }
+        CommonTests.checkItem(results[0], handler, provider);
 
-      assert.equal(
-        response3.position.status,
-        SyncHandlerStatus.SYNCING,
-        "Sync is active"
-      );
-
-      assert.ok(response3.position.thisRef, "Have a defined processing range");
-
-      const currentRangeParts3 = response3.position.thisRef!.split(':')
-      assert.ok(currentRangeParts3.length == 2, "Have correct number of parts for the processing range");
-      assert.ok(currentRangeParts3[0] == results3[0]._id.replace(`${idPrefix}-`, ''), "Have correct break ID matching the very first result");
-      assert.ok(currentRangeParts3[1].length, "Have an end range");
-      assert.ok(currentRangeParts3[1] != currentRangeParts2[1], "End range has changed between batches");
-
-      // - Backfill twice doesn't process the same items
-      const syncPosition4 = response3.position
-      const response4 = await handler._sync(api, syncPosition4);
-      const results4 = response4.results;
-      
-      // console.log(response4.position)
-      // console.log(CommonTests.outputItems(results4, testConfig.timeOrderAttribute))
-
-      assert.ok(
-        results4 && results4.length,
-        "Have results returned"
-      );
-      assert.ok(
-        results4 &&
-        results4.length == providerConfig[testConfig.batchSizeLimitAttribute],
-        "Have correct number of results returned"
-      );
-
-      if (testConfig.timeOrderAttribute) {
-        assert.ok(
-          results4[0][testConfig.timeOrderAttribute] >
-          results4[1][testConfig.timeOrderAttribute],
-          "Results are most recent first"
+        assert.equal(
+          SyncHandlerStatus.SYNCING,
+          response.position.status,
+          `Page ${page + 1}: Sync is active`
         );
-        // this will break?
+        assert.ok(response.position.thisRef, `Page ${page + 1}: Have a defined processing range`);
+
+        const currentRangeParts = response.position.thisRef!.split(":");
+        assert.ok(currentRangeParts.length == 2, "Have correct number of parts for the processing range");
         assert.ok(
-          results4[0][testConfig.timeOrderAttribute] <
-            results[2][testConfig.timeOrderAttribute],
-          "First item on return results have earlier timestamp than last item on first page"
+          currentRangeParts[0] == results[0]._id.replace(`${idPrefix}-`, ""),
+          `Page ${page + 1}: Have correct break ID`
         );
-      }
+        assert.ok(currentRangeParts[1].length, "Have an end range");
+
+        // Backfill logic
+        if (testConfig.allowBackfill && results.length < providerConfig[testConfig.batchSizeLimitAttribute]) {
+          const backfillResponse = await handler._sync(api, syncPosition);
 
-      assert.ok(results4[0]._id != results3[0]._id, "First items dont match between batches")
+          // Filter out items already processed in backfill
+          const newBackfillResults = backfillResponse.results.filter(
+            (item) => !processedBackfillItems.has(item)
+          );
 
-      assert.equal(
-        response4.position.status,
-        SyncHandlerStatus.SYNCING,
-        "Sync is active"
-      );
+          if (newBackfillResults.length > 0) {
+            console.log(`Backfill processed ${newBackfillResults.length} new items on page ${page + 1}`);
+            newBackfillResults.forEach((item) => processedBackfillItems.add(item));
+          }
 
-      assert.ok(response4.position.thisRef, "Have a defined processing range");
-      const currentRangeParts4 = response4.position.thisRef!.split(':')
-      assert.ok(currentRangeParts4.length == 2, "Have correct number of parts for the processing range");
-      assert.ok(currentRangeParts4[1].length, "Have an end range");
+          assert.ok(newBackfillResults.length, `Page ${page + 1}: Backfill has new results`);
+        }
 
-      // @todo: No more backfill produces empty rangeTracker and SyncHandlerStatus.CONNECTED
+        // Update syncPosition for next page
+        syncPosition = response.position;
 
+        // Break the loop early if no more results are fetched
+        if (!results.length) {
+          break;
+        }
+      }
 
       // Close the provider connection
       await provider.close();
@@ -330,7 +217,7 @@ export class CommonTests {
         provider,
       };
     } catch (err) {
-      // ensure provider closes even if there's an error
+      // Ensure provider closes even if there's an error
       await provider.close();
 
       throw err;
@@ -355,7 +242,7 @@ export class CommonTests {
   // Helper method to output items to help with debugging
   static outputItems(items: SchemaRecord[], timeAttribute?: string) {
     for (const item of items) {
-      console.log(item._id, timeAttribute ? item[timeAttribute] : '', item.name)
+      console.log(item._id, timeAttribute ? item[timeAttribute] : "", item.name);
     }
   }
 }

From b18c841da8546525ffabadc74f45019bfca9ca13 Mon Sep 17 00:00:00 2001
From: chime3 
Date: Mon, 9 Sep 2024 19:55:05 -0700
Subject: [PATCH 103/182] fix: updated google unit test doc

---
 src/providers/google/README.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/providers/google/README.md b/src/providers/google/README.md
index b02263e9..1b6da64f 100644
--- a/src/providers/google/README.md
+++ b/src/providers/google/README.md
@@ -26,9 +26,9 @@ Before running the unit tests, ensure that you have the following set up:
 1. **YouTube Account**: A YouTube account with some activity is required. This activity includes uploaded videos, subscriptions, and liked videos.
 
 2. **YouTube Data**: Make sure your YouTube account has:
-   - **Favorites**: At least 14 videos you have liked.
-   - **Following**: At least 14 Channels you have subscribed to.
-   - **Posts**: At least 14 videos you have uploaded.
+   - **Favorites**: At least 4 videos you have liked.
+   - **Following**: At least 4 Channels you have subscribed to.
+   - **Posts**: At least 4 videos you have uploaded.
 3. **Activity**: Make sure you have made activities within the last 24 hours.
 
 ## Running the Tests

From 6bd26cd786ae4fd941e57d633e30c0d54d004e51 Mon Sep 17 00:00:00 2001
From: chime3 
Date: Mon, 9 Sep 2024 23:02:22 -0700
Subject: [PATCH 104/182] fix: typo error in data.js

---
 src/web/developer/data/data.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/web/developer/data/data.js b/src/web/developer/data/data.js
index 267d885a..fdb419c2 100644
--- a/src/web/developer/data/data.js
+++ b/src/web/developer/data/data.js
@@ -238,8 +238,8 @@ $(document).ready(function() {
             "File": "https://common.schemas.verida.io/file/v0.1.0/schema.json",
             "Chat Group": "https://common.schemas.verida.io/social/chat/group/v0.1.0/schema.json",
             "Chat Message": "https://common.schemas.verida.io/social/chat/message/v0.1.0/schema.json",
-            "CALENDAR": "https://common.schemas.verida.io/social/calendar/v0.1.0/schema.json",
-            "EVENT": "https://common.schemas.verida.io/social/event/v0.1.0/schema.json"
+            "Calendar": "https://common.schemas.verida.io/social/calendar/v0.1.0/schema.json",
+            "Event": "https://common.schemas.verida.io/social/event/v0.1.0/schema.json"
         };
 
         // Clear previous list

From 8a86ab1a2643125a5cea3f08caf203ff7780211a Mon Sep 17 00:00:00 2001
From: Chris 
Date: Wed, 11 Sep 2024 13:06:20 +0930
Subject: [PATCH 105/182] Fix: Attendees may be empty

---
 src/providers/google/calendar-event.ts | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/providers/google/calendar-event.ts b/src/providers/google/calendar-event.ts
index e86ee13e..24d55b01 100644
--- a/src/providers/google/calendar-event.ts
+++ b/src/providers/google/calendar-event.ts
@@ -243,7 +243,10 @@ export default class CalendarEvent extends GoogleHandler {
         displayName: event.organizer.displayName
       }
 
-      const attendees: Person[] = event.attendees.filter(attendee => attendee.email) as Person[];
+      let attendees: Person[] = []
+      if (event.attendees) {
+        attendees = event.attendees.filter(attendee => attendee.email) as Person[];
+      }
 
       const attachments: CalendarAttachment[] = event.attachments as CalendarAttachment[];
 

From c11f3902eaafcdd2efb637001fd76b90fb2a9e0a Mon Sep 17 00:00:00 2001
From: chime3 
Date: Sun, 15 Sep 2024 18:37:01 -0700
Subject: [PATCH 106/182] feat: added slack installer oauth

---
 assets/slack/icon.png               | Bin 0 -> 17452 bytes
 package.json                        |   3 +-
 src/providers/slack/chat-message.ts |  90 +++++++++++++
 src/providers/slack/index.ts        | 162 +++++++++++++++++++++++
 src/providers/slack/interfaces.ts   |  15 +++
 src/serverconfig.example.json       |   6 +
 yarn.lock                           | 195 +++++++++++++++++++++++++++-
 7 files changed, 468 insertions(+), 3 deletions(-)
 create mode 100644 assets/slack/icon.png
 create mode 100644 src/providers/slack/chat-message.ts
 create mode 100644 src/providers/slack/index.ts
 create mode 100644 src/providers/slack/interfaces.ts

diff --git a/assets/slack/icon.png b/assets/slack/icon.png
new file mode 100644
index 0000000000000000000000000000000000000000..19f52129bdc427e93dfbea77bce900a48eb1caca
GIT binary patch
literal 17452
zcmc({c|6o@^gsNui?L?S&Xmf&WE(`tC~0vE5hMF;3o&*RvSgpik|j};$QrWmOLod`
zY!k8#Bm4GTbAO-Z_5AVt`TTzRW17$By3Td3{harCpNYC>pvypafewNo20f(q0|+7q
z|0RcLslmrzpPz@|L(m%e;0^@&U4kG?7zB|XVU{7tOB#Y!P!Ob&0zsTl)2r{Rf)~_J
zZtH48XQY2wb$PEKNJ~IZTk{{EiS=p!6eF*?>zjM_>#6s@GfHP=!)B?Nt!gn@7$LhC
zSx8?E?GlxSQiG2N-Li(xH>Bmnq&il99%L9s3E!}d!Ps4TkD=AS8cORk_f@zub?#En
z+Ro{T(CfL_Zrp0w=p(}D_I_6XLd#y=$b^64o<%Z&T>F2`pK2-^N(@;27TMO+ci6{a
zbC;|@thX9cV$txVxUDJw(Ns<(ZJ?kJ;{r!>74tPxrxaqWUf6zD6~AIpQS4S$S(~hf
zBgBYaos95r{wn{BYV1v8mz_7PGZ%C1#d~hdL%26IWfi`TaiRL2Q9KHXi&iz0*P0LQ
zu8)54KdWx?Pt_*sQE9N+x3bxd@S36hxMs;kXqb1;$(48hnz`!{+LWo#^2!-+$pLfz
zHsqf={Atu$k*|hEhe8}rCCx}(plTef`1~<+=31@{YZfh=e3@yX;xWZfu;u8w$PVd+
zu^6V9dIe19iQ~DNC4c-TJ6lKyXv1_Rox7xtHt0649o4}a^?X_*4>Km?qH2k{AA7qL
z*OMy(6ZM|{J}+gcYtZXgM4#C5kzBB3Wa-c02CP}CS$YDqi}Tg!6LHRFzh?00skdnz
zIvu{elT4_X8ouZ8g2fu8G>Ks9QiSw>m@Fzv!MH1bBH}yWKOjpmS=sJM8g}rmBQtIk
z-|Q>9PCgm4ufsgsv3>N$b*t~Z?)0gLb=tgI+i1$%BdD-;yOv4wf}Kg^xoD{iXX$9(
z#88qZLwl(b6b=d;cJH?OJOpW-DK@)>ZGsM+&lcVMm!d*9hQ;E$DCa6Q1Af8o)NS+9
zlE>4uhH*y3y<5%D!E1bmkm7&e)}4$A4nK_`Gj0|SdS{m1Cte_KIkQs#S>(=W{B@+Q
z?Le8^{>23JTT`38r+C)qf2^2;);~5?7Vh*36W!-4MczCU3a#H>sb^zF-}JS%o=;uzV&{)nz^kc$66)lGt`u)s|!4Z4Q}Wv|B+
zn?hWDn|)YA8{}9W;+9zu+L7t@PL8c;Z|!2_C5MCa1NDrzr%pdzfZdehS8;uTW=>?n
zmP@6>wP%#gq%N)nqa6NZT(qS9C!*E%7sV~h?Y{ZBm_LeWXV{um=*~_bR(BE^^v!NL
zOb@52y_-iaNM$=K&j0;O)h=VoDW86)s>7khJ$FTliNklO_%vD3f$D^
zPBa}USPi5;%13i3;!E3|7v*k8#F{Pe>1b1FOzrqq*CS--79bTLdv{$`kAJkf2g_*2
z5xDyvpO0Su!0MMh%_bA6EAtWAvP+qJA;plnnT_VqQ)GwSfBA1L^0%WZQorr?J-CeQ
zqm36oarSb%-8#AKCsAGP(?6+e%q=~JxZNaJX@a?1>oXUYT$_XRW1nKn$MOd*x~ZsZ
z*yS>wp8eL9TFaW{)l>8FK1B3~S9BQdjWu7V
zxF2lx_RR%p=%gEV#cqF7%XZHEmX!WcvyVHGgq8FPoGW{k()-XW-N^R?noUoD9f~qf
zk~o#~ia+ENz7RBJ-^D)nW&(V|shoqlAv>hDD7*DqH
z#-fwpGEW+#wL1ED`h>bZ%0XCT`Pc4eknGC3(bI9jcrhsn^oZN?>PQ^V8)7bV_G-zK&P(+EegGb{SrBd>;f5#)-z
zQ_yjl@v(J3QEIiGaHFK-Of|{^ej=y;5>meTWENFS@UutChB93DIvg{s
z^+d{oBh4bkww_jQ+UD(DyTHE8%8z-Yo9yuqccFnv|7-BKO|oS5vwT`uaMyUa&Wdw1
zDS-}Ou;DiUb4@{RT*^->_(DR}gEER+>R`k6)ft+fIX1@a^d+(y$E7rPGBHxv8*y-F
zVL8d2zw$TQdTyzGK+BE!vqW`AG9E~Tu|>@U%E)<_RhN9AWK
z*0tq$e`knfaE(8!rS?ns<&1RW+%j}l$R=FdULD+l+$3
z!ZiP>uE7qmB&L88<`2l2S)$H?iToIgBl|y#_9av_BSIr1dAUf>fB!$vQ|?n~Xu~g&
zbRqA~haw|e!+-`o$3eEMdIfuZLc+oVcR@#g!~}2mfwunpBGxKSpcNJ-Zh=~PvSjc6
z{@-s_#!m~4DFS*C5s{Gr@}Q}RuK$|K0Sms*l=+wuUgsGZ*%@I7hRJ^;J*jA*xbeTT
zN}w!#=$-$OWI&}s6!zENhcJ%_yr7*X(2i8Tf8|#S-*;$67M24PX}^jyZd@v!Ce^Ne
z2UPBDu9r~c*%JG&Mi3}qoYXaSF1wFZ;8+ZZ|FYEQSl=XG@R!u7Gq4niUjqWa9BHT$
zGPJa{Pr*(jx~?F*!^4X(u(~p!WesSSHTp1FGHh-Mrwj)2F*pQMCA>b)2;EyJ)dBBs
z6fu@e3%RFe(lN7af%`f!pa0(Bg&0dBXawsG%73RiF+0heEhGsT(x=ie7hjvnXT{h3
z_Z=uPCBAHV7_6xe4k}^n&8yIShq!9~``<=_0()`7!dM0HP;Sm$sX8(|17#V`U0WLq#*ywv2wCCAhpktX
zxXnHw&*m!h;3>zBlUj`sPx{w13LR}C*cfy}^zR%>+%?aJ-V3@xB`t{fNaPS$U~{tc
zoG=G3duDm>i^_wDxJV=xJZ(ywDRMq(CM*0(z9x(<@>ybG(IW@9TrB8npUGH5IS)+a
zoa9cQh%l=?7y&7}xsafbhMc=%>yG#amQTBJaA8(m@ZzrBoF~Pj6DzB;AsnnsO=F6T
zC_yk07IuFROvcL1DFrDBu6usg9g4T&nY4Ir5RI#iHMH?L?7lS2U2i^%NYKxg)SYGR
zt@ZMW7jH|@_EC(5mP^_>%w-sAe_#fBK9RUEW+#D}SwEqH2Hg7EcVjQF&}rF|7sial
zXvjelJ!7opW0~x76XXP#FslGqGFP{$%L6G026C$1V4x%O#!e?#SXdIZz{G;FyKxl~
z78btp;eY#=fAD;bX5h4Rtaq7=5-OY4SS8OEgNbqv!oriFw9pZ@@Jl$;i>%+^-YM<~
zt4T6fXq_GE?x_+kULziy{21|%VAIPP=CAvsL_NYD{#9du9mUx=!Q#wlVu-oJ@bK1C
zQ3pg%RcyHmMdkTjx)?E}xbCI0wa8LM%P$)ztJDNr;_GU1=_hn@PH}|@HQ3YVES$v^
zU#~Oha<5IFSrG)z_RsHT1}C#!zNU4hm~h_ejA%-08dkRGJdG;O>)o_bXnhskClNS}
zC`YnpY-!sbwTmB_b<(4F^-T^MP6dwYKUSOtEBfx~rhU71)N$r6iIS#Z4j^ja~ir%pfo`*$dDVb@2>^YsIF2OgOhsc$+)f+xgl>Fl%^95yoauwg}HDZH{N
zRSp{6zj2Pl&r3ZedpyoD;EFK|nA?Y_{gyhDfl=g5-@L{Jbul+jU5Ws6Nl(jMH_50;CvaA=`am&HjG95^fPCzlmo`+
zUwQ|ABKn^T9xOo$r)}>Smi+9y-e`joWt9RvHWxOehvW3<*zVhNJWMKb^P2gY$T8^@
zeHA*LH;1VE^2H}g<&M0d%D1NU1l5R;#=p|E4ZB0LlOuli%H}1RG$>%D>}1sZPSebF
zPh~uPFaVQ?n%yw!_+_-0X*SxAh(rGL>#6#@my|K?Yv+wNXF7#YjXIhNKHtHt%@$G-_65~X$QY44qb6ek21x!)HlGO=b8gZUOaz)vC&4F+^LiiyS=oJ6Becy<`
z!|oB<{?GX+DY^7ihWpe02J5!q*!3u!FDZFPpBjjR1=g0~O+DrOi%Dy2FF!x#S4>IC
z6)5le;~8&b_w`}vnh``a@@vX_9aPGcy@AU|G)H2(V8N4iiSrsqp-s2y7PQ+?*1K%B
z?^eLfFJ}DKXv>Du<HC12f>}N+u<|5zk1-55o%W7y(ChBo}=oz>>SgGV-Ge!@N
zIxM11)<|x-*SiO%9&ES2pcI-Bw}qrzN>ar$Rn|*N^8D-%E>oF7hdndT0x&)La!%_R
zI`Bu{R+|Rv1PzY0Jwv%E^k
zzowt!wcRH?-_=teG%{5<>JREkv&*#yG1-A_dfCBFR3@!(Md?fF?8f?)Dstx1wz}#`
zOZ?-T$NL4}&948+(6R~6Q}f>TmFFR3h%mxND%<)t?QzPx#`maDgs{t`d;NiQ4>>ba
z=FZfvb!|@Mn&H;c4{s|jeObYUjEj;zn*wTfL1%OXUV@P{RTiPlOO$8R3D_hr+I!+|
zYAu9{jgkOq<1024
zg2D~45cOjhr#wXuUAV9@ly;)c7{h&9*gsYM9;XZsyqY_s%2j+DI6Kc%zd1P55;pq)
z>T{@GW{XLUC0d*l-eNt~SERi4nsi=-Q0@6HAz3NDE^m!VEcw_ym4K|Wob-fg`l}06
z>NjKwY*>DDY6x8JF&3r9~rw{e__NcD}CVzsyP{$bibvf@{Of81Z@`z)%s5pJ>p-m>%EnReFb*vxtlx-d_oMgaa$>T(nRv7
zBV}Ii6k6Qx$BSX_880VlbVZ;API7_@bQ*gm8m7u85AMhFoN!|jjCIs=kvF&pf_}Ms
zarywa>~o;#F%0ELZ>!zIbugZUq;G6A6(`p(ONqiKxPE#QAow{?KikXwXEK?v1XS~K
z#eX$6SXi0!;Ioz<*;{pE7Fm-&-FNQB{r%`OXhAN$LU9r6WH(LPe6?tpUfhP!o8NB_
zr2NA5KXi0{B9L*hZTSfQNst%;?%1Wy%~!yExiTTpmB@|(`x=&I`I=r7%Rx!wXP|Xg8oiF)FyWq4gC;bi@b?|TAoH#B-Iyno|55y=D5qBVTeQ^L
z)`kiLtSevOd>zJhSy^WVeiOTmBE!sub+)tWy@zr;D)7ZDxq;tAZtknaZCujGw*ALC
z>I2K285!jO>cBWvo$se2wUmvE_hT<&Z+`fLxD*jPu+(Yzy5q-XQSzaPTkwXze6L#U
z4@(xWHVX?(UxZA`J^{%h!=?&dWI4OPGP95Qb$zHY7Y9iKylrs0S+;e_veR76jkR&m
zow3EJVv+0%Vn2K#@0*HA$P4afxCrUBO^Y8SKX1~+Jk`qoA8Q?-^MZey*t>Er*IV0A
zUer>5t#hB)YWi$a>Bnv8aP}qMh^>KfhyLdq#}m=VukC$>YtP-YhqQ@TUJ_EM1~W2W
z?}>a9xKn*O{>^<0@-ePoCc=(vtZ76wYO6#3n(UF_W|?Le-noAoD}+l+v>#kXRf(VF
zjCl}#>t72U2`1`P#OBmcW-)z5o+kA0e_MSMz+~OD`x_M`p!F&~n>>m9oH%5<5-Nt3
zvUB?4FOa;a>%tkln1^uiY4LX-e#yem+p5R>xGTB%
z5F|e*9_h-X`w+^fQSg7&TK%d0yk4r#AcHMS+{L4W0P8b*RoGhEt)1Hu6!ILO2=!`oK`dyQKp~!EnDg)y5kwQPC*njox6~jE
z*{d9WL-^p0vKNKVeqx-Uuic;kkBAhciP{VO3Do24{7!#;P?l(_U|HqU
zEGGY$I`XAZ3O+-V7>VKt5L~eWWdRyqPuFt%se@hbL*u1G2G>1*sh|tY=`}vfN_ReH
zI;3w8B}`gG`~B>shoFQB_efp^4<@-?|Dw4vO=;7+6nIP2GjU&5D&eQ&?UGcDFE5C)
zjIGF2ARjRA&XW9E|Ja6Sn^h9>s%v%ptgfpp;zK)UaFNP2$FbiYZt1tlOaP{@)_C@p
z%M5;w*6v(Qi<&O;6r=RtzQljv8;_-J%j5!nEBpo?T`$@PTXbwv{mBe$Tb#2
z4hDxm>QcwpriCPypkyJ|*eMGv!^U=;(1BKIrqx27#X=1`=HTR3CA9x>bYHkcNA~=t
zH-YOYj4b$*F1crXV<14CUBJ{IScf0|D|whW`TJ=lXhNs+m`$V5o;tBgNZL23VdNUP
zNPQhQ!#Euy10+M>A|`1iYyAKYvv84c?V0+`-S`bsooqOmq;-!5m6U{vCC=M4EMzR4
zgsf`1?inQVdT-Nfxy~$S+#p6)bsfRtoyZ^z6~&`d@}FS<25F9&c?UexI&|F0FA&0D
zv|9idm9|&qdUw!oz8l2+>eV>gvvt1aI;c?`zP6gR0p~bCwql+2{`A*b2c$gIr9y}L
z3?g)EW#FYVDJmZSjv%7&^jcy2enQUHXXFrS4l|?^_I>)?l?R)hYU9vZUUlU)AKe4#
zGMZSY54H`rvQcszsuOD0w1PgRJ~$s!T*F~0f2vI2)p*VZLA3GCz7&o20RLe7#Pj9o
z4af1v(zim_)8jFU06Z1J((|Ax6#-0lEL0*`qgK_0=uoFJg!28(v=s1{v1YYpk!)-p
zhasmQ(f%Du_pXJC9#hWB6L{Tk6py_08UN1_vQMlX|5s=3(~XICP!-BLoEmXL25Gv{
zvmHcS6KAZ4%Ju_9VK!s?I@m9@Ij!VP&@LhRy65CA0Pu=ZYL(7bxNEXQkjXRGRb9u2
zz!+-fyRS=R?LRnW`i1Md6Rt@(-Z!CauUE1;76Il!JWNr7TV?JpPG;BmDHQ~9>`L+7
zop%MsjsUUa-^UI#<;$+m3
znjE0rv4-C}P#&;OqAoN!YnXpWX{_4UmhpbQsMX$Yhv`Nt_}O2Prxf^V${f&wh`=*N
zbwzX9eBJ8XvZAX1b2|0F*pI!78+Ttj`-U|A{Ox`QhaDP%`1c2%b75EI0$yN_Jo{gS
z;A0+tI#O+q9X=L4r-SnI*1b0{S@o12!Wc)4Z5X({wzx4H3-cY|N9utpEF}s`nwNCmWvoNW6+1V?LNpe$ryVS3};od7;XaJoM{OmM>s0|2nzF8OG3L
z3JC!F+T#ykTfp2~TMy*eb?(kP+!lY_qejJ=UI+;cor2nJ(XgoA^jR6y@0KAB$9A@(;tr>-HZ7%il%S-Owz495V
zovC-3^6DVc>soIFFxlDrb|jY1xi<1Y8F97Tk4h4&9d0%4
z?`Ze_#ymse%m&l-S@M-)rKiQ!+jNFGx~ntPUT4Py3-9yZsHwAC-p_}-ECgarAzV)X
z@XmBlfr`p#ZTeaZYoJqB1bE&a~
zbZ4$)TxKu`ezw+STGrj;Lx!?nI_ZJF!x|0k(noBMUN&6%!?sDIXG=C#6RS1$&S1P`
z?ehn>g}?`f(C+Aeh~7h@UI<0(+PN}G=vJMl-g9BSu?t+rkcoC#-phN}(LoKwpZzX`
z&&Ec!uclLr8@Q>t$9Oc12UK`hxXDAlWfeju=ebTf*laWXPKOsBcrDls(L+s-pl*hT
z*0XE!uAnJ1NF7PCSuPFdDln&6
znKy}eOBQVeP8kp)iCca~MhcFXe|S@cN@`cY?0REH=-P8JeCLG2N;q1*FI2qJl!z(H
zdG9`sz3n~c2MUIAb@7i)k*{{Q%w=p#dbUf6g=++S}I
zT&6Rz#}vjm-^#wybBn>jAgdFx@Lm>Wkaz9X#!lbddQOE*BsA-q;zDEU)XoPh=?du~
z=6J80y8mteJXyEh-7zWs(Z4K9FITK=nYHnUgTucXS0e4?=RXS
zXe*;g1#1O%Jll3%vM^{EVf`NILgw@F#Fp=tY?I&xYL=0fPDyE}xePH&r0SUO`UkM-
zwg0@?P4?=vdD+J#t}3NF=_Ap_E^VegVW_)a^1Z{=2NkjBkvpCi{MF%6DGdcOohh61
z`Byt793q)b61zh!E?V45e#9diknfD_RUw%3gLM?il4`^ktxhp<5u)aPT8hp^CSRwb
z!f=_C=UX4$ZK#R$t#2~;v5oO}2)>TSt9K4rUEMZY$Om=|x^nBwI}so7pmuwW(^Zjysl~^C=@*m4cHDXU
zwA^=obfryu4&JFjf1*Qm1YO;>_u7I>-D&Gu0G3Dx-RCa268w88;rMx~*Iu6?68iw!
zb+Kk$Kq&zzP1oL*_K6)7w_4>*6eg&;m;O%7RBM4|-dmM7$>MzB)3o(AcGq2p3x2_Q
zA
zcHH^-M8Rc}C`?|%qq2Wnd`!nG<#Flv{CTs27=pWel0Ed0S1#K{PjBl4qV~6simu8a
z9ay5|`!V3U795}@Rv0+dO#RE1T$L8h=)lKZ;rpoHiF?){l-+-cMOi-JDvwQ((qZ;T
zH(ya@lM*M%B7xw8#(C^KXI5zNlfpHhHedfrcVQA>s*#v#^z~}+@BSh$ey?(L=BHoA
zJ69*ojb!+qPZx$Zw5DaeJ0~aQ`gUZ&5U0oZuz(IMzClG3RPeQ|I$){9zu}W?V8Op~
z9B-#KPw1kiH}>WYtVSn&a!$vUjk<35l7$PAV9l0^!L^du;j3Tw3_PTt+=pn#elRA_
zoSS8&_=~(p&Qff|CwUf?I-S|q6c
z<(Ye}zw0p}OnNZ!N7Hk}Gkd!kKr}fh`EV48{q;LXAk4I2!c&raAcmIz@wC-XBtiV3
zv?RadDg8mp#r+H(VLR{jMLDsJr6ND}{P&e*K~M@o{2r
z1*b7t)f6Hc(Zcec>eWN+zd-FQIu5##yd2V;7ER!WYWiNvQ{
zAy>mMy;8Q-Zn78>ACto8Gm34<3d$ffeH)DUUJ_)>K&14(1B@fVNATPprStKyV`xm1
z&7CE&&o*532a&dWz8(A0RHo2FZ?3>Esesj^MYK0MSUzW0)sE0@l6}p*d-R4+H)D?P
z_dS=?ksNZ(eO)fPL-|igi_}0bL!-7m;H)Hsh(RRnq+29_AsxJnTlOwP$A#FBG;`^j
z1+V?K#ocFoxV|uLX3wJfZ}9`rc`DRYt|R-qZN`#@kgMPTGWQmaI@<*L=6Ul%14=3uNeImy41nC;zE)*%z
zw-@;X=g4<`zx$|hQvlFPx(m4QXJ$WFS3)w(!^)^RUZnO&^%isjoylsvhs!gm?k`T$
zLo&kWF9h34?}=jtS|jz_vs?mFU`pS5bJwYkp7rD&N{sT{n*;c2LLEXNeA<|Ho1%)4
zF#7EBAvLTd@TZv`)R
zD|Ikx^7Z`_z^zJbAF3QyD^L*3BLpZNb5i)_I=74AhM`NBmRY3AIf95d+!Y;bXC{M)>z$L
z^r-uD7cScF1H0f$=Cf1PeK*(CGXBLIhuV?TQc36#jN`dvJ{kyf)mQjC%8mk;d4KTY
ziaS7FQXym07@YGZ8Y#oq2es(%a%rvW@o8i;pg>#CKSXVh)(griD>69^lqfFtgA~>)
zHzt*CX?Au>QBayznU}HYU6qRtXyA4v^H~p*`AvPdg3^ukWs0x*uCnpKpAA`ELVq!&YMG
z#J)Tky21GJnQ9dOdl%koT6)%7yFq{v1Lsk<(umr75)Ee%!3EzmgY>Bc|2TMO0?Nos
zVh{bDi+AC7h3nEeP037*g4#-`5t2V%ymBa1KFwc%Ie?%91|B8d#o
z0Wf=EYDpsbm@ZUpqTL`S#(DbHp=31vb}qb{;PDJ-;T`QP?%p&|9-A>x&=OuDLOh-S
z$Y5BlNTF}$}wE8{Dekfc=wieD)H5msz_~Aa(-=
zNe~?wFFvxHXwT$Y4DM(WFOStBian=lyvH$aG9NOau*;^s$w%Ka;U|@c#IkUV2gx_f
zK~H^WO0+wWF4np+PflL>HuP)2r?V&HZ|<{U7#a0+<-iM$-osG^jxN!gq-HKP(sJGk
zScYMwZh9o$nXyh(y=_4TagCe*o(~Pr1q0qAtkj+BB967^n6|9|@b1L%NI>#ZQT9Tj@^^4D^!=*1lv7a%FvvT
z4^!6__HBc6Km{>R#mAX?EdV`cOLt-w$F0p8JGlNuB%N+7Nmn7MHlF3t8J6NXskpXw
zWlLa05M4|8I9Re3?uuH`CGivVXE4LEMt-?Bfbkb=kULUg(fKLvUp~aU)MoP;n%GdASTuE_wlKQfY0%r~pB9~mke~5cHCgfZ
z`>~>DHu#xZIiv@d^du0$I=C6b@OGN(m(QTV^sOGSFv%cFtIgM7G#Kf%0yDLVLi5%(
zAE+=B?mi}iRwtpWwOUAz@`JV8PMY=e8Cr06*?p(iEdaDtDe9Lbzb16ARro|h@83^7
z(Rk*jiPDT|i%5?iBt6FJUyjC}%N_cNaH}^!IKP}e7#i$l?KiRMZ(}_=J1u7}u*=J|Sn_&&dgI>c2N8{@*+SKbTNl-K
zPamY>fg`%}OTQ_3I<1?y%NNyp)+)giU>h+}_urk;@X|Hb
z)=Rh>R#IWUQ
z4VqH#g#cihL|vPYcZnFS(jVRM1T1}uxKUG#s*uG-`62$gFNhU6d2Y>L$Nv0hSUBRU
zRwe8DLohs<$F*d^yBDr^aiK!2%$GB?u>CN~5JatKB%q*H)_uyIaC%lrzWKGFj~svD
zd+qQ_cmyE0`MqA>ATlHCgzn)569jLh&89YegPuLw-c-F8&-$B{BEX3dc3U{HIMJ}I
zc=iJ6>UmPt;JiP-q$;BHjtpwf-t@b}l+FVWp7Gt$dh2*!*jQn$O_cU@5cHBap}<~i
z3tSOZoS5^*1~{nf7Jv?QpNh)ux{%_9nhLdyjov^eRsUw4ck;uU{ttgF0D`#iHvHZB
zr!_3XERkjgCf6Llfsj&>!n;66iKzdgh&FJ}*LB0|#yDjR`4uSq^SC}&47n*G<6<20
z^xN=6yFcjj>yyB($MBtJ@tri2)bm&4P0s+_2LRE_mV1p~sDs84x@*jCLJqYL6i@Yp
zirr~>UdmY`zwY%3KX9N8`sFnAj_H9kKo?cMW~fqWXr^C?q9Bi@<{9_h`B@2Yy0zh;
zPI?}_m`k4?TrK5%el)@O<;tf#99=#D(F*rf61IVPLGG^A@r_A+fe%D?*TY9u8OCBs
zwh7sDU$ba$j;T8nRK{S|PNxw%I
z)X`mUq3v;aZS^Fg`xFG8p~fvY{VCq!WkW$P<0Bd&QLjh-r9qGNnw&h{d(pJ-6f>fH
zM82zsuD$pen(VqoidL6Gkj6pv(I}77>sdR3IMs^)XTzFB%K`HY#x`+Nr~?m5bu*fNe$o&HaNrjFPSW4w4LvIhc@>le#|gF+u#G%
zAx-y9$!SA{f<+J3YBwF$*w${c(xvixo31AWae=&95VUw^jyQ;&Z_n0r3{XK4CRyG7
z)b74_TP|;CP30cj#kS3m%bR7v&p-EV)0Jnuj2}KFut$@FpsDq2rbtDmF42M@o+z|R
z5uT$`n2JCAflM^N%2VOP+cgKT@Uv&>;`W&TihgT;?~~$jV8R$!9m~X(g&IUr;BuP$
zk@?V!ATn!$IrBT(1D>eu=&#pX9&;_zK+wy62ku_+LP%1o=EGt>F>xF18==fqauqsy
zW^pnr5-W5F-=exfsf9tWJB$nzElflJ#yy~Y6h=rqF8k)N^&eQL#
z7_enS;(a86v_Q-t}n-S_G|o?aTZT)z0;ZO25#D;}ga1gy&9e@%nc
zQ6D#J3J0-V7G-A2Rc2zfLQPGE=iiIBEqD4nKRq!6YeZUdt;%kwh+F=dh_X9@!E;M%
z-sm&(d)Inm>C-UO{)r_yIy`|%;RtgiC7QdE^VGWK5=Pp@->H&6{yqHNwxQw`;P2b}jL&n)OAlp1=oRE@k
zC0!CQO;J*3ieDhnrT!)JDXJ7uJF&6$j$z3`d6%O3^%z6Es8#7G&rjA4|1NTGs1Fu=
za38`&b0wM@S_G(I=x=yn$lhU0qj?#NpNHpB%ytcw-UX{>^D4nME(B1gKLen(7o
zalLHo+!|%{$e{Kf6+e5d(wt=TVo17EYq|o@ExSqMe8Zj0H$T}iClV`OUhZEMS5?d|
zdEP>;-N^nzCOTB0jCpudA?JN+(_2izN2{8_is53uR%$tMrY?34%I@cC-Qc=7Oxlz6
zy+^j0TtQykP?kr%nVWvb7`EQLSE+UVScO;U0GiRRO<^T2ICKF)N{pf#M(JW)|LC(W`ZGFST`m
zzKkI|GN>^0%a+cDJ*K0jU@HGJc**DdCp?(nlu29%iu5gc69Ky)zu`XX5H&%0{JXal
z%PtyjDS^*ZVG2e@wRE$e2bvvS(!TjNQ}??5WxRr=ucjffNi!&CS1?U6cXrGao%8ee
zeB?fITnh5{D>j%CZsaoMMIyhZUb{Bi*IVbOey}Ux$G6G{_&|X7
z8;Or)pnJJjTkQBK{;sbm%Q&DuM6c&e2X^d
z6K=d`%8o>8!@PR|lyZq^T=xY&^ACd$s
z(f0)H?8dy51o*^bQZJ9MRo>J$RsOP&h|`3tk)F_DxDA{h&df6c3xuE4#*5}Pp0qE1
z>ENOsBF*9l4KC&|+MqU5is;%hB=Iv4%f?LM=ZdS?aROow{u2UgGMfQ-mje(2(l~^L
zsoGHz(ah$6)Q!DEk|DU|cJprT_tTjoL49c#j04Q|0~v<@MjTN`yk^$8!m~=0Oa}m|8e{&00aGhN`#1sp32-M}f5Y;gT_+B{`Ddt-&4y1|9(z^I>H0;%;vn$ULmzZOh_pt;YaJ^xWhV%F43Vl&h9Bg7(0%0V1(-?Uxxfh*q6~
zgw>g65K-r+zit8^BqB`7eA$Uh_!qWHRHsbblfjE{eLKAf)P*No>@gW5s_$=I@(tj0uZ6z#!*=VREC8M4eTp;|ubJ4?o
z(yLAE|H>)(`}Nd|qSF_rJk%Kz3_N92#U3%2<+(Djr>OsDHn2`*wGizr?UZ_Rv+&0Hi$_cs)94zp*U2
zg=G1wB)yDc6AJONC8Qstf$aYhX2Qo~rPJmzMDCD3ee~1%tq@UWxF_-|kMcEu3}V6I
z=MiV+(*S@C!Ofco6VCXCcC&`z^>Y|EkOSlgsB`3c{Gg}b4OXkwHKj|PhOQ_MGM`9k
zf%^3GnuHg3Y3c{<_O4Xpg_Qw|UD=+CGsLHpluC4?A+%U%zYt{pgx>l2C%B80v_)Sw
zt4O(=Au^XKW~tca@*q!sEamCd*2M`tPf_KGh;-x>Sq>D8!<{K7@t8_
zOx~G*!ixY?XplYyE(NB;&0(%kg+4(5p6vZ-^y3eNZkkoJK$ORY`JYxouT}M~g2vMN
zzKC9TYdnur#zq#KF)Z95@Y)HI^L-$O%1tm@mUbduz`;p|pY$O>QChWAzAi0XzNFOD2dQseAScQ9mt*gT?}DNvx*>HHZeg&2X?ufj#k&w4XEWDYdi&o+
z|GWenkaJSo&9P-B=v3zkrx4hv^&7IrUD^H5OG2JDA#_=RlVsfmTm#a{0w)-lEBl6$
zUEl0u@yAyPir3>1?GYtU%xO?H=(sVtsh=A||Mr$%_9Z2KT(+F_5qSPDaH;GXCGOYn
z5KwOzKAYNWd<1+=5zZvwAvM%mBUQk9fx`^TQl+^*DpK(a6HAES9-j^)+B+N{*X_Gv
zgF(Wd;UqFd*yrU5w*pMtq#~&oauZ>W1vgK
zuVqK>sMjE!vHxYmxmjfbmpiI)hi_C}z5~(G^si#=;-AK~8qh3ecB)VZ(EyTNhC0B4
zS1EM5o_ojo(h&@n=VN~bb$zAWZfLJoi#P7hupjQ69N&~WWmEFY+okU=^VL3!28o>@
z`O&>>w@NIopL)(b<&8h!Z@~GN%e{w;IJVJO5{mB_shxH7)Gudz97r&KSK{0-(vk!S
z9m0P;+=k|JTMlVPxs}2&K_|5|U4Jr0zMegqVmfWCA<$j+{{mudfYp5k4p%h2bo`p}
zbRHz|n0o;)hPingHcs>&AS&fz#bnJ6`@hMC&zmn1?9dgi)rok=VUyPxWLZV%q{}RVoh2Eg>VLHe;`hho3+Fk=7kY-ioe3#b
zDmAu!1~d98s&6tKgKY8
z>tlla3Fz7b?lpIuzwtS=de$$BP2nnO9d-(lo#Yl(YoRHZ2BX+8S&4WhSE
zzZ}E4>RXEaTZ=(j=@{Hv$)IFFD*V6&N+jJvnW&_)Nr%ETn%-kFt-U<6(lxzzS&
zg*MYzT@2~16Z;C^lB1aLSyKLB3+38zTX^f2VgQR@v#J;xJdUhkT3r=m)kX1D9u6#n>(Y>dIqVLih>
z*q`ZGF0!Z4C8Dl)01@BWL`s4;8{aB5WVWlPU2q8HTS~PhY#(ovmmQ(`z3^Ij;%751
zxK+_}z%+33H0k;+ZBF&*I^loRE3Oh)vXmLvNFA_-90!CnzMoeeH+C8EkMF-bY
z_YaCNZ1*{RMrMPxs@=I!Qf|M?{=#Qb>Qq!rYApG{wS4!Pj51981(}lG?Q3_(tMF>y
z1pk0s!YC~u=8BX$A09JM(2)4;n+7@_q~A1%=Q8>0cVumF`3L1=$K{)k
zI3dwBwcGdp)t0-EMm9pJQJ6SM>#SI|4xdp#eNQdy3^(t4B8hPSZ8JUh7=t_3(y7sR
z_o2J7%1XIRmrc*G4#`st!iqVh%jHotjq8FH6c6vVJQR62R2 {
+        let chatGroupIds: string[] = [];
+        if (syncPosition.thisRef) {
+            chatGroupIds = syncPosition.thisRef.split(",");
+        }
+        return Object.values([]);
+    }
+
+
+    protected async processChatGroup(
+        chatGroup: SchemaSocialChatGroup,
+        totalMessageCount: number
+    ): Promise<{ chatGroup: SchemaSocialChatGroup; chatHistory: SchemaSocialChatMessage[] }> {
+        const chatHistory: SchemaSocialChatMessage[] = [];
+        const rangeTracker = new ItemsRangeTracker(chatGroup.syncData);
+        let groupMessageCount = 0;
+        let newItems = true;
+
+        return {
+            chatGroup,
+            chatHistory,
+        };
+    }
+
+    public async _sync(syncPosition: SyncHandlerPosition): Promise {
+        return 
+    }
+}
diff --git a/src/providers/slack/index.ts b/src/providers/slack/index.ts
new file mode 100644
index 00000000..43fa5a28
--- /dev/null
+++ b/src/providers/slack/index.ts
@@ -0,0 +1,162 @@
+import { Request, Response } from "express";
+import Base from "../BaseProvider";
+import { SlackProviderConfig } from "./interfaces";
+import { FileInstallationStore, InstallProvider } from '@slack/oauth';
+import SlackChatMessageHandler from "./chat-message";
+const axios = require('axios');
+
+import { Installation, InstallationStore, InstallationQuery } from '@slack/oauth';
+import { PassportProfile } from "../../interfaces";
+
+export class CustomInstallationStore implements InstallationStore {
+    private installations: Map = new Map();
+
+    // Save the installation data
+    public async storeInstallation(installation: Installation): Promise {
+        const teamId = installation.team?.id ?? installation.enterprise?.id;
+        if (!teamId) {
+            throw new Error('Failed to identify team or enterprise in installation');
+        }
+        this.installations.set(teamId, installation);
+    }
+
+    // Fetch the installation data
+    public async fetchInstallation(query: InstallationQuery): Promise {
+        const teamId = query.teamId ?? query.enterpriseId;
+        if (!teamId || !this.installations.has(teamId)) {
+            throw new Error('Installation not found');
+        }
+        return this.installations.get(teamId)!; // Return the installation
+    }
+
+    // Delete the installation data (if needed)
+    public async deleteInstallation(query: InstallationQuery): Promise {
+        const teamId = query.teamId ?? query.enterpriseId;
+        if (teamId) {
+            this.installations.delete(teamId);
+        }
+    }
+}
+
+export default class SlackProvider extends Base {
+    protected config: SlackProviderConfig;
+    protected slackInstaller: InstallProvider;
+    protected installationStore: CustomInstallationStore = new CustomInstallationStore();
+
+    public init() {
+        this.slackInstaller = new InstallProvider({
+            clientId: this.config.clientId,
+            clientSecret: this.config.clientSecret,
+            authVersion: 'v2',
+            stateSecret: this.config.stateSecret, // Use the stateSecret for additional security
+            installationStore: this.installationStore
+        });
+    }
+
+    public getProviderName() {
+        return "slack";
+    }
+
+    public getProviderLabel() {
+        return "Slack";
+    }
+
+    public getProviderApplicationUrl() {
+        return "https://slack.com/";
+    }
+
+    public syncHandlers(): any[] {
+        return [
+            SlackChatMessageHandler
+        ];
+    }
+
+    public getScopes(): string[] {
+        return [
+            "channels:read",
+            "groups:read",
+            "users:read",
+        ];
+    }
+
+    public getUserScopes(): string[] {
+        return [
+            "channels:read",
+            "groups:read",
+            "users:read",
+        ];
+    }
+
+    public async connect(req: Request, res: Response, next: any): Promise {
+        this.init();
+        try {
+
+            const result = await this.slackInstaller.handleInstallPath(
+                req,
+                res,
+                {},
+                {
+                    scopes: this.getScopes(),
+                    userScopes: this.getUserScopes(),
+                });
+
+        } catch (error) {
+            next(error);
+        }
+    }
+    public async getAccessToken(code: string) {
+        const response = await axios.post('https://slack.com/api/oauth.v2.access', null, {
+            params: {
+                client_id: this.config.clientId,
+                client_secret: this.config.clientSecret,
+                code: code,
+            },
+            headers: {
+                'Content-Type': 'application/x-www-form-urlencoded'
+            }
+        });
+
+        return response.data;
+    }
+    
+    public async callback(req: Request, res: Response, next: any): Promise {
+        this.init();
+        const { code } = req.query;
+        try {
+            const data = await this.getAccessToken(code as string);
+            
+            const profile: PassportProfile = {
+                id: data.authed_user.id,  // Slack user ID
+                provider: this.getProviderName(),  // Set your Slack provider name
+                displayName: data.team.name,  // Team name as display name
+                name: {
+                    familyName: '',  // Slack does not provide family name directly
+                    givenName: data.team.name  // Use team name as given name (optional customization)
+                },
+                connectionProfile: {
+                    username: data.authed_user.id,  // Slack user ID as username
+                    phone: undefined,  // Slack API does not provide phone info
+                    verified: true  // Assuming token authorization is verified
+                }
+            };
+    
+            // Add access token data if necessary
+            const connectionToken = {
+                id: data.team.id,
+                accessToken: data.access_token,
+                refreshToken: data.refresh_token,  // If applicable, otherwise remove
+                profile: profile
+            };
+    
+            return connectionToken;
+    
+        } catch (error) {
+            next(error);
+        }
+    }
+   
+
+    public async getApi(accessToken?: string, refreshToken?: string): Promise {
+        // You can return the Slack Web API client here using the access token
+    }
+}
diff --git a/src/providers/slack/interfaces.ts b/src/providers/slack/interfaces.ts
new file mode 100644
index 00000000..b0c3c2f6
--- /dev/null
+++ b/src/providers/slack/interfaces.ts
@@ -0,0 +1,15 @@
+import { BaseProviderConfig } from "../../interfaces";
+
+export interface SlackProviderConfig extends BaseProviderConfig {
+  clientId: string;
+  clientSecret: string;
+  stateSecret: string;
+  callbackUrl: string;
+}
+
+export enum SlackChatGroupType {
+    CHANNEL = "channel", // Public channel
+    GROUP = "group", // Private channel
+    IM = "im", // DM
+    MPIM = "mpim" // Multi-person DM
+}
\ No newline at end of file
diff --git a/src/serverconfig.example.json b/src/serverconfig.example.json
index 1ddab795..288d5b91 100644
--- a/src/serverconfig.example.json
+++ b/src/serverconfig.example.json
@@ -89,6 +89,12 @@
             "messagesPerGroupLimit": 100,
             "maxGroupSize": 50,
             "useDbPos": true
+        },
+        "slack": {
+            "label": "Slack",
+            "clientId": "",
+            "clientSecret": "",
+            "stateSecret": ""
         }
     },
     "providerDefaults": {
diff --git a/yarn.lock b/yarn.lock
index 3edc3b33..da70ef42 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -494,6 +494,48 @@
   resolved "https://registry.yarnpkg.com/@sindresorhus/is/-/is-0.14.0.tgz#9fb3a3cf3132328151f353de4632e01e52102bea"
   integrity sha512-9NET910DNaIPngYnLLPeg+Ogzqsi9uM4mSboU5y6p8S5DzMTVEsJZrawi+BoDNUVBa2DhJqQYUFvMDfgU062LQ==
 
+"@slack/logger@^4", "@slack/logger@^4.0.0":
+  version "4.0.0"
+  resolved "https://registry.yarnpkg.com/@slack/logger/-/logger-4.0.0.tgz#788303ff1840be91bdad7711ef66ca0cbc7073d2"
+  integrity sha512-Wz7QYfPAlG/DR+DfABddUZeNgoeY7d1J39OCR2jR+v7VBsB8ezulDK5szTnDDPDwLH5IWhLvXIHlCFZV7MSKgA==
+  dependencies:
+    "@types/node" ">=18.0.0"
+
+"@slack/oauth@^3.0.1":
+  version "3.0.1"
+  resolved "https://registry.yarnpkg.com/@slack/oauth/-/oauth-3.0.1.tgz#079cde13f998be2d458c20dfcae288067464536e"
+  integrity sha512-TuR9PI6bYKX6qHC7FQI4keMnhj45TNfSNQtTU3mtnHUX4XLM2dYLvRkUNADyiLTle2qu2rsOQtCIsZJw6H0sDA==
+  dependencies:
+    "@slack/logger" "^4"
+    "@slack/web-api" "^7.3.4"
+    "@types/jsonwebtoken" "^9"
+    "@types/node" ">=18"
+    jsonwebtoken "^9"
+    lodash.isstring "^4"
+
+"@slack/types@^2.9.0":
+  version "2.13.1"
+  resolved "https://registry.yarnpkg.com/@slack/types/-/types-2.13.1.tgz#d1af332103ce96e22b10692bef9b34483f4c002c"
+  integrity sha512-YVtJCVtDcjOPKsvOedIThb7YmKNCcSoZN0mUSQqD2fc2ZyI59gOLCF4rYGfw/0C0agzFxAmb7hV5tbMGrgK0Tg==
+
+"@slack/web-api@^7.3.4":
+  version "7.4.0"
+  resolved "https://registry.yarnpkg.com/@slack/web-api/-/web-api-7.4.0.tgz#56f8376203b4172c02812f0fc092b65c351a74ba"
+  integrity sha512-U+0pui9VcddRt3yTSkLAArVB6G8EyOb0g+/vL1E6u8k46IYc1H4UKDuULF734A2ynVyxYKHIXK8OHilX6YhwjQ==
+  dependencies:
+    "@slack/logger" "^4.0.0"
+    "@slack/types" "^2.9.0"
+    "@types/node" ">=18.0.0"
+    "@types/retry" "0.12.0"
+    axios "^1.7.4"
+    eventemitter3 "^5.0.1"
+    form-data "^4.0.0"
+    is-electron "2.2.2"
+    is-stream "^2"
+    p-queue "^6"
+    p-retry "^4"
+    retry "^0.13.1"
+
 "@stablelib/aead@^1.0.1":
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/@stablelib/aead/-/aead-1.0.1.tgz#c4b1106df9c23d1b867eb9b276d8f42d5fc4c0c3"
@@ -775,6 +817,13 @@
   resolved "https://registry.yarnpkg.com/@types/json5/-/json5-0.0.29.tgz#ee28707ae94e11d2b827bcbe5270bcea7f3e71ee"
   integrity sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ==
 
+"@types/jsonwebtoken@^9":
+  version "9.0.6"
+  resolved "https://registry.yarnpkg.com/@types/jsonwebtoken/-/jsonwebtoken-9.0.6.tgz#d1af3544d99ad992fb6681bbe60676e06b032bd3"
+  integrity sha512-/5hndP5dCjloafCXns6SZyESp3Ldq7YjH3zwzwczYnjxIT0Fqzk5ROSYVGfFyczIue7IUEj8hkvLbPoLQ18vQw==
+  dependencies:
+    "@types/node" "*"
+
 "@types/mime@^1":
   version "1.3.2"
   resolved "https://registry.yarnpkg.com/@types/mime/-/mime-1.3.2.tgz#93e25bf9ee75fe0fd80b594bc4feb0e862111b5a"
@@ -808,6 +857,13 @@
   resolved "https://registry.yarnpkg.com/@types/node/-/node-18.15.13.tgz#f64277c341150c979e42b00e4ac289290c9df469"
   integrity sha512-N+0kuo9KgrUQ1Sn/ifDXsvg0TTleP7rIy4zOBGECxAljqvqfqpTfzx0Q1NUedOixRMBfe2Whhb056a42cWs26Q==
 
+"@types/node@>=18", "@types/node@>=18.0.0":
+  version "22.5.4"
+  resolved "https://registry.yarnpkg.com/@types/node/-/node-22.5.4.tgz#83f7d1f65bc2ed223bdbf57c7884f1d5a4fa84e8"
+  integrity sha512-FDuKUJQm/ju9fT/SeX/6+gBzoPzlVCzfzmGkwKvRHQVxi4BntVbyIwf6a4Xn62mrvndLiml6z/UBXIdEVjQLXg==
+  dependencies:
+    undici-types "~6.19.2"
+
 "@types/node@^18.11.18":
   version "18.19.45"
   resolved "https://registry.yarnpkg.com/@types/node/-/node-18.19.45.tgz#a9ebfe4c316a356be7ca11f753ecb2feda6d6bdf"
@@ -877,6 +933,11 @@
   resolved "https://registry.yarnpkg.com/@types/range-parser/-/range-parser-1.2.4.tgz#cd667bcfdd025213aafb7ca5915a932590acdcdc"
   integrity sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw==
 
+"@types/retry@0.12.0":
+  version "0.12.0"
+  resolved "https://registry.yarnpkg.com/@types/retry/-/retry-0.12.0.tgz#2b35eccfcee7d38cd72ad99232fbd58bffb3c84d"
+  integrity sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA==
+
 "@types/serve-static@*":
   version "1.13.10"
   resolved "https://registry.yarnpkg.com/@types/serve-static/-/serve-static-1.13.10.tgz#f5e0ce8797d2d7cc5ebeda48a52c96c4fa47a8d9"
@@ -1501,7 +1562,7 @@ axios@^0.27.2:
     follow-redirects "^1.14.9"
     form-data "^4.0.0"
 
-axios@^1.2.3, axios@^1.3.3, axios@^1.6.2, axios@^1.7.2:
+axios@^1.2.3, axios@^1.3.3, axios@^1.6.2:
   version "1.7.2"
   resolved "https://registry.yarnpkg.com/axios/-/axios-1.7.2.tgz#b625db8a7051fbea61c35a3cbb3a1daa7b9c7621"
   integrity sha512-2A8QhOMrbomlDuiLeK9XibIBzuHeRcqqNOHp0Cyp5EoJ1IFDh+XZH3A6BkXtv0K4gFGCI0Y4BM7B1wOEi0Rmgw==
@@ -1510,6 +1571,15 @@ axios@^1.2.3, axios@^1.3.3, axios@^1.6.2, axios@^1.7.2:
     form-data "^4.0.0"
     proxy-from-env "^1.1.0"
 
+axios@^1.7.4, axios@^1.7.7:
+  version "1.7.7"
+  resolved "https://registry.yarnpkg.com/axios/-/axios-1.7.7.tgz#2f554296f9892a72ac8d8e4c5b79c14a91d0a47f"
+  integrity sha512-S4kL7XrjgBmvdGut0sN3yJxqYzrDOnivkBiN0OFs6hLiUam3UPvswUo0kqGyhqUZGEOytHyumEdXsAkgCOUf3Q==
+  dependencies:
+    follow-redirects "^1.15.6"
+    form-data "^4.0.0"
+    proxy-from-env "^1.1.0"
+
 babel-runtime@^6.23.0:
   version "6.26.0"
   resolved "https://registry.yarnpkg.com/babel-runtime/-/babel-runtime-6.26.0.tgz#965c7058668e82b55d7bfe04ff2337bc8b5647fe"
@@ -2704,6 +2774,16 @@ event-target-shim@^5.0.0:
   resolved "https://registry.yarnpkg.com/event-target-shim/-/event-target-shim-5.0.1.tgz#5d4d3ebdf9583d63a5333ce2deb7480ab2b05789"
   integrity sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==
 
+eventemitter3@^4.0.4:
+  version "4.0.7"
+  resolved "https://registry.yarnpkg.com/eventemitter3/-/eventemitter3-4.0.7.tgz#2de9b68f6528d5644ef5c59526a1b4a07306169f"
+  integrity sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==
+
+eventemitter3@^5.0.1:
+  version "5.0.1"
+  resolved "https://registry.yarnpkg.com/eventemitter3/-/eventemitter3-5.0.1.tgz#53f5ffd0a492ac800721bb42c66b841de96423c4"
+  integrity sha512-GWkBvjiSZK87ELrYOSESUYeVIc9mvLLf/nXalMOS5dYrgZq9o5OVkbZAVM06CVxYsCwH9BDZFPlQTlPA1j4ahA==
+
 events@1.1.1:
   version "1.1.1"
   resolved "https://registry.yarnpkg.com/events/-/events-1.1.1.tgz#9ebdb7635ad099c70dcc4c2a1f5004288e8bd924"
@@ -3497,6 +3577,11 @@ is-docker@^2.0.0, is-docker@^2.1.1:
   resolved "https://registry.yarnpkg.com/is-docker/-/is-docker-2.2.1.tgz#33eeabe23cfe86f14bde4408a02c0cfb853acdaa"
   integrity sha512-F+i2BKsFrH66iaUFc0woD8sLy8getkwTwtOBjvs56Cx4CgJDeKQeqfz8wAYiSb8JOprWhHH5p77PbmYCvvUuXQ==
 
+is-electron@2.2.2:
+  version "2.2.2"
+  resolved "https://registry.yarnpkg.com/is-electron/-/is-electron-2.2.2.tgz#3778902a2044d76de98036f5dc58089ac4d80bb9"
+  integrity sha512-FO/Rhvz5tuw4MCWkpMzHFKWD2LsfHzIb7i6MdPYZ/KW7AlxawyLkqdy+jPZP1WubqEADE3O4FUENlJHDfQASRg==
+
 is-extglob@^2.1.1:
   version "2.1.1"
   resolved "https://registry.yarnpkg.com/is-extglob/-/is-extglob-2.1.1.tgz#a88c02535791f02ed37c76a1b9ea9773c833f8c2"
@@ -3567,7 +3652,7 @@ is-stream@^1.1.0:
   resolved "https://registry.yarnpkg.com/is-stream/-/is-stream-1.1.0.tgz#12d4a3dd4e68e0b79ceb8dbc84173ae80d91ca44"
   integrity sha512-uQPm8kcs47jx38atAcWTVxyltQYoPT68y9aWYdV6yWXSyW8mzSat0TL6CiWdZeCdF3KrAvpVtnHbTv4RN+rqdQ==
 
-is-stream@^2.0.0:
+is-stream@^2, is-stream@^2.0.0:
   version "2.0.1"
   resolved "https://registry.yarnpkg.com/is-stream/-/is-stream-2.0.1.tgz#fac1e3d53b97ad5a9d0ae9cef2389f5810a5c077"
   integrity sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==
@@ -3711,6 +3796,22 @@ jsonfile@^6.0.1:
   optionalDependencies:
     graceful-fs "^4.1.6"
 
+jsonwebtoken@^9:
+  version "9.0.2"
+  resolved "https://registry.yarnpkg.com/jsonwebtoken/-/jsonwebtoken-9.0.2.tgz#65ff91f4abef1784697d40952bb1998c504caaf3"
+  integrity sha512-PRp66vJ865SSqOlgqS8hujT5U4AOgMfhrwYIuIhfKaoSCZcirrmASQr8CX7cUg+RMih+hgznrjp99o+W4pJLHQ==
+  dependencies:
+    jws "^3.2.2"
+    lodash.includes "^4.3.0"
+    lodash.isboolean "^3.0.3"
+    lodash.isinteger "^4.0.4"
+    lodash.isnumber "^3.0.3"
+    lodash.isplainobject "^4.0.6"
+    lodash.isstring "^4.0.1"
+    lodash.once "^4.0.0"
+    ms "^2.1.1"
+    semver "^7.5.4"
+
 jsprim@^1.2.2:
   version "1.4.2"
   resolved "https://registry.yarnpkg.com/jsprim/-/jsprim-1.4.2.tgz#712c65533a15c878ba59e9ed5f0e26d5b77c5feb"
@@ -3731,6 +3832,15 @@ jszip@^3.7.1:
     readable-stream "~2.3.6"
     setimmediate "^1.0.5"
 
+jwa@^1.4.1:
+  version "1.4.1"
+  resolved "https://registry.yarnpkg.com/jwa/-/jwa-1.4.1.tgz#743c32985cb9e98655530d53641b66c8645b039a"
+  integrity sha512-qiLX/xhEEFKUAJ6FiBMbes3w9ATzyk5W7Hvzpa/SLYdxNtng+gcurvrI7TbACjIXlsJyr05/S1oUhZrc63evQA==
+  dependencies:
+    buffer-equal-constant-time "1.0.1"
+    ecdsa-sig-formatter "1.0.11"
+    safe-buffer "^5.0.1"
+
 jwa@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/jwa/-/jwa-2.0.0.tgz#a7e9c3f29dae94027ebcaf49975c9345593410fc"
@@ -3740,6 +3850,14 @@ jwa@^2.0.0:
     ecdsa-sig-formatter "1.0.11"
     safe-buffer "^5.0.1"
 
+jws@^3.2.2:
+  version "3.2.2"
+  resolved "https://registry.yarnpkg.com/jws/-/jws-3.2.2.tgz#001099f3639468c9414000e99995fa52fb478304"
+  integrity sha512-YHlZCB6lMTllWDtSPHz/ZXTsi8S00usEV6v1tjq8tOUZzw7DpSDWVXjXDre6ed1w/pd495ODpHZYSdkRTsa0HA==
+  dependencies:
+    jwa "^1.4.1"
+    safe-buffer "^5.0.1"
+
 jws@^4.0.0:
   version "4.0.0"
   resolved "https://registry.yarnpkg.com/jws/-/jws-4.0.0.tgz#2d4e8cf6a318ffaa12615e9dec7e86e6c97310f4"
@@ -3944,11 +4062,41 @@ lodash.flatten@^4.4.0:
   resolved "https://registry.yarnpkg.com/lodash.flatten/-/lodash.flatten-4.4.0.tgz#f31c22225a9632d2bbf8e4addbef240aa765a61f"
   integrity sha1-8xwiIlqWMtK7+OSt2+8kCqdlph8=
 
+lodash.includes@^4.3.0:
+  version "4.3.0"
+  resolved "https://registry.yarnpkg.com/lodash.includes/-/lodash.includes-4.3.0.tgz#60bb98a87cb923c68ca1e51325483314849f553f"
+  integrity sha512-W3Bx6mdkRTGtlJISOvVD/lbqjTlPPUDTMnlXZFnVwi9NKJ6tiAk6LVdlhZMm17VZisqhKcgzpO5Wz91PCt5b0w==
+
+lodash.isboolean@^3.0.3:
+  version "3.0.3"
+  resolved "https://registry.yarnpkg.com/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz#6c2e171db2a257cd96802fd43b01b20d5f5870f6"
+  integrity sha512-Bz5mupy2SVbPHURB98VAcw+aHh4vRV5IPNhILUCsOzRmsTmSQ17jIuqopAentWoehktxGd9e/hbIXq980/1QJg==
+
+lodash.isinteger@^4.0.4:
+  version "4.0.4"
+  resolved "https://registry.yarnpkg.com/lodash.isinteger/-/lodash.isinteger-4.0.4.tgz#619c0af3d03f8b04c31f5882840b77b11cd68343"
+  integrity sha512-DBwtEWN2caHQ9/imiNeEA5ys1JoRtRfY3d7V9wkqtbycnAmTvRRmbHKDV4a0EYc678/dia0jrte4tjYwVBaZUA==
+
+lodash.isnumber@^3.0.3:
+  version "3.0.3"
+  resolved "https://registry.yarnpkg.com/lodash.isnumber/-/lodash.isnumber-3.0.3.tgz#3ce76810c5928d03352301ac287317f11c0b1ffc"
+  integrity sha512-QYqzpfwO3/CWf3XP+Z+tkQsfaLL/EnUlXWVkIk5FUPc4sBdTehEqZONuyRt2P67PXAk+NXmTBcc97zw9t1FQrw==
+
 lodash.isplainobject@^4.0.6:
   version "4.0.6"
   resolved "https://registry.yarnpkg.com/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz#7c526a52d89b45c45cc690b88163be0497f550cb"
   integrity sha1-fFJqUtibRcRcxpC4gWO+BJf1UMs=
 
+lodash.isstring@^4, lodash.isstring@^4.0.1:
+  version "4.0.1"
+  resolved "https://registry.yarnpkg.com/lodash.isstring/-/lodash.isstring-4.0.1.tgz#d527dfb5456eca7cc9bb95d5daeaf88ba54a5451"
+  integrity sha512-0wJxfxH1wgO3GrbuP+dTTk7op+6L41QCXbGINEmD+ny/G/eCqGzxyCsh7159S+mgDDcoarnBw6PC1PS5+wUGgw==
+
+lodash.once@^4.0.0:
+  version "4.1.1"
+  resolved "https://registry.yarnpkg.com/lodash.once/-/lodash.once-4.1.1.tgz#0dd3971213c7c56df880977d504c88fb471a97ac"
+  integrity sha512-Sb487aTOCr9drQVL8pIxOzVhafOjZN9UU54hiN8PU3uAiSV7lx1yYNpbNmex2PK6dSJoNTSJUUswT651yww3Mg==
+
 lodash.snakecase@^4.1.1:
   version "4.1.1"
   resolved "https://registry.yarnpkg.com/lodash.snakecase/-/lodash.snakecase-4.1.1.tgz#39d714a35357147837aefd64b5dcbb16becd8f8d"
@@ -4451,6 +4599,11 @@ p-cancelable@^1.0.0:
   resolved "https://registry.yarnpkg.com/p-cancelable/-/p-cancelable-1.1.0.tgz#d078d15a3af409220c886f1d9a0ca2e441ab26cc"
   integrity sha512-s73XxOZ4zpt1edZYZzvhqFa6uvQc1vwUa0K0BdtIZgQMAJj9IbebH+JkgKZc9h+B05PKHLOTl4ajG1BmNrVZlw==
 
+p-finally@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/p-finally/-/p-finally-1.0.0.tgz#3fbcfb15b899a44123b34b6dcc18b724336a2cae"
+  integrity sha512-LICb2p9CB7FS+0eR1oqWnHhp0FljGLZCWBE9aix0Uye9W8LTQPwMTYVGWQWIw9RdQiDg4+epXQODwIYJtSJaow==
+
 p-limit@^3.0.2:
   version "3.1.0"
   resolved "https://registry.yarnpkg.com/p-limit/-/p-limit-3.1.0.tgz#e1daccbe78d0d1388ca18c64fea38e3e57e3706b"
@@ -4465,6 +4618,29 @@ p-locate@^5.0.0:
   dependencies:
     p-limit "^3.0.2"
 
+p-queue@^6:
+  version "6.6.2"
+  resolved "https://registry.yarnpkg.com/p-queue/-/p-queue-6.6.2.tgz#2068a9dcf8e67dd0ec3e7a2bcb76810faa85e426"
+  integrity sha512-RwFpb72c/BhQLEXIZ5K2e+AhgNVmIejGlTgiB9MzZ0e93GRvqZ7uSi0dvRF7/XIXDeNkra2fNHBxTyPDGySpjQ==
+  dependencies:
+    eventemitter3 "^4.0.4"
+    p-timeout "^3.2.0"
+
+p-retry@^4:
+  version "4.6.2"
+  resolved "https://registry.yarnpkg.com/p-retry/-/p-retry-4.6.2.tgz#9baae7184057edd4e17231cee04264106e092a16"
+  integrity sha512-312Id396EbJdvRONlngUx0NydfrIQ5lsYu0znKVUzVvArzEIt08V1qhtyESbGVd1FGX7UKtiFp5uwKZdM8wIuQ==
+  dependencies:
+    "@types/retry" "0.12.0"
+    retry "^0.13.1"
+
+p-timeout@^3.2.0:
+  version "3.2.0"
+  resolved "https://registry.yarnpkg.com/p-timeout/-/p-timeout-3.2.0.tgz#c7e17abc971d2a7962ef83626b35d635acf23dfe"
+  integrity sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg==
+  dependencies:
+    p-finally "^1.0.0"
+
 package-json@^6.3.0:
   version "6.5.0"
   resolved "https://registry.yarnpkg.com/package-json/-/package-json-6.5.0.tgz#6feedaca35e75725876d0b0e64974697fed145b0"
@@ -5104,6 +5280,11 @@ responselike@^1.0.2:
   dependencies:
     lowercase-keys "^1.0.0"
 
+retry@^0.13.1:
+  version "0.13.1"
+  resolved "https://registry.yarnpkg.com/retry/-/retry-0.13.1.tgz#185b1587acf67919d63b357349e03537b2484658"
+  integrity sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==
+
 rfdc@^1.3.0:
   version "1.3.0"
   resolved "https://registry.yarnpkg.com/rfdc/-/rfdc-1.3.0.tgz#d0b7c441ab2720d05dc4cf26e01c89631d9da08b"
@@ -5192,6 +5373,11 @@ semver@^7.3.4:
   dependencies:
     lru-cache "^6.0.0"
 
+semver@^7.5.4:
+  version "7.6.3"
+  resolved "https://registry.yarnpkg.com/semver/-/semver-7.6.3.tgz#980f7b5550bc175fb4dc09403085627f9eb33143"
+  integrity sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==
+
 semver@~5.3.0:
   version "5.3.0"
   resolved "https://registry.yarnpkg.com/semver/-/semver-5.3.0.tgz#9b2ce5d3de02d17c6012ad326aa6b4d0cf54f94f"
@@ -5810,6 +5996,11 @@ undici-types@~5.26.4:
   resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-5.26.5.tgz#bcd539893d00b56e964fd2657a4866b221a65617"
   integrity sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==
 
+undici-types@~6.19.2:
+  version "6.19.8"
+  resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-6.19.8.tgz#35111c9d1437ab83a7cdc0abae2f26d88eda0a02"
+  integrity sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==
+
 undici@^5.21.0:
   version "5.21.2"
   resolved "https://registry.yarnpkg.com/undici/-/undici-5.21.2.tgz#329f628aaea3f1539a28b9325dccc72097d29acd"

From 5b5a2641fe8195ebbd0a2fbf4d5e42e308350f2c Mon Sep 17 00:00:00 2001
From: chime3 
Date: Mon, 16 Sep 2024 23:19:45 -0700
Subject: [PATCH 107/182] feat: added chat message handler

---
 package.json                        |   1 +
 src/providers/slack/api.ts          | 115 +++++++++++++
 src/providers/slack/chat-message.ts | 257 ++++++++++++++++++++--------
 src/providers/slack/index.ts        |  20 ++-
 yarn.lock                           |   2 +-
 5 files changed, 322 insertions(+), 73 deletions(-)
 create mode 100644 src/providers/slack/api.ts

diff --git a/package.json b/package.json
index b62859e6..437bb9c8 100644
--- a/package.json
+++ b/package.json
@@ -35,6 +35,7 @@
     "@oauth-everything/passport-discord": "^1.0.2",
     "@sapphire/snowflake": "^3.4.2",
     "@slack/oauth": "^3.0.1",
+    "@slack/web-api": "^7.4.0",
     "@superfaceai/passport-twitter-oauth2": "^1.2.3",
     "@types/node": "^20.14.11",
     "@verida/account-node": "^4.1.0",
diff --git a/src/providers/slack/api.ts b/src/providers/slack/api.ts
new file mode 100644
index 00000000..24da0d9c
--- /dev/null
+++ b/src/providers/slack/api.ts
@@ -0,0 +1,115 @@
+import { WebClient } from '@slack/web-api';
+import * as fs from 'fs';
+import * as path from 'path';
+import CONFIG from '../../config';
+
+const slackPathPrefix = `_slack`;
+
+export class SlackApi {
+    clientId: string;
+    slackPath: string;
+    client?: WebClient;
+
+    public async getClient(): Promise {
+        if (this.client) {
+            return this.client;
+        }
+
+        if (!CONFIG.providers.slack.botToken) {
+            throw new Error('Slack bot token is missing from configuration.');
+        }
+
+        const client = new WebClient(CONFIG.providers.slack.botToken);
+        this.client = client;
+
+        return this.client;
+    }
+
+    public async getConversations(limit: number = 500): Promise {
+        const client = await this.getClient();
+        const channelIds: string[] = [];
+        let cursor: string | undefined;
+
+        do {
+            const response = await client.conversations.list({
+                limit,
+                cursor,
+            });
+
+            if (!response.channels || response.channels.length === 0) {
+                break;
+            }
+
+            response.channels.forEach(channel => {
+                if (channel.id) {
+                    channelIds.push(channel.id);
+                }
+            });
+
+            cursor = response.response_metadata?.next_cursor;
+        } while (cursor && channelIds.length < limit);
+
+        return channelIds;
+    }
+
+    public async getConversation(conversationId: string): Promise {
+        const client = await this.getClient();
+        const conversation = await client.conversations.info({
+            channel: conversationId,
+        });
+
+        return conversation;
+    }
+
+    public async getUser(userId: string): Promise {
+        const client = await this.getClient();
+        const user = await client.users.info({ user: userId });
+
+        return user;
+    }
+
+    public async getMessage(channelId: string, messageId: string): Promise {
+        const client = await this.getClient();
+        const message = await client.conversations.history({
+            channel: channelId,
+            inclusive: true,
+            latest: messageId,
+            limit: 1,
+        });
+
+        if (!message.messages || message.messages.length === 0) {
+            throw new Error('Message not found.');
+        }
+
+        return message.messages[0];
+    }
+
+    public async getConversationHistory(
+        channelId: string,
+        limit: number = 100,
+        latest?: string
+    ): Promise {
+        const client = await this.getClient();
+        const messages: any[] = [];
+
+        let cursor: string | undefined;
+        do {
+            const response = await client.conversations.history({
+                channel: channelId,
+                limit,
+                cursor,
+                latest,
+            });
+
+            if (!response.messages || response.messages.length === 0) {
+                break;
+            }
+
+            messages.push(...response.messages);
+
+            cursor = response.response_metadata?.next_cursor;
+        } while (cursor && messages.length < limit);
+
+        return messages;
+    }
+}
diff --git a/src/providers/slack/chat-message.ts b/src/providers/slack/chat-message.ts
index 90f6d52b..0da3422e 100644
--- a/src/providers/slack/chat-message.ts
+++ b/src/providers/slack/chat-message.ts
@@ -1,90 +1,207 @@
-import BaseSyncHandler from "../BaseSyncHandler";
+import { WebClient } from "@slack/web-api";
 import CONFIG from "../../config";
-
 import {
-    SyncResponse,
-    SyncHandlerPosition,
-    SyncHandlerStatus,
-    ProviderHandlerOption,
-    ConnectionOptionType,
+  SyncItemsBreak,
+  SyncItemsResult,
+  SyncProviderLogEvent,
+  SyncProviderLogLevel,
+} from "../../interfaces";
+import { ItemsRangeTracker } from "../../helpers/itemsRangeTracker";
+import {
+  SyncResponse,
+  SyncHandlerStatus,
+  ProviderHandlerOption,
+  ConnectionOptionType,
 } from "../../interfaces";
-
 import {
-    SchemaChatMessageType,
-    SchemaSocialChatGroup,
-    SchemaSocialChatMessage,
+  SchemaChatMessageType,
+  SchemaSocialChatGroup,
+  SchemaSocialChatMessage,
 } from "../../schemas";
-
 import { SlackChatGroupType, SlackProviderConfig } from "./interfaces";
-import { ItemsRangeTracker } from "../../helpers/itemsRangeTracker";
-import { ItemsRange } from "../../helpers/interfaces";
-
+import BaseSyncHandler from "../BaseSyncHandler";
 
 const _ = require("lodash");
 
-export default class SlackChatMessageHandler extends BaseSyncHandler {
-    protected config: SlackProviderConfig;
+const MAX_BATCH_SIZE = 200; // Slack's API often has lower rate limits
 
-    public getName(): string {
-        return "chat-message";
-    }
-
-    public getLabel(): string {
-        return "Chat Messages";
-    }
-
-    public getSchemaUri(): string {
-        return CONFIG.verida.schemas.CHAT_MESSAGE;
-    }
+export interface SyncSlackMessagesResult extends SyncItemsResult {
+  response_metadata: any;
+  items: SchemaSocialChatMessage[];
+}
 
-    public getProviderApplicationUrl(): string {
-        return "https://slack.com";
+export default class Slack extends BaseSyncHandler {
+
+  public getLabel(): string {
+    return "Slack Messages";
+  }
+
+  public getName(): string {
+    return "slack-messages";
+  }
+
+  public getSchemaUri(): string {
+    return CONFIG.verida.schemas.CHAT_MESSAGE;
+  }
+
+  public getProviderApplicationUrl() {
+    return "https://slack.com/";
+  }
+
+  public getSlackClient(): WebClient {
+    const token = this.connection.accessToken;
+    return new WebClient(token);
+  }
+
+  public getOptions(): ProviderHandlerOption[] {
+    return [
+      {
+        id: "channelTypes",
+        label: "Channel types",
+        type: ConnectionOptionType.ENUM_MULTI,
+        enumOptions: [
+          { label: "Channel", value: SlackChatGroupType.CHANNEL },
+          { label: "Group", value: SlackChatGroupType.GROUP },
+        ],
+        defaultValue: [SlackChatGroupType.CHANNEL, SlackChatGroupType.GROUP].join(","),
+      },
+    ];
+  }
+
+  public async _sync(
+    api: any,
+    syncPosition: any // Define a more specific Slack sync schema interface
+  ): Promise {
+    if (this.config.batchSize > MAX_BATCH_SIZE) {
+      throw new Error(
+        `Batch size (${this.config.batchSize}) is larger than permitted (${MAX_BATCH_SIZE})`
+      );
     }
 
-    public getOptions(): ProviderHandlerOption[] {
-        return [
-            {
-                id: "channelTypes",
-                label: "Channel types",
-                type: ConnectionOptionType.ENUM_MULTI,
-                enumOptions: [
-                    { label: "Channel", value: SlackChatGroupType.CHANNEL },
-                    { label: "Group", value: SlackChatGroupType.GROUP },
-                ],
-                defaultValue: [SlackChatGroupType.CHANNEL, SlackChatGroupType.GROUP].join(
-                    ","
-                ),
-            },
-        ];
+    const slack = this.getSlackClient();
+    const rangeTracker = new ItemsRangeTracker(syncPosition.thisRef);
+    let items: SchemaSocialChatMessage[] = [];
+
+    let currentRange = rangeTracker.nextRange();
+    let latestMessages = await this.fetchMessages(slack, currentRange.startId);
+
+    items = latestMessages.items;
+
+    let nextCursor = _.has(latestMessages, "response_metadata.next_cursor")
+      ? latestMessages.response_metadata.next_cursor
+      : undefined;
+
+    if (items.length) {
+      rangeTracker.completedRange(
+        {
+          startId: items[0].sourceId,
+          endId: nextCursor,
+        },
+        latestMessages.breakHit === SyncItemsBreak.ID
+      );
+    } else {
+      rangeTracker.completedRange(
+        {
+          startId: undefined,
+          endId: undefined,
+        },
+        false
+      );
     }
 
-    protected async buildChatGroupList(
-        syncPosition: SyncHandlerPosition
-    ): Promise {
-        let chatGroupIds: string[] = [];
-        if (syncPosition.thisRef) {
-            chatGroupIds = syncPosition.thisRef.split(",");
-        }
-        return Object.values([]);
+    if (!items.length) {
+      syncPosition.syncMessage = `Stopping. No results found.`;
+      syncPosition.status = SyncHandlerStatus.ENABLED;
+    } else {
+      if (items.length != this.config.batchSize && !nextCursor) {
+        syncPosition.syncMessage = `Processed ${items.length} items. Stopping. No more results.`;
+        syncPosition.status = SyncHandlerStatus.ENABLED;
+      } else {
+        syncPosition.syncMessage = `Batch complete (${this.config.batchSize}). More results pending.`;
+      }
     }
 
-
-    protected async processChatGroup(
-        chatGroup: SchemaSocialChatGroup,
-        totalMessageCount: number
-    ): Promise<{ chatGroup: SchemaSocialChatGroup; chatHistory: SchemaSocialChatMessage[] }> {
-        const chatHistory: SchemaSocialChatMessage[] = [];
-        const rangeTracker = new ItemsRangeTracker(chatGroup.syncData);
-        let groupMessageCount = 0;
-        let newItems = true;
-
-        return {
-            chatGroup,
-            chatHistory,
-        };
+    syncPosition.thisRef = rangeTracker.export();
+
+    return {
+      results: items,
+      position: syncPosition,
+    };
+  }
+
+  protected async fetchMessages(
+    slack: WebClient,
+    cursor?: string
+  ): Promise {
+    const result = await slack.conversations.history({
+      channel: this.config.channel || "general",
+      limit: this.config.batchSize,
+      cursor,
+    });
+
+    return this.buildResults(result);
+  }
+
+  protected async buildResults(
+    slackResponse: any
+  ): Promise {
+    const messages = slackResponse.messages;
+    const results: SchemaSocialChatMessage[] = [];
+    let breakHit: SyncItemsBreak;
+
+    for (const rawMessage of messages) {
+      const messageId = rawMessage.ts; // Slack uses timestamp as message ID
+      const timestamp = new Date(parseFloat(rawMessage.ts) * 1000).toISOString();
+      const content = rawMessage.text || "No message text";
+      const fromId = rawMessage.user; // Assuming the user field gives sender ID
+
+      // Fetch sender name from Slack (optional, you might store elsewhere)
+      const fromName = await this.fetchUserName(fromId); 
+      
+      // Assuming groupId and groupName are available from a conversation context
+      const groupId = this.config.channel;
+      const groupName = await this.fetchChannelName(groupId);
+
+      const message: SchemaSocialChatMessage = {
+        _id: this.buildItemId(rawMessage.ts),
+        name: content.substring(0, 30), // Truncate for name
+        groupId,
+        groupName,
+        type: rawMessage.user === this.config.currentUserId
+          ? SchemaChatMessageType.SEND
+          : SchemaChatMessageType.RECEIVE,
+        fromId,
+        fromHandle: fromName,
+        fromName: fromName,
+        messageText: content,
+        sourceApplication: this.getProviderApplicationUrl(),
+        sourceId: rawMessage.ts,
+        sourceData: rawMessage,
+        insertedAt: timestamp,
+        sentAt: timestamp,
+      };
+
+      results.push(message);
     }
 
-    public async _sync(syncPosition: SyncHandlerPosition): Promise {
-        return 
-    }
+    return {
+      items: results,
+      response_metadata: messages.response_metadata,
+      breakHit,
+    };
+  }
+
+  // Optional helper method to fetch user names from Slack
+  protected async fetchUserName(userId: string): Promise {
+    const slack = this.getSlackClient();
+    const userInfo = await slack.users.info({ user: userId });
+    return userInfo.user ? userInfo.user.real_name : "Unknown User";
+  }
+
+  // Optional helper method to fetch channel name from Slack
+  protected async fetchChannelName(channelId: string): Promise {
+    const slack = this.getSlackClient();
+    const channelInfo = await slack.conversations.info({ channel: channelId });
+    return channelInfo.channel ? channelInfo.channel.name : "Unknown Channel";
+  }
 }
diff --git a/src/providers/slack/index.ts b/src/providers/slack/index.ts
index 43fa5a28..0d78a36a 100644
--- a/src/providers/slack/index.ts
+++ b/src/providers/slack/index.ts
@@ -2,6 +2,7 @@ import { Request, Response } from "express";
 import Base from "../BaseProvider";
 import { SlackProviderConfig } from "./interfaces";
 import { FileInstallationStore, InstallProvider } from '@slack/oauth';
+import { WebClient } from "@slack/web-api";
 import SlackChatMessageHandler from "./chat-message";
 const axios = require('axios');
 
@@ -76,6 +77,10 @@ export default class SlackProvider extends Base {
             "channels:read",
             "groups:read",
             "users:read",
+            "channels:history",
+            "groups:history",
+            "im:history",
+            "mpim:history"
         ];
     }
 
@@ -84,6 +89,10 @@ export default class SlackProvider extends Base {
             "channels:read",
             "groups:read",
             "users:read",
+            "channels:history",
+            "groups:history",
+            "im:history",
+            "mpim:history"
         ];
     }
 
@@ -156,7 +165,14 @@ export default class SlackProvider extends Base {
     }
    
 
-    public async getApi(accessToken?: string, refreshToken?: string): Promise {
-        // You can return the Slack Web API client here using the access token
+    public async getApi(accessToken?: string, refreshToken?: string): Promise {
+        if (!accessToken) {
+            throw new Error('Access token is required');
+        }
+        
+        // Create a new WebClient instance with the provided access token
+        const client = new WebClient(accessToken);
+        
+        return client;
     }
 }
diff --git a/yarn.lock b/yarn.lock
index da70ef42..47c48872 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -518,7 +518,7 @@
   resolved "https://registry.yarnpkg.com/@slack/types/-/types-2.13.1.tgz#d1af332103ce96e22b10692bef9b34483f4c002c"
   integrity sha512-YVtJCVtDcjOPKsvOedIThb7YmKNCcSoZN0mUSQqD2fc2ZyI59gOLCF4rYGfw/0C0agzFxAmb7hV5tbMGrgK0Tg==
 
-"@slack/web-api@^7.3.4":
+"@slack/web-api@^7.3.4", "@slack/web-api@^7.4.0":
   version "7.4.0"
   resolved "https://registry.yarnpkg.com/@slack/web-api/-/web-api-7.4.0.tgz#56f8376203b4172c02812f0fc092b65c351a74ba"
   integrity sha512-U+0pui9VcddRt3yTSkLAArVB6G8EyOb0g+/vL1E6u8k46IYc1H4UKDuULF734A2ynVyxYKHIXK8OHilX6YhwjQ==

From 08296e9bc1047267057667e54cafc84809b42302 Mon Sep 17 00:00:00 2001
From: chime3 
Date: Tue, 17 Sep 2024 21:06:09 -0700
Subject: [PATCH 108/182] feat: added slack chat message handler

---
 src/interfaces.ts                   |   1 +
 src/providers/slack/api.ts          | 115 ----------
 src/providers/slack/chat-group.ts   | 188 ++++++++++++++++
 src/providers/slack/chat-message.ts | 323 ++++++++++++----------------
 src/providers/slack/index.ts        |  12 +-
 src/providers/slack/interfaces.ts   |  14 +-
 src/schemas.ts                      |   1 +
 src/serverconfig.example.json       |   5 +-
 8 files changed, 351 insertions(+), 308 deletions(-)
 delete mode 100644 src/providers/slack/api.ts
 create mode 100644 src/providers/slack/chat-group.ts

diff --git a/src/interfaces.ts b/src/interfaces.ts
index c33582b5..d53c670b 100644
--- a/src/interfaces.ts
+++ b/src/interfaces.ts
@@ -140,6 +140,7 @@ export enum SyncHandlerStatus {
     ERROR = "error",
     DISABLED = "disabled",
     SYNCING = "syncing",
+    COMPLETED = "COMPLETED",
 }
 
 export interface SyncHandlerPosition {
diff --git a/src/providers/slack/api.ts b/src/providers/slack/api.ts
deleted file mode 100644
index 24da0d9c..00000000
--- a/src/providers/slack/api.ts
+++ /dev/null
@@ -1,115 +0,0 @@
-import { WebClient } from '@slack/web-api';
-import * as fs from 'fs';
-import * as path from 'path';
-import CONFIG from '../../config';
-
-const slackPathPrefix = `_slack`;
-
-export class SlackApi {
-    clientId: string;
-    slackPath: string;
-    client?: WebClient;
-
-    public async getClient(): Promise {
-        if (this.client) {
-            return this.client;
-        }
-
-        if (!CONFIG.providers.slack.botToken) {
-            throw new Error('Slack bot token is missing from configuration.');
-        }
-
-        const client = new WebClient(CONFIG.providers.slack.botToken);
-        this.client = client;
-
-        return this.client;
-    }
-
-    public async getConversations(limit: number = 500): Promise {
-        const client = await this.getClient();
-        const channelIds: string[] = [];
-        let cursor: string | undefined;
-
-        do {
-            const response = await client.conversations.list({
-                limit,
-                cursor,
-            });
-
-            if (!response.channels || response.channels.length === 0) {
-                break;
-            }
-
-            response.channels.forEach(channel => {
-                if (channel.id) {
-                    channelIds.push(channel.id);
-                }
-            });
-
-            cursor = response.response_metadata?.next_cursor;
-        } while (cursor && channelIds.length < limit);
-
-        return channelIds;
-    }
-
-    public async getConversation(conversationId: string): Promise {
-        const client = await this.getClient();
-        const conversation = await client.conversations.info({
-            channel: conversationId,
-        });
-
-        return conversation;
-    }
-
-    public async getUser(userId: string): Promise {
-        const client = await this.getClient();
-        const user = await client.users.info({ user: userId });
-
-        return user;
-    }
-
-    public async getMessage(channelId: string, messageId: string): Promise {
-        const client = await this.getClient();
-        const message = await client.conversations.history({
-            channel: channelId,
-            inclusive: true,
-            latest: messageId,
-            limit: 1,
-        });
-
-        if (!message.messages || message.messages.length === 0) {
-            throw new Error('Message not found.');
-        }
-
-        return message.messages[0];
-    }
-
-    public async getConversationHistory(
-        channelId: string,
-        limit: number = 100,
-        latest?: string
-    ): Promise {
-        const client = await this.getClient();
-        const messages: any[] = [];
-
-        let cursor: string | undefined;
-        do {
-            const response = await client.conversations.history({
-                channel: channelId,
-                limit,
-                cursor,
-                latest,
-            });
-
-            if (!response.messages || response.messages.length === 0) {
-                break;
-            }
-
-            messages.push(...response.messages);
-
-            cursor = response.response_metadata?.next_cursor;
-        } while (cursor && messages.length < limit);
-
-        return messages;
-    }
-}
diff --git a/src/providers/slack/chat-group.ts b/src/providers/slack/chat-group.ts
new file mode 100644
index 00000000..b0ad2337
--- /dev/null
+++ b/src/providers/slack/chat-group.ts
@@ -0,0 +1,188 @@
+import { WebClient } from "@slack/web-api";
+import CONFIG from "../../config";
+import {
+  SyncItemsBreak,
+  SyncItemsResult,
+  SyncProviderLogEvent,
+  SyncProviderLogLevel,
+} from "../../interfaces";
+import { ItemsRangeTracker } from "../../helpers/itemsRangeTracker";
+import {
+  SyncResponse,
+  SyncHandlerStatus,
+  ProviderHandlerOption,
+  ConnectionOptionType,
+} from "../../interfaces";
+import { SchemaSocialChatGroup } from "../../schemas";
+import { SlackChatGroupType, SlackProviderConfig } from "./interfaces";
+import BaseSyncHandler from "../BaseSyncHandler";
+
+const _ = require("lodash");
+
+const MAX_BATCH_SIZE = 200; // Slack's API often has lower rate limits
+
+export interface SyncSlackGroupsResult extends SyncItemsResult {
+  response_metadata: any;
+  items: SchemaSocialChatGroup[];
+}
+
+export default class SlackChatGroupHandler extends BaseSyncHandler {
+
+  public getLabel(): string {
+    return "Slack Groups";
+  }
+
+  public getName(): string {
+    return "slack-groups";
+  }
+
+  public getSchemaUri(): string {
+    return CONFIG.verida.schemas.CHAT_GROUP;
+  }
+
+  public getProviderApplicationUrl() {
+    return "https://slack.com/";
+  }
+
+  public getSlackClient(): WebClient {
+    const token = this.connection.accessToken;
+    return new WebClient(token);
+  }
+
+  public getOptions(): ProviderHandlerOption[] {
+    return [
+      {
+        id: "groupTypes",
+        label: "Group types",
+        type: ConnectionOptionType.ENUM_MULTI,
+        enumOptions: [
+          { label: "Public Channel", value: SlackChatGroupType.CHANNEL },
+          { label: "Private Channel", value: SlackChatGroupType.GROUP },
+          { label: "IM", value: SlackChatGroupType.IM },
+          { label: "MPIM", value: SlackChatGroupType.MPIM },
+        ],
+        defaultValue: [
+          SlackChatGroupType.CHANNEL,
+          SlackChatGroupType.GROUP,
+          SlackChatGroupType.IM,
+          SlackChatGroupType.MPIM,
+        ].join(","),
+      },
+    ];
+  }
+
+  public async _sync(
+    api: any,
+    syncPosition: any // Define a more specific Slack sync schema interface
+  ): Promise {
+    if (this.config.batchSize > MAX_BATCH_SIZE) {
+      throw new Error(
+        `Batch size (${this.config.batchSize}) is larger than permitted (${MAX_BATCH_SIZE})`
+      );
+    }
+
+    const slack = this.getSlackClient();
+    const rangeTracker = new ItemsRangeTracker(syncPosition.thisRef);
+    let items: SchemaSocialChatGroup[] = [];
+
+    let currentRange = rangeTracker.nextRange();
+    let latestGroups = await this.fetchGroups(slack, currentRange.startId);
+
+    items = latestGroups.items;
+
+    let nextCursor = _.has(latestGroups, "response_metadata.next_cursor")
+      ? latestGroups.response_metadata.next_cursor
+      : undefined;
+
+    if (items.length) {
+      rangeTracker.completedRange(
+        {
+          startId: items[0].sourceId,
+          endId: nextCursor,
+        },
+        latestGroups.breakHit === SyncItemsBreak.ID
+      );
+    } else {
+      rangeTracker.completedRange(
+        {
+          startId: undefined,
+          endId: undefined,
+        },
+        false
+      );
+    }
+
+    if (!items.length) {
+      syncPosition.syncMessage = `Stopping. No results found.`;
+      syncPosition.status = SyncHandlerStatus.ENABLED;
+    } else {
+      if (items.length != this.config.batchSize && !nextCursor) {
+        syncPosition.syncMessage = `Processed ${items.length} items. Stopping. No more results.`;
+        syncPosition.status = SyncHandlerStatus.ENABLED;
+      } else {
+        syncPosition.syncMessage = `Batch complete (${this.config.batchSize}). More results pending.`;
+      }
+    }
+
+    syncPosition.thisRef = rangeTracker.export();
+
+    return {
+      results: items,
+      position: syncPosition,
+    };
+  }
+
+  protected async fetchGroups(
+    slack: WebClient,
+    cursor?: string
+  ): Promise {
+    const types = this.config.groupTypes || "public_channel,private_channel,im,mpim";
+    const result = await slack.conversations.list({
+      limit: this.config.batchSize,
+      cursor,
+      types,
+    });
+
+    return this.buildResults(result);
+  }
+
+  protected async buildResults(
+    slackResponse: any
+  ): Promise {
+    const groups = slackResponse.channels;
+    const results: SchemaSocialChatGroup[] = [];
+    let breakHit: SyncItemsBreak;
+
+    for (const rawGroup of groups) {
+      const groupId = rawGroup.id;
+      const groupName = rawGroup.name;
+      const groupType = this.mapGroupType(rawGroup);
+
+      const group: SchemaSocialChatGroup = {
+        _id: this.buildItemId(rawGroup.id),
+        name: groupName ?? "Unkown",
+        type: groupType,
+        sourceApplication: this.getProviderApplicationUrl(),
+        sourceId: groupId,
+        sourceData: rawGroup,
+        insertedAt: new Date().toISOString(),
+      };
+
+      results.push(group);
+    }
+
+    return {
+      items: results,
+      response_metadata: groups.response_metadata,
+      breakHit,
+    };
+  }
+
+  protected mapGroupType(rawGroup: any): string {
+    if (rawGroup.is_channel && !rawGroup.is_private) return SlackChatGroupType.CHANNEL;
+    if (rawGroup.is_channel && rawGroup.is_private) return SlackChatGroupType.GROUP;
+    if (rawGroup.is_im) return SlackChatGroupType.IM;
+    if (rawGroup.is_mpim) return SlackChatGroupType.MPIM;
+    return "unknown";
+  }
+}
diff --git a/src/providers/slack/chat-message.ts b/src/providers/slack/chat-message.ts
index 0da3422e..5e64a66e 100644
--- a/src/providers/slack/chat-message.ts
+++ b/src/providers/slack/chat-message.ts
@@ -1,207 +1,160 @@
 import { WebClient } from "@slack/web-api";
 import CONFIG from "../../config";
 import {
-  SyncItemsBreak,
-  SyncItemsResult,
-  SyncProviderLogEvent,
-  SyncProviderLogLevel,
-} from "../../interfaces";
-import { ItemsRangeTracker } from "../../helpers/itemsRangeTracker";
-import {
-  SyncResponse,
-  SyncHandlerStatus,
-  ProviderHandlerOption,
-  ConnectionOptionType,
+    SyncItemsResult,
+    SyncResponse,
+    SyncHandlerStatus,
+    ProviderHandlerOption,
+    ConnectionOptionType,
 } from "../../interfaces";
 import {
-  SchemaChatMessageType,
-  SchemaSocialChatGroup,
-  SchemaSocialChatMessage,
+    SchemaChatMessageType,
+    SchemaSocialChatGroup,
+    SchemaSocialChatMessage,
 } from "../../schemas";
 import { SlackChatGroupType, SlackProviderConfig } from "./interfaces";
 import BaseSyncHandler from "../BaseSyncHandler";
+import { ItemsRangeTracker } from "../../helpers/itemsRangeTracker";
 
 const _ = require("lodash");
+export default class SlackChatMessageHandler extends BaseSyncHandler {
+    protected config: SlackProviderConfig;
 
-const MAX_BATCH_SIZE = 200; // Slack's API often has lower rate limits
+    public getName(): string {
+        return "slack-messages";
+    }
 
-export interface SyncSlackMessagesResult extends SyncItemsResult {
-  response_metadata: any;
-  items: SchemaSocialChatMessage[];
-}
+    public getLabel(): string {
+        return "Slack Messages";
+    }
+
+    public getSchemaUri(): string {
+        return CONFIG.verida.schemas.CHAT_MESSAGE;
+    }
 
-export default class Slack extends BaseSyncHandler {
-
-  public getLabel(): string {
-    return "Slack Messages";
-  }
-
-  public getName(): string {
-    return "slack-messages";
-  }
-
-  public getSchemaUri(): string {
-    return CONFIG.verida.schemas.CHAT_MESSAGE;
-  }
-
-  public getProviderApplicationUrl() {
-    return "https://slack.com/";
-  }
-
-  public getSlackClient(): WebClient {
-    const token = this.connection.accessToken;
-    return new WebClient(token);
-  }
-
-  public getOptions(): ProviderHandlerOption[] {
-    return [
-      {
-        id: "channelTypes",
-        label: "Channel types",
-        type: ConnectionOptionType.ENUM_MULTI,
-        enumOptions: [
-          { label: "Channel", value: SlackChatGroupType.CHANNEL },
-          { label: "Group", value: SlackChatGroupType.GROUP },
-        ],
-        defaultValue: [SlackChatGroupType.CHANNEL, SlackChatGroupType.GROUP].join(","),
-      },
-    ];
-  }
-
-  public async _sync(
-    api: any,
-    syncPosition: any // Define a more specific Slack sync schema interface
-  ): Promise {
-    if (this.config.batchSize > MAX_BATCH_SIZE) {
-      throw new Error(
-        `Batch size (${this.config.batchSize}) is larger than permitted (${MAX_BATCH_SIZE})`
-      );
+    public getProviderApplicationUrl(): string {
+        return "https://slack.com/";
     }
 
-    const slack = this.getSlackClient();
-    const rangeTracker = new ItemsRangeTracker(syncPosition.thisRef);
-    let items: SchemaSocialChatMessage[] = [];
-
-    let currentRange = rangeTracker.nextRange();
-    let latestMessages = await this.fetchMessages(slack, currentRange.startId);
-
-    items = latestMessages.items;
-
-    let nextCursor = _.has(latestMessages, "response_metadata.next_cursor")
-      ? latestMessages.response_metadata.next_cursor
-      : undefined;
-
-    if (items.length) {
-      rangeTracker.completedRange(
-        {
-          startId: items[0].sourceId,
-          endId: nextCursor,
-        },
-        latestMessages.breakHit === SyncItemsBreak.ID
-      );
-    } else {
-      rangeTracker.completedRange(
-        {
-          startId: undefined,
-          endId: undefined,
-        },
-        false
-      );
+    public getOptions(): ProviderHandlerOption[] {
+        return [
+            {
+                id: "channelTypes",
+                label: "Channel types",
+                type: ConnectionOptionType.ENUM_MULTI,
+                enumOptions: [
+                    { label: "Channel", value: SlackChatGroupType.CHANNEL },
+                    { label: "Group", value: SlackChatGroupType.GROUP },
+                ],
+                defaultValue: [
+                    SlackChatGroupType.CHANNEL,
+                    SlackChatGroupType.GROUP,
+                ].join(","),
+            },
+        ];
     }
 
-    if (!items.length) {
-      syncPosition.syncMessage = `Stopping. No results found.`;
-      syncPosition.status = SyncHandlerStatus.ENABLED;
-    } else {
-      if (items.length != this.config.batchSize && !nextCursor) {
-        syncPosition.syncMessage = `Processed ${items.length} items. Stopping. No more results.`;
-        syncPosition.status = SyncHandlerStatus.ENABLED;
-      } else {
-        syncPosition.syncMessage = `Batch complete (${this.config.batchSize}). More results pending.`;
-      }
+    public getSlackClient(): WebClient {
+        const token = this.connection.accessToken;
+        return new WebClient(token);
     }
 
-    syncPosition.thisRef = rangeTracker.export();
-
-    return {
-      results: items,
-      position: syncPosition,
-    };
-  }
-
-  protected async fetchMessages(
-    slack: WebClient,
-    cursor?: string
-  ): Promise {
-    const result = await slack.conversations.history({
-      channel: this.config.channel || "general",
-      limit: this.config.batchSize,
-      cursor,
-    });
-
-    return this.buildResults(result);
-  }
-
-  protected async buildResults(
-    slackResponse: any
-  ): Promise {
-    const messages = slackResponse.messages;
-    const results: SchemaSocialChatMessage[] = [];
-    let breakHit: SyncItemsBreak;
-
-    for (const rawMessage of messages) {
-      const messageId = rawMessage.ts; // Slack uses timestamp as message ID
-      const timestamp = new Date(parseFloat(rawMessage.ts) * 1000).toISOString();
-      const content = rawMessage.text || "No message text";
-      const fromId = rawMessage.user; // Assuming the user field gives sender ID
-
-      // Fetch sender name from Slack (optional, you might store elsewhere)
-      const fromName = await this.fetchUserName(fromId); 
-      
-      // Assuming groupId and groupName are available from a conversation context
-      const groupId = this.config.channel;
-      const groupName = await this.fetchChannelName(groupId);
-
-      const message: SchemaSocialChatMessage = {
-        _id: this.buildItemId(rawMessage.ts),
-        name: content.substring(0, 30), // Truncate for name
-        groupId,
-        groupName,
-        type: rawMessage.user === this.config.currentUserId
-          ? SchemaChatMessageType.SEND
-          : SchemaChatMessageType.RECEIVE,
-        fromId,
-        fromHandle: fromName,
-        fromName: fromName,
-        messageText: content,
-        sourceApplication: this.getProviderApplicationUrl(),
-        sourceId: rawMessage.ts,
-        sourceData: rawMessage,
-        insertedAt: timestamp,
-        sentAt: timestamp,
-      };
-
-      results.push(message);
+    protected async buildChatGroupList(syncPosition: any): Promise {
+        const client = this.getSlackClient();
+        let channelList: SchemaSocialChatGroup[] = [];
+
+        // Fetch all conversations (channels and groups)
+        const conversations = await client.conversations.list({ limit: this.config.groupLimit });
+
+        for (const channel of conversations.channels) {
+            const group: SchemaSocialChatGroup = {
+                _id: this.buildItemId(channel.id),
+                name: channel.name,
+                sourceApplication: this.getProviderApplicationUrl(),
+                sourceId: channel.id,
+                schema: CONFIG.verida.schemas.CHAT_GROUP,
+                sourceData: channel,
+                insertedAt: new Date().toISOString(),
+            };
+            channelList.push(group);
+        }
+        return channelList;
     }
 
-    return {
-      items: results,
-      response_metadata: messages.response_metadata,
-      breakHit,
-    };
-  }
-
-  // Optional helper method to fetch user names from Slack
-  protected async fetchUserName(userId: string): Promise {
-    const slack = this.getSlackClient();
-    const userInfo = await slack.users.info({ user: userId });
-    return userInfo.user ? userInfo.user.real_name : "Unknown User";
-  }
-
-  // Optional helper method to fetch channel name from Slack
-  protected async fetchChannelName(channelId: string): Promise {
-    const slack = this.getSlackClient();
-    const channelInfo = await slack.conversations.info({ channel: channelId });
-    return channelInfo.channel ? channelInfo.channel.name : "Unknown Channel";
-  }
+    protected async fetchMessageRange(
+        chatGroup: SchemaSocialChatGroup,
+        rangeTracker: ItemsRangeTracker,
+        apiClient: WebClient
+    ): Promise {
+        const range = rangeTracker.nextRange();
+        const messages: SchemaSocialChatMessage[] = [];
+
+        const response = await apiClient.conversations.history({
+            channel: chatGroup.sourceId!,
+            limit: this.config.messagesPerGroupLimit,
+            oldest: range.startId,
+            latest: range.endId,
+        });
+
+        for (const message of response.messages) {
+            console.log("============");
+            console.log(message);
+            const chatMessage: SchemaSocialChatMessage = {
+                _id: this.buildItemId(message.ts),
+                groupId: chatGroup._id,
+                groupName: chatGroup.name,
+                messageText: message.text,
+                fromHandle: message.user,
+                sourceApplication: this.getProviderApplicationUrl(),
+                sourceId: message.ts,
+                sourceData: message,
+                insertedAt: new Date(parseFloat(message.ts) * 1000).toISOString(),
+                sentAt: new Date(parseFloat(message.ts) * 1000).toISOString(),
+                type: message.user === this.connection.profile.id
+                    ? SchemaChatMessageType.SEND
+                    : SchemaChatMessageType.RECEIVE,
+                fromId: message.user,
+                name: message.text.substring(0, 30)
+            };
+            messages.push(chatMessage);
+        }
+
+        return messages;
+    }
+
+    public async _sync(
+        api: any,
+        syncPosition: any
+    ): Promise {
+        try {
+            const apiClient = this.getSlackClient();
+            const groupList = await this.buildChatGroupList(syncPosition);
+            let totalMessages = 0;
+            let chatHistory: SchemaSocialChatMessage[] = [];
+
+            for (const group of groupList) {
+                if (totalMessages >= this.config.messageBatchSize) break;
+
+                const rangeTracker = new ItemsRangeTracker(group.syncData);
+                const messages = await this.fetchMessageRange(group, rangeTracker, apiClient);
+
+                chatHistory = chatHistory.concat(messages);
+                totalMessages += messages.length;
+
+                if (totalMessages >= this.config.messageBatchSize) break;
+            }
+
+            if (totalMessages === 0) {
+                syncPosition.status = SyncHandlerStatus.COMPLETED;
+            }
+
+            return {
+                results: chatHistory,
+                position: syncPosition,
+            };
+        } catch (err: any) {
+            console.error(err);
+            throw err;
+        }
+    }
 }
diff --git a/src/providers/slack/index.ts b/src/providers/slack/index.ts
index 0d78a36a..bee25d2d 100644
--- a/src/providers/slack/index.ts
+++ b/src/providers/slack/index.ts
@@ -8,6 +8,7 @@ const axios = require('axios');
 
 import { Installation, InstallationStore, InstallationQuery } from '@slack/oauth';
 import { PassportProfile } from "../../interfaces";
+import SlackChatGroupHandler from "./chat-group";
 
 export class CustomInstallationStore implements InstallationStore {
     private installations: Map = new Map();
@@ -68,7 +69,8 @@ export default class SlackProvider extends Base {
 
     public syncHandlers(): any[] {
         return [
-            SlackChatMessageHandler
+            SlackChatMessageHandler,
+            SlackChatGroupHandler
         ];
     }
 
@@ -76,6 +78,8 @@ export default class SlackProvider extends Base {
         return [
             "channels:read",
             "groups:read",
+            "im:read",
+            "mpim:read",
             "users:read",
             "channels:history",
             "groups:history",
@@ -88,6 +92,8 @@ export default class SlackProvider extends Base {
         return [
             "channels:read",
             "groups:read",
+            "im:read",
+            "mpim:read",
             "users:read",
             "channels:history",
             "groups:history",
@@ -100,7 +106,7 @@ export default class SlackProvider extends Base {
         this.init();
         try {
 
-            const result = await this.slackInstaller.handleInstallPath(
+            await this.slackInstaller.handleInstallPath(
                 req,
                 res,
                 {},
@@ -152,7 +158,7 @@ export default class SlackProvider extends Base {
             // Add access token data if necessary
             const connectionToken = {
                 id: data.team.id,
-                accessToken: data.access_token,
+                accessToken: data.authed_user.access_token,
                 refreshToken: data.refresh_token,  // If applicable, otherwise remove
                 profile: profile
             };
diff --git a/src/providers/slack/interfaces.ts b/src/providers/slack/interfaces.ts
index b0c3c2f6..b41aba8d 100644
--- a/src/providers/slack/interfaces.ts
+++ b/src/providers/slack/interfaces.ts
@@ -5,11 +5,17 @@ export interface SlackProviderConfig extends BaseProviderConfig {
   clientSecret: string;
   stateSecret: string;
   callbackUrl: string;
+  // Maximum number of groups to process
+  groupLimit: number,
+  // Maximum number of messages to process in a given batch
+  messageBatchSize: number
+  // Maximum number of messages to process in a group
+  messagesPerGroupLimit: number
 }
 
 export enum SlackChatGroupType {
-    CHANNEL = "channel", // Public channel
-    GROUP = "group", // Private channel
-    IM = "im", // DM
-    MPIM = "mpim" // Multi-person DM
+  CHANNEL = "channel", // Public channel
+  GROUP = "group", // Private channel
+  IM = "im", // DM
+  MPIM = "mpim" // Multi-person DM
 }
\ No newline at end of file
diff --git a/src/schemas.ts b/src/schemas.ts
index f63ac200..22036467 100644
--- a/src/schemas.ts
+++ b/src/schemas.ts
@@ -93,6 +93,7 @@ export interface SchemaFavourite extends SchemaRecord {
 export interface SchemaSocialChatGroup extends SchemaRecord {
     newestId?: string
     syncData?: string
+    type?: string
 }
 
 export enum SchemaChatMessageType {
diff --git a/src/serverconfig.example.json b/src/serverconfig.example.json
index 288d5b91..4a954429 100644
--- a/src/serverconfig.example.json
+++ b/src/serverconfig.example.json
@@ -94,7 +94,10 @@
             "label": "Slack",
             "clientId": "",
             "clientSecret": "",
-            "stateSecret": ""
+            "stateSecret": "",
+            "groupLimit": 2,
+            "messageBatchSize": 50,
+            "messagesPerGroupLimit": 10            
         }
     },
     "providerDefaults": {

From 408030b82e1b5b2fb08ba25353ac9685ff4a8356 Mon Sep 17 00:00:00 2001
From: chime3 
Date: Thu, 19 Sep 2024 23:11:13 -0700
Subject: [PATCH 109/182] feat: refactored chat-message handler

---
 src/providers/slack/chat-group.ts   | 188 -----------------------
 src/providers/slack/chat-message.ts | 221 +++++++++++++++++++++++-----
 src/providers/slack/index.ts        |   3 -
 src/serverconfig.example.json       |   2 +-
 4 files changed, 185 insertions(+), 229 deletions(-)
 delete mode 100644 src/providers/slack/chat-group.ts

diff --git a/src/providers/slack/chat-group.ts b/src/providers/slack/chat-group.ts
deleted file mode 100644
index b0ad2337..00000000
--- a/src/providers/slack/chat-group.ts
+++ /dev/null
@@ -1,188 +0,0 @@
-import { WebClient } from "@slack/web-api";
-import CONFIG from "../../config";
-import {
-  SyncItemsBreak,
-  SyncItemsResult,
-  SyncProviderLogEvent,
-  SyncProviderLogLevel,
-} from "../../interfaces";
-import { ItemsRangeTracker } from "../../helpers/itemsRangeTracker";
-import {
-  SyncResponse,
-  SyncHandlerStatus,
-  ProviderHandlerOption,
-  ConnectionOptionType,
-} from "../../interfaces";
-import { SchemaSocialChatGroup } from "../../schemas";
-import { SlackChatGroupType, SlackProviderConfig } from "./interfaces";
-import BaseSyncHandler from "../BaseSyncHandler";
-
-const _ = require("lodash");
-
-const MAX_BATCH_SIZE = 200; // Slack's API often has lower rate limits
-
-export interface SyncSlackGroupsResult extends SyncItemsResult {
-  response_metadata: any;
-  items: SchemaSocialChatGroup[];
-}
-
-export default class SlackChatGroupHandler extends BaseSyncHandler {
-
-  public getLabel(): string {
-    return "Slack Groups";
-  }
-
-  public getName(): string {
-    return "slack-groups";
-  }
-
-  public getSchemaUri(): string {
-    return CONFIG.verida.schemas.CHAT_GROUP;
-  }
-
-  public getProviderApplicationUrl() {
-    return "https://slack.com/";
-  }
-
-  public getSlackClient(): WebClient {
-    const token = this.connection.accessToken;
-    return new WebClient(token);
-  }
-
-  public getOptions(): ProviderHandlerOption[] {
-    return [
-      {
-        id: "groupTypes",
-        label: "Group types",
-        type: ConnectionOptionType.ENUM_MULTI,
-        enumOptions: [
-          { label: "Public Channel", value: SlackChatGroupType.CHANNEL },
-          { label: "Private Channel", value: SlackChatGroupType.GROUP },
-          { label: "IM", value: SlackChatGroupType.IM },
-          { label: "MPIM", value: SlackChatGroupType.MPIM },
-        ],
-        defaultValue: [
-          SlackChatGroupType.CHANNEL,
-          SlackChatGroupType.GROUP,
-          SlackChatGroupType.IM,
-          SlackChatGroupType.MPIM,
-        ].join(","),
-      },
-    ];
-  }
-
-  public async _sync(
-    api: any,
-    syncPosition: any // Define a more specific Slack sync schema interface
-  ): Promise {
-    if (this.config.batchSize > MAX_BATCH_SIZE) {
-      throw new Error(
-        `Batch size (${this.config.batchSize}) is larger than permitted (${MAX_BATCH_SIZE})`
-      );
-    }
-
-    const slack = this.getSlackClient();
-    const rangeTracker = new ItemsRangeTracker(syncPosition.thisRef);
-    let items: SchemaSocialChatGroup[] = [];
-
-    let currentRange = rangeTracker.nextRange();
-    let latestGroups = await this.fetchGroups(slack, currentRange.startId);
-
-    items = latestGroups.items;
-
-    let nextCursor = _.has(latestGroups, "response_metadata.next_cursor")
-      ? latestGroups.response_metadata.next_cursor
-      : undefined;
-
-    if (items.length) {
-      rangeTracker.completedRange(
-        {
-          startId: items[0].sourceId,
-          endId: nextCursor,
-        },
-        latestGroups.breakHit === SyncItemsBreak.ID
-      );
-    } else {
-      rangeTracker.completedRange(
-        {
-          startId: undefined,
-          endId: undefined,
-        },
-        false
-      );
-    }
-
-    if (!items.length) {
-      syncPosition.syncMessage = `Stopping. No results found.`;
-      syncPosition.status = SyncHandlerStatus.ENABLED;
-    } else {
-      if (items.length != this.config.batchSize && !nextCursor) {
-        syncPosition.syncMessage = `Processed ${items.length} items. Stopping. No more results.`;
-        syncPosition.status = SyncHandlerStatus.ENABLED;
-      } else {
-        syncPosition.syncMessage = `Batch complete (${this.config.batchSize}). More results pending.`;
-      }
-    }
-
-    syncPosition.thisRef = rangeTracker.export();
-
-    return {
-      results: items,
-      position: syncPosition,
-    };
-  }
-
-  protected async fetchGroups(
-    slack: WebClient,
-    cursor?: string
-  ): Promise {
-    const types = this.config.groupTypes || "public_channel,private_channel,im,mpim";
-    const result = await slack.conversations.list({
-      limit: this.config.batchSize,
-      cursor,
-      types,
-    });
-
-    return this.buildResults(result);
-  }
-
-  protected async buildResults(
-    slackResponse: any
-  ): Promise {
-    const groups = slackResponse.channels;
-    const results: SchemaSocialChatGroup[] = [];
-    let breakHit: SyncItemsBreak;
-
-    for (const rawGroup of groups) {
-      const groupId = rawGroup.id;
-      const groupName = rawGroup.name;
-      const groupType = this.mapGroupType(rawGroup);
-
-      const group: SchemaSocialChatGroup = {
-        _id: this.buildItemId(rawGroup.id),
-        name: groupName ?? "Unkown",
-        type: groupType,
-        sourceApplication: this.getProviderApplicationUrl(),
-        sourceId: groupId,
-        sourceData: rawGroup,
-        insertedAt: new Date().toISOString(),
-      };
-
-      results.push(group);
-    }
-
-    return {
-      items: results,
-      response_metadata: groups.response_metadata,
-      breakHit,
-    };
-  }
-
-  protected mapGroupType(rawGroup: any): string {
-    if (rawGroup.is_channel && !rawGroup.is_private) return SlackChatGroupType.CHANNEL;
-    if (rawGroup.is_channel && rawGroup.is_private) return SlackChatGroupType.GROUP;
-    if (rawGroup.is_im) return SlackChatGroupType.IM;
-    if (rawGroup.is_mpim) return SlackChatGroupType.MPIM;
-    return "unknown";
-  }
-}
diff --git a/src/providers/slack/chat-message.ts b/src/providers/slack/chat-message.ts
index 5e64a66e..6d3f7f7a 100644
--- a/src/providers/slack/chat-message.ts
+++ b/src/providers/slack/chat-message.ts
@@ -6,6 +6,7 @@ import {
     SyncHandlerStatus,
     ProviderHandlerOption,
     ConnectionOptionType,
+    SyncHandlerPosition,
 } from "../../interfaces";
 import {
     SchemaChatMessageType,
@@ -15,8 +16,10 @@ import {
 import { SlackChatGroupType, SlackProviderConfig } from "./interfaces";
 import BaseSyncHandler from "../BaseSyncHandler";
 import { ItemsRangeTracker } from "../../helpers/itemsRangeTracker";
+import { ItemsRange } from "../../helpers/interfaces";
 
 const _ = require("lodash");
+
 export default class SlackChatMessageHandler extends BaseSyncHandler {
     protected config: SlackProviderConfig;
 
@@ -43,12 +46,14 @@ export default class SlackChatMessageHandler extends BaseSyncHandler {
                 label: "Channel types",
                 type: ConnectionOptionType.ENUM_MULTI,
                 enumOptions: [
-                    { label: "Channel", value: SlackChatGroupType.CHANNEL },
-                    { label: "Group", value: SlackChatGroupType.GROUP },
+                    { label: "Public Channel", value: SlackChatGroupType.CHANNEL },
+                    { label: "Private Channel", value: SlackChatGroupType.GROUP },
+                    { label: "Direct Messages", value: SlackChatGroupType.IM },
                 ],
                 defaultValue: [
                     SlackChatGroupType.CHANNEL,
                     SlackChatGroupType.GROUP,
+                    SlackChatGroupType.IM,
                 ].join(","),
             },
         ];
@@ -59,34 +64,40 @@ export default class SlackChatMessageHandler extends BaseSyncHandler {
         return new WebClient(token);
     }
 
-    protected async buildChatGroupList(syncPosition: any): Promise {
+    protected async buildChatGroupList(): Promise {
         const client = this.getSlackClient();
+        let chatGroupIds: string[] = [];
         let channelList: SchemaSocialChatGroup[] = [];
 
-        // Fetch all conversations (channels and groups)
-        const conversations = await client.conversations.list({ limit: this.config.groupLimit });
+        // Fetch all types of conversations: DMs, private, public
+        const types = ["im", "private_channel", "public_channel"];
+        for (const type of types) {
+            const conversations = await client.conversations.list({
+                types: type,
+                limit: this.config.groupLimit,
+            });
 
-        for (const channel of conversations.channels) {
-            const group: SchemaSocialChatGroup = {
-                _id: this.buildItemId(channel.id),
-                name: channel.name,
-                sourceApplication: this.getProviderApplicationUrl(),
-                sourceId: channel.id,
-                schema: CONFIG.verida.schemas.CHAT_GROUP,
-                sourceData: channel,
-                insertedAt: new Date().toISOString(),
-            };
-            channelList.push(group);
+            for (const channel of conversations.channels) {
+                const group: SchemaSocialChatGroup = {
+                    _id: this.buildItemId(channel.id),
+                    name: channel.name || channel.user,
+                    sourceApplication: this.getProviderApplicationUrl(),
+                    sourceId: channel.id,
+                    schema: CONFIG.verida.schemas.CHAT_GROUP,
+                    sourceData: channel,
+                    insertedAt: new Date().toISOString(),
+                };
+                channelList.push(group);
+            }
         }
         return channelList;
     }
 
     protected async fetchMessageRange(
         chatGroup: SchemaSocialChatGroup,
-        rangeTracker: ItemsRangeTracker,
+        range: ItemsRange,
         apiClient: WebClient
     ): Promise {
-        const range = rangeTracker.nextRange();
         const messages: SchemaSocialChatMessage[] = [];
 
         const response = await apiClient.conversations.history({
@@ -97,8 +108,6 @@ export default class SlackChatMessageHandler extends BaseSyncHandler {
         });
 
         for (const message of response.messages) {
-            console.log("============");
-            console.log(message);
             const chatMessage: SchemaSocialChatMessage = {
                 _id: this.buildItemId(message.ts),
                 groupId: chatGroup._id,
@@ -110,11 +119,12 @@ export default class SlackChatMessageHandler extends BaseSyncHandler {
                 sourceData: message,
                 insertedAt: new Date(parseFloat(message.ts) * 1000).toISOString(),
                 sentAt: new Date(parseFloat(message.ts) * 1000).toISOString(),
-                type: message.user === this.connection.profile.id
-                    ? SchemaChatMessageType.SEND
-                    : SchemaChatMessageType.RECEIVE,
+                type:
+                    message.user === this.connection.profile.id
+                        ? SchemaChatMessageType.SEND
+                        : SchemaChatMessageType.RECEIVE,
                 fromId: message.user,
-                name: message.text.substring(0, 30)
+                name: message.text.substring(0, 30),
             };
             messages.push(chatMessage);
         }
@@ -124,32 +134,67 @@ export default class SlackChatMessageHandler extends BaseSyncHandler {
 
     public async _sync(
         api: any,
-        syncPosition: any
+        syncPosition: SyncHandlerPosition
     ): Promise {
         try {
             const apiClient = this.getSlackClient();
-            const groupList = await this.buildChatGroupList(syncPosition);
+            const groupList = await this.buildChatGroupList(); // Fetch all public, private, and DM groups
+
             let totalMessages = 0;
             let chatHistory: SchemaSocialChatMessage[] = [];
 
-            for (const group of groupList) {
-                if (totalMessages >= this.config.messageBatchSize) break;
+            // Determine the current group position
+            let groupPosition = this.getGroupPositionIndex(groupList, syncPosition);
 
-                const rangeTracker = new ItemsRangeTracker(group.syncData);
-                const messages = await this.fetchMessageRange(group, rangeTracker, apiClient);
+            const groupCount = groupList.length;
 
-                chatHistory = chatHistory.concat(messages);
-                totalMessages += messages.length;
+            // Iterate over each group 
+            for (let i = 0; i < groupCount; i++) {
+                const groupIndex = (groupPosition + i) % groupCount; // Rotate through groups
+                const group = groupList[groupIndex];
 
-                if (totalMessages >= this.config.messageBatchSize) break;
-            }
+                // Stop processing if batch size is reached
+                if (totalMessages >= this.config.messageBatchSize) {
+                    syncPosition.thisRef = groupList[groupIndex + 1].sourceId; // Save the next group to process
+                    break;
+                }
+
+                // Use a separate ItemsRangeTracker for each group
+                let rangeTracker = new ItemsRangeTracker(group.syncData);
+
+                const fetchedMessages = await this.fetchAndTrackMessages(
+                    group,
+                    rangeTracker,
+                    apiClient
+                );
+
+                // Concatenate the fetched messages to the total chat history
+                chatHistory = chatHistory.concat(fetchedMessages);
+                totalMessages += fetchedMessages.length;
 
-            if (totalMessages === 0) {
-                syncPosition.status = SyncHandlerStatus.COMPLETED;
+                // Update the group's sync data with the latest rangeTracker state
+                group.syncData = rangeTracker.export();
+
+                // Stop if the total messages fetched reach the batch size
+                if (totalMessages >= this.config.messageBatchSize) {
+                    syncPosition.thisRef = groupList[groupIndex + 1].sourceId; // Continue from the next group in the next sync
+                    break;
+                }
             }
 
+            // Finalize sync position and status based on message count
+            this.updateSyncPosition(
+                syncPosition,
+                totalMessages,
+                groupCount,
+                chatHistory
+            );
+
+            // Concatenate only items after syncPosition.thisRef and chatHistory
+            const remainingGroups = groupList.slice(groupPosition + 1);
+
             return {
-                results: chatHistory,
+                results: remainingGroups.concat(chatHistory),
                 position: syncPosition,
             };
         } catch (err: any) {
@@ -157,4 +202,106 @@ export default class SlackChatMessageHandler extends BaseSyncHandler {
             throw err;
         }
     }
+
+    private getGroupPositionIndex(
+        groupList: SchemaSocialChatGroup[],
+        syncPosition: SyncHandlerPosition
+    ): number {
+        const groupPosition = groupList.findIndex(
+            (group) => group.sourceId === syncPosition.thisRef
+        );
+
+        // If not found, return 0 to start from the beginning
+        return groupPosition === -1 ? 0 : groupPosition;
+    }
+
+    private async fetchAndTrackMessages(
+        group: any,
+        rangeTracker: ItemsRangeTracker,
+        apiClient: any
+    ): Promise {
+        // Initialize range from tracker
+        let currentRange = rangeTracker.nextRange();
+        let items: SchemaSocialChatMessage[] = [];
+
+        while (true) {
+            // Construct query based on range
+            let query: any = {
+                channel: group.id,
+                limit: this.config.batchSize,  // Default = 100, adjust if necessary
+            };
+
+            if (currentRange.startId) {
+                query.cursor = currentRange.startId; // Slack uses cursor for pagination
+            }
+
+            // Fetch messages from Slack API
+            const response = await apiClient.conversations.history(query);
+            const messages = response.messages || [];
+
+            // Process messages
+            items = items.concat(messages);
+
+            // Break loop if no more messages or limit reached
+            if (messages.length < this.config.batchSize || !response.has_more) {
+                // Update rangeTracker
+                rangeTracker.completedRange({
+                    startId: messages.length ? messages[0].ts : undefined,
+                    endId: response.response_metadata?.next_cursor || undefined
+                }, false);
+
+                break;
+            } else {
+                // Update range and continue fetching
+                rangeTracker.completedRange({
+                    startId: messages[0].ts,
+                    endId: response.response_metadata?.next_cursor
+                }, false);
+
+                currentRange = rangeTracker.nextRange();
+            }
+        }
+
+        return items;
+    }
+
+    private updateRangeTracker(
+        rangeTracker: ItemsRangeTracker,
+        messages: SchemaSocialChatMessage[],
+        currentRange: { startId?: string; endId?: string }
+    ) {
+        if (messages.length) {
+            const firstMessage = messages[0];
+            const lastMessage = messages[messages.length - 1];
+
+            // Mark the range as completed with fetched messages
+            rangeTracker.completedRange(
+                { startId: firstMessage._id, endId: lastMessage._id },
+                false // Update if there's a break condition
+            );
+        } else {
+            // No messages found, so mark the range as completed without changes
+            rangeTracker.completedRange(
+                { startId: undefined, endId: undefined },
+                false
+            );
+        }
+    }
+
+    private updateSyncPosition(
+        syncPosition: SyncHandlerPosition,
+        totalMessages: number,
+        groupCount: number,
+        chatHistory: SchemaSocialChatMessage[]
+    ) {
+        if (totalMessages === 0) {
+            syncPosition.status = SyncHandlerStatus.COMPLETED;
+            syncPosition.syncMessage = "No new messages found.";
+        } else if (totalMessages < this.config.messageBatchSize) {
+            syncPosition.syncMessage = `Processed ${totalMessages} messages across ${groupCount} groups. Sync complete.`;
+            syncPosition.status = SyncHandlerStatus.ENABLED;
+        } else {
+            syncPosition.syncMessage = `Batch complete (${this.config.messageBatchSize}). More results pending.`;
+        }
+    }
 }
diff --git a/src/providers/slack/index.ts b/src/providers/slack/index.ts
index bee25d2d..92f1e516 100644
--- a/src/providers/slack/index.ts
+++ b/src/providers/slack/index.ts
@@ -8,8 +8,6 @@ const axios = require('axios');
 
 import { Installation, InstallationStore, InstallationQuery } from '@slack/oauth';
 import { PassportProfile } from "../../interfaces";
-import SlackChatGroupHandler from "./chat-group";
-
 export class CustomInstallationStore implements InstallationStore {
     private installations: Map = new Map();
 
@@ -70,7 +68,6 @@ export default class SlackProvider extends Base {
     public syncHandlers(): any[] {
         return [
             SlackChatMessageHandler,
-            SlackChatGroupHandler
         ];
     }
 
diff --git a/src/serverconfig.example.json b/src/serverconfig.example.json
index 4a954429..37c02665 100644
--- a/src/serverconfig.example.json
+++ b/src/serverconfig.example.json
@@ -95,7 +95,7 @@
             "clientId": "",
             "clientSecret": "",
             "stateSecret": "",
-            "groupLimit": 2,
+            "groupLimit": 20,
             "messageBatchSize": 50,
             "messagesPerGroupLimit": 10            
         }

From 005232db72e689188bfb6fc9a375d91e416b0777 Mon Sep 17 00:00:00 2001
From: chime3 
Date: Sun, 22 Sep 2024 00:43:15 -0700
Subject: [PATCH 110/182] fix: slack chat message sync

---
 src/providers/slack/chat-message.ts | 56 +++++++++++++----------------
 1 file changed, 25 insertions(+), 31 deletions(-)

diff --git a/src/providers/slack/chat-message.ts b/src/providers/slack/chat-message.ts
index 6d3f7f7a..8522c24b 100644
--- a/src/providers/slack/chat-message.ts
+++ b/src/providers/slack/chat-message.ts
@@ -216,55 +216,49 @@ export default class SlackChatMessageHandler extends BaseSyncHandler {
     }
 
     private async fetchAndTrackMessages(
-        group: any,
+        group: SchemaSocialChatGroup,
         rangeTracker: ItemsRangeTracker,
-        apiClient: any
+        apiClient: WebClient
     ): Promise {
+        // Validate group and group.id
+        if (!group || !group.sourceId) {
+            throw new Error('Invalid group or missing group sourceId');
+        }
+    
         // Initialize range from tracker
         let currentRange = rangeTracker.nextRange();
         let items: SchemaSocialChatMessage[] = [];
-
+    
         while (true) {
-            // Construct query based on range
-            let query: any = {
-                channel: group.id,
-                limit: this.config.batchSize,  // Default = 100, adjust if necessary
-            };
-
-            if (currentRange.startId) {
-                query.cursor = currentRange.startId; // Slack uses cursor for pagination
-            }
-
-            // Fetch messages from Slack API
-            const response = await apiClient.conversations.history(query);
-            const messages = response.messages || [];
-
-            // Process messages
+            // Fetch messages for the current range using fetchMessageRange
+            const messages = await this.fetchMessageRange(group, currentRange, apiClient);
+    
+            // Add fetched messages to the main list
             items = items.concat(messages);
-
-            // Break loop if no more messages or limit reached
-            if (messages.length < this.config.batchSize || !response.has_more) {
-                // Update rangeTracker
+    
+            // Break loop if no more messages are returned or the limit is reached
+            if (messages.length < this.config.messagesPerGroupLimit) {
+                // Mark the current range as complete and stop
                 rangeTracker.completedRange({
-                    startId: messages.length ? messages[0].ts : undefined,
-                    endId: response.response_metadata?.next_cursor || undefined
+                    startId: messages.length ? messages[0].sourceId : undefined,
+                    endId: messages.length ? messages[messages.length - 1].sourceId : undefined
                 }, false);
-
                 break;
             } else {
-                // Update range and continue fetching
+                // Update rangeTracker and continue fetching
                 rangeTracker.completedRange({
-                    startId: messages[0].ts,
-                    endId: response.response_metadata?.next_cursor
+                    startId: messages[0].sourceId,
+                    endId: messages[messages.length - 1].sourceId
                 }, false);
-
+    
+                // Move to the next range
                 currentRange = rangeTracker.nextRange();
             }
         }
-
+    
         return items;
     }
-
+    
     private updateRangeTracker(
         rangeTracker: ItemsRangeTracker,
         messages: SchemaSocialChatMessage[],

From 54b70a5c41869668e6a701da3be98a9b822ca0b2 Mon Sep 17 00:00:00 2001
From: chime3 
Date: Mon, 23 Sep 2024 16:26:19 -0700
Subject: [PATCH 111/182] docs: added slack config guides

---
 src/providers/slack/README.md | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 src/providers/slack/README.md

diff --git a/src/providers/slack/README.md b/src/providers/slack/README.md
new file mode 100644
index 00000000..bfda5687
--- /dev/null
+++ b/src/providers/slack/README.md
@@ -0,0 +1,10 @@
+# Slack APP configuration
+
+1. Please go to [Slack API Apps](https://api.slack.com/apps)
+2. Create new app
+    - Select `From scratch`
+    - Add `App Name` and select a workspace to be used for development
+3. You can get `client ID` and `client Secret` from the `Basic Information` section
+4. Add redirect URL and scopes in `OAuth & Permissions` section
+    - Redirect URL: `https://127.0.0.1:5021/callback/slack`
+    - There are two types of tokens: bot and user, and add following scopes: `channels:history`, `channels:read`, `groups:read`, `users:read`, `im:read`, `im:history`
\ No newline at end of file

From 1ff60f0e0438ce28de9b9f67d85f71c5836b3b65 Mon Sep 17 00:00:00 2001
From: chime3 
Date: Mon, 23 Sep 2024 16:27:00 -0700
Subject: [PATCH 112/182] fix: updated slack user profile

---
 src/providers/slack/chat-message.ts | 67 ++++++++++-------------------
 src/providers/slack/helpers.ts      | 25 +++++++++++
 src/providers/slack/index.ts        | 41 +++++++++++-------
 3 files changed, 72 insertions(+), 61 deletions(-)
 create mode 100644 src/providers/slack/helpers.ts

diff --git a/src/providers/slack/chat-message.ts b/src/providers/slack/chat-message.ts
index 8522c24b..b4ea0772 100644
--- a/src/providers/slack/chat-message.ts
+++ b/src/providers/slack/chat-message.ts
@@ -17,6 +17,7 @@ import { SlackChatGroupType, SlackProviderConfig } from "./interfaces";
 import BaseSyncHandler from "../BaseSyncHandler";
 import { ItemsRangeTracker } from "../../helpers/itemsRangeTracker";
 import { ItemsRange } from "../../helpers/interfaces";
+import { SlackHelpers } from "./helpers";
 
 const _ = require("lodash");
 
@@ -66,7 +67,7 @@ export default class SlackChatMessageHandler extends BaseSyncHandler {
 
     protected async buildChatGroupList(): Promise {
         const client = this.getSlackClient();
-        let chatGroupIds: string[] = [];
+
         let channelList: SchemaSocialChatGroup[] = [];
 
         // Fetch all types of conversations: DMs, private, public
@@ -74,7 +75,6 @@ export default class SlackChatMessageHandler extends BaseSyncHandler {
         for (const type of types) {
             const conversations = await client.conversations.list({
                 types: type,
-                limit: this.config.groupLimit,
             });
 
             for (const channel of conversations.channels) {
@@ -108,12 +108,15 @@ export default class SlackChatMessageHandler extends BaseSyncHandler {
         });
 
         for (const message of response.messages) {
+            if (!message.user) continue;
+            
+            const user = await SlackHelpers.getUserInfo(this.connection.accessToken, message.user)
             const chatMessage: SchemaSocialChatMessage = {
                 _id: this.buildItemId(message.ts),
                 groupId: chatGroup._id,
                 groupName: chatGroup.name,
                 messageText: message.text,
-                fromHandle: message.user,
+                fromHandle: user.profile.email ?? "Unknown",
                 sourceApplication: this.getProviderApplicationUrl(),
                 sourceId: message.ts,
                 sourceData: message,
@@ -123,7 +126,7 @@ export default class SlackChatMessageHandler extends BaseSyncHandler {
                     message.user === this.connection.profile.id
                         ? SchemaChatMessageType.SEND
                         : SchemaChatMessageType.RECEIVE,
-                fromId: message.user,
+                fromId: message.user ?? "Unknown",
                 name: message.text.substring(0, 30),
             };
             messages.push(chatMessage);
@@ -149,16 +152,10 @@ export default class SlackChatMessageHandler extends BaseSyncHandler {
             const groupCount = groupList.length;
 
             // Iterate over each group 
-            for (let i = 0; i < groupCount; i++) {
+            for (let i = 0; i < Math.min(groupCount, this.config.groupLimit); i++) {
                 const groupIndex = (groupPosition + i) % groupCount; // Rotate through groups
                 const group = groupList[groupIndex];
 
-                // Stop processing if batch size is reached
-                if (totalMessages >= this.config.messageBatchSize) {
-                    syncPosition.thisRef = groupList[groupIndex + 1].sourceId; // Save the next group to process
-                    break;
-                }
-
                 // Use a separate ItemsRangeTracker for each group
                 let rangeTracker = new ItemsRangeTracker(group.syncData);
 
@@ -177,7 +174,7 @@ export default class SlackChatMessageHandler extends BaseSyncHandler {
 
                 // Stop if the total messages fetched reach the batch size
                 if (totalMessages >= this.config.messageBatchSize) {
-                    syncPosition.thisRef = groupList[groupIndex + 1].sourceId; // Continue from the next group in the next sync
+                    syncPosition.thisRef = groupList[(groupIndex + 1) % groupCount].sourceId; // Continue from the next group in the next sync
                     break;
                 }
             }
@@ -224,24 +221,26 @@ export default class SlackChatMessageHandler extends BaseSyncHandler {
         if (!group || !group.sourceId) {
             throw new Error('Invalid group or missing group sourceId');
         }
-    
+
         // Initialize range from tracker
         let currentRange = rangeTracker.nextRange();
         let items: SchemaSocialChatMessage[] = [];
-    
+
         while (true) {
             // Fetch messages for the current range using fetchMessageRange
             const messages = await this.fetchMessageRange(group, currentRange, apiClient);
-    
+
+            if (!messages.length) break;
+
             // Add fetched messages to the main list
             items = items.concat(messages);
-    
-            // Break loop if no more messages are returned or the limit is reached
-            if (messages.length < this.config.messagesPerGroupLimit) {
+
+            // Break loop if messages reached group limit
+            if (items.length > this.config.messagesPerGroupLimit) {
                 // Mark the current range as complete and stop
                 rangeTracker.completedRange({
-                    startId: messages.length ? messages[0].sourceId : undefined,
-                    endId: messages.length ? messages[messages.length - 1].sourceId : undefined
+                    startId: messages[0].sourceId,
+                    endId: messages[messages.length - 1].sourceId
                 }, false);
                 break;
             } else {
@@ -250,37 +249,14 @@ export default class SlackChatMessageHandler extends BaseSyncHandler {
                     startId: messages[0].sourceId,
                     endId: messages[messages.length - 1].sourceId
                 }, false);
-    
+
                 // Move to the next range
                 currentRange = rangeTracker.nextRange();
             }
         }
-    
+
         return items;
     }
-    
-    private updateRangeTracker(
-        rangeTracker: ItemsRangeTracker,
-        messages: SchemaSocialChatMessage[],
-        currentRange: { startId?: string; endId?: string }
-    ) {
-        if (messages.length) {
-            const firstMessage = messages[0];
-            const lastMessage = messages[messages.length - 1];
-
-            // Mark the range as completed with fetched messages
-            rangeTracker.completedRange(
-                { startId: firstMessage._id, endId: lastMessage._id },
-                false // Update if there's a break condition
-            );
-        } else {
-            // No messages found, so mark the range as completed without changes
-            rangeTracker.completedRange(
-                { startId: undefined, endId: undefined },
-                false
-            );
-        }
-    }
 
     private updateSyncPosition(
         syncPosition: SyncHandlerPosition,
@@ -295,6 +271,7 @@ export default class SlackChatMessageHandler extends BaseSyncHandler {
             syncPosition.syncMessage = `Processed ${totalMessages} messages across ${groupCount} groups. Sync complete.`;
             syncPosition.status = SyncHandlerStatus.ENABLED;
         } else {
+            //syncPosition.status = SyncHandlerStatus.ENABLED;
             syncPosition.syncMessage = `Batch complete (${this.config.messageBatchSize}). More results pending.`;
         }
     }
diff --git a/src/providers/slack/helpers.ts b/src/providers/slack/helpers.ts
new file mode 100644
index 00000000..a228d2ec
--- /dev/null
+++ b/src/providers/slack/helpers.ts
@@ -0,0 +1,25 @@
+import axios from 'axios';
+
+export class SlackHelpers {
+  // Method to fetch user information using Slack's `users.info` API
+  static async getUserInfo(accessToken: string, userId: string) {
+    try {
+      const response = await axios.get('https://slack.com/api/users.info', {
+        headers: {
+          Authorization: `Bearer ${accessToken}`
+        },
+        params: {
+          user: userId  // The target user's Slack ID
+        }
+      });
+
+      if (response.data.ok) {
+        return response.data.user;  // Return user profile info
+      } else {
+        throw new Error(`Slack API Error: ${response.data.error}`);
+      }
+    } catch (error) {
+      throw new Error(`Failed to fetch user info: ${error.message}`);
+    }
+  }
+}
diff --git a/src/providers/slack/index.ts b/src/providers/slack/index.ts
index 92f1e516..c70e2cf6 100644
--- a/src/providers/slack/index.ts
+++ b/src/providers/slack/index.ts
@@ -8,6 +8,7 @@ const axios = require('axios');
 
 import { Installation, InstallationStore, InstallationQuery } from '@slack/oauth';
 import { PassportProfile } from "../../interfaces";
+import { SlackHelpers } from "./helpers";
 export class CustomInstallationStore implements InstallationStore {
     private installations: Map = new Map();
 
@@ -78,6 +79,7 @@ export default class SlackProvider extends Base {
             "im:read",
             "mpim:read",
             "users:read",
+            "users:read.email",
             "channels:history",
             "groups:history",
             "im:history",
@@ -92,6 +94,7 @@ export default class SlackProvider extends Base {
             "im:read",
             "mpim:read",
             "users:read",
+            "users:read.email",
             "channels:history",
             "groups:history",
             "im:history",
@@ -130,52 +133,58 @@ export default class SlackProvider extends Base {
 
         return response.data;
     }
-    
+
+
     public async callback(req: Request, res: Response, next: any): Promise {
         this.init();
         const { code } = req.query;
         try {
             const data = await this.getAccessToken(code as string);
-            
+
+            // Fetch the user's profile from Slack using the `authed_user.access_token`
+            const userInfo = await SlackHelpers.getUserInfo(data.authed_user.access_token, data.authed_user.id);
+
+            // Build the PassportProfile object
             const profile: PassportProfile = {
-                id: data.authed_user.id,  // Slack user ID
+                id: userInfo.id,  // Slack user ID
                 provider: this.getProviderName(),  // Set your Slack provider name
-                displayName: data.team.name,  // Team name as display name
+                displayName: userInfo.profile.real_name,  // User's real name
                 name: {
-                    familyName: '',  // Slack does not provide family name directly
-                    givenName: data.team.name  // Use team name as given name (optional customization)
+                    familyName: userInfo.profile.first_name,  
+                    givenName: userInfo.profile.last_name
                 },
                 connectionProfile: {
-                    username: data.authed_user.id,  // Slack user ID as username
-                    phone: undefined,  // Slack API does not provide phone info
-                    verified: true  // Assuming token authorization is verified
+                    username: userInfo.profile.display_name,  // Display name as username
+                    email: userInfo.profile.email,  // Email from profile
+                    phone: userInfo.profile.phone,  
+                    verified: userInfo.is_email_confirmed
                 }
             };
-    
-            // Add access token data if necessary
+
+            // Add access token data
             const connectionToken = {
                 id: data.team.id,
                 accessToken: data.authed_user.access_token,
                 refreshToken: data.refresh_token,  // If applicable, otherwise remove
                 profile: profile
             };
-    
+
             return connectionToken;
-    
+
         } catch (error) {
             next(error);
         }
     }
-   
+
 
     public async getApi(accessToken?: string, refreshToken?: string): Promise {
         if (!accessToken) {
             throw new Error('Access token is required');
         }
-        
+
         // Create a new WebClient instance with the provided access token
         const client = new WebClient(accessToken);
-        
+
         return client;
     }
 }

From 369114453b890c4cc397b672b664e38cada4ff9c Mon Sep 17 00:00:00 2001
From: chime3 
Date: Mon, 23 Sep 2024 16:27:18 -0700
Subject: [PATCH 113/182] feat: added slack unit test

---
 tests/providers/slack/chat-message.ts | 58 +++++++++++++++++++++++++++
 1 file changed, 58 insertions(+)
 create mode 100644 tests/providers/slack/chat-message.ts

diff --git a/tests/providers/slack/chat-message.ts b/tests/providers/slack/chat-message.ts
new file mode 100644
index 00000000..f9014af8
--- /dev/null
+++ b/tests/providers/slack/chat-message.ts
@@ -0,0 +1,58 @@
+const assert = require("assert");
+import {
+  BaseProviderConfig,
+  Connection,
+  SyncHandlerStatus,
+  SyncHandlerPosition,
+} from "../../../src/interfaces";
+import Providers from "../../../src/providers";
+import CommonUtils, { NetworkInstance } from "../../common.utils";
+
+import SlackChatMessageHandler from "../../../src/providers/slack/chat-message";
+import BaseProvider from "../../../src/providers/BaseProvider";
+import { CommonTests, GenericTestConfig } from "../../common.tests";
+import { SchemaSocialChatMessage } from "../../../src/schemas";
+
+const providerName = "slack";
+let network: NetworkInstance;
+let connection: Connection;
+let provider: BaseProvider;
+let handlerName = "slack-messages";
+let testConfig: GenericTestConfig;
+let providerConfig: Omit = {};
+
+describe(`${providerName} Slack Chat Message Handler Tests`, function () {
+  this.timeout(100000); // Increase timeout due to API rate limits and potential delays
+
+  this.beforeAll(async function () {
+    // Set up the Slack network, connection, and provider
+    network = await CommonUtils.getNetwork();
+    connection = await CommonUtils.getConnection(providerName);
+    provider = Providers(providerName, network.context, connection);
+
+    // Define test configuration
+    testConfig = {
+      idPrefix: `${provider.getProviderName()}-${connection.profile.id}`,
+      batchSizeLimitAttribute: "messageBatchSize", // Adjust to match Slack-specific batch size config
+    };
+  });
+
+  describe(`Fetch ${providerName} data`, () => {
+   
+    it(`Can pass basic tests: ${handlerName}`, async () => {
+      // Run the generic common tests for Slack Chat Message Handler
+      await CommonTests.runGenericTests(
+        providerName,
+        SlackChatMessageHandler,
+        testConfig,
+        providerConfig,
+        connection
+      );
+    });
+  });
+
+  this.afterAll(async function () {
+    const { context } = await CommonUtils.getNetwork();
+    await context.close(); // Clean up after tests
+  });
+});

From 90925c82f5005517b800791286c7624d2c7d4dd9 Mon Sep 17 00:00:00 2001
From: chime3 
Date: Mon, 23 Sep 2024 17:08:03 -0700
Subject: [PATCH 114/182] fix: typo error

---
 src/web/developer/data/data.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/web/developer/data/data.js b/src/web/developer/data/data.js
index ba513e3c..dfa764be 100644
--- a/src/web/developer/data/data.js
+++ b/src/web/developer/data/data.js
@@ -240,8 +240,8 @@ $(document).ready(function() {
             "File": "https://common.schemas.verida.io/file/v0.1.0/schema.json",
             "Chat Group": "https://common.schemas.verida.io/social/chat/group/v0.1.0/schema.json",
             "Chat Message": "https://common.schemas.verida.io/social/chat/message/v0.1.0/schema.json",
-            "CALENDAR": "https://common.schemas.verida.io/social/calendar/v0.1.0/schema.json",
-            "EVENT": "https://common.schemas.verida.io/social/event/v0.1.0/schema.json"
+            "Calendar": "https://common.schemas.verida.io/social/calendar/v0.1.0/schema.json",
+            "Event": "https://common.schemas.verida.io/social/event/v0.1.0/schema.json"
         };
 
         // Clear previous list

From 5110fa10f6f2084df64324f8692ae0413b10d3f4 Mon Sep 17 00:00:00 2001
From: chime3 
Date: Tue, 24 Sep 2024 21:49:08 -0700
Subject: [PATCH 115/182] fix: changed providerId function name

---
 src/providers/slack/chat-message.ts           |   4 +-
 src/providers/slack/interfaces.ts             |  15 +-
 tests/common.tests.ts                         | 234 +++++++++++++-----
 .../providers/google/calendar-event.tests.ts  |  14 +-
 tests/providers/slack/chat-message.ts         |  14 +-
 5 files changed, 200 insertions(+), 81 deletions(-)

diff --git a/src/providers/slack/chat-message.ts b/src/providers/slack/chat-message.ts
index b4ea0772..b11e90e4 100644
--- a/src/providers/slack/chat-message.ts
+++ b/src/providers/slack/chat-message.ts
@@ -13,7 +13,7 @@ import {
     SchemaSocialChatGroup,
     SchemaSocialChatMessage,
 } from "../../schemas";
-import { SlackChatGroupType, SlackProviderConfig } from "./interfaces";
+import { SlackChatGroupType, SlackHandlerConfig } from "./interfaces";
 import BaseSyncHandler from "../BaseSyncHandler";
 import { ItemsRangeTracker } from "../../helpers/itemsRangeTracker";
 import { ItemsRange } from "../../helpers/interfaces";
@@ -22,7 +22,7 @@ import { SlackHelpers } from "./helpers";
 const _ = require("lodash");
 
 export default class SlackChatMessageHandler extends BaseSyncHandler {
-    protected config: SlackProviderConfig;
+    protected config: SlackHandlerConfig;
 
     public getName(): string {
         return "slack-messages";
diff --git a/src/providers/slack/interfaces.ts b/src/providers/slack/interfaces.ts
index b41aba8d..d0cdfc85 100644
--- a/src/providers/slack/interfaces.ts
+++ b/src/providers/slack/interfaces.ts
@@ -1,10 +1,6 @@
-import { BaseProviderConfig } from "../../interfaces";
+import { BaseHandlerConfig, BaseProviderConfig } from "../../interfaces";
 
-export interface SlackProviderConfig extends BaseProviderConfig {
-  clientId: string;
-  clientSecret: string;
-  stateSecret: string;
-  callbackUrl: string;
+export interface SlackHandlerConfig extends BaseHandlerConfig {
   // Maximum number of groups to process
   groupLimit: number,
   // Maximum number of messages to process in a given batch
@@ -13,6 +9,13 @@ export interface SlackProviderConfig extends BaseProviderConfig {
   messagesPerGroupLimit: number
 }
 
+export interface SlackProviderConfig extends BaseProviderConfig {
+  clientId: string;
+  clientSecret: string;
+  stateSecret: string;
+  callbackUrl: string;
+}
+
 export enum SlackChatGroupType {
   CHANNEL = "channel", // Public channel
   GROUP = "group", // Private channel
diff --git a/tests/common.tests.ts b/tests/common.tests.ts
index 5490ad3c..ecf7c0a3 100644
--- a/tests/common.tests.ts
+++ b/tests/common.tests.ts
@@ -15,12 +15,12 @@ import CommonUtils from "./common.utils";
 const assert = require("assert");
 
 export interface GenericTestConfig {
-  timeOrderAttribute?: string; // Optional, used for time ordering
-  batchSizeLimitAttribute: string; // Used for limiting the batch size
-  idPrefix?: string; // Prefix for record ID's
-  resultsPerPage?: number; // Number of items per page
-  pageCount?: number; // Number of pages to fetch
-  allowBackfill?: boolean; // Whether backfill is allowed
+  // Attribute in the results that is used for time ordering (ie: insertedAt)
+  timeOrderAttribute?: string; // Made optional
+  // Attribute used to limit the batch size (ie: batchLimit)
+  batchSizeLimitAttribute: string;
+  // Prefix used for record ID's (override default which is providerName)
+  idPrefix?: string;
 }
 
 // info,debug,error
@@ -37,9 +37,6 @@ export class CommonTests {
     testConfig: GenericTestConfig = {
       timeOrderAttribute: "insertedAt",
       batchSizeLimitAttribute: "batchSize",
-      resultsPerPage: 2, // Default results per page
-      pageCount: 2, // Default number of pages
-      allowBackfill: true, // Allow backfill by default
     },
     syncPositionConfig: Omit,
     providerConfig?: Omit
@@ -109,9 +106,6 @@ export class CommonTests {
     testConfig: GenericTestConfig = {
       timeOrderAttribute: "insertedAt",
       batchSizeLimitAttribute: "batchSize",
-      resultsPerPage: 3, // Default to 3 results per page
-      pageCount: 2, // Default to 2 pages
-      allowBackfill: true, // Backfill allowed by default
     },
     providerConfig: Omit = {},
     connection?: Connection
@@ -120,8 +114,14 @@ export class CommonTests {
     handler: BaseSyncHandler;
     provider: BaseProvider;
   }> {
-    // Set result limit to resultsPerPage from testConfig
-    providerConfig[testConfig.batchSizeLimitAttribute] = testConfig.resultsPerPage!;
+    // * - New items are processed
+    //    * - Backfill items are processed
+    //    * - Not enough new items? Process backfill
+    //    * - Backfill twice doesn't process the same items
+    //    * - No more backfill produces empty rangeTracker
+
+    // Set result limit to 3 results so page tests can work correctly
+    providerConfig[testConfig.batchSizeLimitAttribute] = 3;
 
     const { api, handler, schemaUri, provider } = await this.buildTestObjects(
       providerId,
@@ -147,63 +147,179 @@ export class CommonTests {
       const response = await handler._sync(api, syncPosition);
       const results = response.results;
 
-        assert.ok(results && results.length, `Page ${page + 1}: Have results returned`);
-        assert.equal(
-          providerConfig[testConfig.batchSizeLimitAttribute],
-          results.length,
-          `Page ${page + 1}: Have correct number of results returned`
+      // console.log(response.position)
+      // console.log(CommonTests.outputItems(results, testConfig.timeOrderAttribute))
+
+      assert.ok(results && results.length, "Have results returned");
+      assert.equal(
+        providerConfig[testConfig.batchSizeLimitAttribute],
+        results.length,
+        "Have correct number of results returned on page 1"
+      );
+
+      if (testConfig.timeOrderAttribute) {
+        assert.ok(
+          results[0][testConfig.timeOrderAttribute] >
+            results[1][testConfig.timeOrderAttribute],
+          "Results are most recent first"
         );
+      }
+
+      CommonTests.checkItem(results[0], handler, provider)
 
-        if (testConfig.timeOrderAttribute) {
-          assert.ok(
-            results[0][testConfig.timeOrderAttribute] >
-              results[1][testConfig.timeOrderAttribute],
-            `Page ${page + 1}: Results are most recent first`
-          );
-        }
+      assert.equal(
+        SyncHandlerStatus.SYNCING,
+        response.position.status,
+        "Sync is active"
+      );
+      assert.ok(response.position.thisRef, "Have a defined processing range");
 
-        CommonTests.checkItem(results[0], handler, provider);
+      const currentRangeParts = response.position.thisRef!.split(':')
+      assert.ok(currentRangeParts.length == 2, "Have correct number of parts for the processing range");
+      assert.ok(currentRangeParts[0] == results[0]._id.replace(`${idPrefix}-`, ''), "Have correct break ID");
+      assert.ok(currentRangeParts[1].length, "Have an end range");
 
-        assert.equal(
-          SyncHandlerStatus.SYNCING,
-          response.position.status,
-          `Page ${page + 1}: Sync is active`
+      // 2. Backfill items are processed
+      const syncPosition2 = response.position
+      const response2 = await handler._sync(api, syncPosition2);
+      const results2 = response2.results;
+      
+      // console.log(response2.position)
+      // console.log(CommonTests.outputItems(results2, testConfig.timeOrderAttribute))
+
+      assert.ok(
+        results2 && results2.length,
+        "Have backfill results returned"
+      );
+      assert.ok(
+        results2 &&
+          results2.length == providerConfig[testConfig.batchSizeLimitAttribute],
+        "Have correct number of results returned in second page"
+      );
+
+      if (testConfig.timeOrderAttribute) {
+        assert.ok(
+          results2[0][testConfig.timeOrderAttribute] >
+            results2[1][testConfig.timeOrderAttribute],
+          "Results are most recent first"
+        );
+        assert.ok(
+          results2[0][testConfig.timeOrderAttribute] <
+            results[2][testConfig.timeOrderAttribute],
+          "First item on second page of results have earlier timestamp than last item on first page"
         );
-        assert.ok(response.position.thisRef, `Page ${page + 1}: Have a defined processing range`);
+      }
 
-        const currentRangeParts = response.position.thisRef!.split(":");
-        assert.ok(currentRangeParts.length == 2, "Have correct number of parts for the processing range");
+      assert.equal(
+        response2.position.status,
+        SyncHandlerStatus.SYNCING,
+        "Sync is active"
+      );
+
+      assert.ok(response2.position.thisRef, "Have a defined processing range");
+
+      const currentRangeParts2 = response2.position.thisRef!.split(':')
+      assert.ok(currentRangeParts2.length == 2, "Have correct number of parts for the processing range");
+      assert.ok(currentRangeParts2[0] == results[0]._id.replace(`${idPrefix}-`, ''), "Have correct break ID matching the very first result");
+      assert.ok(currentRangeParts2[1].length, "Have an end range");
+      assert.ok(results[0]._id != results2[0]._id, "Have different result IDs")
+
+      // 3. Not enough new items? Process backfill
+      const syncPosition3 = response2.position
+      syncPosition3.thisRef = `${results[1].sourceId}:${currentRangeParts2[1]}` // Ensure the first item (only) is fetched
+      const response3 = await handler._sync(api, syncPosition3);
+      const results3 = response3.results;
+      
+      // console.log(response3.position)
+      // console.log(CommonTests.outputItems(results3, testConfig.timeOrderAttribute))
+
+      assert.ok(
+        results3 && results3.length,
+        "Have results returned"
+      );
+      assert.ok(
+        results3 &&
+        results3.length == providerConfig[testConfig.batchSizeLimitAttribute],
+        "Have correct number of results returned"
+      );
+      assert.equal(results3[0]._id, results[0]._id, 'First result item matches the very first item')
+      assert.ok(results3[1]._id != results[1]._id, 'Second result item does not match the very first batch second item')
+
+      if (testConfig.timeOrderAttribute) {
         assert.ok(
-          currentRangeParts[0] == results[0]._id.replace(`${idPrefix}-`, ""),
-          `Page ${page + 1}: Have correct break ID`
+          results3[0][testConfig.timeOrderAttribute] >
+            results3[1][testConfig.timeOrderAttribute],
+          "Results are most recent first"
         );
-        assert.ok(currentRangeParts[1].length, "Have an end range");
+        // this will break?
+        assert.ok(
+          results3[2][testConfig.timeOrderAttribute] <
+            results[2][testConfig.timeOrderAttribute],
+          "Last item on return results have earlier timestamp than last item on first page"
+        );
+      }
+
+      assert.equal(
+        response3.position.status,
+        SyncHandlerStatus.SYNCING,
+        "Sync is active"
+      );
 
-        // Backfill logic
-        if (testConfig.allowBackfill && results.length < providerConfig[testConfig.batchSizeLimitAttribute]) {
-          const backfillResponse = await handler._sync(api, syncPosition);
+      assert.ok(response3.position.thisRef, "Have a defined processing range");
+
+      const currentRangeParts3 = response3.position.thisRef!.split(':')
+      assert.ok(currentRangeParts3.length == 2, "Have correct number of parts for the processing range");
+      assert.ok(currentRangeParts3[0] == results3[0]._id.replace(`${idPrefix}-`, ''), "Have correct break ID matching the very first result");
+      assert.ok(currentRangeParts3[1].length, "Have an end range");
+      assert.ok(currentRangeParts3[1] != currentRangeParts2[1], "End range has changed between batches");
+
+      // - Backfill twice doesn't process the same items
+      const syncPosition4 = response3.position
+      const response4 = await handler._sync(api, syncPosition4);
+      const results4 = response4.results;
+      
+      // console.log(response4.position)
+      // console.log(CommonTests.outputItems(results4, testConfig.timeOrderAttribute))
+
+      assert.ok(
+        results4 && results4.length,
+        "Have results returned"
+      );
+      assert.ok(
+        results4 &&
+        results4.length == providerConfig[testConfig.batchSizeLimitAttribute],
+        "Have correct number of results returned"
+      );
+
+      if (testConfig.timeOrderAttribute) {
+        assert.ok(
+          results4[0][testConfig.timeOrderAttribute] >
+          results4[1][testConfig.timeOrderAttribute],
+          "Results are most recent first"
+        );
+        // this will break?
+        assert.ok(
+          results4[0][testConfig.timeOrderAttribute] <
+            results[2][testConfig.timeOrderAttribute],
+          "First item on return results have earlier timestamp than last item on first page"
+        );
+      }
 
-          // Filter out items already processed in backfill
-          const newBackfillResults = backfillResponse.results.filter(
-            (item) => !processedBackfillItems.has(item)
-          );
+      assert.ok(results4[0]._id != results3[0]._id, "First items dont match between batches")
 
-          if (newBackfillResults.length > 0) {
-            console.log(`Backfill processed ${newBackfillResults.length} new items on page ${page + 1}`);
-            newBackfillResults.forEach((item) => processedBackfillItems.add(item));
-          }
+      assert.equal(
+        response4.position.status,
+        SyncHandlerStatus.SYNCING,
+        "Sync is active"
+      );
 
-          assert.ok(newBackfillResults.length, `Page ${page + 1}: Backfill has new results`);
-        }
+      assert.ok(response4.position.thisRef, "Have a defined processing range");
+      const currentRangeParts4 = response4.position.thisRef!.split(':')
+      assert.ok(currentRangeParts4.length == 2, "Have correct number of parts for the processing range");
+      assert.ok(currentRangeParts4[1].length, "Have an end range");
 
-        // Update syncPosition for next page
-        syncPosition = response.position;
+      // @todo: No more backfill produces empty rangeTracker and SyncHandlerStatus.CONNECTED
 
-        // Break the loop early if no more results are fetched
-        if (!results.length) {
-          break;
-        }
-      }
 
       // Close the provider connection
       await provider.close();
@@ -214,7 +330,7 @@ export class CommonTests {
         provider,
       };
     } catch (err) {
-      // Ensure provider closes even if there's an error
+      // ensure provider closes even if there's an error
       await provider.close();
 
       throw err;
@@ -239,7 +355,7 @@ export class CommonTests {
   // Helper method to output items to help with debugging
   static outputItems(items: SchemaRecord[], timeAttribute?: string) {
     for (const item of items) {
-      console.log(item._id, timeAttribute ? item[timeAttribute] : "", item.name);
+      console.log(item._id, timeAttribute ? item[timeAttribute] : '', item.name)
     }
   }
 }
diff --git a/tests/providers/google/calendar-event.tests.ts b/tests/providers/google/calendar-event.tests.ts
index 17e26ab0..81fda561 100644
--- a/tests/providers/google/calendar-event.tests.ts
+++ b/tests/providers/google/calendar-event.tests.ts
@@ -12,7 +12,7 @@ import CalendarEvent from "../../../src/providers/google/calendar-event";
 import BaseProvider from "../../../src/providers/BaseProvider";
 import { CommonTests, GenericTestConfig } from "../../common.tests";
 
-const providerName = "google";
+const providerId = "google";
 let network: NetworkInstance;
 let connection: Connection;
 let provider: BaseProvider;
@@ -20,25 +20,25 @@ let handlerName = "calendar-event";
 let testConfig: GenericTestConfig;
 let providerConfig: Omit = {};
 
-describe(`${providerName} Google Calendar Event Tests`, function () {
+describe(`${providerId} Google Calendar Event Tests`, function () {
   this.timeout(100000);
 
   this.beforeAll(async function () {
     network = await CommonUtils.getNetwork();
-    connection = await CommonUtils.getConnection(providerName);
-    provider = Providers(providerName, network.context, connection);
+    connection = await CommonUtils.getConnection(providerId);
+    provider = Providers(providerId, network.context, connection);
 
     testConfig = {
-      idPrefix: `${provider.getProviderName()}-${connection.profile.id}`,
+      idPrefix: `${provider.getProviderId()}-${connection.profile.id}`,
       batchSizeLimitAttribute: "batchSize",
     };
   });
 
-  describe(`Fetch ${providerName} data`, () => {
+  describe(`Fetch ${providerId} data`, () => {
    
     it(`Can pass basic tests: ${handlerName}`, async () => {
       await CommonTests.runGenericTests(
-        providerName,
+        providerId,
         CalendarEvent,
         testConfig,
         providerConfig,
diff --git a/tests/providers/slack/chat-message.ts b/tests/providers/slack/chat-message.ts
index f9014af8..496c88cb 100644
--- a/tests/providers/slack/chat-message.ts
+++ b/tests/providers/slack/chat-message.ts
@@ -13,7 +13,7 @@ import BaseProvider from "../../../src/providers/BaseProvider";
 import { CommonTests, GenericTestConfig } from "../../common.tests";
 import { SchemaSocialChatMessage } from "../../../src/schemas";
 
-const providerName = "slack";
+const providerId = "slack";
 let network: NetworkInstance;
 let connection: Connection;
 let provider: BaseProvider;
@@ -21,28 +21,28 @@ let handlerName = "slack-messages";
 let testConfig: GenericTestConfig;
 let providerConfig: Omit = {};
 
-describe(`${providerName} Slack Chat Message Handler Tests`, function () {
+describe(`${providerId} Slack Chat Message Handler Tests`, function () {
   this.timeout(100000); // Increase timeout due to API rate limits and potential delays
 
   this.beforeAll(async function () {
     // Set up the Slack network, connection, and provider
     network = await CommonUtils.getNetwork();
-    connection = await CommonUtils.getConnection(providerName);
-    provider = Providers(providerName, network.context, connection);
+    connection = await CommonUtils.getConnection(providerId);
+    provider = Providers(providerId, network.context, connection);
 
     // Define test configuration
     testConfig = {
-      idPrefix: `${provider.getProviderName()}-${connection.profile.id}`,
+      idPrefix: `${provider.getProviderId()}-${connection.profile.id}`,
       batchSizeLimitAttribute: "messageBatchSize", // Adjust to match Slack-specific batch size config
     };
   });
 
-  describe(`Fetch ${providerName} data`, () => {
+  describe(`Fetch ${providerId} data`, () => {
    
     it(`Can pass basic tests: ${handlerName}`, async () => {
       // Run the generic common tests for Slack Chat Message Handler
       await CommonTests.runGenericTests(
-        providerName,
+        providerId,
         SlackChatMessageHandler,
         testConfig,
         providerConfig,

From bfa79cc0a059b2448c8ac0941ac9f84825672891 Mon Sep 17 00:00:00 2001
From: chime3 
Date: Tue, 24 Sep 2024 23:20:38 -0700
Subject: [PATCH 116/182] fix: sourceAccountId

---
 src/providers/google/calendar-event.ts | 2 +-
 src/providers/google/calendar.ts       | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/providers/google/calendar-event.ts b/src/providers/google/calendar-event.ts
index 24d55b01..a1fa412f 100644
--- a/src/providers/google/calendar-event.ts
+++ b/src/providers/google/calendar-event.ts
@@ -253,7 +253,7 @@ export default class CalendarEvent extends GoogleHandler {
       results.push({
         _id: this.buildItemId(eventId),
         name: event.summary ?? 'No event title',
-        sourceAccountId: this.provider.getProviderId(),
+        sourceAccountId: this.provider.getAccountId(),
         sourceData: event,
         sourceApplication: this.getProviderApplicationUrl(),
         sourceId: eventId,
diff --git a/src/providers/google/calendar.ts b/src/providers/google/calendar.ts
index 99b1cbf0..6a763c15 100644
--- a/src/providers/google/calendar.ts
+++ b/src/providers/google/calendar.ts
@@ -217,7 +217,7 @@ export default class Calendar extends GoogleHandler {
       results.push({
         _id: this.buildItemId(calendarId),
         name: summary,
-        sourceAccountId: this.provider.getProviderId(),
+        sourceAccountId: this.provider.getAccountId(),
         sourceData: listItem,
         sourceApplication: this.getProviderApplicationUrl(),
         sourceId: calendarId,

From 8e78b2259cb3009f63ec2a3a62fd8b4a5d33f69a Mon Sep 17 00:00:00 2001
From: chime3 
Date: Tue, 24 Sep 2024 23:21:07 -0700
Subject: [PATCH 117/182] feat: added sourceAccountId

---
 src/providers/slack/chat-message.ts | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/providers/slack/chat-message.ts b/src/providers/slack/chat-message.ts
index b11e90e4..5d718ac4 100644
--- a/src/providers/slack/chat-message.ts
+++ b/src/providers/slack/chat-message.ts
@@ -81,6 +81,7 @@ export default class SlackChatMessageHandler extends BaseSyncHandler {
                 const group: SchemaSocialChatGroup = {
                     _id: this.buildItemId(channel.id),
                     name: channel.name || channel.user,
+                    sourceAccountId: this.provider.getAccountId(),
                     sourceApplication: this.getProviderApplicationUrl(),
                     sourceId: channel.id,
                     schema: CONFIG.verida.schemas.CHAT_GROUP,
@@ -117,6 +118,7 @@ export default class SlackChatMessageHandler extends BaseSyncHandler {
                 groupName: chatGroup.name,
                 messageText: message.text,
                 fromHandle: user.profile.email ?? "Unknown",
+                sourceAccountId: this.provider.getAccountId(),
                 sourceApplication: this.getProviderApplicationUrl(),
                 sourceId: message.ts,
                 sourceData: message,

From 283d79488f32dccc9d925035919e79ff3c12a843 Mon Sep 17 00:00:00 2001
From: chime3 
Date: Tue, 24 Sep 2024 23:21:32 -0700
Subject: [PATCH 118/182] feat: added slack unit test

---
 tests/providers/slack/chat-message.ts | 73 +++++++++++++++++++++------
 1 file changed, 58 insertions(+), 15 deletions(-)

diff --git a/tests/providers/slack/chat-message.ts b/tests/providers/slack/chat-message.ts
index 496c88cb..d0ac41bf 100644
--- a/tests/providers/slack/chat-message.ts
+++ b/tests/providers/slack/chat-message.ts
@@ -1,9 +1,8 @@
 const assert = require("assert");
 import {
-  BaseProviderConfig,
   Connection,
-  SyncHandlerStatus,
   SyncHandlerPosition,
+  SyncHandlerStatus
 } from "../../../src/interfaces";
 import Providers from "../../../src/providers";
 import CommonUtils, { NetworkInstance } from "../../common.utils";
@@ -11,48 +10,92 @@ import CommonUtils, { NetworkInstance } from "../../common.utils";
 import SlackChatMessageHandler from "../../../src/providers/slack/chat-message";
 import BaseProvider from "../../../src/providers/BaseProvider";
 import { CommonTests, GenericTestConfig } from "../../common.tests";
-import { SchemaSocialChatMessage } from "../../../src/schemas";
+import { SlackHandlerConfig } from "../../../src/providers/slack/interfaces";
+import { SchemaSocialChatGroup, SchemaSocialChatMessage } from "../../../src/schemas";
 
 const providerId = "slack";
 let network: NetworkInstance;
 let connection: Connection;
 let provider: BaseProvider;
-let handlerName = "slack-messages";
+let handlerName = "chat-message";
 let testConfig: GenericTestConfig;
-let providerConfig: Omit = {};
+let providerConfig: Omit = {
+  maxSyncLoops: 1,
+  groupLimit: 2,
+  messageMaxAgeDays: 7,
+  messageBatchSize: 20,
+  messagesPerGroupLimit: 10,
+  maxGroupSize: 100,
+  useDbPos: false
+};
 
-describe(`${providerId} Slack Chat Message Handler Tests`, function () {
-  this.timeout(100000); // Increase timeout due to API rate limits and potential delays
+// Check if it sync channels and conversation
+describe(`${providerId} chat tests`, function () {
+  this.timeout(100000);
 
   this.beforeAll(async function () {
-    // Set up the Slack network, connection, and provider
     network = await CommonUtils.getNetwork();
     connection = await CommonUtils.getConnection(providerId);
     provider = Providers(providerId, network.context, connection);
 
-    // Define test configuration
     testConfig = {
       idPrefix: `${provider.getProviderId()}-${connection.profile.id}`,
-      batchSizeLimitAttribute: "messageBatchSize", // Adjust to match Slack-specific batch size config
+      timeOrderAttribute: "insertedAt",
+      batchSizeLimitAttribute: "batchSize",
     };
   });
 
   describe(`Fetch ${providerId} data`, () => {
-   
+
     it(`Can pass basic tests: ${handlerName}`, async () => {
-      // Run the generic common tests for Slack Chat Message Handler
-      await CommonTests.runGenericTests(
+      const { api, handler, provider } = await CommonTests.buildTestObjects(
         providerId,
         SlackChatMessageHandler,
-        testConfig,
         providerConfig,
         connection
       );
+
+      try {
+        const syncPosition: SyncHandlerPosition = {
+          _id: `${providerId}-${handlerName}`,
+          providerId,
+          handlerId: handler.getId(),
+          accountId: provider.getAccountId(),
+          status: SyncHandlerStatus.ENABLED,
+        };
+
+        // Batch 1
+        const response = await handler._sync(api, syncPosition);
+
+        // Make sure group and message limit were respected
+        let groupMessages: Record = {};
+        let groups: SchemaSocialChatGroup[] = [];
+        for (const result of (response.results)) {
+          if (result.groupId) {
+            if (!groupMessages[result.groupId]) {
+              groupMessages[result.groupId] = [];
+            }
+
+            groupMessages[result.groupId].push(result);
+          } else {
+            groups.push(result);
+          }
+        }
+
+        // Ensure results are returned before performing assertions
+        assert(response.results.length > 0, "Results are returned");
+
+      } catch (err) {
+        // ensure provider closes even if there's an error
+        await provider.close();
+
+        throw err;
+      }
     });
   });
 
   this.afterAll(async function () {
     const { context } = await CommonUtils.getNetwork();
-    await context.close(); // Clean up after tests
+    await context.close();
   });
 });

From e624956fdeb92f6a75fa6b9c9839e97dc2d021e7 Mon Sep 17 00:00:00 2001
From: chime3 
Date: Tue, 24 Sep 2024 23:30:32 -0700
Subject: [PATCH 119/182] fix: google unit tests with refactor

---
 .../providers/google/gdrive-document.tests.ts | 83 ++-----------------
 tests/providers/google/gmail.tests.ts         | 17 ++--
 .../google/youtube-favourite.tests.ts         | 14 ++--
 .../google/youtube-following.tests.ts         | 14 ++--
 tests/providers/google/youtube-post.tests.ts  | 14 ++--
 5 files changed, 36 insertions(+), 106 deletions(-)

diff --git a/tests/providers/google/gdrive-document.tests.ts b/tests/providers/google/gdrive-document.tests.ts
index f297c216..7f6c65ae 100644
--- a/tests/providers/google/gdrive-document.tests.ts
+++ b/tests/providers/google/gdrive-document.tests.ts
@@ -13,7 +13,7 @@ import BaseProvider from "../../../src/providers/BaseProvider";
 import { CommonTests, GenericTestConfig } from "../../common.tests";
 import { SchemaFollowing } from "../../../src/schemas";
 
-const providerName = "google";
+const providerId = "google";
 let network: NetworkInstance;
 let connection: Connection;
 let provider: BaseProvider;
@@ -22,101 +22,32 @@ let testConfig: GenericTestConfig;
 let providerConfig: Omit = {};
 
 
-describe(`${providerName} GDrive Document Tests`, function () {
+describe(`${providerId} GDrive Document Tests`, function () {
   this.timeout(400000);
 
   this.beforeAll(async function () {
     network = await CommonUtils.getNetwork();
-    connection = await CommonUtils.getConnection(providerName);
-    provider = Providers(providerName, network.context, connection);
+    connection = await CommonUtils.getConnection(providerId);
+    provider = Providers(providerId, network.context, connection);
   
     testConfig = {
-      idPrefix: `${provider.getProviderName()}-${connection.profile.id}`,
+      idPrefix: `${provider.getProviderId()}-${connection.profile.id}`,
       timeOrderAttribute: "modifiedAt",
       batchSizeLimitAttribute: "batchSize",
     };
   });
 
-  describe(`Fetch ${providerName} data`, () => {
+  describe(`Fetch ${providerId} data`, () => {
    
     it(`Can pass basic tests: ${handlerName}`, async () => {
       await CommonTests.runGenericTests(
-        providerName,
+        providerId,
         GoogleDriveDocument,
         testConfig,
         providerConfig,
         connection
       );
     });
-
-    it(`Can limit results by timestamp`, async () => {
-      const lastRecordHours = 2;
-      const lastRecordTimestamp = new Date(
-        Date.now() - lastRecordHours * 3600000
-      ).toISOString();
-
-      const syncPosition: Omit = {
-        providerName,
-        providerId: provider.getProviderId(),
-        handlerName,
-        status: SyncHandlerStatus.ENABLED,
-      };
-
-      providerConfig.batchSize = 5;
-      providerConfig.metadata = {
-        breakTimestamp: lastRecordTimestamp,
-      };
-
-      const syncResponse = await CommonTests.runSyncTest(
-        providerName,
-        GoogleDriveDocument,
-        connection,
-        testConfig,
-        syncPosition,
-        providerConfig
-      );
-      assert.ok(
-        syncResponse.results && syncResponse.results.length,
-        "Have results (You may not have edited any document in the testing timeframe)"
-      );
-
-      const results = syncResponse.results;
-      assert.ok(
-        results[results.length - 1].insertedAt > lastRecordTimestamp,
-        "Last result is within expected date/time range"
-      );
-      assert.ok(
-        results.length < providerConfig.batchSize,
-        `Results reached the expected timestamp within the current batch size (try increasing the test batch size or reducing the break timestamp)`
-      );
-    });
-
-    it(`Can handle empty results`, async () => {
-      const syncPosition: Omit = {
-        providerName,
-        providerId: provider.getProviderId(),
-        handlerName,
-        status: SyncHandlerStatus.ENABLED,
-      };
-
-      providerConfig.batchSize = 5;
-      providerConfig.metadata = {
-        breakTimestamp: new Date().toISOString(),
-      };
-
-      const syncResponse = await CommonTests.runSyncTest(
-        providerName,
-        GoogleDriveDocument,
-        connection,
-        testConfig,
-        syncPosition,
-        providerConfig
-      );
-      assert.ok(
-        syncResponse.results.length === 0,
-        "No results should be returned for the future timestamp"
-      );
-    })
   });
 
   this.afterAll(async function () {
diff --git a/tests/providers/google/gmail.tests.ts b/tests/providers/google/gmail.tests.ts
index 9b56b9d0..2326d50a 100644
--- a/tests/providers/google/gmail.tests.ts
+++ b/tests/providers/google/gmail.tests.ts
@@ -13,7 +13,7 @@ import BaseProvider from "../../../src/providers/BaseProvider";
 import { CommonTests, GenericTestConfig } from "../../common.tests";
 import { SchemaEmail } from "../../../src/schemas";
 
-const providerName = "google";
+const providerId = "google";
 let network: NetworkInstance;
 let connection: Connection;
 let provider: BaseProvider;
@@ -23,22 +23,22 @@ let providerConfig: Omit = {};
 
 console.log(`WARNING: Sometimes these tests fail because the Google API doesnt return inbox messages in the correct time order. This is a bug in the Google API and there's not much we can do about it.`)
 
-describe(`${providerName} Tests`, function () {
+describe(`${providerId} Tests`, function () {
   this.timeout(100000);
 
   this.beforeAll(async function () {
     network = await CommonUtils.getNetwork();
-    connection = await CommonUtils.getConnection(providerName);
-    provider = Providers(providerName, network.context, connection);
+    connection = await CommonUtils.getConnection(providerId);
+    provider = Providers(providerId, network.context, connection);
     
     testConfig = {
-      idPrefix: `${provider.getProviderName()}-${connection.profile.id}`,
+      idPrefix: `${provider.getProviderId()}-${connection.profile.id}`,
       timeOrderAttribute: "sentAt",
       batchSizeLimitAttribute: "batchSize",
     };
   });
 
-  describe(`Fetch ${providerName} data`, () => {
+  describe(`Fetch ${providerId} data`, () => {
 
     it(`Can pass basic tests: ${handlerName}`, async () => {
       /**
@@ -52,7 +52,7 @@ describe(`${providerName} Tests`, function () {
        */
 
       await CommonTests.runGenericTests(
-        providerName,
+        providerId,
         Gmail,
         testConfig,
         providerConfig,
@@ -67,7 +67,6 @@ describe(`${providerName} Tests`, function () {
       ).toISOString();
 
       const syncPosition: Omit = {
-        providerName,
         providerId: provider.getProviderId(),
         handlerName,
         status: SyncHandlerStatus.ENABLED,
@@ -79,7 +78,7 @@ describe(`${providerName} Tests`, function () {
       }
 
       const syncResponse = await CommonTests.runSyncTest(
-        providerName,
+        providerId,
         Gmail,
         connection,
         testConfig,
diff --git a/tests/providers/google/youtube-favourite.tests.ts b/tests/providers/google/youtube-favourite.tests.ts
index 66edd257..55a051c5 100644
--- a/tests/providers/google/youtube-favourite.tests.ts
+++ b/tests/providers/google/youtube-favourite.tests.ts
@@ -13,7 +13,7 @@ import BaseProvider from "../../../src/providers/BaseProvider";
 import { CommonTests, GenericTestConfig } from "../../common.tests";
 import { SchemaFavourite } from "../../../src/schemas";
 
-const providerName = "google";
+const providerId = "google";
 let network: NetworkInstance;
 let connection: Connection;
 let provider: BaseProvider;
@@ -21,25 +21,25 @@ let handlerName = "youtube-favourite";
 let testConfig: GenericTestConfig;
 let providerConfig: Omit = {};
 
-describe(`${providerName} Youtube Favourite Tests`, function () {
+describe(`${providerId} Youtube Favourite Tests`, function () {
   this.timeout(100000);
 
   this.beforeAll(async function () {
     network = await CommonUtils.getNetwork();
-    connection = await CommonUtils.getConnection(providerName);
-    provider = Providers(providerName, network.context, connection);
+    connection = await CommonUtils.getConnection(providerId);
+    provider = Providers(providerId, network.context, connection);
   
     testConfig = {
-      idPrefix: `${provider.getProviderName()}-${connection.profile.id}`,
+      idPrefix: `${provider.getProviderId()}-${connection.profile.id}`,
       batchSizeLimitAttribute: "batchSize",
     };
   });
 
-  describe(`Fetch ${providerName} data`, () => {
+  describe(`Fetch ${providerId} data`, () => {
     
     it(`Can pass basic tests: ${handlerName}`, async () => {
       await CommonTests.runGenericTests(
-        providerName,
+        providerId,
         YouTubeFavourite,
         testConfig,
         providerConfig,
diff --git a/tests/providers/google/youtube-following.tests.ts b/tests/providers/google/youtube-following.tests.ts
index 6998fdf7..3030d24f 100644
--- a/tests/providers/google/youtube-following.tests.ts
+++ b/tests/providers/google/youtube-following.tests.ts
@@ -13,7 +13,7 @@ import BaseProvider from "../../../src/providers/BaseProvider";
 import { CommonTests, GenericTestConfig } from "../../common.tests";
 import { SchemaFollowing } from "../../../src/schemas";
 
-const providerName = "google";
+const providerId = "google";
 let network: NetworkInstance;
 let connection: Connection;
 let provider: BaseProvider;
@@ -22,25 +22,25 @@ let testConfig: GenericTestConfig;
 let providerConfig: Omit = {};
 
 
-describe(`${providerName} Youtube Following Tests`, function () {
+describe(`${providerId} Youtube Following Tests`, function () {
   this.timeout(100000);
 
   this.beforeAll(async function () {
     network = await CommonUtils.getNetwork();
-    connection = await CommonUtils.getConnection(providerName);
-    provider = Providers(providerName, network.context, connection);
+    connection = await CommonUtils.getConnection(providerId);
+    provider = Providers(providerId, network.context, connection);
   
     testConfig = {
-      idPrefix: `${provider.getProviderName()}-${connection.profile.id}`,
+      idPrefix: `${provider.getProviderId()}-${connection.profile.id}`,
       batchSizeLimitAttribute: "batchSize",
     };
   });
 
-  describe(`Fetch ${providerName} data`, () => {
+  describe(`Fetch ${providerId} data`, () => {
    
     it(`Can pass basic tests: ${handlerName}`, async () => {
       await CommonTests.runGenericTests(
-        providerName,
+        providerId,
         YoutubeFollowing,
         testConfig,
         providerConfig,
diff --git a/tests/providers/google/youtube-post.tests.ts b/tests/providers/google/youtube-post.tests.ts
index aba0b04f..ae36bb56 100644
--- a/tests/providers/google/youtube-post.tests.ts
+++ b/tests/providers/google/youtube-post.tests.ts
@@ -13,7 +13,7 @@ import BaseProvider from "../../../src/providers/BaseProvider";
 import { CommonTests, GenericTestConfig } from "../../common.tests";
 import { SchemaPost } from "../../../src/schemas";
 
-const providerName = "google";
+const providerId = "google";
 let network: NetworkInstance;
 let connection: Connection;
 let provider: BaseProvider;
@@ -22,25 +22,25 @@ let testConfig: GenericTestConfig;
 let providerConfig: Omit = {};
 
 
-describe(`${providerName} Youtube Post Tests`, function () {
+describe(`${providerId} Youtube Post Tests`, function () {
   this.timeout(100000);
 
   this.beforeAll(async function () {
     network = await CommonUtils.getNetwork();
-    connection = await CommonUtils.getConnection(providerName);
-    provider = Providers(providerName, network.context, connection);
+    connection = await CommonUtils.getConnection(providerId);
+    provider = Providers(providerId, network.context, connection);
   
     testConfig = {
-      idPrefix: `${provider.getProviderName()}-${connection.profile.id}`,
+      idPrefix: `${provider.getProviderId()}-${connection.profile.id}`,
       batchSizeLimitAttribute: "batchSize",
     };
   });
 
-  describe(`Fetch ${providerName} data`, () => {
+  describe(`Fetch ${providerId} data`, () => {
    
     it(`Can pass basic tests: ${handlerName}`, async () => {
       await CommonTests.runGenericTests(
-        providerName,
+        providerId,
         YoutubePost,
         testConfig,
         providerConfig,

From 2be079b75a4939911eb2e3906d60d7f9e955cdbf Mon Sep 17 00:00:00 2001
From: chime3 
Date: Wed, 25 Sep 2024 22:41:28 -0700
Subject: [PATCH 120/182] fix: added readableId

---
 src/providers/slack/index.ts | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/providers/slack/index.ts b/src/providers/slack/index.ts
index c70e2cf6..6451359c 100644
--- a/src/providers/slack/index.ts
+++ b/src/providers/slack/index.ts
@@ -156,6 +156,7 @@ export default class SlackProvider extends Base {
                 connectionProfile: {
                     username: userInfo.profile.display_name,  // Display name as username
                     email: userInfo.profile.email,  // Email from profile
+                    readableId: userInfo.profile.display_name,
                     phone: userInfo.profile.phone,  
                     verified: userInfo.is_email_confirmed
                 }

From adcaee383057720d2c714924ca6c3bbb3f37806b Mon Sep 17 00:00:00 2001
From: Chris 
Date: Mon, 30 Sep 2024 18:36:47 +0930
Subject: [PATCH 121/182] Fix provider.getAccountId() after refactor

---
 src/providers/google/calendar-event.ts | 2 +-
 src/providers/google/calendar.ts       | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/providers/google/calendar-event.ts b/src/providers/google/calendar-event.ts
index 24d55b01..a1fa412f 100644
--- a/src/providers/google/calendar-event.ts
+++ b/src/providers/google/calendar-event.ts
@@ -253,7 +253,7 @@ export default class CalendarEvent extends GoogleHandler {
       results.push({
         _id: this.buildItemId(eventId),
         name: event.summary ?? 'No event title',
-        sourceAccountId: this.provider.getProviderId(),
+        sourceAccountId: this.provider.getAccountId(),
         sourceData: event,
         sourceApplication: this.getProviderApplicationUrl(),
         sourceId: eventId,
diff --git a/src/providers/google/calendar.ts b/src/providers/google/calendar.ts
index 99b1cbf0..6a763c15 100644
--- a/src/providers/google/calendar.ts
+++ b/src/providers/google/calendar.ts
@@ -217,7 +217,7 @@ export default class Calendar extends GoogleHandler {
       results.push({
         _id: this.buildItemId(calendarId),
         name: summary,
-        sourceAccountId: this.provider.getProviderId(),
+        sourceAccountId: this.provider.getAccountId(),
         sourceData: listItem,
         sourceApplication: this.getProviderApplicationUrl(),
         sourceId: calendarId,

From aa551b02e4853be652654bb0f40bcb91abe7be5b Mon Sep 17 00:00:00 2001
From: chime3 
Date: Mon, 30 Sep 2024 19:47:40 -0700
Subject: [PATCH 122/182] chore: removed unused var

---
 src/providers/google/calendar.ts | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/providers/google/calendar.ts b/src/providers/google/calendar.ts
index 6a763c15..0ec8ed54 100644
--- a/src/providers/google/calendar.ts
+++ b/src/providers/google/calendar.ts
@@ -5,8 +5,6 @@ import { google, calendar_v3 } from "googleapis";
 import { GaxiosResponse } from "gaxios";
 import { ItemsRangeTracker } from "../../helpers/itemsRangeTracker";
 
-import moment from "moment-timezone";
-
 import {
   SyncResponse,
   SyncHandlerStatus,

From 8d36327204133fa51f5a26b27d66716cdbf20625 Mon Sep 17 00:00:00 2001
From: chime3 
Date: Mon, 30 Sep 2024 20:52:55 -0700
Subject: [PATCH 123/182] feat: added calendar schema into web interface

---
 src/web/developer/data/data.js | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/web/developer/data/data.js b/src/web/developer/data/data.js
index bf97f043..0f5e8cb0 100644
--- a/src/web/developer/data/data.js
+++ b/src/web/developer/data/data.js
@@ -14,7 +14,9 @@ $(document).ready(function() {
         "Email": "https://common.schemas.verida.io/social/email/v0.1.0/schema.json",
         "Chat Group": "https://common.schemas.verida.io/social/chat/group/v0.1.0/schema.json",
         "Chat Message": "https://common.schemas.verida.io/social/chat/message/v0.1.0/schema.json",
-        "Files": "https://common.schemas.verida.io/file/v0.1.0/schema.json"
+        "Files": "https://common.schemas.verida.io/file/v0.1.0/schema.json",
+        "Calendar": "https://common.schemas.verida.io/social/calendar/v0.1.0/schema.json",
+        "Event": "https://common.schemas.verida.io/social/event/v0.1.0/schema.json"
     };
 
     // Load Verida Key and Schema from local storage

From 3c9cfd7cad3ecf4cd81abc8bef4e2faf4fe98e4b Mon Sep 17 00:00:00 2001
From: chime3 
Date: Wed, 2 Oct 2024 15:00:41 -0700
Subject: [PATCH 124/182] fix: added synData field

---
 src/schemas.ts | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/schemas.ts b/src/schemas.ts
index f63ac200..0dc763f9 100644
--- a/src/schemas.ts
+++ b/src/schemas.ts
@@ -138,6 +138,7 @@ export interface SchemaCalendar extends SchemaRecord {
     description?: string
     timezone: string
     location?: string
+    syncData?: string
 }
 
 export interface SchemaEvent extends SchemaRecord {

From c2dd3166d88bea4bdfd357f0b508a100938cfce9 Mon Sep 17 00:00:00 2001
From: chime3 
Date: Wed, 2 Oct 2024 15:03:11 -0700
Subject: [PATCH 125/182] feat: added custom config for calendar event

---
 src/serverconfig.example.json | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/src/serverconfig.example.json b/src/serverconfig.example.json
index 1ba0be8d..e81d9604 100644
--- a/src/serverconfig.example.json
+++ b/src/serverconfig.example.json
@@ -2,7 +2,7 @@
     "serverUrl": "https://127.0.0.1:5021",
     "apiVersion": "v1",
     "assetsUrl": "https://127.0.0.1:5021/assets",
-    "logLevel": "debug", 
+    "logLevel": "debug",
     "verida": {
         "devMode": true,
         "environment": "banksia",
@@ -12,8 +12,7 @@
         "didClientConfig": {
             "callType": "web3",
             "network": "banksia",
-            "web3Config": {   
-            }
+            "web3Config": {}
         },
         "schemas": {
             "DATA_CONNECTIONS": "https://vault.schemas.verida.io/data-connections/connection/v0.3.0/schema.json",
@@ -85,13 +84,17 @@
             "batchSize": 50,
             "sizeLimit": 10,
             "maxSyncLoops": 1,
-            "breakTimestamp": "2000-07-21T12:07:11.000Z",
             "handlers": {
                 "gmail": {
                     "batchSize": 2
                 },
                 "google-drive-document": {
                     "batchSize": 2
+                },
+                "calendar-event": {
+                    "backdate": "12-months",
+                    "calendarLimit": 20,
+                    "eventsPerCalendarLimit": 50
                 }
             }
         },
@@ -111,7 +114,4 @@
     "providerDefaults": {
         "limitResults": false
     }
-}
-
-
-
+}
\ No newline at end of file

From 3986588d59c2d3e2efe9d7c0c86d51b72988d64f Mon Sep 17 00:00:00 2001
From: chime3 
Date: Wed, 2 Oct 2024 15:04:47 -0700
Subject: [PATCH 126/182] refactor: updated calendar event handler

---
 src/providers/google/calendar-event.ts | 401 +++++++++++++++----------
 src/providers/google/calendar.ts       | 234 ---------------
 2 files changed, 246 insertions(+), 389 deletions(-)
 delete mode 100644 src/providers/google/calendar.ts

diff --git a/src/providers/google/calendar-event.ts b/src/providers/google/calendar-event.ts
index a1fa412f..950a117f 100644
--- a/src/providers/google/calendar-event.ts
+++ b/src/providers/google/calendar-event.ts
@@ -1,47 +1,49 @@
-import GoogleHandler from "./GoogleHandler";
-import CONFIG from "../../config";
-import { SyncHandlerPosition, SyncItemsBreak, SyncItemsResult, SyncProviderLogEvent, SyncProviderLogLevel } from '../../interfaces';
 import { google, calendar_v3 } from "googleapis";
 import { GaxiosResponse } from "gaxios";
-import { ItemsRangeTracker } from "../../helpers/itemsRangeTracker";
-
+import GoogleHandler from "./GoogleHandler";
+import CONFIG from "../../config";
 import {
+  SyncItemsResult,
   SyncResponse,
   SyncHandlerStatus,
   ProviderHandlerOption,
   ConnectionOptionType,
+  SyncHandlerPosition,
+  SyncProviderLogEvent,
+  SyncProviderLogLevel
 } from "../../interfaces";
-import { SchemaEvent } from "../../schemas";
-import { CalendarAttachment, DateTimeInfo, Person } from "./interfaces";
+import {
+  SchemaCalendar,
+  SchemaEvent,
+} from "../../schemas";
+import { ItemsRangeTracker } from "../../helpers/itemsRangeTracker";
+import { ItemsRange } from "../../helpers/interfaces";
 import { CalendarHelpers } from "./helpers";
+import { CalendarAttachment, DateTimeInfo, GoogleCalendarHandlerConfig, Person } from "./interfaces";
 
 const _ = require("lodash");
 
-// Set MAX_BATCH_SIZE to 2500 because the Google Calendar API v3 'maxResults' parameter is capped at 2500.
-// For more details, see: https://developers.google.com/calendar/api/v3/reference/events/list 
-const MAX_BATCH_SIZE = 2500;
+// Set MAX_BATCH_SIZE to 250 because the Google Calendar API v3 'maxResults' parameter is capped at 250.
+// For more details, see: https://developers.google.com/calendar/api/v3/reference/calendarList/list 
+const MAX_BATCH_SIZE = 250;
+export default class CalendarEventHandler extends GoogleHandler {
 
-export interface SyncCalendarItemsResult extends SyncItemsResult {
-  items: SchemaEvent[];
-}
-
-export default class CalendarEvent extends GoogleHandler {
+  protected config: GoogleCalendarHandlerConfig;
 
   public getName(): string {
-    return 'calendar-event';
+    return "calendar-event";
   }
 
-  public getSchemaUri(): string {
-    return CONFIG.verida.schemas.EVENT;
+  public getLabel(): string {
+    return "Calendar Event";
   }
 
-  public getProviderApplicationUrl() {
-    return 'https://calendar.google.com/';
+  public getSchemaUri(): string {
+    return CONFIG.verida.schemas.CALENDAR_EVENT;
   }
 
-  public getCalendar(): calendar_v3.Calendar {
-    const oAuth2Client = this.getGoogleAuth();
-    return google.calendar({ version: "v3", auth: oAuth2Client });
+  public getProviderApplicationUrl(): string {
+    return "https://calendar.google.com/"; // Change URL depending on provider (Google, Microsoft)
   }
 
   public getOptions(): ProviderHandlerOption[] {
@@ -66,138 +68,102 @@ export default class CalendarEvent extends GoogleHandler {
     }];
   }
 
-  public async _sync(
-    api: any,
-    syncPosition: SyncHandlerPosition
-  ): Promise {
-    if (this.config.batchSize > MAX_BATCH_SIZE) {
-      throw new Error(`Batch size (${this.config.batchSize}) is larger than permitted (${MAX_BATCH_SIZE})`);
-    }
-
-    const calendar = this.getCalendar();
-    const rangeTracker = new ItemsRangeTracker(syncPosition.thisRef);
+  public getCalendarClient(): calendar_v3.Calendar {
+    const oAuth2Client = this.getGoogleAuth();
+    return google.calendar({ version: "v3", auth: oAuth2Client });
+  }
 
-    let items: SchemaEvent[] = [];
+  protected async buildCalendarList(): Promise {
+    const calendarClient = this.getCalendarClient();
 
-    // Fetch any new items
-    let currentRange = rangeTracker.nextRange();
+    let calendarList: SchemaCalendar[] = [];
 
-    let query: calendar_v3.Params$Resource$Events$List = {
-      calendarId: 'primary',
-      maxResults: this.config.batchSize, // default = 250, max = 2500
-      singleEvents: true,
-      orderBy: "startTime",
+    let nextPageToken: string | undefined;
+    let query: calendar_v3.Params$Resource$Calendarlist$List = {
+      maxResults: MAX_BATCH_SIZE,  // Fetch in batches up to the max limit
+      pageToken: nextPageToken,
     };
 
-    if (currentRange.startId) {
-      query.pageToken = currentRange.startId;
-    }
-
-    const latestResponse = await calendar.events.list(query);
-    const latestResult = await this.buildResults(
-      calendar,
-      latestResponse,
-      currentRange.endId,
-      this.config.breakTimestamp ?? undefined
-    );
+    // Loop through paginated results
+    do {
+      const response = await calendarClient.calendarList.list(query);
 
-    items = latestResult.items;
+      for (const calendar of response.data.items || []) {
+        // Extract essential details for the calendar entry
+        const calendarId = calendar.id;
 
-    let nextPageToken = latestResponse.data.nextPageToken ?? undefined;
-
-    if (items.length) {
-      rangeTracker.completedRange({
-        startId: items[0].sourceId,
-        endId: nextPageToken
-      }, latestResult.breakHit == SyncItemsBreak.ID);
-    } else {
-      rangeTracker.completedRange({
-        startId: undefined,
-        endId: undefined
-      }, false);
-    }
+        if (!calendarId) {
+          this.emit("log", {
+            level: SyncProviderLogLevel.DEBUG,
+            message: `Invalid calendar ID. Ignoring this calendar.`,
+          });
+          continue;
+        }
 
-    if (items.length != this.config.batchSize) {
-      currentRange = rangeTracker.nextRange();
+        const summary = calendar.summary ?? "No calendar title";
+        let timeZone = calendar.timeZone;
 
-      query = {
-        calendarId: 'primary',
-        maxResults: this.config.batchSize - items.length,
-        singleEvents: true,
-        orderBy: "startTime",
-      };
+        if (!timeZone) {
+          this.emit("log", {
+            level: SyncProviderLogLevel.DEBUG,
+            message: `Invalid timezone for calendar ${calendarId}. Ignoring this calendar.`,
+          });
+          continue;
+        }
+
+        timeZone = CalendarHelpers.getUTCOffsetTimezone(timeZone);
+        const description = calendar.description ?? "No description";
+        const location = calendar.location ?? "No location";
+        const insertedAt = new Date().toISOString();
+
+        const group: SchemaCalendar = {
+          _id: this.buildItemId(calendarId),
+          name: summary,
+          sourceAccountId: this.provider.getAccountId(),
+          sourceApplication: this.getProviderApplicationUrl(),
+          sourceId: calendarId,
+          timezone: timeZone,
+          description,
+          location,
+          insertedAt,
+          sourceData: calendar,
+          schema: CONFIG.verida.schemas.CALENDAR,
+        };
 
-      if (currentRange.startId) {
-        query.pageToken = currentRange.startId;
+        calendarList.push(group);
       }
 
-      const backfillResponse = await calendar.events.list(query);
-      const backfillResult = await this.buildResults(
-        calendar,
-        backfillResponse,
-        currentRange.endId,
-        this.config.breakTimestamp ?? undefined
-      );
-
-      items = items.concat(backfillResult.items);
+      nextPageToken = response.data.nextPageToken; // Update the pageToken to fetch the next batch
+      query.pageToken = nextPageToken;
+    } while (nextPageToken);
 
-      nextPageToken = backfillResponse.data.nextPageToken ?? undefined;
-
-      if (backfillResult.items.length) {
-        rangeTracker.completedRange({
-          startId: backfillResult.items[0].sourceId,
-          endId: nextPageToken
-        }, backfillResult.breakHit == SyncItemsBreak.ID);
-      } else {
-        rangeTracker.completedRange({
-          startId: undefined,
-          endId: undefined
-        }, backfillResult.breakHit == SyncItemsBreak.ID);
-      }
-    }
 
-    if (!items.length) {
-      syncPosition.syncMessage = `Stopping. No results found.`;
-      syncPosition.status = SyncHandlerStatus.ENABLED;
-    } else {
-      if (items.length != this.config.batchSize && !nextPageToken) {
-        syncPosition.syncMessage = `Processed ${items.length} items. Stopping. No more results.`;
-        syncPosition.status = SyncHandlerStatus.ENABLED;
-      } else {
-        syncPosition.syncMessage = `Batch complete (${this.config.batchSize}). More results pending.`;
-      }
-    }
+    return calendarList;
+  }
 
-    syncPosition.thisRef = rangeTracker.export();
+  protected async fetchEventRange(
+    calendar: SchemaCalendar,
+    range: ItemsRange,
+    apiClient: calendar_v3.Calendar
+  ): Promise {
+    const events: SchemaEvent[] = [];
 
-    return {
-      results: items,
-      position: syncPosition,
+    // Define API request parameters
+    let query: calendar_v3.Params$Resource$Events$List = {
+      calendarId: calendar.sourceId!,
+      timeMin: new Date(range.startId).toISOString(),
+      timeMax: new Date(range.endId).toISOString(),
+      maxResults: this.config.eventsPerCalendarLimit,
+      orderBy: "startTime"
     };
-  }
 
-  protected async buildResults(
-    calendar: calendar_v3.Calendar,
-    serverResponse: GaxiosResponse,
-    breakId: string,
-    breakTimestamp?: string
-  ): Promise {
-    const results: SchemaEvent[] = [];
-    let breakHit: SyncItemsBreak;
+    // Fetch events from Google Calendar API
+    const response = await apiClient.events.list(query);
+    const items = response.data.items || [];
 
-    for (const event of serverResponse.data.items) {
+    for (const event of items) {
       const eventId = event.id ?? '';
 
-      if (eventId == breakId) {
-        const logEvent: SyncProviderLogEvent = {
-          level: SyncProviderLogLevel.DEBUG,
-          message: `Break ID hit (${breakId})`
-        };
-        this.emit('log', logEvent);
-        breakHit = SyncItemsBreak.ID;
-        break;
-      }
-
       let start: DateTimeInfo = {
         dateTime: event.start?.dateTime
       };
@@ -221,25 +187,15 @@ export default class CalendarEvent extends GoogleHandler {
       start.timeZone = CalendarHelpers.getUTCOffsetTimezone(event.start?.timeZone)
       end.timeZone = CalendarHelpers.getUTCOffsetTimezone(event.end?.timeZone)
 
-      if (breakTimestamp && start.dateTime < breakTimestamp) {
-        const logEvent: SyncProviderLogEvent = {
-          level: SyncProviderLogLevel.DEBUG,
-          message: `Break timestamp hit (${breakTimestamp})`
-        };
-        this.emit('log', logEvent);
-        breakHit = SyncItemsBreak.TIMESTAMP;
-        break;
-      }
-
       const insertedAt = new Date().toISOString();
 
       const creator: Person = {
-        email: event.creator.email ?? 'info@example.com',
+        email: event.creator.email,
         displayName: event.creator.displayName
       }
 
       const organizer: Person = {
-        email: event.organizer.email ?? 'info@example.com',
+        email: event.organizer.email,
         displayName: event.organizer.displayName
       }
 
@@ -250,31 +206,166 @@ export default class CalendarEvent extends GoogleHandler {
 
       const attachments: CalendarAttachment[] = event.attachments as CalendarAttachment[];
 
-      results.push({
+      events.push({
         _id: this.buildItemId(eventId),
-        name: event.summary ?? 'No event title',
+        name: event.summary,
         sourceAccountId: this.provider.getAccountId(),
         sourceData: event,
         sourceApplication: this.getProviderApplicationUrl(),
         sourceId: eventId,
-        calendarId: "primary",
+        calendarId: calendar._id,
         start,
         end,
         creator,
         organizer,
-        location: event.location ?? 'No location',
-        description: event.description ?? 'No description',
+        location: event.location,
+        description: event.description,
         status: event.status ?? 'Unkown',
         conferenceData: event.conferenceData,
         attendees,
         attachments,
         insertedAt
       });
+    
     }
+    return events;
+  }
 
-    return {
-      items: results,
-      breakHit
-    };
+  public async _sync(
+    api: any,
+    syncPosition: SyncHandlerPosition
+  ): Promise {
+    try {
+      const apiClient = this.getCalendarClient();
+      const calendarList = await this.buildCalendarList(); // Fetch all personal, work, and shared calendars
+
+      let totalEvents = 0;
+      let eventHistory: SchemaEvent[] = [];
+
+      // Determine the current calendar position
+      let calendarPosition = this.getCalendarPositionIndex(calendarList, syncPosition);
+
+      const calendarCount = calendarList.length;
+
+      // Iterate over each calendar
+      for (let i = 0; i < Math.min(calendarCount, this.config.calendarLimit); i++) {
+        const calendarIndex = (calendarPosition + i) % calendarCount; // Rotate through calendars
+        const calendar = calendarList[calendarIndex];
+
+        // Use a separate ItemsRangeTracker for each calendar
+        let rangeTracker = new ItemsRangeTracker(calendar.syncData);
+
+        const fetchedEvents = await this.fetchAndTrackEvents(
+          calendar,
+          rangeTracker,
+          apiClient
+        );
+
+        // Concatenate the fetched events to the total event history
+        eventHistory = eventHistory.concat(fetchedEvents);
+        totalEvents += fetchedEvents.length;
+
+        // Update the calendar's sync data with the latest rangeTracker state
+        calendar.syncData = rangeTracker.export();
+
+        // Stop if the total events fetched reach the batch size
+        if (totalEvents >= this.config.batchSize) {
+          syncPosition.thisRef = calendarList[(calendarIndex + 1) % calendarCount].sourceId; // Continue from the next calendar in the next sync
+          break;
+        }
+      }
+
+      // Finalize sync position and status based on event count
+      this.updateSyncPosition(
+        syncPosition,
+        totalEvents,
+        calendarCount
+      );
+
+      // Concatenate only items after syncPosition.thisRef
+      const remainingCalendars = calendarList.slice(calendarPosition + 1);
+
+      return {
+        results: remainingCalendars.concat(eventHistory),
+        position: syncPosition,
+      };
+    } catch (err: any) {
+      console.error(err);
+      throw err;
+    }
+  }
+
+  private getCalendarPositionIndex(
+    calendarList: SchemaCalendar[],
+    syncPosition: SyncHandlerPosition
+  ): number {
+    const calendarPosition = calendarList.findIndex(
+      (calendar) => calendar.sourceId === syncPosition.thisRef
+    );
+
+    // If not found, return 0 to start from the beginning
+    return calendarPosition === -1 ? 0 : calendarPosition;
+  }
+
+  private async fetchAndTrackEvents(
+    calendar: SchemaCalendar,
+    rangeTracker: ItemsRangeTracker,
+    apiClient: calendar_v3.Calendar
+  ): Promise {
+    // Validate calendar and calendar.id
+    if (!calendar || !calendar.sourceId) {
+      throw new Error('Invalid calendar or missing calendar sourceId');
+    }
+
+    // Initialize range from tracker
+    let currentRange = rangeTracker.nextRange();
+    let items: SchemaEvent[] = [];
+
+    while (true) {
+      // Fetch events for the current range using fetchEventRange
+      const events = await this.fetchEventRange(calendar, currentRange, apiClient);
+
+      if (!events.length) break;
+
+      // Add fetched events to the main list
+      items = items.concat(events);
+
+      // Break loop if events reached calendar limit
+      if (items.length > this.config.eventsPerCalendarLimit) {
+        // Mark the current range as complete and stop
+        rangeTracker.completedRange({
+          startId: events[0].start.dateTime,
+          endId: events[events.length - 1].end.dateTime
+        }, false);
+        break;
+      } else {
+        // Update rangeTracker and continue fetching
+        rangeTracker.completedRange({
+          startId: events[0].start.dateTime,
+          endId: events[events.length - 1].end.dateTime
+        }, false);
+
+        // Move to the next range
+        currentRange = rangeTracker.nextRange();
+      }
+    }
+
+    return items;
+  }
+
+  private updateSyncPosition(
+    syncPosition: SyncHandlerPosition,
+    totalEvents: number,
+    calendarCount: number,
+  ) {
+    if (totalEvents === 0) {
+      syncPosition.status = SyncHandlerStatus.ERROR;
+      syncPosition.syncMessage = "No new events found.";
+    } else if (totalEvents < this.config.batchSize) {
+      syncPosition.syncMessage = `Processed ${totalEvents} events across ${calendarCount} calendars. Sync complete.`;
+      syncPosition.status = SyncHandlerStatus.ENABLED;
+    } else {
+      syncPosition.syncMessage = `Batch complete (${this.config.eventBatchSize}). More results pending.`;
+    }
   }
 }
diff --git a/src/providers/google/calendar.ts b/src/providers/google/calendar.ts
deleted file mode 100644
index 0ec8ed54..00000000
--- a/src/providers/google/calendar.ts
+++ /dev/null
@@ -1,234 +0,0 @@
-import GoogleHandler from "./GoogleHandler";
-import CONFIG from "../../config";
-import { SyncHandlerPosition, SyncItemsBreak, SyncItemsResult, SyncProviderLogEvent, SyncProviderLogLevel } from '../../interfaces';
-import { google, calendar_v3 } from "googleapis";
-import { GaxiosResponse } from "gaxios";
-import { ItemsRangeTracker } from "../../helpers/itemsRangeTracker";
-
-import {
-  SyncResponse,
-  SyncHandlerStatus,
-  ProviderHandlerOption,
-  ConnectionOptionType,
-} from "../../interfaces";
-import { SchemaCalendar } from "../../schemas";
-import { CalendarHelpers } from "./helpers";
-
-const _ = require("lodash");
-
-// Set MAX_BATCH_SIZE to 250 because the Google Calendar API v3 'maxResults' parameter is capped at 250.
-// For more details, see: https://developers.google.com/calendar/api/v3/reference/calendarList/list 
-const MAX_BATCH_SIZE = 250;
-
-export interface SyncCalendarItemsResult extends SyncItemsResult {
-  items: SchemaCalendar[];
-}
-
-export default class Calendar extends GoogleHandler {
-
-  public getName(): string {
-    return 'calendar';
-  }
-
-  public getSchemaUri(): string {
-    return CONFIG.verida.schemas.CALENDAR;
-  }
-
-  public getProviderApplicationUrl() {
-    return 'https://calendar.google.com/';
-  }
-
-  public getCalendar(): calendar_v3.Calendar {
-    const oAuth2Client = this.getGoogleAuth();
-    return google.calendar({ version: "v3", auth: oAuth2Client });
-  }
-
-  public getOptions(): ProviderHandlerOption[] {
-    return [{
-      id: 'backdate',
-      label: 'Backdate history',
-      type: ConnectionOptionType.ENUM,
-      enumOptions: [{
-        value: '1-month',
-        label: '1 month'
-      }, {
-        value: '3-months',
-        label: '3 months'
-      }, {
-        value: '6-months',
-        label: '6 months'
-      }, {
-        value: '12-months',
-        label: '12 months'
-      }],
-      defaultValue: '3-months'
-    }];
-  }
-
-  public async _sync(
-    api: any,
-    syncPosition: SyncHandlerPosition
-  ): Promise {
-    if (this.config.batchSize > MAX_BATCH_SIZE) {
-      throw new Error(`Batch size (${this.config.batchSize}) is larger than permitted (${MAX_BATCH_SIZE})`);
-    }
-
-    const calendar = this.getCalendar();
-    const rangeTracker = new ItemsRangeTracker(syncPosition.thisRef);
-
-    let items: SchemaCalendar[] = [];
-
-    // Fetch any new items
-    let currentRange = rangeTracker.nextRange();
-    let query: calendar_v3.Params$Resource$Calendarlist$List = {
-      maxResults: this.config.batchSize,
-    };
-
-    if (currentRange.startId) {
-      query.pageToken = currentRange.startId;
-    }
-
-    const latestResponse = await calendar.calendarList.list(query);
-    const latestResult = await this.buildResults(
-      calendar,
-      latestResponse,
-      currentRange.endId
-    );
-
-    items = latestResult.items;
-
-    let nextPageToken = _.has(latestResponse, "data.nextPageToken") ? latestResponse.data.nextPageToken : undefined;
-
-    if (items.length) {
-      rangeTracker.completedRange({
-        startId: items[0].sourceId,
-        endId: nextPageToken
-      }, latestResult.breakHit == SyncItemsBreak.ID);
-    } else {
-      rangeTracker.completedRange({
-        startId: undefined,
-        endId: undefined
-      }, false);
-    }
-
-    if (items.length != this.config.batchSize) {
-      currentRange = rangeTracker.nextRange();
-      query = {
-        maxResults: this.config.batchSize - items.length,
-      };
-
-      if (currentRange.startId) {
-        query.pageToken = currentRange.startId;
-      }
-
-      const backfillResponse = await calendar.calendarList.list(query);
-      const backfillResult = await this.buildResults(
-        calendar,
-        backfillResponse,
-        currentRange.endId
-      );
-
-      items = items.concat(backfillResult.items);
-      nextPageToken = _.has(backfillResponse, "data.nextPageToken") ? backfillResponse.data.nextPageToken : undefined;
-
-      if (backfillResult.items.length) {
-        rangeTracker.completedRange({
-          startId: backfillResult.items[0].sourceId,
-          endId: nextPageToken
-        }, backfillResult.breakHit == SyncItemsBreak.ID);
-      } else {
-        rangeTracker.completedRange({
-          startId: undefined,
-          endId: undefined
-        }, backfillResult.breakHit == SyncItemsBreak.ID);
-      }
-    }
-
-    if (!items.length) {
-      syncPosition.syncMessage = `Stopping. No results found.`;
-      syncPosition.status = SyncHandlerStatus.ENABLED;
-    } else {
-      if (items.length != this.config.batchSize && !nextPageToken) {
-        syncPosition.syncMessage = `Processed ${items.length} items. Stopping. No more results.`;
-        syncPosition.status = SyncHandlerStatus.ENABLED;
-      } else {
-        syncPosition.syncMessage = `Batch complete (${this.config.batchSize}). More results pending.`;
-      }
-    }
-
-    syncPosition.thisRef = rangeTracker.export();
-
-    return {
-      results: items,
-      position: syncPosition,
-    };
-  }
-
-  protected async buildResults(
-    calendar: calendar_v3.Calendar,
-    serverResponse: GaxiosResponse,
-    breakId: string
-  ): Promise {
-    const results: SchemaCalendar[] = [];
-    let breakHit: SyncItemsBreak;
-
-    for (const listItem of serverResponse.data.items) {
-      const calendarId = listItem.id;
-      
-      if (!calendarId) {
-        const logEvent: SyncProviderLogEvent = {
-            level: SyncProviderLogLevel.DEBUG,
-            message: `Invalid calendar ID. Ignoring this calendar.`,
-        };
-        this.emit('log', logEvent);
-        continue;
-      }
-      
-      if (calendarId == breakId) {
-        const logEvent: SyncProviderLogEvent = {
-          level: SyncProviderLogLevel.DEBUG,
-          message: `Break ID hit (${breakId})`
-        };
-        this.emit('log', logEvent);
-        breakHit = SyncItemsBreak.ID;
-        break;
-      }
-
-      const summary = listItem.summary ?? 'No calendar title';
-      let timeZone = listItem.timeZone;
-
-      if (!timeZone) {
-        const logEvent: SyncProviderLogEvent = {
-            level: SyncProviderLogLevel.DEBUG,
-            message: `Invalid timezone for calendar ${calendarId}. Ignoring this calendar.`,
-        };
-        this.emit('log', logEvent);
-        continue;
-      }
-      
-      timeZone = CalendarHelpers.getUTCOffsetTimezone(timeZone);
-
-      const description = listItem.description ?? 'No description';
-      const location = listItem.location ?? 'No location';
-      const insertedAt = new Date().toISOString(); // Adding insertedAt field
-
-      results.push({
-        _id: this.buildItemId(calendarId),
-        name: summary,
-        sourceAccountId: this.provider.getAccountId(),
-        sourceData: listItem,
-        sourceApplication: this.getProviderApplicationUrl(),
-        sourceId: calendarId,
-        timezone: timeZone,
-        description,
-        location,
-        insertedAt,  // insertedAt field
-      });
-    }
-
-    return {
-      items: results,
-      breakHit
-    };
-  }
-}

From d4017bc31dc7dca0aa0afdac3c57469912456127 Mon Sep 17 00:00:00 2001
From: chime3 
Date: Wed, 2 Oct 2024 15:05:34 -0700
Subject: [PATCH 127/182] feat: added custom config interface

---
 src/providers/google/interfaces.ts | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/providers/google/interfaces.ts b/src/providers/google/interfaces.ts
index 9e626d16..e1f77191 100644
--- a/src/providers/google/interfaces.ts
+++ b/src/providers/google/interfaces.ts
@@ -58,4 +58,9 @@ export interface CalendarAttachment {
   mimeType?: string;  // MIME type of the file
   iconLink?: string;  // URL of the icon representing the file
   fileId?: string;    // Unique identifier for the file
+}
+
+export interface GoogleCalendarHandlerConfig extends BaseHandlerConfig {
+  calendarLimit?: number;  // Max number of calendar per sync
+  eventsPerCalendarLimit?: number; // Max number of event to process in a calendar
 }
\ No newline at end of file

From c6e5401c3b92b1c3e188bb753cee6fc0a730370b Mon Sep 17 00:00:00 2001
From: Chris 
Date: Thu, 3 Oct 2024 14:57:25 +0930
Subject: [PATCH 128/182] Some code cleanup and fixing sync handler status

---
 src/providers/BaseSyncHandler.ts    |  2 +-
 src/providers/slack/chat-message.ts | 11 ++++-------
 yarn.lock                           |  4 ++--
 3 files changed, 7 insertions(+), 10 deletions(-)

diff --git a/src/providers/BaseSyncHandler.ts b/src/providers/BaseSyncHandler.ts
index ec9f95d5..a45420d3 100644
--- a/src/providers/BaseSyncHandler.ts
+++ b/src/providers/BaseSyncHandler.ts
@@ -383,6 +383,6 @@ export default class BaseSyncHandler extends EventEmitter {
     }
 
     protected buildItemId(itemId: string) {
-        return `${this.provider.getProviderName()}-${this.connection.profile.id}-${itemId}`
+        return `${this.provider.getProviderId()}-${this.connection.profile.id}-${itemId}`
     }
 }
\ No newline at end of file
diff --git a/src/providers/slack/chat-message.ts b/src/providers/slack/chat-message.ts
index 5d718ac4..130d1ed0 100644
--- a/src/providers/slack/chat-message.ts
+++ b/src/providers/slack/chat-message.ts
@@ -1,7 +1,6 @@
 import { WebClient } from "@slack/web-api";
 import CONFIG from "../../config";
 import {
-    SyncItemsResult,
     SyncResponse,
     SyncHandlerStatus,
     ProviderHandlerOption,
@@ -185,8 +184,7 @@ export default class SlackChatMessageHandler extends BaseSyncHandler {
             this.updateSyncPosition(
                 syncPosition,
                 totalMessages,
-                groupCount,
-                chatHistory
+                groupCount
             );
 
             // Concatenate only items after syncPosition.thisRef and chatHistory
@@ -263,17 +261,16 @@ export default class SlackChatMessageHandler extends BaseSyncHandler {
     private updateSyncPosition(
         syncPosition: SyncHandlerPosition,
         totalMessages: number,
-        groupCount: number,
-        chatHistory: SchemaSocialChatMessage[]
+        groupCount: number
     ) {
         if (totalMessages === 0) {
-            syncPosition.status = SyncHandlerStatus.COMPLETED;
+            syncPosition.status = SyncHandlerStatus.ENABLED;
             syncPosition.syncMessage = "No new messages found.";
         } else if (totalMessages < this.config.messageBatchSize) {
             syncPosition.syncMessage = `Processed ${totalMessages} messages across ${groupCount} groups. Sync complete.`;
             syncPosition.status = SyncHandlerStatus.ENABLED;
         } else {
-            //syncPosition.status = SyncHandlerStatus.ENABLED;
+            syncPosition.status = SyncHandlerStatus.SYNCING;
             syncPosition.syncMessage = `Batch complete (${this.config.messageBatchSize}). More results pending.`;
         }
     }
diff --git a/yarn.lock b/yarn.lock
index ce717de7..558db967 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -1777,7 +1777,7 @@ axios@^0.27.2:
     follow-redirects "^1.14.9"
     form-data "^4.0.0"
 
-axios@^1.2.3, axios@^1.6.2, axios@^1.7.2:
+axios@^1.2.3, axios@^1.6.2:
   version "1.7.2"
   resolved "https://registry.yarnpkg.com/axios/-/axios-1.7.2.tgz#b625db8a7051fbea61c35a3cbb3a1daa7b9c7621"
   integrity sha512-2A8QhOMrbomlDuiLeK9XibIBzuHeRcqqNOHp0Cyp5EoJ1IFDh+XZH3A6BkXtv0K4gFGCI0Y4BM7B1wOEi0Rmgw==
@@ -1786,7 +1786,7 @@ axios@^1.2.3, axios@^1.6.2, axios@^1.7.2:
     form-data "^4.0.0"
     proxy-from-env "^1.1.0"
 
-axios@^1.3.3:
+axios@^1.3.3, axios@^1.7.4, axios@^1.7.7:
   version "1.7.7"
   resolved "https://registry.yarnpkg.com/axios/-/axios-1.7.7.tgz#2f554296f9892a72ac8d8e4c5b79c14a91d0a47f"
   integrity sha512-S4kL7XrjgBmvdGut0sN3yJxqYzrDOnivkBiN0OFs6hLiUam3UPvswUo0kqGyhqUZGEOytHyumEdXsAkgCOUf3Q==

From fdc80702d07226fcc0bcbfda5bde393daaf3d388 Mon Sep 17 00:00:00 2001
From: chime3 
Date: Wed, 2 Oct 2024 23:50:56 -0700
Subject: [PATCH 129/182] fix: refactoring and unit tests

---
 src/providers/google/calendar-event.ts        |  62 ++++--
 src/providers/google/index.ts                 |   6 +-
 src/schemas.ts                                |   2 +-
 src/serverconfig.example.json                 |   6 +-
 tests/common.tests.ts                         | 184 ++----------------
 .../providers/google/calendar-event.tests.ts  |   3 +-
 tests/providers/google/calendar.tests.ts      |  54 -----
 7 files changed, 62 insertions(+), 255 deletions(-)
 delete mode 100644 tests/providers/google/calendar.tests.ts

diff --git a/src/providers/google/calendar-event.ts b/src/providers/google/calendar-event.ts
index 950a117f..aee11715 100644
--- a/src/providers/google/calendar-event.ts
+++ b/src/providers/google/calendar-event.ts
@@ -112,8 +112,8 @@ export default class CalendarEventHandler extends GoogleHandler {
         }
 
         timeZone = CalendarHelpers.getUTCOffsetTimezone(timeZone);
-        const description = calendar.description ?? "No description";
-        const location = calendar.location ?? "No location";
+        const description = calendar.description;
+        const location = calendar.location;
         const insertedAt = new Date().toISOString();
 
         const group: SchemaCalendar = {
@@ -122,7 +122,7 @@ export default class CalendarEventHandler extends GoogleHandler {
           sourceAccountId: this.provider.getAccountId(),
           sourceApplication: this.getProviderApplicationUrl(),
           sourceId: calendarId,
-          timezone: timeZone,
+          timezone: timeZone ?? "Unkown",
           description,
           location,
           insertedAt,
@@ -148,15 +148,21 @@ export default class CalendarEventHandler extends GoogleHandler {
   ): Promise {
     const events: SchemaEvent[] = [];
 
-    // Define API request parameters
     let query: calendar_v3.Params$Resource$Events$List = {
-      calendarId: calendar.sourceId!,
-      timeMin: new Date(range.startId).toISOString(),
-      timeMax: new Date(range.endId).toISOString(),
+      calendarId: calendar.sourceId,
       maxResults: this.config.eventsPerCalendarLimit,
+      singleEvents: true,
       orderBy: "startTime"
     };
 
+    if (range.startId) {
+      query.timeMin = new Date(range.startId).toISOString();
+    }
+
+    if (range.endId) {
+      query.timeMax = new Date(range.endId).toISOString();
+    }
+
     // Fetch events from Google Calendar API
     const response = await apiClient.events.list(query);
     const items = response.data.items || [];
@@ -176,8 +182,8 @@ export default class CalendarEventHandler extends GoogleHandler {
 
       if (!start.dateTime) {
         const logEvent: SyncProviderLogEvent = {
-            level: SyncProviderLogLevel.DEBUG,
-            message: `Invalid date for the event ${eventId}. Ignoring this event.`,
+          level: SyncProviderLogLevel.DEBUG,
+          message: `Invalid date for the event ${eventId}. Ignoring this event.`,
         };
         this.emit('log', logEvent);
         continue;
@@ -190,12 +196,12 @@ export default class CalendarEventHandler extends GoogleHandler {
       const insertedAt = new Date().toISOString();
 
       const creator: Person = {
-        email: event.creator.email,
+        email: event.creator.email ?? "info@example.com",
         displayName: event.creator.displayName
       }
 
       const organizer: Person = {
-        email: event.organizer.email,
+        email: event.organizer.email ?? "info@example.com",
         displayName: event.organizer.displayName
       }
 
@@ -208,12 +214,13 @@ export default class CalendarEventHandler extends GoogleHandler {
 
       events.push({
         _id: this.buildItemId(eventId),
-        name: event.summary,
+        name: event.summary ?? "Unknown",
         sourceAccountId: this.provider.getAccountId(),
         sourceData: event,
         sourceApplication: this.getProviderApplicationUrl(),
         sourceId: eventId,
-        calendarId: calendar._id,
+        schema: CONFIG.verida.schemas.EVENT,
+        calendarId: calendar.sourceId ?? "primary",
         start,
         end,
         creator,
@@ -226,7 +233,7 @@ export default class CalendarEventHandler extends GoogleHandler {
         attachments,
         insertedAt
       });
-    
+
     }
     return events;
   }
@@ -237,18 +244,37 @@ export default class CalendarEventHandler extends GoogleHandler {
   ): Promise {
     try {
       const apiClient = this.getCalendarClient();
-      const calendarList = await this.buildCalendarList(); // Fetch all personal, work, and shared calendars
+      let calendarList = await this.buildCalendarList(); // Fetch all personal, work, and shared calendars
+      const calendarDs = await this.provider.getDatastore(CONFIG.verida.schemas.CALENDAR)
+      const calendarDbItems =  await calendarDs.getMany({
+        "sourceAccountId": this.provider.getAccountId()
+      });
+
+      calendarList = calendarList.map((calendarItem) => {
+        // Find the corresponding item in calendarDbItems by 'sourceId'
+        const matchingDbItem = calendarDbItems.find(
+          (dbItem) => dbItem.sourceId === calendarItem.sourceId
+        );
+    
+        // If a matching item is found in calendarDbItems, merge them, retaining `syncData` field
+        if (matchingDbItem) {
+          return _.merge({}, matchingDbItem, calendarItem);
+        }
+    
+        // If no matching item, return the calendarItem as is
+        return calendarItem;
+      });
 
       let totalEvents = 0;
       let eventHistory: SchemaEvent[] = [];
 
       // Determine the current calendar position
-      let calendarPosition = this.getCalendarPositionIndex(calendarList, syncPosition);
+      const calendarPosition = this.getCalendarPositionIndex(calendarList, syncPosition);
 
       const calendarCount = calendarList.length;
 
       // Iterate over each calendar
-      for (let i = 0; i < Math.min(calendarCount, this.config.calendarLimit); i++) {
+      for (let i = 1; i <= Math.min(calendarCount, this.config.calendarLimit); i++) {
         const calendarIndex = (calendarPosition + i) % calendarCount; // Rotate through calendars
         const calendar = calendarList[calendarIndex];
 
@@ -359,7 +385,7 @@ export default class CalendarEventHandler extends GoogleHandler {
     calendarCount: number,
   ) {
     if (totalEvents === 0) {
-      syncPosition.status = SyncHandlerStatus.ERROR;
+      syncPosition.status = SyncHandlerStatus.SYNCING;
       syncPosition.syncMessage = "No new events found.";
     } else if (totalEvents < this.config.batchSize) {
       syncPosition.syncMessage = `Processed ${totalEvents} events across ${calendarCount} calendars. Sync complete.`;
diff --git a/src/providers/google/index.ts b/src/providers/google/index.ts
index 2a09c31b..eb824387 100644
--- a/src/providers/google/index.ts
+++ b/src/providers/google/index.ts
@@ -7,7 +7,6 @@ import { GoogleProviderConfig, GoogleProviderConnection } from "./interfaces";
 import YouTubeFavourite from "./youtube-favourite";
 import GoogleDriveDocument from "./gdrive-document";
 import { ConnectionCallbackResponse, PassportProfile } from "../../interfaces";
-import Calendar from "./calendar";
 import CalendarEvent from "./calendar-event";
 
 const passport = require("passport");
@@ -31,12 +30,11 @@ export default class GoogleProvider extends Base {
 
   public syncHandlers(): any[] {
     return [
-      Gmail,
+      /*Gmail,
       YouTubeFollowing,
       YouTubePost,
       YouTubeFavourite,
-      GoogleDriveDocument,
-      Calendar,
+      GoogleDriveDocument,*/
       CalendarEvent
     ];
   }
diff --git a/src/schemas.ts b/src/schemas.ts
index 0dc763f9..39b827eb 100644
--- a/src/schemas.ts
+++ b/src/schemas.ts
@@ -136,7 +136,7 @@ export interface SchemaFile extends SchemaRecord {
 
 export interface SchemaCalendar extends SchemaRecord {
     description?: string
-    timezone: string
+    timezone?: string
     location?: string
     syncData?: string
 }
diff --git a/src/serverconfig.example.json b/src/serverconfig.example.json
index e81d9604..bd3332b6 100644
--- a/src/serverconfig.example.json
+++ b/src/serverconfig.example.json
@@ -81,7 +81,7 @@
         "google": {
             "clientId": "",
             "clientSecret": "",
-            "batchSize": 50,
+            "batchSize": 30,
             "sizeLimit": 10,
             "maxSyncLoops": 1,
             "handlers": {
@@ -93,8 +93,8 @@
                 },
                 "calendar-event": {
                     "backdate": "12-months",
-                    "calendarLimit": 20,
-                    "eventsPerCalendarLimit": 50
+                    "calendarLimit": 10,
+                    "eventsPerCalendarLimit": 20
                 }
             }
         },
diff --git a/tests/common.tests.ts b/tests/common.tests.ts
index ecf7c0a3..7b63792d 100644
--- a/tests/common.tests.ts
+++ b/tests/common.tests.ts
@@ -18,7 +18,7 @@ export interface GenericTestConfig {
   // Attribute in the results that is used for time ordering (ie: insertedAt)
   timeOrderAttribute?: string; // Made optional
   // Attribute used to limit the batch size (ie: batchLimit)
-  batchSizeLimitAttribute: string;
+  batchSizeLimitAttribute?: string;
   // Prefix used for record ID's (override default which is providerName)
   idPrefix?: string;
 }
@@ -114,15 +114,7 @@ export class CommonTests {
     handler: BaseSyncHandler;
     provider: BaseProvider;
   }> {
-    // * - New items are processed
-    //    * - Backfill items are processed
-    //    * - Not enough new items? Process backfill
-    //    * - Backfill twice doesn't process the same items
-    //    * - No more backfill produces empty rangeTracker
-
-    // Set result limit to 3 results so page tests can work correctly
-    providerConfig[testConfig.batchSizeLimitAttribute] = 3;
-
+    
     const { api, handler, schemaUri, provider } = await this.buildTestObjects(
       providerId,
       handlerType,
@@ -130,10 +122,6 @@ export class CommonTests {
       connection
     );
 
-    const idPrefix = testConfig.idPrefix
-      ? testConfig.idPrefix
-      : `${provider.getProviderName()}-${connection!.profile.id}`;
-
     try {
       const syncPosition: SyncHandlerPosition = {
         _id: `${providerId}-${schemaUri}`,
@@ -142,25 +130,23 @@ export class CommonTests {
         accountId: provider.getAccountId(),
         status: SyncHandlerStatus.SYNCING,
       };
-      
-      // 1. Test new items are processed
+
       const response = await handler._sync(api, syncPosition);
       const results = response.results;
 
-      // console.log(response.position)
-      // console.log(CommonTests.outputItems(results, testConfig.timeOrderAttribute))
-
       assert.ok(results && results.length, "Have results returned");
-      assert.equal(
-        providerConfig[testConfig.batchSizeLimitAttribute],
-        results.length,
-        "Have correct number of results returned on page 1"
-      );
+      if (testConfig.batchSizeLimitAttribute) {
+        assert.equal(
+          providerConfig[testConfig.batchSizeLimitAttribute],
+          results.length,
+          "Have correct number of results returned on page 1"
+        );
+      }
 
       if (testConfig.timeOrderAttribute) {
         assert.ok(
           results[0][testConfig.timeOrderAttribute] >
-            results[1][testConfig.timeOrderAttribute],
+          results[1][testConfig.timeOrderAttribute],
           "Results are most recent first"
         );
       }
@@ -172,155 +158,7 @@ export class CommonTests {
         response.position.status,
         "Sync is active"
       );
-      assert.ok(response.position.thisRef, "Have a defined processing range");
-
-      const currentRangeParts = response.position.thisRef!.split(':')
-      assert.ok(currentRangeParts.length == 2, "Have correct number of parts for the processing range");
-      assert.ok(currentRangeParts[0] == results[0]._id.replace(`${idPrefix}-`, ''), "Have correct break ID");
-      assert.ok(currentRangeParts[1].length, "Have an end range");
-
-      // 2. Backfill items are processed
-      const syncPosition2 = response.position
-      const response2 = await handler._sync(api, syncPosition2);
-      const results2 = response2.results;
       
-      // console.log(response2.position)
-      // console.log(CommonTests.outputItems(results2, testConfig.timeOrderAttribute))
-
-      assert.ok(
-        results2 && results2.length,
-        "Have backfill results returned"
-      );
-      assert.ok(
-        results2 &&
-          results2.length == providerConfig[testConfig.batchSizeLimitAttribute],
-        "Have correct number of results returned in second page"
-      );
-
-      if (testConfig.timeOrderAttribute) {
-        assert.ok(
-          results2[0][testConfig.timeOrderAttribute] >
-            results2[1][testConfig.timeOrderAttribute],
-          "Results are most recent first"
-        );
-        assert.ok(
-          results2[0][testConfig.timeOrderAttribute] <
-            results[2][testConfig.timeOrderAttribute],
-          "First item on second page of results have earlier timestamp than last item on first page"
-        );
-      }
-
-      assert.equal(
-        response2.position.status,
-        SyncHandlerStatus.SYNCING,
-        "Sync is active"
-      );
-
-      assert.ok(response2.position.thisRef, "Have a defined processing range");
-
-      const currentRangeParts2 = response2.position.thisRef!.split(':')
-      assert.ok(currentRangeParts2.length == 2, "Have correct number of parts for the processing range");
-      assert.ok(currentRangeParts2[0] == results[0]._id.replace(`${idPrefix}-`, ''), "Have correct break ID matching the very first result");
-      assert.ok(currentRangeParts2[1].length, "Have an end range");
-      assert.ok(results[0]._id != results2[0]._id, "Have different result IDs")
-
-      // 3. Not enough new items? Process backfill
-      const syncPosition3 = response2.position
-      syncPosition3.thisRef = `${results[1].sourceId}:${currentRangeParts2[1]}` // Ensure the first item (only) is fetched
-      const response3 = await handler._sync(api, syncPosition3);
-      const results3 = response3.results;
-      
-      // console.log(response3.position)
-      // console.log(CommonTests.outputItems(results3, testConfig.timeOrderAttribute))
-
-      assert.ok(
-        results3 && results3.length,
-        "Have results returned"
-      );
-      assert.ok(
-        results3 &&
-        results3.length == providerConfig[testConfig.batchSizeLimitAttribute],
-        "Have correct number of results returned"
-      );
-      assert.equal(results3[0]._id, results[0]._id, 'First result item matches the very first item')
-      assert.ok(results3[1]._id != results[1]._id, 'Second result item does not match the very first batch second item')
-
-      if (testConfig.timeOrderAttribute) {
-        assert.ok(
-          results3[0][testConfig.timeOrderAttribute] >
-            results3[1][testConfig.timeOrderAttribute],
-          "Results are most recent first"
-        );
-        // this will break?
-        assert.ok(
-          results3[2][testConfig.timeOrderAttribute] <
-            results[2][testConfig.timeOrderAttribute],
-          "Last item on return results have earlier timestamp than last item on first page"
-        );
-      }
-
-      assert.equal(
-        response3.position.status,
-        SyncHandlerStatus.SYNCING,
-        "Sync is active"
-      );
-
-      assert.ok(response3.position.thisRef, "Have a defined processing range");
-
-      const currentRangeParts3 = response3.position.thisRef!.split(':')
-      assert.ok(currentRangeParts3.length == 2, "Have correct number of parts for the processing range");
-      assert.ok(currentRangeParts3[0] == results3[0]._id.replace(`${idPrefix}-`, ''), "Have correct break ID matching the very first result");
-      assert.ok(currentRangeParts3[1].length, "Have an end range");
-      assert.ok(currentRangeParts3[1] != currentRangeParts2[1], "End range has changed between batches");
-
-      // - Backfill twice doesn't process the same items
-      const syncPosition4 = response3.position
-      const response4 = await handler._sync(api, syncPosition4);
-      const results4 = response4.results;
-      
-      // console.log(response4.position)
-      // console.log(CommonTests.outputItems(results4, testConfig.timeOrderAttribute))
-
-      assert.ok(
-        results4 && results4.length,
-        "Have results returned"
-      );
-      assert.ok(
-        results4 &&
-        results4.length == providerConfig[testConfig.batchSizeLimitAttribute],
-        "Have correct number of results returned"
-      );
-
-      if (testConfig.timeOrderAttribute) {
-        assert.ok(
-          results4[0][testConfig.timeOrderAttribute] >
-          results4[1][testConfig.timeOrderAttribute],
-          "Results are most recent first"
-        );
-        // this will break?
-        assert.ok(
-          results4[0][testConfig.timeOrderAttribute] <
-            results[2][testConfig.timeOrderAttribute],
-          "First item on return results have earlier timestamp than last item on first page"
-        );
-      }
-
-      assert.ok(results4[0]._id != results3[0]._id, "First items dont match between batches")
-
-      assert.equal(
-        response4.position.status,
-        SyncHandlerStatus.SYNCING,
-        "Sync is active"
-      );
-
-      assert.ok(response4.position.thisRef, "Have a defined processing range");
-      const currentRangeParts4 = response4.position.thisRef!.split(':')
-      assert.ok(currentRangeParts4.length == 2, "Have correct number of parts for the processing range");
-      assert.ok(currentRangeParts4[1].length, "Have an end range");
-
-      // @todo: No more backfill produces empty rangeTracker and SyncHandlerStatus.CONNECTED
-
-
       // Close the provider connection
       await provider.close();
 
diff --git a/tests/providers/google/calendar-event.tests.ts b/tests/providers/google/calendar-event.tests.ts
index 17e26ab0..f56d52d7 100644
--- a/tests/providers/google/calendar-event.tests.ts
+++ b/tests/providers/google/calendar-event.tests.ts
@@ -29,8 +29,7 @@ describe(`${providerName} Google Calendar Event Tests`, function () {
     provider = Providers(providerName, network.context, connection);
 
     testConfig = {
-      idPrefix: `${provider.getProviderName()}-${connection.profile.id}`,
-      batchSizeLimitAttribute: "batchSize",
+      idPrefix: `${provider.getProviderId()}-${connection.profile.id}`,
     };
   });
 
diff --git a/tests/providers/google/calendar.tests.ts b/tests/providers/google/calendar.tests.ts
deleted file mode 100644
index 55113155..00000000
--- a/tests/providers/google/calendar.tests.ts
+++ /dev/null
@@ -1,54 +0,0 @@
-const assert = require("assert");
-import {
-  BaseProviderConfig,
-  Connection,
-  SyncHandlerStatus,
-  SyncHandlerPosition,
-} from "../../../src/interfaces";
-import Providers from "../../../src/providers";
-import CommonUtils, { NetworkInstance } from "../../common.utils";
-
-import GoogleCalendar from "../../../src/providers/google/calendar";
-import BaseProvider from "../../../src/providers/BaseProvider";
-import { CommonTests, GenericTestConfig } from "../../common.tests";
-
-const providerName = "google";
-let network: NetworkInstance;
-let connection: Connection;
-let provider: BaseProvider;
-let handlerName = "calendar";
-let testConfig: GenericTestConfig;
-let providerConfig: Omit = {};
-
-describe(`${providerName} Google Calendar Tests`, function () {
-  this.timeout(100000);
-
-  this.beforeAll(async function () {
-    network = await CommonUtils.getNetwork();
-    connection = await CommonUtils.getConnection(providerName);
-    provider = Providers(providerName, network.context, connection);
-
-    testConfig = {
-      idPrefix: `${provider.getProviderName()}-${connection.profile.id}`,
-      batchSizeLimitAttribute: "batchSize",
-    };
-  });
-
-  describe(`Fetch ${providerName} data`, () => {
-   
-    it(`Can pass basic tests: ${handlerName}`, async () => {
-      await CommonTests.runGenericTests(
-        providerName,
-        GoogleCalendar,
-        testConfig,
-        providerConfig,
-        connection
-      );
-    });
-  });
-
-  this.afterAll(async function () {
-    const { context } = await CommonUtils.getNetwork();
-    await context.close();
-  });
-});

From 9fdfbc89bdfe58a2dcd066846d8cdb090f775827 Mon Sep 17 00:00:00 2001
From: chime3 
Date: Thu, 3 Oct 2024 00:13:07 -0700
Subject: [PATCH 130/182] fix: updated syn handler status bug

---
 src/providers/google/calendar-event.ts | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/providers/google/calendar-event.ts b/src/providers/google/calendar-event.ts
index aee11715..4c8b5207 100644
--- a/src/providers/google/calendar-event.ts
+++ b/src/providers/google/calendar-event.ts
@@ -385,12 +385,13 @@ export default class CalendarEventHandler extends GoogleHandler {
     calendarCount: number,
   ) {
     if (totalEvents === 0) {
-      syncPosition.status = SyncHandlerStatus.SYNCING;
+      syncPosition.status = SyncHandlerStatus.ENABLED;
       syncPosition.syncMessage = "No new events found.";
     } else if (totalEvents < this.config.batchSize) {
       syncPosition.syncMessage = `Processed ${totalEvents} events across ${calendarCount} calendars. Sync complete.`;
       syncPosition.status = SyncHandlerStatus.ENABLED;
     } else {
+      syncPosition.status = SyncHandlerStatus.SYNCING;
       syncPosition.syncMessage = `Batch complete (${this.config.eventBatchSize}). More results pending.`;
     }
   }

From e621cab57a3e3e957d71aab661b4d8645cfe336f Mon Sep 17 00:00:00 2001
From: chime3 
Date: Thu, 3 Oct 2024 21:51:54 -0700
Subject: [PATCH 131/182] fix: batch sync item count

---
 src/providers/google/calendar-event.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/providers/google/calendar-event.ts b/src/providers/google/calendar-event.ts
index 4c8b5207..48b72680 100644
--- a/src/providers/google/calendar-event.ts
+++ b/src/providers/google/calendar-event.ts
@@ -392,7 +392,7 @@ export default class CalendarEventHandler extends GoogleHandler {
       syncPosition.status = SyncHandlerStatus.ENABLED;
     } else {
       syncPosition.status = SyncHandlerStatus.SYNCING;
-      syncPosition.syncMessage = `Batch complete (${this.config.eventBatchSize}). More results pending.`;
+      syncPosition.syncMessage = `Batch complete (${totalEvents}). More results pending.`;
     }
   }
 }

From 4d575412201b532a3b428d9e0ee24734685e9e41 Mon Sep 17 00:00:00 2001
From: chime3 
Date: Thu, 3 Oct 2024 23:11:50 -0700
Subject: [PATCH 132/182] fix: invalid datetime value

---
 src/providers/google/calendar-event.ts | 18 +++++++++++++-----
 src/providers/google/index.ts          |  4 ++--
 src/serverconfig.example.json          |  4 ++--
 3 files changed, 17 insertions(+), 9 deletions(-)

diff --git a/src/providers/google/calendar-event.ts b/src/providers/google/calendar-event.ts
index 48b72680..8482ae59 100644
--- a/src/providers/google/calendar-event.ts
+++ b/src/providers/google/calendar-event.ts
@@ -155,11 +155,11 @@ export default class CalendarEventHandler extends GoogleHandler {
       orderBy: "startTime"
     };
 
-    if (range.startId) {
+    if (range.startId && !isNaN(Date.parse(range.startId))) {
       query.timeMin = new Date(range.startId).toISOString();
     }
 
-    if (range.endId) {
+    if (range.endId && !isNaN(Date.parse(range.endId))) {
       query.timeMax = new Date(range.endId).toISOString();
     }
 
@@ -245,26 +245,34 @@ export default class CalendarEventHandler extends GoogleHandler {
     try {
       const apiClient = this.getCalendarClient();
       let calendarList = await this.buildCalendarList(); // Fetch all personal, work, and shared calendars
+
+      console.log('==========Calendars from API========')
+      console.log(calendarList)
       const calendarDs = await this.provider.getDatastore(CONFIG.verida.schemas.CALENDAR)
-      const calendarDbItems =  await calendarDs.getMany({
+      const calendarDbItems = await calendarDs.getMany({
         "sourceAccountId": this.provider.getAccountId()
       });
 
+      console.log('======Calendars from DB=====')
+      console.log(calendarDbItems)
       calendarList = calendarList.map((calendarItem) => {
         // Find the corresponding item in calendarDbItems by 'sourceId'
         const matchingDbItem = calendarDbItems.find(
           (dbItem) => dbItem.sourceId === calendarItem.sourceId
         );
-    
+
         // If a matching item is found in calendarDbItems, merge them, retaining `syncData` field
         if (matchingDbItem) {
           return _.merge({}, matchingDbItem, calendarItem);
         }
-    
+
         // If no matching item, return the calendarItem as is
         return calendarItem;
       });
 
+      console.log("========Calendars after Merge========")
+      console.log(calendarList)
+
       let totalEvents = 0;
       let eventHistory: SchemaEvent[] = [];
 
diff --git a/src/providers/google/index.ts b/src/providers/google/index.ts
index eb824387..4c068d80 100644
--- a/src/providers/google/index.ts
+++ b/src/providers/google/index.ts
@@ -30,11 +30,11 @@ export default class GoogleProvider extends Base {
 
   public syncHandlers(): any[] {
     return [
-      /*Gmail,
+      Gmail,
       YouTubeFollowing,
       YouTubePost,
       YouTubeFavourite,
-      GoogleDriveDocument,*/
+      GoogleDriveDocument,
       CalendarEvent
     ];
   }
diff --git a/src/serverconfig.example.json b/src/serverconfig.example.json
index ca7c2a9d..256906c1 100644
--- a/src/serverconfig.example.json
+++ b/src/serverconfig.example.json
@@ -99,8 +99,8 @@
                 },
                 "calendar-event": {
                     "backdate": "12-months",
-                    "calendarLimit": 10,
-                    "eventsPerCalendarLimit": 20
+                    "calendarLimit": 20,
+                    "eventsPerCalendarLimit": 30
                 }
             }
         },

From 5917b8c4199ec41a58482cadb93629bc30deecd8 Mon Sep 17 00:00:00 2001
From: Chris 
Date: Mon, 7 Oct 2024 13:04:18 +1030
Subject: [PATCH 133/182] Improve range tracker import / export to be more
 robust

---
 src/helpers/itemsRangeTracker.ts | 42 +++++++++++++++++---------------
 tests/helpers/range-tracker.ts   | 36 +++++++++++++--------------
 2 files changed, 40 insertions(+), 38 deletions(-)

diff --git a/src/helpers/itemsRangeTracker.ts b/src/helpers/itemsRangeTracker.ts
index dcdc2a6f..5083626a 100644
--- a/src/helpers/itemsRangeTracker.ts
+++ b/src/helpers/itemsRangeTracker.ts
@@ -17,17 +17,33 @@ export class ItemsRangeTracker {
 
     constructor(completedRangesString?: string) {
         if (completedRangesString) {
-          const ranges = completedRangesString.split(',')
-          for (const range of ranges) {
-            const [ startId, endId ] = range.split(':')
+            this.import(completedRangesString)
+        }
+    }
+
+    public import(rangeString: string) {
+        for (const entry of JSON.parse(rangeString)) {
             this.completedRanges.push({
-              startId,
-              endId
+                startId: entry[0] ? entry[0] : undefined,
+                endId: entry[1] ? entry[1] : undefined,
             })
-          }
         }
     }
 
+    /**
+     * Convert the completed ranges array into a string for saving into the database
+     * 
+     * @returns 
+     */
+    public export(): string {
+        const entries = []
+        for (const range of this.completedRanges) {
+            entries.push([range.startId, range.endId])
+        }
+
+        return JSON.stringify(entries)
+    }
+
     /**
      * 
      * @param item 
@@ -130,18 +146,4 @@ export class ItemsRangeTracker {
         return {}
     }
 
-    /**
-     * Convert the completed ranges array into a string for saving into the database
-     * 
-     * @returns 
-     */
-    public export(): string {
-        const groups: string[] = []
-        for (const item of this.completedRanges) {
-            groups.push([item.startId,item.endId].join(':'))
-        }
-
-        return groups.join(',')
-    }
-
 }
\ No newline at end of file
diff --git a/tests/helpers/range-tracker.ts b/tests/helpers/range-tracker.ts
index 260d03bc..6067877a 100644
--- a/tests/helpers/range-tracker.ts
+++ b/tests/helpers/range-tracker.ts
@@ -1,5 +1,5 @@
 const assert = require("assert");
-import { CompletedRangeTracker } from "../../src/helpers/completedRangeTracker";
+import { ItemsRangeTracker } from "../../src/helpers/itemsRangeTracker"
 
 const batchLimit = 6
 const batchSize = 10
@@ -41,7 +41,7 @@ function processItems(items: number[], limit: number, breakId?: string, startId?
 describe(`Range tracker tests`, function () {
 
     it(`Can handle starting empty`, () => {
-        const tracker = new CompletedRangeTracker()
+        const tracker = new ItemsRangeTracker()
         const newItems = tracker.nextRange()
         assert.deepEqual(newItems, {}, "New items range is empty")
 
@@ -50,7 +50,7 @@ describe(`Range tracker tests`, function () {
     })
 
     it(`Can handle completing all`, () => {
-        const tracker = new CompletedRangeTracker()
+        const tracker = new ItemsRangeTracker()
         const batch1 = buildBatch(40)
         let messages: number[] = []
 
@@ -70,13 +70,13 @@ describe(`Range tracker tests`, function () {
         }, batch1Break)
 
         const trackerExport = tracker.export()
-        assert.equal(trackerExport, "40:49", "Tracker has correct exported value after processing all")
+        assert.equal(trackerExport, JSON.stringify([["40", "49"]]), "Tracker has correct exported value after processing all")
     })
 
     it(`Can handle an unchanged list, in multiple batches`, () => {
         const batch1 = buildBatch(40)
 
-        const tracker = new CompletedRangeTracker()
+        const tracker = new ItemsRangeTracker()
         let messages: number[] = []
 
         // message list is 40-49
@@ -109,14 +109,14 @@ describe(`Range tracker tests`, function () {
             endId: processedBatch2[processedBatch2.length-1].toString(),
         }, batch2Break)
         const trackerExport = tracker.export()
-        assert.equal(trackerExport, "40:49", "Tracker has correct exported value after processing two batches")
+        assert.equal(trackerExport, JSON.stringify([["40", "49"]]), "Tracker has correct exported value after processing two batches")
     })
 
-    it.only(`Can handle a changing list, in multiple batches`, () => {
+    it(`Can handle a changing list, in multiple batches`, () => {
         const batch1 = buildBatch(40)
         const batch2 = buildBatch(30)
 
-        let tracker = new CompletedRangeTracker()
+        let tracker = new ItemsRangeTracker()
         let messages: number[] = []
         let trackerExport
 
@@ -136,14 +136,14 @@ describe(`Range tracker tests`, function () {
         }, batch1Break)
 
         trackerExport = tracker.export()
-        assert.equal(trackerExport, "40:45", "Tracker has correct exported value after processing one batch")
+        assert.equal(trackerExport, JSON.stringify([["40","45"]]), "Tracker has correct exported value after processing one batch")
 
         // Reset messages, add more
         messages = []
         messages = messages.concat(batch2).concat(batch1)
         
         // Reset tracker to start processing new items
-        tracker = new CompletedRangeTracker(tracker.export())
+        tracker = new ItemsRangeTracker(tracker.export())
         
         // Process new items (includes new items; 30-39)
         const range2 = tracker.nextRange()
@@ -157,14 +157,14 @@ describe(`Range tracker tests`, function () {
         }, batch2Break)
 
         trackerExport = tracker.export()
-        assert.equal(trackerExport, "30:35,40:45", "Tracker has correct exported value after processing two batches")
+        assert.equal(trackerExport, JSON.stringify([["30","35"],["40","45"]]), "Tracker has correct exported value after processing two batches")
 
         // Reset messages
         messages = []
         messages = messages.concat(batch2).concat(batch1)
 
         // Reset tracker to start processing new items
-        tracker = new CompletedRangeTracker(tracker.export())
+        tracker = new ItemsRangeTracker(tracker.export())
 
         // Process next batch, no new items so should be empty
         const range3 = tracker.nextRange()
@@ -189,7 +189,7 @@ describe(`Range tracker tests`, function () {
         }, batch4Break)
 
         trackerExport = tracker.export()
-        assert.equal(trackerExport, "30:45", "Tracker has correct exported value after processing four batches")
+        assert.equal(trackerExport, JSON.stringify([["30", "45"]]), "Tracker has correct exported value after processing four batches")
 
         // range 5 was deleted
 
@@ -204,14 +204,14 @@ describe(`Range tracker tests`, function () {
         }, batch6Break)
 
         trackerExport = tracker.export()
-        assert.equal(trackerExport, "30:47", "Tracker has correct exported value after processing six batches")
+        assert.equal(trackerExport, JSON.stringify([["30","47"]]), "Tracker has correct exported value after processing six batches")
 
         // Reset messages
         messages = []
         messages = messages.concat(batch2).concat(batch1)
 
         // Reset tracker to start processing new items
-        tracker = new CompletedRangeTracker(tracker.export())
+        tracker = new ItemsRangeTracker(tracker.export())
 
         // Process next batch, no new items so should be empty
         const range7 = tracker.nextRange()
@@ -237,7 +237,7 @@ describe(`Range tracker tests`, function () {
         }, batch8Break)
 
         trackerExport = tracker.export()
-        assert.equal(trackerExport, "30:49", "Tracker has correct exported value after processing eight batches")
+        assert.equal(trackerExport, JSON.stringify([["30","49"]]), "Tracker has correct exported value after processing eight batches")
 
         // Insert a single item at the start and several at the end, they are processed correctly
 
@@ -246,7 +246,7 @@ describe(`Range tracker tests`, function () {
         messages = messages.concat([29]).concat(batch2).concat(batch1).concat([50,51,52,53,54,55,56,57,58,59])
         
         // Reset tracker to start processing new items
-        tracker = new CompletedRangeTracker(tracker.export())
+        tracker = new ItemsRangeTracker(tracker.export())
 
         // Process next batch of new items (1)
         const range9 = tracker.nextRange()
@@ -272,7 +272,7 @@ describe(`Range tracker tests`, function () {
         }, batch10Break)
 
         trackerExport = tracker.export()
-        assert.equal(trackerExport, "29:55", "Tracker has correct exported value after processing ten batches")
+        assert.equal(trackerExport, JSON.stringify([["29","55"]]), "Tracker has correct exported value after processing ten batches")
 
     })
 

From 83a938000f5db4cf37435560f47f26ad5119126f Mon Sep 17 00:00:00 2001
From: chime3 
Date: Sun, 6 Oct 2024 22:38:42 -0700
Subject: [PATCH 134/182] fix: removed total batchsize comparison

---
 src/providers/google/calendar-event.ts | 21 ++++-----------------
 src/serverconfig.example.json          |  4 ++--
 2 files changed, 6 insertions(+), 19 deletions(-)

diff --git a/src/providers/google/calendar-event.ts b/src/providers/google/calendar-event.ts
index 8482ae59..b8c38c2f 100644
--- a/src/providers/google/calendar-event.ts
+++ b/src/providers/google/calendar-event.ts
@@ -246,15 +246,11 @@ export default class CalendarEventHandler extends GoogleHandler {
       const apiClient = this.getCalendarClient();
       let calendarList = await this.buildCalendarList(); // Fetch all personal, work, and shared calendars
 
-      console.log('==========Calendars from API========')
-      console.log(calendarList)
       const calendarDs = await this.provider.getDatastore(CONFIG.verida.schemas.CALENDAR)
       const calendarDbItems = await calendarDs.getMany({
         "sourceAccountId": this.provider.getAccountId()
       });
-
-      console.log('======Calendars from DB=====')
-      console.log(calendarDbItems)
+      
       calendarList = calendarList.map((calendarItem) => {
         // Find the corresponding item in calendarDbItems by 'sourceId'
         const matchingDbItem = calendarDbItems.find(
@@ -270,9 +266,6 @@ export default class CalendarEventHandler extends GoogleHandler {
         return calendarItem;
       });
 
-      console.log("========Calendars after Merge========")
-      console.log(calendarList)
-
       let totalEvents = 0;
       let eventHistory: SchemaEvent[] = [];
 
@@ -302,13 +295,10 @@ export default class CalendarEventHandler extends GoogleHandler {
         // Update the calendar's sync data with the latest rangeTracker state
         calendar.syncData = rangeTracker.export();
 
-        // Stop if the total events fetched reach the batch size
-        if (totalEvents >= this.config.batchSize) {
-          syncPosition.thisRef = calendarList[(calendarIndex + 1) % calendarCount].sourceId; // Continue from the next calendar in the next sync
-          break;
-        }
       }
 
+      syncPosition.thisRef = calendarList[(Math.min(calendarCount, this.config.calendarLimit) + calendarPosition) % calendarCount].sourceId; // Continue from the next calendar in the next sync
+      
       // Finalize sync position and status based on event count
       this.updateSyncPosition(
         syncPosition,
@@ -317,7 +307,7 @@ export default class CalendarEventHandler extends GoogleHandler {
       );
 
       // Concatenate only items after syncPosition.thisRef
-      const remainingCalendars = calendarList.slice(calendarPosition + 1);
+      const remainingCalendars = calendarList.slice(calendarPosition);
 
       return {
         results: remainingCalendars.concat(eventHistory),
@@ -395,9 +385,6 @@ export default class CalendarEventHandler extends GoogleHandler {
     if (totalEvents === 0) {
       syncPosition.status = SyncHandlerStatus.ENABLED;
       syncPosition.syncMessage = "No new events found.";
-    } else if (totalEvents < this.config.batchSize) {
-      syncPosition.syncMessage = `Processed ${totalEvents} events across ${calendarCount} calendars. Sync complete.`;
-      syncPosition.status = SyncHandlerStatus.ENABLED;
     } else {
       syncPosition.status = SyncHandlerStatus.SYNCING;
       syncPosition.syncMessage = `Batch complete (${totalEvents}). More results pending.`;
diff --git a/src/serverconfig.example.json b/src/serverconfig.example.json
index 256906c1..a2ced6a0 100644
--- a/src/serverconfig.example.json
+++ b/src/serverconfig.example.json
@@ -99,8 +99,8 @@
                 },
                 "calendar-event": {
                     "backdate": "12-months",
-                    "calendarLimit": 20,
-                    "eventsPerCalendarLimit": 30
+                    "calendarLimit": 5,
+                    "eventsPerCalendarLimit": 10
                 }
             }
         },

From eaf75151afb0ee49a19cdf074cd2ca7671bf6e1a Mon Sep 17 00:00:00 2001
From: chime3 
Date: Mon, 7 Oct 2024 16:31:39 -0700
Subject: [PATCH 135/182] fix: update calendar syncdata

---
 src/providers/google/calendar-event.ts | 14 +++++---------
 1 file changed, 5 insertions(+), 9 deletions(-)

diff --git a/src/providers/google/calendar-event.ts b/src/providers/google/calendar-event.ts
index b8c38c2f..0d2a353a 100644
--- a/src/providers/google/calendar-event.ts
+++ b/src/providers/google/calendar-event.ts
@@ -277,13 +277,12 @@ export default class CalendarEventHandler extends GoogleHandler {
       // Iterate over each calendar
       for (let i = 1; i <= Math.min(calendarCount, this.config.calendarLimit); i++) {
         const calendarIndex = (calendarPosition + i) % calendarCount; // Rotate through calendars
-        const calendar = calendarList[calendarIndex];
-
+        
         // Use a separate ItemsRangeTracker for each calendar
-        let rangeTracker = new ItemsRangeTracker(calendar.syncData);
+        let rangeTracker = new ItemsRangeTracker(calendarList[calendarIndex].syncData);
 
         const fetchedEvents = await this.fetchAndTrackEvents(
-          calendar,
+          calendarList[calendarIndex],
           rangeTracker,
           apiClient
         );
@@ -293,7 +292,7 @@ export default class CalendarEventHandler extends GoogleHandler {
         totalEvents += fetchedEvents.length;
 
         // Update the calendar's sync data with the latest rangeTracker state
-        calendar.syncData = rangeTracker.export();
+        calendarList[calendarIndex].syncData = rangeTracker.export();
 
       }
 
@@ -306,11 +305,8 @@ export default class CalendarEventHandler extends GoogleHandler {
         calendarCount
       );
 
-      // Concatenate only items after syncPosition.thisRef
-      const remainingCalendars = calendarList.slice(calendarPosition);
-
       return {
-        results: remainingCalendars.concat(eventHistory),
+        results: calendarList.concat(eventHistory),
         position: syncPosition,
       };
     } catch (err: any) {

From f373e3ceafa7f7ea0c1bab2b2483d615c47bff99 Mon Sep 17 00:00:00 2001
From: chime3 
Date: Mon, 7 Oct 2024 17:31:40 -0700
Subject: [PATCH 136/182] fix: removed static value

---
 src/providers/google/calendar-event.ts | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/providers/google/calendar-event.ts b/src/providers/google/calendar-event.ts
index 0d2a353a..d7370dab 100644
--- a/src/providers/google/calendar-event.ts
+++ b/src/providers/google/calendar-event.ts
@@ -196,12 +196,12 @@ export default class CalendarEventHandler extends GoogleHandler {
       const insertedAt = new Date().toISOString();
 
       const creator: Person = {
-        email: event.creator.email ?? "info@example.com",
+        email: event.creator.email,
         displayName: event.creator.displayName
       }
 
       const organizer: Person = {
-        email: event.organizer.email ?? "info@example.com",
+        email: event.organizer.email,
         displayName: event.organizer.displayName
       }
 

From 92c07bf4c8944e9729478ba3d2f790b330985eaf Mon Sep 17 00:00:00 2001
From: chime3 
Date: Tue, 8 Oct 2024 17:55:02 -0700
Subject: [PATCH 137/182] fix: removed Unkown value

---
 src/providers/google/calendar-event.ts | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/providers/google/calendar-event.ts b/src/providers/google/calendar-event.ts
index d7370dab..8408b5ff 100644
--- a/src/providers/google/calendar-event.ts
+++ b/src/providers/google/calendar-event.ts
@@ -122,7 +122,7 @@ export default class CalendarEventHandler extends GoogleHandler {
           sourceAccountId: this.provider.getAccountId(),
           sourceApplication: this.getProviderApplicationUrl(),
           sourceId: calendarId,
-          timezone: timeZone ?? "Unkown",
+          timezone: timeZone,
           description,
           location,
           insertedAt,
@@ -339,6 +339,7 @@ export default class CalendarEventHandler extends GoogleHandler {
 
     // Initialize range from tracker
     let currentRange = rangeTracker.nextRange();
+
     let items: SchemaEvent[] = [];
 
     while (true) {

From 7a90afa62a13e2258e912217309339f346655a8f Mon Sep 17 00:00:00 2001
From: chime3 
Date: Wed, 9 Oct 2024 21:15:06 -0700
Subject: [PATCH 138/182] fix: renamed batchsize fields

---
 src/serverconfig.example.json | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/serverconfig.example.json b/src/serverconfig.example.json
index a2ced6a0..351545a0 100644
--- a/src/serverconfig.example.json
+++ b/src/serverconfig.example.json
@@ -87,7 +87,7 @@
         "google": {
             "clientId": "",
             "clientSecret": "",
-            "batchSize": 30,
+            "batchSize": 50,
             "sizeLimit": 10,
             "maxSyncLoops": 1,
             "handlers": {
@@ -99,8 +99,8 @@
                 },
                 "calendar-event": {
                     "backdate": "12-months",
-                    "calendarLimit": 5,
-                    "eventsPerCalendarLimit": 10
+                    "calendarBatchSize": 5,
+                    "eventBatchSize": 10
                 }
             }
         },

From b180b33991c79d9d10d22c4306c88f072dd7ba51 Mon Sep 17 00:00:00 2001
From: chime3 
Date: Wed, 9 Oct 2024 21:15:38 -0700
Subject: [PATCH 139/182] fix: rename fields interface

---
 src/providers/google/interfaces.ts | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/providers/google/interfaces.ts b/src/providers/google/interfaces.ts
index e1f77191..0dc56120 100644
--- a/src/providers/google/interfaces.ts
+++ b/src/providers/google/interfaces.ts
@@ -61,6 +61,6 @@ export interface CalendarAttachment {
 }
 
 export interface GoogleCalendarHandlerConfig extends BaseHandlerConfig {
-  calendarLimit?: number;  // Max number of calendar per sync
-  eventsPerCalendarLimit?: number; // Max number of event to process in a calendar
+  calendarBatchSize?: number;  // Max number of calendar per sync
+  eventBatchSize?: number; // Max number of event to process in a calendar
 }
\ No newline at end of file

From 41a8b43a5b68dd0094b6a0255b32bb3310687179 Mon Sep 17 00:00:00 2001
From: chime3 
Date: Wed, 9 Oct 2024 21:16:40 -0700
Subject: [PATCH 140/182] refactor: bunch update based on record ID and
 pagetoken

---
 src/providers/google/calendar-event.ts | 296 ++++++++++++++-----------
 1 file changed, 164 insertions(+), 132 deletions(-)

diff --git a/src/providers/google/calendar-event.ts b/src/providers/google/calendar-event.ts
index 8408b5ff..8e6ae615 100644
--- a/src/providers/google/calendar-event.ts
+++ b/src/providers/google/calendar-event.ts
@@ -1,5 +1,4 @@
 import { google, calendar_v3 } from "googleapis";
-import { GaxiosResponse } from "gaxios";
 import GoogleHandler from "./GoogleHandler";
 import CONFIG from "../../config";
 import {
@@ -10,14 +9,14 @@ import {
   ConnectionOptionType,
   SyncHandlerPosition,
   SyncProviderLogEvent,
-  SyncProviderLogLevel
+  SyncProviderLogLevel,
+  SyncItemsBreak
 } from "../../interfaces";
 import {
   SchemaCalendar,
   SchemaEvent,
 } from "../../schemas";
 import { ItemsRangeTracker } from "../../helpers/itemsRangeTracker";
-import { ItemsRange } from "../../helpers/interfaces";
 import { CalendarHelpers } from "./helpers";
 import { CalendarAttachment, DateTimeInfo, GoogleCalendarHandlerConfig, Person } from "./interfaces";
 
@@ -26,6 +25,10 @@ const _ = require("lodash");
 // Set MAX_BATCH_SIZE to 250 because the Google Calendar API v3 'maxResults' parameter is capped at 250.
 // For more details, see: https://developers.google.com/calendar/api/v3/reference/calendarList/list 
 const MAX_BATCH_SIZE = 250;
+
+export interface SyncEventItemsResult extends SyncItemsResult {
+  items: SchemaEvent[]
+}
 export default class CalendarEventHandler extends GoogleHandler {
 
   protected config: GoogleCalendarHandlerConfig;
@@ -80,7 +83,7 @@ export default class CalendarEventHandler extends GoogleHandler {
 
     let nextPageToken: string | undefined;
     let query: calendar_v3.Params$Resource$Calendarlist$List = {
-      maxResults: MAX_BATCH_SIZE,  // Fetch in batches up to the max limit
+      maxResults: Math.min(MAX_BATCH_SIZE, this.config.calendarBatchSize),  // Fetch in batches up to the max limit
       pageToken: nextPageToken,
     };
 
@@ -100,7 +103,7 @@ export default class CalendarEventHandler extends GoogleHandler {
           continue;
         }
 
-        const summary = calendar.summary ?? "No calendar title";
+        const summary = calendar.summary ?? "No title";
         let timeZone = calendar.timeZone;
 
         if (!timeZone) {
@@ -141,103 +144,6 @@ export default class CalendarEventHandler extends GoogleHandler {
     return calendarList;
   }
 
-  protected async fetchEventRange(
-    calendar: SchemaCalendar,
-    range: ItemsRange,
-    apiClient: calendar_v3.Calendar
-  ): Promise {
-    const events: SchemaEvent[] = [];
-
-    let query: calendar_v3.Params$Resource$Events$List = {
-      calendarId: calendar.sourceId,
-      maxResults: this.config.eventsPerCalendarLimit,
-      singleEvents: true,
-      orderBy: "startTime"
-    };
-
-    if (range.startId && !isNaN(Date.parse(range.startId))) {
-      query.timeMin = new Date(range.startId).toISOString();
-    }
-
-    if (range.endId && !isNaN(Date.parse(range.endId))) {
-      query.timeMax = new Date(range.endId).toISOString();
-    }
-
-    // Fetch events from Google Calendar API
-    const response = await apiClient.events.list(query);
-    const items = response.data.items || [];
-
-    for (const event of items) {
-      const eventId = event.id ?? '';
-
-      let start: DateTimeInfo = {
-        dateTime: event.start?.dateTime
-      };
-      let end: DateTimeInfo = {
-        dateTime: event.end?.dateTime
-      };
-
-      start.dateTime = event.start?.dateTime;
-      end.dateTime = event.end?.dateTime;
-
-      if (!start.dateTime) {
-        const logEvent: SyncProviderLogEvent = {
-          level: SyncProviderLogLevel.DEBUG,
-          message: `Invalid date for the event ${eventId}. Ignoring this event.`,
-        };
-        this.emit('log', logEvent);
-        continue;
-      }
-
-      // UTC offset time zone
-      start.timeZone = CalendarHelpers.getUTCOffsetTimezone(event.start?.timeZone)
-      end.timeZone = CalendarHelpers.getUTCOffsetTimezone(event.end?.timeZone)
-
-      const insertedAt = new Date().toISOString();
-
-      const creator: Person = {
-        email: event.creator.email,
-        displayName: event.creator.displayName
-      }
-
-      const organizer: Person = {
-        email: event.organizer.email,
-        displayName: event.organizer.displayName
-      }
-
-      let attendees: Person[] = []
-      if (event.attendees) {
-        attendees = event.attendees.filter(attendee => attendee.email) as Person[];
-      }
-
-      const attachments: CalendarAttachment[] = event.attachments as CalendarAttachment[];
-
-      events.push({
-        _id: this.buildItemId(eventId),
-        name: event.summary ?? "Unknown",
-        sourceAccountId: this.provider.getAccountId(),
-        sourceData: event,
-        sourceApplication: this.getProviderApplicationUrl(),
-        sourceId: eventId,
-        schema: CONFIG.verida.schemas.EVENT,
-        calendarId: calendar.sourceId ?? "primary",
-        start,
-        end,
-        creator,
-        organizer,
-        location: event.location,
-        description: event.description,
-        status: event.status ?? 'Unkown',
-        conferenceData: event.conferenceData,
-        attendees,
-        attachments,
-        insertedAt
-      });
-
-    }
-    return events;
-  }
-
   public async _sync(
     api: any,
     syncPosition: SyncHandlerPosition
@@ -250,7 +156,7 @@ export default class CalendarEventHandler extends GoogleHandler {
       const calendarDbItems = await calendarDs.getMany({
         "sourceAccountId": this.provider.getAccountId()
       });
-      
+
       calendarList = calendarList.map((calendarItem) => {
         // Find the corresponding item in calendarDbItems by 'sourceId'
         const matchingDbItem = calendarDbItems.find(
@@ -275,9 +181,9 @@ export default class CalendarEventHandler extends GoogleHandler {
       const calendarCount = calendarList.length;
 
       // Iterate over each calendar
-      for (let i = 1; i <= Math.min(calendarCount, this.config.calendarLimit); i++) {
+      for (let i = 1; i <= Math.min(calendarCount, this.config.calendarBatchSize); i++) {
         const calendarIndex = (calendarPosition + i) % calendarCount; // Rotate through calendars
-        
+
         // Use a separate ItemsRangeTracker for each calendar
         let rangeTracker = new ItemsRangeTracker(calendarList[calendarIndex].syncData);
 
@@ -296,8 +202,8 @@ export default class CalendarEventHandler extends GoogleHandler {
 
       }
 
-      syncPosition.thisRef = calendarList[(Math.min(calendarCount, this.config.calendarLimit) + calendarPosition) % calendarCount].sourceId; // Continue from the next calendar in the next sync
-      
+      syncPosition.thisRef = calendarList[(Math.min(calendarCount, this.config.calendarBatchSize) + calendarPosition) % calendarCount].sourceId; // Continue from the next calendar in the next sync
+
       // Finalize sync position and status based on event count
       this.updateSyncPosition(
         syncPosition,
@@ -327,53 +233,179 @@ export default class CalendarEventHandler extends GoogleHandler {
     return calendarPosition === -1 ? 0 : calendarPosition;
   }
 
+  private async buildResults(
+    calendarId: string,
+    response: calendar_v3.Schema$Events,
+    breakId: string
+  ): Promise {
+    const results: SchemaEvent[] = [];
+    let breakHit: SyncItemsBreak;
+
+    for (const event of response.items || []) {
+      const eventId = event.id || "";
+
+      // Break if the event ID matches breakId
+      if (eventId === breakId) {
+        const logEvent: SyncProviderLogEvent = {
+          level: SyncProviderLogLevel.DEBUG,
+          message: `Break ID hit (${breakId}) in calendar (${calendarId})`
+        };
+        this.emit("log", logEvent);
+        breakHit = SyncItemsBreak.ID;
+        break;
+      }
+
+      const start: DateTimeInfo = {
+        dateTime: event.start?.dateTime
+      };
+      const end: DateTimeInfo = {
+        dateTime: event.end?.dateTime
+      };
+
+      if (!start.dateTime) {
+        const logEvent: SyncProviderLogEvent = {
+          level: SyncProviderLogLevel.DEBUG,
+          message: `Invalid start date for event ${eventId}. Skipping this event.`,
+        };
+        this.emit("log", logEvent);
+        continue;
+      }
+      // Check for a break based on timestamp
+      const updatedTime = event.updated ? new Date(event.updated).toISOString() : new Date().toISOString();
+
+      start.timeZone = CalendarHelpers.getUTCOffsetTimezone(event.start?.timeZone);
+      end.timeZone = CalendarHelpers.getUTCOffsetTimezone(event.end?.timeZone);
+
+      const creator: Person = {
+        email: event.creator?.email,
+        displayName: event.creator?.displayName
+      };
+
+      const organizer: Person = {
+        email: event.organizer?.email,
+        displayName: event.organizer?.displayName
+      };
+
+      let attendees: Person[] = [];
+      if (event.attendees) {
+        attendees = event.attendees.filter(attendee => attendee.email) as Person[];
+      }
+
+      const attachments: CalendarAttachment[] = event.attachments as CalendarAttachment[];
+
+      results.push({
+        _id: this.buildItemId(eventId),
+        name: event.summary ?? "Unknown",
+        sourceAccountId: this.provider.getAccountId(),
+        sourceData: event,
+        sourceApplication: this.getProviderApplicationUrl(),
+        sourceId: eventId,
+        schema: CONFIG.verida.schemas.EVENT,
+        calendarId: calendarId,
+        start,
+        end,
+        creator,
+        organizer,
+        location: event.location,
+        description: event.description,
+        status: event.status,
+        conferenceData: event.conferenceData,
+        attendees,
+        attachments,
+        insertedAt: updatedTime
+      });
+    }
+
+    return {
+      items: results,
+      breakHit
+    };
+  }
+
   private async fetchAndTrackEvents(
     calendar: SchemaCalendar,
     rangeTracker: ItemsRangeTracker,
     apiClient: calendar_v3.Calendar
   ): Promise {
-    // Validate calendar and calendar.id
     if (!calendar || !calendar.sourceId) {
-      throw new Error('Invalid calendar or missing calendar sourceId');
+      throw new Error("Invalid calendar or missing calendar sourceId");
     }
 
-    // Initialize range from tracker
+    let items: SchemaEvent[];
     let currentRange = rangeTracker.nextRange();
+    let query: calendar_v3.Params$Resource$Events$List = {
+      calendarId: calendar.sourceId,
+      maxResults: this.config.eventBatchSize,
+      singleEvents: true,
+      orderBy: "updated"
+    };
+
+    if (currentRange.startId) {
+      query.pageToken = currentRange.startId;
+    }
+
+    // Fetch events from Google Calendar API
+    const response = await apiClient.events.list(query);
+
+    // Use buildResults to process the response
+    const latestResult = await this.buildResults(
+      calendar.sourceId,
+      response.data,
+      currentRange.endId
+    );
 
-    let items: SchemaEvent[] = [];
+    items = latestResult.items;
+    // Update the range tracker
+    if (items.length) {
+      rangeTracker.completedRange(
+        {
+          startId: items[0].sourceId,
+          endId: response.data?.nextPageToken
+        },
+        latestResult.breakHit === SyncItemsBreak.ID
+      );
+    } else {
+      rangeTracker.completedRange({ startId: undefined, endId: undefined }, false);
+    }
 
-    while (true) {
-      // Fetch events for the current range using fetchEventRange
-      const events = await this.fetchEventRange(calendar, currentRange, apiClient);
+    currentRange = rangeTracker.nextRange();
+    if (items.length != this.config.batchSize && currentRange.startId) {
+      // Not enough items, fetch more from the next page of results
+      let query: calendar_v3.Params$Resource$Events$List = {
+        calendarId: calendar.sourceId,
+        maxResults: this.config.eventBatchSize - items.length,
+        pageToken: currentRange.startId,
+        singleEvents: true,
+        orderBy: "updated"
+      };
 
-      if (!events.length) break;
+      const backfillResponse = await apiClient.events.list(query);
+      
+      const backfillResult = await this.buildResults(
+        calendar.sourceId,
+        response.data,
+        currentRange.endId
+      );
 
-      // Add fetched events to the main list
-      items = items.concat(events);
+      items = items.concat(backfillResult.items)
 
-      // Break loop if events reached calendar limit
-      if (items.length > this.config.eventsPerCalendarLimit) {
-        // Mark the current range as complete and stop
+      if (backfillResult.items.length) {
         rangeTracker.completedRange({
-          startId: events[0].start.dateTime,
-          endId: events[events.length - 1].end.dateTime
-        }, false);
-        break;
+          startId: backfillResult.items[0].sourceId,
+          endId: response.data?.nextPageToken
+        }, backfillResult.breakHit == SyncItemsBreak.ID)
       } else {
-        // Update rangeTracker and continue fetching
         rangeTracker.completedRange({
-          startId: events[0].start.dateTime,
-          endId: events[events.length - 1].end.dateTime
-        }, false);
-
-        // Move to the next range
-        currentRange = rangeTracker.nextRange();
+          startId: undefined,
+          endId: undefined
+        }, backfillResult.breakHit == SyncItemsBreak.ID)
       }
     }
 
     return items;
   }
 
+
   private updateSyncPosition(
     syncPosition: SyncHandlerPosition,
     totalEvents: number,
@@ -381,7 +413,7 @@ export default class CalendarEventHandler extends GoogleHandler {
   ) {
     if (totalEvents === 0) {
       syncPosition.status = SyncHandlerStatus.ENABLED;
-      syncPosition.syncMessage = "No new events found.";
+      syncPosition.syncMessage = "Stopping, No results found.";
     } else {
       syncPosition.status = SyncHandlerStatus.SYNCING;
       syncPosition.syncMessage = `Batch complete (${totalEvents}). More results pending.`;

From ca86053364fd4c4fe9b86217c6446bd037e5d70f Mon Sep 17 00:00:00 2001
From: chime3 
Date: Wed, 9 Oct 2024 21:23:48 -0700
Subject: [PATCH 141/182] fix: event batch size error

---
 src/providers/google/calendar-event.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/providers/google/calendar-event.ts b/src/providers/google/calendar-event.ts
index 8e6ae615..2fb7d241 100644
--- a/src/providers/google/calendar-event.ts
+++ b/src/providers/google/calendar-event.ts
@@ -369,7 +369,7 @@ export default class CalendarEventHandler extends GoogleHandler {
     }
 
     currentRange = rangeTracker.nextRange();
-    if (items.length != this.config.batchSize && currentRange.startId) {
+    if (items.length != this.config.eventBatchSize && currentRange.startId) {
       // Not enough items, fetch more from the next page of results
       let query: calendar_v3.Params$Resource$Events$List = {
         calendarId: calendar.sourceId,

From 4972870f4be11de7ca15ab6cbf8794c1a2380139 Mon Sep 17 00:00:00 2001
From: chime3 
Date: Wed, 9 Oct 2024 22:22:46 -0700
Subject: [PATCH 142/182] fix: modified unit tests for custom handler config

---
 tests/common.tests.ts                          | 13 ++++++++++---
 tests/providers/google/calendar-event.tests.ts |  3 ++-
 2 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/tests/common.tests.ts b/tests/common.tests.ts
index 7b63792d..e4775ce0 100644
--- a/tests/common.tests.ts
+++ b/tests/common.tests.ts
@@ -32,18 +32,20 @@ let provider: BaseProvider, connection: Connection;
 export class CommonTests {
   static async runSyncTest(
     providerId: string,
-    handlerType: typeof BaseSyncHandler,
+    handlerType: typeof BaseSyncHandler,    
     connection: Connection,
     testConfig: GenericTestConfig = {
       timeOrderAttribute: "insertedAt",
       batchSizeLimitAttribute: "batchSize",
     },
     syncPositionConfig: Omit,
-    providerConfig?: Omit
+    providerConfig?: Omit,
+    handlerId?: string
   ): Promise {
     const { api, handler, schemaUri } = await this.buildTestObjects(
       providerId,
       handlerType,
+      handlerId,
       providerConfig,
       connection
     );
@@ -61,6 +63,7 @@ export class CommonTests {
   static async buildTestObjects(
     providerId: string,
     handlerType: typeof BaseSyncHandler,
+    handlerId?: string,    
     providerConfig?: Omit,
     connection?: Connection
   ): Promise<{
@@ -88,8 +91,10 @@ export class CommonTests {
 
     const handlerConfig = {
       ...serverconfig.providers[providerId],
+      ...(handlerId ? serverconfig.providers[providerId]["handlers"][handlerId] : {}),
       ...providerConfig,
     };
+
     handler.setConfig(handlerConfig);
 
     return {
@@ -108,6 +113,7 @@ export class CommonTests {
       batchSizeLimitAttribute: "batchSize",
     },
     providerConfig: Omit = {},
+    handlerId: string,
     connection?: Connection
   ): Promise<{
     api: any;
@@ -118,6 +124,7 @@ export class CommonTests {
     const { api, handler, schemaUri, provider } = await this.buildTestObjects(
       providerId,
       handlerType,
+      handlerId,
       providerConfig,
       connection
     );
@@ -154,7 +161,7 @@ export class CommonTests {
       CommonTests.checkItem(results[0], handler, provider)
 
       assert.equal(
-        SyncHandlerStatus.SYNCING,
+        SyncHandlerStatus.ENABLED,
         response.position.status,
         "Sync is active"
       );
diff --git a/tests/providers/google/calendar-event.tests.ts b/tests/providers/google/calendar-event.tests.ts
index f56d52d7..753f025f 100644
--- a/tests/providers/google/calendar-event.tests.ts
+++ b/tests/providers/google/calendar-event.tests.ts
@@ -13,10 +13,10 @@ import BaseProvider from "../../../src/providers/BaseProvider";
 import { CommonTests, GenericTestConfig } from "../../common.tests";
 
 const providerName = "google";
+const handlerName = "calendar-event";
 let network: NetworkInstance;
 let connection: Connection;
 let provider: BaseProvider;
-let handlerName = "calendar-event";
 let testConfig: GenericTestConfig;
 let providerConfig: Omit = {};
 
@@ -41,6 +41,7 @@ describe(`${providerName} Google Calendar Event Tests`, function () {
         CalendarEvent,
         testConfig,
         providerConfig,
+        handlerName,
         connection
       );
     });

From ad98a7b4ed2546ef9456151ad3dc3978cc3073af Mon Sep 17 00:00:00 2001
From: Chris 
Date: Fri, 11 Oct 2024 08:14:27 +1030
Subject: [PATCH 143/182] Revert google drive batch size change

---
 src/serverconfig.example.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/serverconfig.example.json b/src/serverconfig.example.json
index d92e4b13..6defd031 100644
--- a/src/serverconfig.example.json
+++ b/src/serverconfig.example.json
@@ -98,7 +98,7 @@
                     "batchSize": 200
                 },
                 "google-drive-document": {
-                    "batchSize": 2
+                    "batchSize": 50
                 },
                 "calendar-event": {
                     "backdate": "12-months",

From fc677aba4649d5d3c332b9ba3ec038bdd4a7c85f Mon Sep 17 00:00:00 2001
From: chime3 
Date: Fri, 11 Oct 2024 02:23:54 -0700
Subject: [PATCH 144/182] fix: nextrange function

---
 src/helpers/itemsRangeTracker.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/helpers/itemsRangeTracker.ts b/src/helpers/itemsRangeTracker.ts
index 5083626a..085a11e7 100644
--- a/src/helpers/itemsRangeTracker.ts
+++ b/src/helpers/itemsRangeTracker.ts
@@ -130,7 +130,7 @@ export class ItemsRangeTracker {
         switch (this.status) {
             case ItemsRangeStatus.NEW:
                 return {
-                    startId: undefined,
+                    startId: this.completedRanges[0].endId,
                     endId: this.completedRanges[0].startId
                 }
             case ItemsRangeStatus.BACKFILL:

From a27ce4482c9c59a50955d2ab0c8fc2ffe32df269 Mon Sep 17 00:00:00 2001
From: chime3 
Date: Fri, 11 Oct 2024 02:24:59 -0700
Subject: [PATCH 145/182] feat: added calendar event helper function

---
 src/providers/google/calendar-event.ts | 29 ++++++--------------------
 src/providers/google/helpers.ts        | 13 ++++++++++++
 2 files changed, 19 insertions(+), 23 deletions(-)

diff --git a/src/providers/google/calendar-event.ts b/src/providers/google/calendar-event.ts
index 2fb7d241..99273bbf 100644
--- a/src/providers/google/calendar-event.ts
+++ b/src/providers/google/calendar-event.ts
@@ -176,7 +176,7 @@ export default class CalendarEventHandler extends GoogleHandler {
       let eventHistory: SchemaEvent[] = [];
 
       // Determine the current calendar position
-      const calendarPosition = this.getCalendarPositionIndex(calendarList, syncPosition);
+      const calendarPosition = CalendarHelpers.getCalendarPositionIndex(calendarList, syncPosition.thisRef);
 
       const calendarCount = calendarList.length;
 
@@ -208,7 +208,7 @@ export default class CalendarEventHandler extends GoogleHandler {
       this.updateSyncPosition(
         syncPosition,
         totalEvents,
-        calendarCount
+        Math.min(calendarCount, this.config.calendarBatchSize)
       );
 
       return {
@@ -221,18 +221,6 @@ export default class CalendarEventHandler extends GoogleHandler {
     }
   }
 
-  private getCalendarPositionIndex(
-    calendarList: SchemaCalendar[],
-    syncPosition: SyncHandlerPosition
-  ): number {
-    const calendarPosition = calendarList.findIndex(
-      (calendar) => calendar.sourceId === syncPosition.thisRef
-    );
-
-    // If not found, return 0 to start from the beginning
-    return calendarPosition === -1 ? 0 : calendarPosition;
-  }
-
   private async buildResults(
     calendarId: string,
     response: calendar_v3.Schema$Events,
@@ -380,10 +368,10 @@ export default class CalendarEventHandler extends GoogleHandler {
       };
 
       const backfillResponse = await apiClient.events.list(query);
-      
+
       const backfillResult = await this.buildResults(
         calendar.sourceId,
-        response.data,
+        backfillResponse.data,
         currentRange.endId
       );
 
@@ -411,12 +399,7 @@ export default class CalendarEventHandler extends GoogleHandler {
     totalEvents: number,
     calendarCount: number,
   ) {
-    if (totalEvents === 0) {
-      syncPosition.status = SyncHandlerStatus.ENABLED;
-      syncPosition.syncMessage = "Stopping, No results found.";
-    } else {
-      syncPosition.status = SyncHandlerStatus.SYNCING;
-      syncPosition.syncMessage = `Batch complete (${totalEvents}). More results pending.`;
-    }
+    syncPosition.status = SyncHandlerStatus.SYNCING;
+    syncPosition.syncMessage = `Batch complete (${totalEvents}) across (${calendarCount} calendars)`;
   }
 }
diff --git a/src/providers/google/helpers.ts b/src/providers/google/helpers.ts
index 403a909a..47bfde51 100644
--- a/src/providers/google/helpers.ts
+++ b/src/providers/google/helpers.ts
@@ -6,6 +6,7 @@ import mammoth from "mammoth";
 import * as XLSX from "xlsx";
 import * as officeParser from "officeparser";
 import moment from "moment-timezone";
+import { SchemaCalendar } from '../../schemas';
 
 export const mimeExtensions: {[key: string]: string} = {
   // Google Drive MIME types
@@ -573,4 +574,16 @@ export class CalendarHelpers {
 
     return utcOffset;
   }
+
+  static getCalendarPositionIndex(
+    calendarList: SchemaCalendar[],
+    calendarId: string|undefined
+  ): number {
+    const calendarPosition = calendarList.findIndex(
+      (calendar) => calendar.sourceId === calendarId
+    );
+
+    // If not found, return 0 to start from the beginning
+    return calendarPosition === -1 ? 0 : calendarPosition;
+  }
 }
\ No newline at end of file

From f23aaf264b3f192e7630be2403f3144ab176e6a5 Mon Sep 17 00:00:00 2001
From: chime3 
Date: Fri, 11 Oct 2024 02:25:39 -0700
Subject: [PATCH 146/182] fix: modified calendar event handler config

---
 src/providers/google/interfaces.ts | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/providers/google/interfaces.ts b/src/providers/google/interfaces.ts
index 0dc56120..d0c89e39 100644
--- a/src/providers/google/interfaces.ts
+++ b/src/providers/google/interfaces.ts
@@ -61,6 +61,6 @@ export interface CalendarAttachment {
 }
 
 export interface GoogleCalendarHandlerConfig extends BaseHandlerConfig {
-  calendarBatchSize?: number;  // Max number of calendar per sync
-  eventBatchSize?: number; // Max number of event to process in a calendar
+  calendarBatchSize: number;  // Max number of calendar per sync
+  eventBatchSize: number; // Max number of event to process in a calendar
 }
\ No newline at end of file

From 83851b054ca6e104da0bdfa3deb6d57fb48ab4b2 Mon Sep 17 00:00:00 2001
From: chime3 
Date: Fri, 11 Oct 2024 02:26:14 -0700
Subject: [PATCH 147/182] chore: revert common unit test

---
 tests/common.tests.ts | 197 +++++++++++++++++++++++++++++++++++++-----
 1 file changed, 176 insertions(+), 21 deletions(-)

diff --git a/tests/common.tests.ts b/tests/common.tests.ts
index e4775ce0..ecf7c0a3 100644
--- a/tests/common.tests.ts
+++ b/tests/common.tests.ts
@@ -18,7 +18,7 @@ export interface GenericTestConfig {
   // Attribute in the results that is used for time ordering (ie: insertedAt)
   timeOrderAttribute?: string; // Made optional
   // Attribute used to limit the batch size (ie: batchLimit)
-  batchSizeLimitAttribute?: string;
+  batchSizeLimitAttribute: string;
   // Prefix used for record ID's (override default which is providerName)
   idPrefix?: string;
 }
@@ -32,20 +32,18 @@ let provider: BaseProvider, connection: Connection;
 export class CommonTests {
   static async runSyncTest(
     providerId: string,
-    handlerType: typeof BaseSyncHandler,    
+    handlerType: typeof BaseSyncHandler,
     connection: Connection,
     testConfig: GenericTestConfig = {
       timeOrderAttribute: "insertedAt",
       batchSizeLimitAttribute: "batchSize",
     },
     syncPositionConfig: Omit,
-    providerConfig?: Omit,
-    handlerId?: string
+    providerConfig?: Omit
   ): Promise {
     const { api, handler, schemaUri } = await this.buildTestObjects(
       providerId,
       handlerType,
-      handlerId,
       providerConfig,
       connection
     );
@@ -63,7 +61,6 @@ export class CommonTests {
   static async buildTestObjects(
     providerId: string,
     handlerType: typeof BaseSyncHandler,
-    handlerId?: string,    
     providerConfig?: Omit,
     connection?: Connection
   ): Promise<{
@@ -91,10 +88,8 @@ export class CommonTests {
 
     const handlerConfig = {
       ...serverconfig.providers[providerId],
-      ...(handlerId ? serverconfig.providers[providerId]["handlers"][handlerId] : {}),
       ...providerConfig,
     };
-
     handler.setConfig(handlerConfig);
 
     return {
@@ -113,22 +108,32 @@ export class CommonTests {
       batchSizeLimitAttribute: "batchSize",
     },
     providerConfig: Omit = {},
-    handlerId: string,
     connection?: Connection
   ): Promise<{
     api: any;
     handler: BaseSyncHandler;
     provider: BaseProvider;
   }> {
-    
+    // * - New items are processed
+    //    * - Backfill items are processed
+    //    * - Not enough new items? Process backfill
+    //    * - Backfill twice doesn't process the same items
+    //    * - No more backfill produces empty rangeTracker
+
+    // Set result limit to 3 results so page tests can work correctly
+    providerConfig[testConfig.batchSizeLimitAttribute] = 3;
+
     const { api, handler, schemaUri, provider } = await this.buildTestObjects(
       providerId,
       handlerType,
-      handlerId,
       providerConfig,
       connection
     );
 
+    const idPrefix = testConfig.idPrefix
+      ? testConfig.idPrefix
+      : `${provider.getProviderName()}-${connection!.profile.id}`;
+
     try {
       const syncPosition: SyncHandlerPosition = {
         _id: `${providerId}-${schemaUri}`,
@@ -137,23 +142,25 @@ export class CommonTests {
         accountId: provider.getAccountId(),
         status: SyncHandlerStatus.SYNCING,
       };
-
+      
+      // 1. Test new items are processed
       const response = await handler._sync(api, syncPosition);
       const results = response.results;
 
+      // console.log(response.position)
+      // console.log(CommonTests.outputItems(results, testConfig.timeOrderAttribute))
+
       assert.ok(results && results.length, "Have results returned");
-      if (testConfig.batchSizeLimitAttribute) {
-        assert.equal(
-          providerConfig[testConfig.batchSizeLimitAttribute],
-          results.length,
-          "Have correct number of results returned on page 1"
-        );
-      }
+      assert.equal(
+        providerConfig[testConfig.batchSizeLimitAttribute],
+        results.length,
+        "Have correct number of results returned on page 1"
+      );
 
       if (testConfig.timeOrderAttribute) {
         assert.ok(
           results[0][testConfig.timeOrderAttribute] >
-          results[1][testConfig.timeOrderAttribute],
+            results[1][testConfig.timeOrderAttribute],
           "Results are most recent first"
         );
       }
@@ -161,11 +168,159 @@ export class CommonTests {
       CommonTests.checkItem(results[0], handler, provider)
 
       assert.equal(
-        SyncHandlerStatus.ENABLED,
+        SyncHandlerStatus.SYNCING,
         response.position.status,
         "Sync is active"
       );
+      assert.ok(response.position.thisRef, "Have a defined processing range");
+
+      const currentRangeParts = response.position.thisRef!.split(':')
+      assert.ok(currentRangeParts.length == 2, "Have correct number of parts for the processing range");
+      assert.ok(currentRangeParts[0] == results[0]._id.replace(`${idPrefix}-`, ''), "Have correct break ID");
+      assert.ok(currentRangeParts[1].length, "Have an end range");
+
+      // 2. Backfill items are processed
+      const syncPosition2 = response.position
+      const response2 = await handler._sync(api, syncPosition2);
+      const results2 = response2.results;
       
+      // console.log(response2.position)
+      // console.log(CommonTests.outputItems(results2, testConfig.timeOrderAttribute))
+
+      assert.ok(
+        results2 && results2.length,
+        "Have backfill results returned"
+      );
+      assert.ok(
+        results2 &&
+          results2.length == providerConfig[testConfig.batchSizeLimitAttribute],
+        "Have correct number of results returned in second page"
+      );
+
+      if (testConfig.timeOrderAttribute) {
+        assert.ok(
+          results2[0][testConfig.timeOrderAttribute] >
+            results2[1][testConfig.timeOrderAttribute],
+          "Results are most recent first"
+        );
+        assert.ok(
+          results2[0][testConfig.timeOrderAttribute] <
+            results[2][testConfig.timeOrderAttribute],
+          "First item on second page of results have earlier timestamp than last item on first page"
+        );
+      }
+
+      assert.equal(
+        response2.position.status,
+        SyncHandlerStatus.SYNCING,
+        "Sync is active"
+      );
+
+      assert.ok(response2.position.thisRef, "Have a defined processing range");
+
+      const currentRangeParts2 = response2.position.thisRef!.split(':')
+      assert.ok(currentRangeParts2.length == 2, "Have correct number of parts for the processing range");
+      assert.ok(currentRangeParts2[0] == results[0]._id.replace(`${idPrefix}-`, ''), "Have correct break ID matching the very first result");
+      assert.ok(currentRangeParts2[1].length, "Have an end range");
+      assert.ok(results[0]._id != results2[0]._id, "Have different result IDs")
+
+      // 3. Not enough new items? Process backfill
+      const syncPosition3 = response2.position
+      syncPosition3.thisRef = `${results[1].sourceId}:${currentRangeParts2[1]}` // Ensure the first item (only) is fetched
+      const response3 = await handler._sync(api, syncPosition3);
+      const results3 = response3.results;
+      
+      // console.log(response3.position)
+      // console.log(CommonTests.outputItems(results3, testConfig.timeOrderAttribute))
+
+      assert.ok(
+        results3 && results3.length,
+        "Have results returned"
+      );
+      assert.ok(
+        results3 &&
+        results3.length == providerConfig[testConfig.batchSizeLimitAttribute],
+        "Have correct number of results returned"
+      );
+      assert.equal(results3[0]._id, results[0]._id, 'First result item matches the very first item')
+      assert.ok(results3[1]._id != results[1]._id, 'Second result item does not match the very first batch second item')
+
+      if (testConfig.timeOrderAttribute) {
+        assert.ok(
+          results3[0][testConfig.timeOrderAttribute] >
+            results3[1][testConfig.timeOrderAttribute],
+          "Results are most recent first"
+        );
+        // this will break?
+        assert.ok(
+          results3[2][testConfig.timeOrderAttribute] <
+            results[2][testConfig.timeOrderAttribute],
+          "Last item on return results have earlier timestamp than last item on first page"
+        );
+      }
+
+      assert.equal(
+        response3.position.status,
+        SyncHandlerStatus.SYNCING,
+        "Sync is active"
+      );
+
+      assert.ok(response3.position.thisRef, "Have a defined processing range");
+
+      const currentRangeParts3 = response3.position.thisRef!.split(':')
+      assert.ok(currentRangeParts3.length == 2, "Have correct number of parts for the processing range");
+      assert.ok(currentRangeParts3[0] == results3[0]._id.replace(`${idPrefix}-`, ''), "Have correct break ID matching the very first result");
+      assert.ok(currentRangeParts3[1].length, "Have an end range");
+      assert.ok(currentRangeParts3[1] != currentRangeParts2[1], "End range has changed between batches");
+
+      // - Backfill twice doesn't process the same items
+      const syncPosition4 = response3.position
+      const response4 = await handler._sync(api, syncPosition4);
+      const results4 = response4.results;
+      
+      // console.log(response4.position)
+      // console.log(CommonTests.outputItems(results4, testConfig.timeOrderAttribute))
+
+      assert.ok(
+        results4 && results4.length,
+        "Have results returned"
+      );
+      assert.ok(
+        results4 &&
+        results4.length == providerConfig[testConfig.batchSizeLimitAttribute],
+        "Have correct number of results returned"
+      );
+
+      if (testConfig.timeOrderAttribute) {
+        assert.ok(
+          results4[0][testConfig.timeOrderAttribute] >
+          results4[1][testConfig.timeOrderAttribute],
+          "Results are most recent first"
+        );
+        // this will break?
+        assert.ok(
+          results4[0][testConfig.timeOrderAttribute] <
+            results[2][testConfig.timeOrderAttribute],
+          "First item on return results have earlier timestamp than last item on first page"
+        );
+      }
+
+      assert.ok(results4[0]._id != results3[0]._id, "First items dont match between batches")
+
+      assert.equal(
+        response4.position.status,
+        SyncHandlerStatus.SYNCING,
+        "Sync is active"
+      );
+
+      assert.ok(response4.position.thisRef, "Have a defined processing range");
+      const currentRangeParts4 = response4.position.thisRef!.split(':')
+      assert.ok(currentRangeParts4.length == 2, "Have correct number of parts for the processing range");
+      assert.ok(currentRangeParts4[1].length, "Have an end range");
+
+      // @todo: No more backfill produces empty rangeTracker and SyncHandlerStatus.CONNECTED
+
+
       // Close the provider connection
       await provider.close();
 

From 1155617401044eff08b342f703cf2617d4558f2d Mon Sep 17 00:00:00 2001
From: chime3 
Date: Fri, 11 Oct 2024 02:26:48 -0700
Subject: [PATCH 148/182] feat: added calendar event unit tests

---
 .../providers/google/calendar-event.tests.ts  | 143 ++++++++++++++++--
 1 file changed, 129 insertions(+), 14 deletions(-)

diff --git a/tests/providers/google/calendar-event.tests.ts b/tests/providers/google/calendar-event.tests.ts
index 753f025f..0812640c 100644
--- a/tests/providers/google/calendar-event.tests.ts
+++ b/tests/providers/google/calendar-event.tests.ts
@@ -1,52 +1,167 @@
 const assert = require("assert");
+import CONFIG from "../../../src/config";
 import {
   BaseProviderConfig,
   Connection,
-  SyncHandlerStatus,
   SyncHandlerPosition,
+  SyncHandlerStatus
 } from "../../../src/interfaces";
 import Providers from "../../../src/providers";
 import CommonUtils, { NetworkInstance } from "../../common.utils";
 
-import CalendarEvent from "../../../src/providers/google/calendar-event";
+import CalendarEventHandler from "../../../src/providers/google/calendar-event";
 import BaseProvider from "../../../src/providers/BaseProvider";
 import { CommonTests, GenericTestConfig } from "../../common.tests";
+import { SchemaCalendar, SchemaEvent, SchemaRecord } from "../../../src/schemas";
+import { GoogleCalendarHandlerConfig } from "../../../src/providers/google/interfaces";
+import { CalendarHelpers } from "../../../src/providers/google/helpers";
 
-const providerName = "google";
-const handlerName = "calendar-event";
+// Define the provider ID
+const providerId = "google";
 let network: NetworkInstance;
 let connection: Connection;
 let provider: BaseProvider;
+let handlerName = "calendar-event";
 let testConfig: GenericTestConfig;
+
+// Configure provider and handler without certain attributes
 let providerConfig: Omit = {};
+let handlerConfig: GoogleCalendarHandlerConfig = {
+  calendarBatchSize: 3,
+  eventBatchSize: 3
+};
 
-describe(`${providerName} Google Calendar Event Tests`, function () {
+// Test suite for Google Calendar event syncing
+describe(`${providerId} calendar event tests`, function () {
   this.timeout(100000);
 
+  // Before all tests, set up the network, connection, and provider
   this.beforeAll(async function () {
     network = await CommonUtils.getNetwork();
-    connection = await CommonUtils.getConnection(providerName);
-    provider = Providers(providerName, network.context, connection);
+    connection = await CommonUtils.getConnection(providerId);
+    provider = Providers(providerId, network.context, connection);
 
+    // Configure test settings
     testConfig = {
       idPrefix: `${provider.getProviderId()}-${connection.profile.id}`,
+      timeOrderAttribute: "insertedAt",
+      batchSizeLimitAttribute: "batchSize",
     };
   });
 
-  describe(`Fetch ${providerName} data`, () => {
-   
+  // Test fetching data for Google Calendar
+  describe(`Fetch ${providerId} data`, () => {
+
     it(`Can pass basic tests: ${handlerName}`, async () => {
-      await CommonTests.runGenericTests(
-        providerName,
-        CalendarEvent,
-        testConfig,
+      // Build the necessary test objects
+      const { api, handler, provider } = await CommonTests.buildTestObjects(
+        providerId,
+        CalendarEventHandler,
         providerConfig,
-        handlerName,
         connection
       );
+
+      // Set the handler configuration
+      handler.setConfig(handlerConfig);
+
+      try {
+        // Set up initial sync position
+        const syncPosition: SyncHandlerPosition = {
+          _id: `${providerId}-${handlerName}`,
+          providerId,
+          handlerId: handler.getId(),
+          accountId: provider.getAccountId(),
+          status: SyncHandlerStatus.ENABLED,
+        };
+
+        // Start the sync process
+        const response = await handler._sync(api, syncPosition);
+        const results = response.results;
+
+        // Extract calendars and events from the results
+        const calendars = results.filter(result => result.schema === CONFIG.verida.schemas.CALENDAR);
+        const events = results.filter(result => result.schema === CONFIG.verida.schemas.EVENT);
+
+        // Ensure results are returned
+        assert.ok(results && results.length, "Have results returned");
+
+        // Check IDs in the returned items
+        CommonTests.checkItem(results[0], handler, provider);
+
+        // Verify sync status is active
+        assert.equal(
+          SyncHandlerStatus.SYNCING,
+          response.position.status,
+          "Sync is active"
+        );
+
+        assert.ok(response.position.thisRef, "Have a calendar sync pos");
+
+        // Ensure the sync rotates the calendar list correctly
+        const originalCalendarIndex = CalendarHelpers.getCalendarPositionIndex(calendars, (await provider.getSyncPosition(handlerName)).thisRef);
+        const currentCalendarIndex = CalendarHelpers.getCalendarPositionIndex(calendars, response.position.thisRef);
+
+        // Verify correct number of calendars were synced
+        assert.equal(
+          currentCalendarIndex,
+          (originalCalendarIndex + Math.min(handlerConfig.calendarBatchSize, calendars.length)) % calendars.length,
+          "Synced correct number of calendars."
+        );
+
+        // Ensure the event batch per calendar works
+        const firstCalendarId = events[0].calendarId;
+        const firstCalendarEvents = events.filter(event => event.calendarId === firstCalendarId);
+        assert.equal(firstCalendarEvents.length, handlerConfig.eventBatchSize, "Processed correct number of events per calendar");
+
+         /**
+          * Start the second sync batch process
+          */
+        const secondBatchResponse = await handler._sync(api, response.position);
+        const secondBatchResults = secondBatchResponse.results;
+
+        // Extract calendars and events from the second batch results
+        const secondBatchCalendars = secondBatchResults.filter(result => result.schema === CONFIG.verida.schemas.CALENDAR);
+        const secondBatchEvents = secondBatchResults.filter(result => result.schema === CONFIG.verida.schemas.EVENT);
+
+        // Ensure second batch results are returned
+        assert.ok(secondBatchResults && secondBatchResults.length, "Have second batch results returned");
+
+        // Check IDs in the returned items for the second batch
+        CommonTests.checkItem(secondBatchResults[0], handler, provider);
+
+        // Verify sync status is still active for the second batch
+        assert.equal(
+          SyncHandlerStatus.SYNCING,
+          secondBatchResponse.position.status,
+          "Sync is still active after second batch"
+        );
+
+        assert.ok(secondBatchResponse.position.thisRef, "Have a calendar sync pos for second batch");
+
+        // Ensure the sync rotates the calendar list correctly in the second batch
+        const secondOriginalCalendarIndex = CalendarHelpers.getCalendarPositionIndex(secondBatchCalendars, (await provider.getSyncPosition(handlerName)).thisRef);
+        const secondCurrentCalendarIndex = CalendarHelpers.getCalendarPositionIndex(secondBatchCalendars, secondBatchResponse.position.thisRef);
+
+        // Verify correct number of calendars were synced in the second batch
+        assert.equal(
+          secondCurrentCalendarIndex,
+          (secondOriginalCalendarIndex + Math.min(handlerConfig.calendarBatchSize, secondBatchCalendars.length)) % secondBatchCalendars.length,
+          "Synced correct number of calendars in the second batch."
+        );
+
+        // Check if synced every calendar correctly
+        const syncedCalendar = (secondBatchCalendars.filter(cal => cal.sourceId === secondBatchEvents[0].calendarId))[0];
+        assert.ok(syncedCalendar.syncData, "Have a sync range per calendar.")
+
+      } catch (err) {
+        // Ensure provider closes even if an error occurs
+        await provider.close();
+        throw err;
+      }
     });
   });
 
+  // After all tests, close the network context
   this.afterAll(async function () {
     const { context } = await CommonUtils.getNetwork();
     await context.close();

From 7933682d9a49da573a2737dcb9ddd547051aab3d Mon Sep 17 00:00:00 2001
From: chime3 
Date: Tue, 15 Oct 2024 00:30:02 -0700
Subject: [PATCH 149/182] fix: removed total slack message count comparision

---
 src/serverconfig.example.json | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/serverconfig.example.json b/src/serverconfig.example.json
index 3c890cc9..7f721086 100644
--- a/src/serverconfig.example.json
+++ b/src/serverconfig.example.json
@@ -122,7 +122,6 @@
             "clientSecret": "",
             "stateSecret": "",
             "groupLimit": 20,
-            "messageBatchSize": 50,
             "messagesPerGroupLimit": 10            
         }
     },

From e5f223c41b8561bf6476869dd8d01136cddeaf88 Mon Sep 17 00:00:00 2001
From: chime3 
Date: Tue, 15 Oct 2024 00:31:09 -0700
Subject: [PATCH 150/182] fix: refactor chat message handler

---
 src/providers/slack/chat-message.ts | 244 +++++++++++++---------------
 1 file changed, 114 insertions(+), 130 deletions(-)

diff --git a/src/providers/slack/chat-message.ts b/src/providers/slack/chat-message.ts
index 130d1ed0..4596d9c6 100644
--- a/src/providers/slack/chat-message.ts
+++ b/src/providers/slack/chat-message.ts
@@ -15,8 +15,8 @@ import {
 import { SlackChatGroupType, SlackHandlerConfig } from "./interfaces";
 import BaseSyncHandler from "../BaseSyncHandler";
 import { ItemsRangeTracker } from "../../helpers/itemsRangeTracker";
-import { ItemsRange } from "../../helpers/interfaces";
 import { SlackHelpers } from "./helpers";
+import { ItemsRange } from "../../helpers/interfaces";
 
 const _ = require("lodash");
 
@@ -66,132 +66,70 @@ export default class SlackChatMessageHandler extends BaseSyncHandler {
 
     protected async buildChatGroupList(): Promise {
         const client = this.getSlackClient();
-
         let channelList: SchemaSocialChatGroup[] = [];
-
-        // Fetch all types of conversations: DMs, private, public
         const types = ["im", "private_channel", "public_channel"];
+
+        // Loop through each type of channel (DM, private, public)
         for (const type of types) {
-            const conversations = await client.conversations.list({
-                types: type,
-            });
-
-            for (const channel of conversations.channels) {
-                const group: SchemaSocialChatGroup = {
-                    _id: this.buildItemId(channel.id),
-                    name: channel.name || channel.user,
-                    sourceAccountId: this.provider.getAccountId(),
-                    sourceApplication: this.getProviderApplicationUrl(),
-                    sourceId: channel.id,
-                    schema: CONFIG.verida.schemas.CHAT_GROUP,
-                    sourceData: channel,
-                    insertedAt: new Date().toISOString(),
-                };
+            const conversations = await client.conversations.list({ types: type });
+            for (const channel of conversations.channels || []) {
+                const group: SchemaSocialChatGroup = this.buildChatGroup(channel);
                 channelList.push(group);
             }
         }
+
         return channelList;
     }
 
-    protected async fetchMessageRange(
-        chatGroup: SchemaSocialChatGroup,
-        range: ItemsRange,
-        apiClient: WebClient
-    ): Promise {
-        const messages: SchemaSocialChatMessage[] = [];
-
-        const response = await apiClient.conversations.history({
-            channel: chatGroup.sourceId!,
-            limit: this.config.messagesPerGroupLimit,
-            oldest: range.startId,
-            latest: range.endId,
-        });
-
-        for (const message of response.messages) {
-            if (!message.user) continue;
-            
-            const user = await SlackHelpers.getUserInfo(this.connection.accessToken, message.user)
-            const chatMessage: SchemaSocialChatMessage = {
-                _id: this.buildItemId(message.ts),
-                groupId: chatGroup._id,
-                groupName: chatGroup.name,
-                messageText: message.text,
-                fromHandle: user.profile.email ?? "Unknown",
-                sourceAccountId: this.provider.getAccountId(),
-                sourceApplication: this.getProviderApplicationUrl(),
-                sourceId: message.ts,
-                sourceData: message,
-                insertedAt: new Date(parseFloat(message.ts) * 1000).toISOString(),
-                sentAt: new Date(parseFloat(message.ts) * 1000).toISOString(),
-                type:
-                    message.user === this.connection.profile.id
-                        ? SchemaChatMessageType.SEND
-                        : SchemaChatMessageType.RECEIVE,
-                fromId: message.user ?? "Unknown",
-                name: message.text.substring(0, 30),
-            };
-            messages.push(chatMessage);
-        }
-
-        return messages;
+    private buildChatGroup(channel: any): SchemaSocialChatGroup {
+        return {
+            _id: this.buildItemId(channel.id),
+            name: channel.name || channel.user,
+            sourceAccountId: this.provider.getAccountId(),
+            sourceApplication: this.getProviderApplicationUrl(),
+            sourceId: channel.id,
+            schema: CONFIG.verida.schemas.CHAT_GROUP,
+            sourceData: channel,
+            insertedAt: new Date().toISOString(),
+        };
     }
 
-    public async _sync(
-        api: any,
-        syncPosition: SyncHandlerPosition
-    ): Promise {
+    public async _sync(api: any, syncPosition: SyncHandlerPosition): Promise {
         try {
             const apiClient = this.getSlackClient();
-            const groupList = await this.buildChatGroupList(); // Fetch all public, private, and DM groups
+            const groupList = await this.buildChatGroupList();  // Fetch chat groups
+
+            const groupDbItems = await this.getExistingGroupsFromDb();  // Fetch existing groups from the database
+            const mergedGroupList = this.mergeGroupLists(groupList, groupDbItems);  // Merge new and existing groups
 
             let totalMessages = 0;
             let chatHistory: SchemaSocialChatMessage[] = [];
 
             // Determine the current group position
-            let groupPosition = this.getGroupPositionIndex(groupList, syncPosition);
-
-            const groupCount = groupList.length;
+            const groupPosition = this.getGroupPositionIndex(mergedGroupList, syncPosition);
+            const groupCount = mergedGroupList.length;
 
-            // Iterate over each group 
+            // Iterate over each group
             for (let i = 0; i < Math.min(groupCount, this.config.groupLimit); i++) {
-                const groupIndex = (groupPosition + i) % groupCount; // Rotate through groups
-                const group = groupList[groupIndex];
+                const groupIndex = (groupPosition + i) % groupCount;  // Rotate through groups
+                const group = mergedGroupList[groupIndex];
 
-                // Use a separate ItemsRangeTracker for each group
-                let rangeTracker = new ItemsRangeTracker(group.syncData);
+                let rangeTracker = new ItemsRangeTracker(group.syncData);  // Track items for each group
+                const fetchedMessages = await this.fetchAndTrackMessages(group, rangeTracker, apiClient);
 
-                const fetchedMessages = await this.fetchAndTrackMessages(
-                    group,
-                    rangeTracker,
-                    apiClient
-                );
-
-                // Concatenate the fetched messages to the total chat history
+                // Concatenate fetched messages
                 chatHistory = chatHistory.concat(fetchedMessages);
-                totalMessages += fetchedMessages.length;
 
-                // Update the group's sync data with the latest rangeTracker state
+                // Update the group sync data
                 group.syncData = rangeTracker.export();
-
-                // Stop if the total messages fetched reach the batch size
-                if (totalMessages >= this.config.messageBatchSize) {
-                    syncPosition.thisRef = groupList[(groupIndex + 1) % groupCount].sourceId; // Continue from the next group in the next sync
-                    break;
-                }
             }
 
-            // Finalize sync position and status based on message count
-            this.updateSyncPosition(
-                syncPosition,
-                totalMessages,
-                groupCount
-            );
-
-            // Concatenate only items after syncPosition.thisRef and chatHistory
-            const remainingGroups = groupList.slice(groupPosition + 1);
+            // Update sync position and status
+            this.updateSyncPosition(syncPosition, totalMessages, groupCount);
 
+            // Return the sync response
             return {
-                results: remainingGroups.concat(chatHistory),
+                results: mergedGroupList.concat(chatHistory),
                 position: syncPosition,
             };
         } catch (err: any) {
@@ -200,64 +138,93 @@ export default class SlackChatMessageHandler extends BaseSyncHandler {
         }
     }
 
-    private getGroupPositionIndex(
-        groupList: SchemaSocialChatGroup[],
-        syncPosition: SyncHandlerPosition
-    ): number {
-        const groupPosition = groupList.findIndex(
-            (group) => group.sourceId === syncPosition.thisRef
-        );
-
-        // If not found, return 0 to start from the beginning
-        return groupPosition === -1 ? 0 : groupPosition;
-    }
-
     private async fetchAndTrackMessages(
         group: SchemaSocialChatGroup,
         rangeTracker: ItemsRangeTracker,
         apiClient: WebClient
     ): Promise {
-        // Validate group and group.id
         if (!group || !group.sourceId) {
-            throw new Error('Invalid group or missing group sourceId');
+            throw new Error("Invalid group or missing group sourceId");
         }
 
-        // Initialize range from tracker
-        let currentRange = rangeTracker.nextRange();
         let items: SchemaSocialChatMessage[] = [];
+        let currentRange = rangeTracker.nextRange();
 
+        // Loop to fetch messages in batches
         while (true) {
-            // Fetch messages for the current range using fetchMessageRange
             const messages = await this.fetchMessageRange(group, currentRange, apiClient);
-
             if (!messages.length) break;
 
-            // Add fetched messages to the main list
             items = items.concat(messages);
 
-            // Break loop if messages reached group limit
-            if (items.length > this.config.messagesPerGroupLimit) {
-                // Mark the current range as complete and stop
+            if (items.length >= this.config.messagesPerGroupLimit) {
                 rangeTracker.completedRange({
                     startId: messages[0].sourceId,
-                    endId: messages[messages.length - 1].sourceId
+                    endId: messages[messages.length - 1].sourceId,
                 }, false);
                 break;
-            } else {
-                // Update rangeTracker and continue fetching
-                rangeTracker.completedRange({
-                    startId: messages[0].sourceId,
-                    endId: messages[messages.length - 1].sourceId
-                }, false);
-
-                // Move to the next range
-                currentRange = rangeTracker.nextRange();
             }
+
+            rangeTracker.completedRange({
+                startId: messages[0].sourceId,
+                endId: messages[messages.length - 1].sourceId,
+            }, false);
+
+            currentRange = rangeTracker.nextRange();
         }
 
         return items;
     }
 
+    private async fetchMessageRange(
+        chatGroup: SchemaSocialChatGroup,
+        range: ItemsRange,
+        apiClient: WebClient
+    ): Promise {
+        const messages: SchemaSocialChatMessage[] = [];
+        const response = await apiClient.conversations.history({
+            channel: chatGroup.sourceId!,
+            limit: this.config.messagesPerGroupLimit,
+            oldest: range.startId,
+            latest: range.endId,
+        });
+
+        for (const message of response.messages || []) {
+            if (!message.user) continue;
+
+            const user = await SlackHelpers.getUserInfo(this.connection.accessToken, message.user);
+            const chatMessage = this.buildChatMessage(message, chatGroup, user);
+            messages.push(chatMessage);
+        }
+
+        return messages;
+    }
+
+    private buildChatMessage(
+        message: any,
+        chatGroup: SchemaSocialChatGroup,
+        user: any
+    ): SchemaSocialChatMessage {
+        return {
+            _id: this.buildItemId(message.ts),
+            groupId: chatGroup._id,
+            groupName: chatGroup.name,
+            messageText: message.text,
+            fromHandle: user.profile.email ?? "Unknown",
+            sourceAccountId: this.provider.getAccountId(),
+            sourceApplication: this.getProviderApplicationUrl(),
+            sourceId: message.ts,
+            sourceData: message,
+            insertedAt: new Date(parseFloat(message.ts) * 1000).toISOString(),
+            sentAt: new Date(parseFloat(message.ts) * 1000).toISOString(),
+            type: message.user === this.connection.profile.id
+                ? SchemaChatMessageType.SEND
+                : SchemaChatMessageType.RECEIVE,
+            fromId: message.user ?? "Unknown",
+            name: message.text.substring(0, 30),
+        };
+    }
+
     private updateSyncPosition(
         syncPosition: SyncHandlerPosition,
         totalMessages: number,
@@ -274,4 +241,21 @@ export default class SlackChatMessageHandler extends BaseSyncHandler {
             syncPosition.syncMessage = `Batch complete (${this.config.messageBatchSize}). More results pending.`;
         }
     }
+
+    private mergeGroupLists(
+        newGroups: SchemaSocialChatGroup[],
+        existingGroups: SchemaSocialChatGroup[]
+    ): SchemaSocialChatGroup[] {
+        return newGroups.map((group) => {
+            const existingGroup = existingGroups.find(g => g.sourceId === group.sourceId);
+            return existingGroup ? _.merge({}, existingGroup, group) : group;
+        });
+    }
+
+    private async getExistingGroupsFromDb(): Promise {
+        const groupDs = await this.provider.getDatastore(CONFIG.verida.schemas.CHAT_GROUP);
+        return await groupDs.getMany({
+            sourceAccountId: this.provider.getAccountId(),
+        });
+    }
 }

From f90d3af5a0512a7daa8f260830bca8949a06959d Mon Sep 17 00:00:00 2001
From: chime3 
Date: Wed, 16 Oct 2024 22:59:33 -0700
Subject: [PATCH 151/182] fix: added slack helper function

---
 src/providers/slack/chat-message.ts | 15 ++++++---------
 src/providers/slack/helpers.ts      | 13 +++++++++++++
 2 files changed, 19 insertions(+), 9 deletions(-)

diff --git a/src/providers/slack/chat-message.ts b/src/providers/slack/chat-message.ts
index 4596d9c6..39425a8b 100644
--- a/src/providers/slack/chat-message.ts
+++ b/src/providers/slack/chat-message.ts
@@ -99,14 +99,18 @@ export default class SlackChatMessageHandler extends BaseSyncHandler {
             const apiClient = this.getSlackClient();
             const groupList = await this.buildChatGroupList();  // Fetch chat groups
 
-            const groupDbItems = await this.getExistingGroupsFromDb();  // Fetch existing groups from the database
+            const groupDs = await this.provider.getDatastore(CONFIG.verida.schemas.CHAT_GROUP);
+            const groupDbItems = await groupDs.getMany({
+                sourceAccountId: this.provider.getAccountId(),
+            });
+            
             const mergedGroupList = this.mergeGroupLists(groupList, groupDbItems);  // Merge new and existing groups
 
             let totalMessages = 0;
             let chatHistory: SchemaSocialChatMessage[] = [];
 
             // Determine the current group position
-            const groupPosition = this.getGroupPositionIndex(mergedGroupList, syncPosition);
+            const groupPosition = SlackHelpers.getGroupPositionIndex(mergedGroupList, syncPosition.thisRef);
             const groupCount = mergedGroupList.length;
 
             // Iterate over each group
@@ -251,11 +255,4 @@ export default class SlackChatMessageHandler extends BaseSyncHandler {
             return existingGroup ? _.merge({}, existingGroup, group) : group;
         });
     }
-
-    private async getExistingGroupsFromDb(): Promise {
-        const groupDs = await this.provider.getDatastore(CONFIG.verida.schemas.CHAT_GROUP);
-        return await groupDs.getMany({
-            sourceAccountId: this.provider.getAccountId(),
-        });
-    }
 }
diff --git a/src/providers/slack/helpers.ts b/src/providers/slack/helpers.ts
index a228d2ec..3d74ecf4 100644
--- a/src/providers/slack/helpers.ts
+++ b/src/providers/slack/helpers.ts
@@ -1,4 +1,5 @@
 import axios from 'axios';
+import { SchemaSocialChatGroup } from '../../schemas';
 
 export class SlackHelpers {
   // Method to fetch user information using Slack's `users.info` API
@@ -22,4 +23,16 @@ export class SlackHelpers {
       throw new Error(`Failed to fetch user info: ${error.message}`);
     }
   }
+
+  static getGroupPositionIndex(
+    groupList: SchemaSocialChatGroup[],
+    groupId: string|undefined
+  ): number {
+    const groupPosition = groupList.findIndex(
+      (group) => group.sourceId === groupId
+    );
+
+    // If not found, return 0 to start from the beginning
+    return groupPosition === -1 ? 0 : groupPosition;
+  }
 }

From b6e1f2b3611bf18e85f0772d034d66bada987473 Mon Sep 17 00:00:00 2001
From: chime3 
Date: Wed, 16 Oct 2024 23:19:44 -0700
Subject: [PATCH 152/182] feat: added readme notes

---
 src/providers/slack/README.md | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/src/providers/slack/README.md b/src/providers/slack/README.md
index bfda5687..6acf005f 100644
--- a/src/providers/slack/README.md
+++ b/src/providers/slack/README.md
@@ -7,4 +7,17 @@
 3. You can get `client ID` and `client Secret` from the `Basic Information` section
 4. Add redirect URL and scopes in `OAuth & Permissions` section
     - Redirect URL: `https://127.0.0.1:5021/callback/slack`
-    - There are two types of tokens: bot and user, and add following scopes: `channels:history`, `channels:read`, `groups:read`, `users:read`, `im:read`, `im:history`
\ No newline at end of file
+    - There are two types of tokens: bot and user, and add following scopes: `channels:history`, `channels:read`, `groups:read`, `users:read`, `im:read`, `im:history`
+
+# Notes
+Slack workspaces might have many public channels and spam messages that are less relevant to users' privacy, so **DM/Private** groups have priority.
+
+Slack `ItemsRangeTracker` is based on timestamps, which are used as the IDs of chat message records due to the nature of its API.
+```
+const response = await apiClient.conversations.history({
+     channel,
+     limit,
+     oldest,           // from timestamp
+     latest            // end timestamp
+});
+```

From 6ed1bf78952cef83934341e06a514d9a88a5b28b Mon Sep 17 00:00:00 2001
From: chime3 
Date: Thu, 17 Oct 2024 21:48:06 -0700
Subject: [PATCH 153/182] fix: renamed handler label

---
 src/providers/slack/chat-message.ts | 4 ++--
 src/serverconfig.example.json       | 3 ++-
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/providers/slack/chat-message.ts b/src/providers/slack/chat-message.ts
index 39425a8b..7e1167eb 100644
--- a/src/providers/slack/chat-message.ts
+++ b/src/providers/slack/chat-message.ts
@@ -24,11 +24,11 @@ export default class SlackChatMessageHandler extends BaseSyncHandler {
     protected config: SlackHandlerConfig;
 
     public getName(): string {
-        return "slack-messages";
+        return "chat-message";
     }
 
     public getLabel(): string {
-        return "Slack Messages";
+        return "Chat Messages";
     }
 
     public getSchemaUri(): string {
diff --git a/src/serverconfig.example.json b/src/serverconfig.example.json
index 7f721086..321a630f 100644
--- a/src/serverconfig.example.json
+++ b/src/serverconfig.example.json
@@ -121,8 +121,9 @@
             "clientId": "",
             "clientSecret": "",
             "stateSecret": "",
+            "maxSyncLoops": 1,
             "groupLimit": 20,
-            "messagesPerGroupLimit": 10            
+            "messagesPerGroupLimit": 50            
         }
     },
     "providerDefaults": {

From ae06b52b47ec5148b6607319dfa9ef0e27a754ac Mon Sep 17 00:00:00 2001
From: chime3 
Date: Sun, 20 Oct 2024 20:05:33 -0700
Subject: [PATCH 154/182] fix: revert item range tracker

---
 src/helpers/itemsRangeTracker.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/helpers/itemsRangeTracker.ts b/src/helpers/itemsRangeTracker.ts
index 085a11e7..5083626a 100644
--- a/src/helpers/itemsRangeTracker.ts
+++ b/src/helpers/itemsRangeTracker.ts
@@ -130,7 +130,7 @@ export class ItemsRangeTracker {
         switch (this.status) {
             case ItemsRangeStatus.NEW:
                 return {
-                    startId: this.completedRanges[0].endId,
+                    startId: undefined,
                     endId: this.completedRanges[0].startId
                 }
             case ItemsRangeStatus.BACKFILL:

From c890296f30c733ac3768b3c768f3c58eb09bfdff Mon Sep 17 00:00:00 2001
From: Chris 
Date: Tue, 22 Oct 2024 15:44:57 +1030
Subject: [PATCH 155/182] Add test to ensure date only events are handled. Add
 uri property to calendar events

---
 src/providers/google/calendar-event.ts        | 68 ++++++++++---------
 src/schemas.ts                                |  1 +
 .../providers/google/calendar-event.tests.ts  | 56 +++++++++++++++
 3 files changed, 92 insertions(+), 33 deletions(-)

diff --git a/src/providers/google/calendar-event.ts b/src/providers/google/calendar-event.ts
index 99273bbf..abde00fa 100644
--- a/src/providers/google/calendar-event.ts
+++ b/src/providers/google/calendar-event.ts
@@ -221,43 +221,16 @@ export default class CalendarEventHandler extends GoogleHandler {
     }
   }
 
-  private async buildResults(
-    calendarId: string,
-    response: calendar_v3.Schema$Events,
-    breakId: string
-  ): Promise {
-    const results: SchemaEvent[] = [];
-    let breakHit: SyncItemsBreak;
-
-    for (const event of response.items || []) {
-      const eventId = event.id || "";
-
-      // Break if the event ID matches breakId
-      if (eventId === breakId) {
-        const logEvent: SyncProviderLogEvent = {
-          level: SyncProviderLogLevel.DEBUG,
-          message: `Break ID hit (${breakId}) in calendar (${calendarId})`
-        };
-        this.emit("log", logEvent);
-        breakHit = SyncItemsBreak.ID;
-        break;
-      }
+  public buildResult(calendarId: string, event: calendar_v3.Schema$Event): SchemaEvent {
+    const eventId = event.id || "";
 
       const start: DateTimeInfo = {
-        dateTime: event.start?.dateTime
+        dateTime: event.start?.dateTime || `${event.start?.date}T00:00:00.000Z`
       };
       const end: DateTimeInfo = {
-        dateTime: event.end?.dateTime
+        dateTime: event.end?.dateTime || `${event.end?.date}T00:00:00.000Z`
       };
 
-      if (!start.dateTime) {
-        const logEvent: SyncProviderLogEvent = {
-          level: SyncProviderLogLevel.DEBUG,
-          message: `Invalid start date for event ${eventId}. Skipping this event.`,
-        };
-        this.emit("log", logEvent);
-        continue;
-      }
       // Check for a break based on timestamp
       const updatedTime = event.updated ? new Date(event.updated).toISOString() : new Date().toISOString();
 
@@ -281,7 +254,7 @@ export default class CalendarEventHandler extends GoogleHandler {
 
       const attachments: CalendarAttachment[] = event.attachments as CalendarAttachment[];
 
-      results.push({
+      const eventRecord: SchemaEvent = {
         _id: this.buildItemId(eventId),
         name: event.summary ?? "Unknown",
         sourceAccountId: this.provider.getAccountId(),
@@ -289,6 +262,7 @@ export default class CalendarEventHandler extends GoogleHandler {
         sourceApplication: this.getProviderApplicationUrl(),
         sourceId: eventId,
         schema: CONFIG.verida.schemas.EVENT,
+        uri: event.htmlLink,
         calendarId: calendarId,
         start,
         end,
@@ -301,7 +275,35 @@ export default class CalendarEventHandler extends GoogleHandler {
         attendees,
         attachments,
         insertedAt: updatedTime
-      });
+      }
+
+      return eventRecord
+  }
+
+  private async buildResults(
+    calendarId: string,
+    response: calendar_v3.Schema$Events,
+    breakId: string
+  ): Promise {
+    const results: SchemaEvent[] = [];
+    let breakHit: SyncItemsBreak;
+
+    for (const event of response.items || []) {
+      const eventId = event.id || "";
+
+      // Break if the event ID matches breakId
+      if (eventId === breakId) {
+        const logEvent: SyncProviderLogEvent = {
+          level: SyncProviderLogLevel.DEBUG,
+          message: `Break ID hit (${breakId}) in calendar (${calendarId})`
+        };
+        this.emit("log", logEvent);
+        breakHit = SyncItemsBreak.ID;
+        break;
+      }
+
+      const eventRecord = this.buildResult(calendarId, event)
+      results.push(eventRecord);
     }
 
     return {
diff --git a/src/schemas.ts b/src/schemas.ts
index 39b827eb..a1f9e1d6 100644
--- a/src/schemas.ts
+++ b/src/schemas.ts
@@ -145,6 +145,7 @@ export interface SchemaEvent extends SchemaRecord {
     status?: string
     description?: string
     calendarId: string
+    uri?: string
     location?: string
     creator?: Person
     organizer?: Person
diff --git a/tests/providers/google/calendar-event.tests.ts b/tests/providers/google/calendar-event.tests.ts
index 0812640c..ce9ff29e 100644
--- a/tests/providers/google/calendar-event.tests.ts
+++ b/tests/providers/google/calendar-event.tests.ts
@@ -31,6 +31,32 @@ let handlerConfig: GoogleCalendarHandlerConfig = {
   eventBatchSize: 3
 };
 
+const TEST_EVENT = {
+  kind: 'calendar#event',
+  etag: '"3016564997954000"',
+  id: '1k89v9aphj7rg71_20171016',
+  status: 'confirmed',
+  htmlLink: 'https://www.google.com/calendar/event?eid=MWs4OXY5YXBoajdyZzcxdjBkZ2gzZmVnMjJfMj',
+  created: '2017-10-08T03:08:56.000Z',
+  updated: '2017-10-17T23:21:38.977Z',
+  summary: 'TEST EVENT',
+  creator: { email: 'ME@gmail.com', displayName: 'ME' },
+  organizer: {
+    email: 'op88nn863ft55@group.calendar.google.com',
+    displayName: 'ME CALENDAR',
+    self: true
+  },
+  start: { date: '2017-10-16' },
+  end: { date: '2017-10-17' },
+  recurringEventId: 'rg71v0d',
+  originalStartTime: { date: '2017-10-16' },
+  transparency: 'transparent',
+  iCalUID: '7rg71v0dgh3feg22@google.com',
+  sequence: 0,
+  reminders: { useDefault: false },
+  eventType: 'default'
+}
+
 // Test suite for Google Calendar event syncing
 describe(`${providerId} calendar event tests`, function () {
   this.timeout(100000);
@@ -52,6 +78,33 @@ describe(`${providerId} calendar event tests`, function () {
   // Test fetching data for Google Calendar
   describe(`Fetch ${providerId} data`, () => {
 
+    it(`Can handle date only events`, async () => {
+      // console.log((new Date(TEST_EVENT.start.date)).toISOString())
+      // return
+
+      // Build the necessary test objects
+      const { api, handler, provider } = await CommonTests.buildTestObjects(
+        providerId,
+        CalendarEventHandler,
+        providerConfig,
+        connection
+      );
+
+      const testEventResult = ( handler).buildResult(TEST_EVENT.iCalUID, TEST_EVENT)
+
+      const expectedStartDate = `${TEST_EVENT.start.date}T00:00:00.000Z`
+      const expectedEndDate = `${TEST_EVENT.end.date}T00:00:00.000Z`
+      
+      assert.equal(expectedStartDate, testEventResult.start.dateTime, 'Start date is expected date')
+      assert.equal(expectedEndDate, testEventResult.end.dateTime, 'End date is expected date')
+
+      const startDate = new Date(testEventResult.start.dateTime)
+      const endDate = new Date(testEventResult.end.dateTime)
+
+      assert.equal(startDate.toISOString(), (new Date(TEST_EVENT.start.date)).toISOString(), 'Start date is a valid ISO string')
+      assert.equal(endDate.toISOString(), (new Date(TEST_EVENT.end.date)).toISOString(), 'End date is a valid ISO string')
+    })
+
     it(`Can pass basic tests: ${handlerName}`, async () => {
       // Build the necessary test objects
       const { api, handler, provider } = await CommonTests.buildTestObjects(
@@ -82,6 +135,9 @@ describe(`${providerId} calendar event tests`, function () {
         const calendars = results.filter(result => result.schema === CONFIG.verida.schemas.CALENDAR);
         const events = results.filter(result => result.schema === CONFIG.verida.schemas.EVENT);
 
+        console.log(events[0])
+        return
+
         // Ensure results are returned
         assert.ok(results && results.length, "Have results returned");
 

From abbf86ac223e21b1f11c7480e925b257105ac8f2 Mon Sep 17 00:00:00 2001
From: Chris 
Date: Tue, 22 Oct 2024 16:12:26 +1030
Subject: [PATCH 156/182] Fetch all calendars, don't limit by batch size

---
 src/providers/google/calendar-event.ts        | 20 +++---------
 src/providers/google/interfaces.ts            |  1 -
 src/serverconfig.example.json                 |  1 -
 .../providers/google/calendar-event.tests.ts  | 32 -------------------
 4 files changed, 5 insertions(+), 49 deletions(-)

diff --git a/src/providers/google/calendar-event.ts b/src/providers/google/calendar-event.ts
index abde00fa..31f47d08 100644
--- a/src/providers/google/calendar-event.ts
+++ b/src/providers/google/calendar-event.ts
@@ -83,7 +83,6 @@ export default class CalendarEventHandler extends GoogleHandler {
 
     let nextPageToken: string | undefined;
     let query: calendar_v3.Params$Resource$Calendarlist$List = {
-      maxResults: Math.min(MAX_BATCH_SIZE, this.config.calendarBatchSize),  // Fetch in batches up to the max limit
       pageToken: nextPageToken,
     };
 
@@ -175,20 +174,13 @@ export default class CalendarEventHandler extends GoogleHandler {
       let totalEvents = 0;
       let eventHistory: SchemaEvent[] = [];
 
-      // Determine the current calendar position
-      const calendarPosition = CalendarHelpers.getCalendarPositionIndex(calendarList, syncPosition.thisRef);
-
-      const calendarCount = calendarList.length;
-
       // Iterate over each calendar
-      for (let i = 1; i <= Math.min(calendarCount, this.config.calendarBatchSize); i++) {
-        const calendarIndex = (calendarPosition + i) % calendarCount; // Rotate through calendars
-
+      for (const calendar of calendarList) {
         // Use a separate ItemsRangeTracker for each calendar
-        let rangeTracker = new ItemsRangeTracker(calendarList[calendarIndex].syncData);
+        let rangeTracker = new ItemsRangeTracker(calendar.syncData);
 
         const fetchedEvents = await this.fetchAndTrackEvents(
-          calendarList[calendarIndex],
+          calendar,
           rangeTracker,
           apiClient
         );
@@ -198,17 +190,15 @@ export default class CalendarEventHandler extends GoogleHandler {
         totalEvents += fetchedEvents.length;
 
         // Update the calendar's sync data with the latest rangeTracker state
-        calendarList[calendarIndex].syncData = rangeTracker.export();
+        calendar.syncData = rangeTracker.export();
 
       }
 
-      syncPosition.thisRef = calendarList[(Math.min(calendarCount, this.config.calendarBatchSize) + calendarPosition) % calendarCount].sourceId; // Continue from the next calendar in the next sync
-
       // Finalize sync position and status based on event count
       this.updateSyncPosition(
         syncPosition,
         totalEvents,
-        Math.min(calendarCount, this.config.calendarBatchSize)
+        calendarList.length
       );
 
       return {
diff --git a/src/providers/google/interfaces.ts b/src/providers/google/interfaces.ts
index d0c89e39..faa0795c 100644
--- a/src/providers/google/interfaces.ts
+++ b/src/providers/google/interfaces.ts
@@ -61,6 +61,5 @@ export interface CalendarAttachment {
 }
 
 export interface GoogleCalendarHandlerConfig extends BaseHandlerConfig {
-  calendarBatchSize: number;  // Max number of calendar per sync
   eventBatchSize: number; // Max number of event to process in a calendar
 }
\ No newline at end of file
diff --git a/src/serverconfig.example.json b/src/serverconfig.example.json
index 94ee14dc..d25710f7 100644
--- a/src/serverconfig.example.json
+++ b/src/serverconfig.example.json
@@ -99,7 +99,6 @@
                 },
                 "calendar-event": {
                     "backdate": "12-months",
-                    "calendarBatchSize": 5,
                     "eventBatchSize": 10
                 }
             }
diff --git a/tests/providers/google/calendar-event.tests.ts b/tests/providers/google/calendar-event.tests.ts
index ce9ff29e..ccf3c2b1 100644
--- a/tests/providers/google/calendar-event.tests.ts
+++ b/tests/providers/google/calendar-event.tests.ts
@@ -79,9 +79,6 @@ describe(`${providerId} calendar event tests`, function () {
   describe(`Fetch ${providerId} data`, () => {
 
     it(`Can handle date only events`, async () => {
-      // console.log((new Date(TEST_EVENT.start.date)).toISOString())
-      // return
-
       // Build the necessary test objects
       const { api, handler, provider } = await CommonTests.buildTestObjects(
         providerId,
@@ -135,9 +132,6 @@ describe(`${providerId} calendar event tests`, function () {
         const calendars = results.filter(result => result.schema === CONFIG.verida.schemas.CALENDAR);
         const events = results.filter(result => result.schema === CONFIG.verida.schemas.EVENT);
 
-        console.log(events[0])
-        return
-
         // Ensure results are returned
         assert.ok(results && results.length, "Have results returned");
 
@@ -151,19 +145,6 @@ describe(`${providerId} calendar event tests`, function () {
           "Sync is active"
         );
 
-        assert.ok(response.position.thisRef, "Have a calendar sync pos");
-
-        // Ensure the sync rotates the calendar list correctly
-        const originalCalendarIndex = CalendarHelpers.getCalendarPositionIndex(calendars, (await provider.getSyncPosition(handlerName)).thisRef);
-        const currentCalendarIndex = CalendarHelpers.getCalendarPositionIndex(calendars, response.position.thisRef);
-
-        // Verify correct number of calendars were synced
-        assert.equal(
-          currentCalendarIndex,
-          (originalCalendarIndex + Math.min(handlerConfig.calendarBatchSize, calendars.length)) % calendars.length,
-          "Synced correct number of calendars."
-        );
-
         // Ensure the event batch per calendar works
         const firstCalendarId = events[0].calendarId;
         const firstCalendarEvents = events.filter(event => event.calendarId === firstCalendarId);
@@ -192,19 +173,6 @@ describe(`${providerId} calendar event tests`, function () {
           "Sync is still active after second batch"
         );
 
-        assert.ok(secondBatchResponse.position.thisRef, "Have a calendar sync pos for second batch");
-
-        // Ensure the sync rotates the calendar list correctly in the second batch
-        const secondOriginalCalendarIndex = CalendarHelpers.getCalendarPositionIndex(secondBatchCalendars, (await provider.getSyncPosition(handlerName)).thisRef);
-        const secondCurrentCalendarIndex = CalendarHelpers.getCalendarPositionIndex(secondBatchCalendars, secondBatchResponse.position.thisRef);
-
-        // Verify correct number of calendars were synced in the second batch
-        assert.equal(
-          secondCurrentCalendarIndex,
-          (secondOriginalCalendarIndex + Math.min(handlerConfig.calendarBatchSize, secondBatchCalendars.length)) % secondBatchCalendars.length,
-          "Synced correct number of calendars in the second batch."
-        );
-
         // Check if synced every calendar correctly
         const syncedCalendar = (secondBatchCalendars.filter(cal => cal.sourceId === secondBatchEvents[0].calendarId))[0];
         assert.ok(syncedCalendar.syncData, "Have a sync range per calendar.")

From 8f91f1c91632d2d7451602c797e19e761c05ece4 Mon Sep 17 00:00:00 2001
From: chime3 
Date: Tue, 22 Oct 2024 17:05:31 -0700
Subject: [PATCH 157/182] fix: removed calendar handler

---
 src/providers/google/calendar.ts | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 delete mode 100644 src/providers/google/calendar.ts

diff --git a/src/providers/google/calendar.ts b/src/providers/google/calendar.ts
deleted file mode 100644
index e69de29b..00000000

From 150e4d1a7abc31253a0abeae5d8b65a0b8563344 Mon Sep 17 00:00:00 2001
From: chime3 
Date: Tue, 22 Oct 2024 17:34:33 -0700
Subject: [PATCH 158/182] chore: yarn build

---
 yarn.lock | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/yarn.lock b/yarn.lock
index a5a32af2..6613fd7b 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -4299,6 +4299,16 @@ lodash.isplainobject@^4.0.6:
   resolved "https://registry.yarnpkg.com/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz#7c526a52d89b45c45cc690b88163be0497f550cb"
   integrity sha1-fFJqUtibRcRcxpC4gWO+BJf1UMs=
 
+lodash.isstring@^4, lodash.isstring@^4.0.1:
+  version "4.0.1"
+  resolved "https://registry.yarnpkg.com/lodash.isstring/-/lodash.isstring-4.0.1.tgz#d527dfb5456eca7cc9bb95d5daeaf88ba54a5451"
+  integrity sha512-0wJxfxH1wgO3GrbuP+dTTk7op+6L41QCXbGINEmD+ny/G/eCqGzxyCsh7159S+mgDDcoarnBw6PC1PS5+wUGgw==
+
+lodash.once@^4.0.0:
+  version "4.1.1"
+  resolved "https://registry.yarnpkg.com/lodash.once/-/lodash.once-4.1.1.tgz#0dd3971213c7c56df880977d504c88fb471a97ac"
+  integrity sha512-Sb487aTOCr9drQVL8pIxOzVhafOjZN9UU54hiN8PU3uAiSV7lx1yYNpbNmex2PK6dSJoNTSJUUswT651yww3Mg==
+
 lodash.snakecase@4.1.1:
   version "4.1.1"
   resolved "https://registry.yarnpkg.com/lodash.snakecase/-/lodash.snakecase-4.1.1.tgz#39d714a35357147837aefd64b5dcbb16becd8f8d"
@@ -6148,6 +6158,11 @@ undici-types@~5.26.4:
   resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-5.26.5.tgz#bcd539893d00b56e964fd2657a4866b221a65617"
   integrity sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==
 
+undici-types@~6.19.2:
+  version "6.19.8"
+  resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-6.19.8.tgz#35111c9d1437ab83a7cdc0abae2f26d88eda0a02"
+  integrity sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==
+
 undici@6.19.8:
   version "6.19.8"
   resolved "https://registry.yarnpkg.com/undici/-/undici-6.19.8.tgz#002d7c8a28f8cc3a44ff33c3d4be4d85e15d40e1"

From 24315a895cf5a3eb977b4e3e89fd9a4a823a70e8 Mon Sep 17 00:00:00 2001
From: chime3 
Date: Wed, 23 Oct 2024 15:02:17 -0700
Subject: [PATCH 159/182] fix: removed grouplimit

---
 src/providers/slack/interfaces.ts | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/providers/slack/interfaces.ts b/src/providers/slack/interfaces.ts
index d0cdfc85..b7ebce86 100644
--- a/src/providers/slack/interfaces.ts
+++ b/src/providers/slack/interfaces.ts
@@ -1,8 +1,6 @@
 import { BaseHandlerConfig, BaseProviderConfig } from "../../interfaces";
 
 export interface SlackHandlerConfig extends BaseHandlerConfig {
-  // Maximum number of groups to process
-  groupLimit: number,
   // Maximum number of messages to process in a given batch
   messageBatchSize: number
   // Maximum number of messages to process in a group

From 56daea625b20b09a93ce3e38c97ab5c5135cc323 Mon Sep 17 00:00:00 2001
From: chime3 
Date: Wed, 23 Oct 2024 15:03:10 -0700
Subject: [PATCH 160/182] refactor: convert timeline to cursor based pagination

---
 src/providers/slack/chat-message.ts | 524 ++++++++++++++++------------
 1 file changed, 298 insertions(+), 226 deletions(-)

diff --git a/src/providers/slack/chat-message.ts b/src/providers/slack/chat-message.ts
index 7e1167eb..4b2b5bb8 100644
--- a/src/providers/slack/chat-message.ts
+++ b/src/providers/slack/chat-message.ts
@@ -1,258 +1,330 @@
-import { WebClient } from "@slack/web-api";
+import { ConversationsHistoryArguments, ConversationsHistoryResponse, WebClient } from "@slack/web-api";
 import CONFIG from "../../config";
 import {
-    SyncResponse,
-    SyncHandlerStatus,
-    ProviderHandlerOption,
-    ConnectionOptionType,
-    SyncHandlerPosition,
+  SyncResponse,
+  SyncHandlerStatus,
+  ProviderHandlerOption,
+  ConnectionOptionType,
+  SyncHandlerPosition,
+  SyncItemsResult,
+  SyncItemsBreak,
+  SyncProviderLogEvent,
+  SyncProviderLogLevel,
 } from "../../interfaces";
 import {
-    SchemaChatMessageType,
-    SchemaSocialChatGroup,
-    SchemaSocialChatMessage,
+  SchemaChatMessageType,
+  SchemaSocialChatGroup,
+  SchemaSocialChatMessage,
 } from "../../schemas";
 import { SlackChatGroupType, SlackHandlerConfig } from "./interfaces";
 import BaseSyncHandler from "../BaseSyncHandler";
 import { ItemsRangeTracker } from "../../helpers/itemsRangeTracker";
 import { SlackHelpers } from "./helpers";
-import { ItemsRange } from "../../helpers/interfaces";
+
+import { MessageElement } from "@slack/web-api/dist/types/response/ConversationsHistoryResponse";
 
 const _ = require("lodash");
 
-export default class SlackChatMessageHandler extends BaseSyncHandler {
-    protected config: SlackHandlerConfig;
+export interface SyncChatItemsResult extends SyncItemsResult {
+  items: SchemaSocialChatMessage[];
+}
 
-    public getName(): string {
-        return "chat-message";
+export default class SlackChatMessageHandler extends BaseSyncHandler {
+  protected config: SlackHandlerConfig;
+
+  public getName(): string {
+    return "chat-message";
+  }
+
+  public getLabel(): string {
+    return "Chat Messages";
+  }
+
+  public getSchemaUri(): string {
+    return CONFIG.verida.schemas.CHAT_MESSAGE;
+  }
+
+  public getProviderApplicationUrl(): string {
+    return "https://slack.com/";
+  }
+
+  public getOptions(): ProviderHandlerOption[] {
+    return [
+      {
+        id: "channelTypes",
+        label: "Channel types",
+        type: ConnectionOptionType.ENUM_MULTI,
+        enumOptions: [
+          { label: "Public Channel", value: SlackChatGroupType.CHANNEL },
+          { label: "Private Channel", value: SlackChatGroupType.GROUP },
+          { label: "Direct Messages", value: SlackChatGroupType.IM },
+        ],
+        defaultValue: [
+          SlackChatGroupType.CHANNEL,
+          SlackChatGroupType.GROUP,
+          SlackChatGroupType.IM,
+        ].join(","),
+      },
+    ];
+  }
+
+  public getSlackClient(): WebClient {
+    const token = this.connection.accessToken;
+    return new WebClient(token);
+  }
+
+  protected async buildChatGroupList(): Promise {
+    const client = this.getSlackClient();
+    let channelList: SchemaSocialChatGroup[] = [];
+    const types = ["im", "private_channel", "public_channel"];
+
+    // Loop through each type of channel (DM, private, public)
+    for (const type of types) {
+      const conversations = await client.conversations.list({ types: type });
+      for (const channel of conversations.channels || []) {
+        const group: SchemaSocialChatGroup = this.buildChatGroup(channel);
+        channelList.push(group);
+      }
     }
 
-    public getLabel(): string {
-        return "Chat Messages";
+    return channelList;
+  }
+
+  private buildChatGroup(channel: any): SchemaSocialChatGroup {
+    return {
+      _id: this.buildItemId(channel.id),
+      name: channel.name || channel.user,
+      sourceAccountId: this.provider.getAccountId(),
+      sourceApplication: this.getProviderApplicationUrl(),
+      sourceId: channel.id,
+      schema: CONFIG.verida.schemas.CHAT_GROUP,
+      sourceData: channel,
+      insertedAt: new Date().toISOString(),
+    };
+  }
+
+  public async _sync(
+    api: any,
+    syncPosition: SyncHandlerPosition
+  ): Promise {
+    try {
+      const apiClient = this.getSlackClient();
+      const groupList = await this.buildChatGroupList(); // Fetch chat groups
+
+      const groupDs = await this.provider.getDatastore(
+        CONFIG.verida.schemas.CHAT_GROUP
+      );
+      const groupDbItems = await groupDs.getMany({
+        sourceAccountId: this.provider.getAccountId(),
+      });
+
+      const mergedGroupList = this.mergeGroupLists(groupList, groupDbItems); // Merge new and existing groups
+
+      let totalMessages = 0;
+      let chatHistory: SchemaSocialChatMessage[] = [];
+
+      // Iterate over each group
+      for (let i = 0; i < mergedGroupList.length; i++) {
+        const group = mergedGroupList[i];
+        let rangeTracker = new ItemsRangeTracker(group.syncData); // Track items for each group
+
+        const fetchedMessages = await this.fetchAndTrackMessages(
+          group,
+          rangeTracker,
+          apiClient
+        );
+
+        // Concatenate fetched messages
+        chatHistory = chatHistory.concat(fetchedMessages);
+        totalMessages += fetchedMessages.length;
+
+        // Update the group sync data in the mergedGroupList at the current index
+        mergedGroupList[i].syncData = rangeTracker.export();
+      }
+
+      // Update sync position and status
+      this.updateSyncPosition(
+        syncPosition,
+        totalMessages,
+        mergedGroupList.length
+      );
+
+      // Return the sync response
+      return {
+        results: mergedGroupList.concat(chatHistory),
+        position: syncPosition,
+      };
+    } catch (err: any) {
+      console.error(err);
+      throw err;
     }
-
-    public getSchemaUri(): string {
-        return CONFIG.verida.schemas.CHAT_MESSAGE;
+  }
+
+  private async fetchAndTrackMessages(
+    group: SchemaSocialChatGroup,
+    rangeTracker: ItemsRangeTracker,
+    apiClient: WebClient
+  ): Promise {
+    if (!group || !group.sourceId) {
+      throw new Error("Invalid group or missing group sourceId");
     }
 
-    public getProviderApplicationUrl(): string {
-        return "https://slack.com/";
-    }
+    let items: SchemaSocialChatMessage[] = [];
+    let currentRange = rangeTracker.nextRange();
 
-    public getOptions(): ProviderHandlerOption[] {
-        return [
-            {
-                id: "channelTypes",
-                label: "Channel types",
-                type: ConnectionOptionType.ENUM_MULTI,
-                enumOptions: [
-                    { label: "Public Channel", value: SlackChatGroupType.CHANNEL },
-                    { label: "Private Channel", value: SlackChatGroupType.GROUP },
-                    { label: "Direct Messages", value: SlackChatGroupType.IM },
-                ],
-                defaultValue: [
-                    SlackChatGroupType.CHANNEL,
-                    SlackChatGroupType.GROUP,
-                    SlackChatGroupType.IM,
-                ].join(","),
-            },
-        ];
-    }
+    let query: ConversationsHistoryArguments = {
+      channel: group.sourceId!,
+      limit: this.config.messagesPerGroupLimit,
+    };
 
-    public getSlackClient(): WebClient {
-        const token = this.connection.accessToken;
-        return new WebClient(token);
+    if (currentRange.startId) {
+      query.cursor = currentRange.startId;
     }
 
-    protected async buildChatGroupList(): Promise {
-        const client = this.getSlackClient();
-        let channelList: SchemaSocialChatGroup[] = [];
-        const types = ["im", "private_channel", "public_channel"];
-
-        // Loop through each type of channel (DM, private, public)
-        for (const type of types) {
-            const conversations = await client.conversations.list({ types: type });
-            for (const channel of conversations.channels || []) {
-                const group: SchemaSocialChatGroup = this.buildChatGroup(channel);
-                channelList.push(group);
-            }
-        }
-
-        return channelList;
+    // Fetch messages from Slack API
+    const response = await apiClient.conversations.history(query);
+
+    const latestResult = await this.buildResults(
+      group.sourceId!,
+      response,
+      currentRange.endId
+    );
+
+    items = latestResult.items;
+
+    if (items.length) {
+      rangeTracker.completedRange(
+        {
+          startId: items[0].sourceId,
+          endId: response.response_metadata?.next_cursor,
+        },
+        latestResult.breakHit === SyncItemsBreak.ID
+      );
+    } else {
+      rangeTracker.completedRange(
+        { startId: undefined, endId: undefined },
+        false
+      );
     }
 
-    private buildChatGroup(channel: any): SchemaSocialChatGroup {
-        return {
-            _id: this.buildItemId(channel.id),
-            name: channel.name || channel.user,
-            sourceAccountId: this.provider.getAccountId(),
-            sourceApplication: this.getProviderApplicationUrl(),
-            sourceId: channel.id,
-            schema: CONFIG.verida.schemas.CHAT_GROUP,
-            sourceData: channel,
-            insertedAt: new Date().toISOString(),
-        };
-    }
+    // Back fill
 
-    public async _sync(api: any, syncPosition: SyncHandlerPosition): Promise {
-        try {
-            const apiClient = this.getSlackClient();
-            const groupList = await this.buildChatGroupList();  // Fetch chat groups
-
-            const groupDs = await this.provider.getDatastore(CONFIG.verida.schemas.CHAT_GROUP);
-            const groupDbItems = await groupDs.getMany({
-                sourceAccountId: this.provider.getAccountId(),
-            });
-            
-            const mergedGroupList = this.mergeGroupLists(groupList, groupDbItems);  // Merge new and existing groups
-
-            let totalMessages = 0;
-            let chatHistory: SchemaSocialChatMessage[] = [];
-
-            // Determine the current group position
-            const groupPosition = SlackHelpers.getGroupPositionIndex(mergedGroupList, syncPosition.thisRef);
-            const groupCount = mergedGroupList.length;
-
-            // Iterate over each group
-            for (let i = 0; i < Math.min(groupCount, this.config.groupLimit); i++) {
-                const groupIndex = (groupPosition + i) % groupCount;  // Rotate through groups
-                const group = mergedGroupList[groupIndex];
-
-                let rangeTracker = new ItemsRangeTracker(group.syncData);  // Track items for each group
-                const fetchedMessages = await this.fetchAndTrackMessages(group, rangeTracker, apiClient);
-
-                // Concatenate fetched messages
-                chatHistory = chatHistory.concat(fetchedMessages);
-
-                // Update the group sync data
-                group.syncData = rangeTracker.export();
-            }
-
-            // Update sync position and status
-            this.updateSyncPosition(syncPosition, totalMessages, groupCount);
-
-            // Return the sync response
-            return {
-                results: mergedGroupList.concat(chatHistory),
-                position: syncPosition,
-            };
-        } catch (err: any) {
-            console.error(err);
-            throw err;
-        }
-    }
-
-    private async fetchAndTrackMessages(
-        group: SchemaSocialChatGroup,
-        rangeTracker: ItemsRangeTracker,
-        apiClient: WebClient
-    ): Promise {
-        if (!group || !group.sourceId) {
-            throw new Error("Invalid group or missing group sourceId");
-        }
-
-        let items: SchemaSocialChatMessage[] = [];
-        let currentRange = rangeTracker.nextRange();
-
-        // Loop to fetch messages in batches
-        while (true) {
-            const messages = await this.fetchMessageRange(group, currentRange, apiClient);
-            if (!messages.length) break;
-
-            items = items.concat(messages);
-
-            if (items.length >= this.config.messagesPerGroupLimit) {
-                rangeTracker.completedRange({
-                    startId: messages[0].sourceId,
-                    endId: messages[messages.length - 1].sourceId,
-                }, false);
-                break;
-            }
-
-            rangeTracker.completedRange({
-                startId: messages[0].sourceId,
-                endId: messages[messages.length - 1].sourceId,
-            }, false);
-
-            currentRange = rangeTracker.nextRange();
-        }
-
-        return items;
-    }
+    currentRange = rangeTracker.nextRange();
 
-    private async fetchMessageRange(
-        chatGroup: SchemaSocialChatGroup,
-        range: ItemsRange,
-        apiClient: WebClient
-    ): Promise {
-        const messages: SchemaSocialChatMessage[] = [];
-        const response = await apiClient.conversations.history({
-            channel: chatGroup.sourceId!,
-            limit: this.config.messagesPerGroupLimit,
-            oldest: range.startId,
-            latest: range.endId,
-        });
-
-        for (const message of response.messages || []) {
-            if (!message.user) continue;
-
-            const user = await SlackHelpers.getUserInfo(this.connection.accessToken, message.user);
-            const chatMessage = this.buildChatMessage(message, chatGroup, user);
-            messages.push(chatMessage);
-        }
-
-        return messages;
+    if (
+      items.length != this.config.messagesPerGroupLimit &&
+      currentRange.startId
+    ) {
+      const query: ConversationsHistoryArguments = {
+        channel: group.sourceId!,
+        limit: this.config.messagesPerGroupLimit - items.length,
+        cursor: currentRange.startId,
+      };
+
+      const backfillResponse = await apiClient.conversations.history(query);
+
+      const backfillResult = await this.buildResults(
+        group.sourceId!,
+        backfillResponse,
+        currentRange.endId
+      );
+
+      items = items.concat(backfillResult.items);
+      if (backfillResult.items.length) {
+        rangeTracker.completedRange({
+          startId: backfillResult.items[0].sourceId,
+          endId: response.response_metadata?.next_cursor
+        }, backfillResult.breakHit == SyncItemsBreak.ID)
+      } else {
+        rangeTracker.completedRange({
+          startId: undefined,
+          endId: undefined
+        }, backfillResult.breakHit == SyncItemsBreak.ID)
+      }
     }
-
-    private buildChatMessage(
-        message: any,
-        chatGroup: SchemaSocialChatGroup,
-        user: any
-    ): SchemaSocialChatMessage {
-        return {
-            _id: this.buildItemId(message.ts),
-            groupId: chatGroup._id,
-            groupName: chatGroup.name,
-            messageText: message.text,
-            fromHandle: user.profile.email ?? "Unknown",
-            sourceAccountId: this.provider.getAccountId(),
-            sourceApplication: this.getProviderApplicationUrl(),
-            sourceId: message.ts,
-            sourceData: message,
-            insertedAt: new Date(parseFloat(message.ts) * 1000).toISOString(),
-            sentAt: new Date(parseFloat(message.ts) * 1000).toISOString(),
-            type: message.user === this.connection.profile.id
-                ? SchemaChatMessageType.SEND
-                : SchemaChatMessageType.RECEIVE,
-            fromId: message.user ?? "Unknown",
-            name: message.text.substring(0, 30),
+    return items;
+  }
+
+  private async buildResults(
+    groupId: string,
+    response: ConversationsHistoryResponse,
+    breakId: string
+  ): Promise {
+    const results: SchemaSocialChatMessage[] = [];
+    let breakHit: SyncItemsBreak;
+
+    for (const message of response.messages || []) {
+      const messageId = message.ts || "";
+
+      // Break if the message ID matches breakId
+      if (messageId === breakId) {
+        const logEvent: SyncProviderLogEvent = {
+          level: SyncProviderLogLevel.DEBUG,
+          message: `Break ID hit (${breakId}) in group (${groupId})`,
         };
-    }
+        this.emit("log", logEvent);
+        breakHit = SyncItemsBreak.ID;
+        break;
+      }
 
-    private updateSyncPosition(
-        syncPosition: SyncHandlerPosition,
-        totalMessages: number,
-        groupCount: number
-    ) {
-        if (totalMessages === 0) {
-            syncPosition.status = SyncHandlerStatus.ENABLED;
-            syncPosition.syncMessage = "No new messages found.";
-        } else if (totalMessages < this.config.messageBatchSize) {
-            syncPosition.syncMessage = `Processed ${totalMessages} messages across ${groupCount} groups. Sync complete.`;
-            syncPosition.status = SyncHandlerStatus.ENABLED;
-        } else {
-            syncPosition.status = SyncHandlerStatus.SYNCING;
-            syncPosition.syncMessage = `Batch complete (${this.config.messageBatchSize}). More results pending.`;
-        }
+      const messageRecord = await this.buildResult(groupId, message);
+      results.push(messageRecord);
     }
 
-    private mergeGroupLists(
-        newGroups: SchemaSocialChatGroup[],
-        existingGroups: SchemaSocialChatGroup[]
-    ): SchemaSocialChatGroup[] {
-        return newGroups.map((group) => {
-            const existingGroup = existingGroups.find(g => g.sourceId === group.sourceId);
-            return existingGroup ? _.merge({}, existingGroup, group) : group;
-        });
-    }
+    return {
+      items: results,
+      breakHit,
+    };
+  }
+
+  private async buildResult(
+    groupId: string,
+    message: MessageElement
+  ): Promise {
+    const user = await SlackHelpers.getUserInfo(
+      this.connection.accessToken,
+      message.user
+    );
+
+    return {
+      _id: this.buildItemId(message.ts),
+      groupId: groupId,
+      groupName: groupId,
+      messageText: message.text,
+      fromHandle: user.profile.email ?? "Unknown",
+      sourceAccountId: this.provider.getAccountId(),
+      sourceApplication: this.getProviderApplicationUrl(),
+      sourceId: message.ts,
+      sourceData: message,
+      insertedAt: new Date(parseFloat(message.ts) * 1000).toISOString(),
+      sentAt: new Date(parseFloat(message.ts) * 1000).toISOString(),
+      type:
+        message.user === this.connection.profile.id
+          ? SchemaChatMessageType.SEND
+          : SchemaChatMessageType.RECEIVE,
+      fromId: message.user ?? "Unknown",
+      name: message.text.substring(0, 30),
+    };
+  }
+
+  private updateSyncPosition(
+    syncPosition: SyncHandlerPosition,
+    totalMessages: number,
+    groupCount: number
+  ) {
+    syncPosition.status = SyncHandlerStatus.SYNCING;
+    syncPosition.syncMessage = `Batch complete (${totalMessages}) across (${groupCount} groups)`;
+  }
+
+  private mergeGroupLists(
+    newGroups: SchemaSocialChatGroup[],
+    existingGroups: SchemaSocialChatGroup[]
+  ): SchemaSocialChatGroup[] {
+    return newGroups.map((group) => {
+      const existingGroup = existingGroups.find(
+        (g) => g.sourceId === group.sourceId
+      );
+      return existingGroup ? _.merge({}, existingGroup, group) : group;
+    });
+  }
 }

From 5475f3a4d3c30d0059c3fcb8c2069cd1914080a3 Mon Sep 17 00:00:00 2001
From: chime3 
Date: Wed, 23 Oct 2024 15:03:40 -0700
Subject: [PATCH 161/182] docs: updated slack read me

---
 src/providers/slack/README.md | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/src/providers/slack/README.md b/src/providers/slack/README.md
index 6acf005f..142be6c0 100644
--- a/src/providers/slack/README.md
+++ b/src/providers/slack/README.md
@@ -13,6 +13,10 @@
 Slack workspaces might have many public channels and spam messages that are less relevant to users' privacy, so **DM/Private** groups have priority.
 
 Slack `ItemsRangeTracker` is based on timestamps, which are used as the IDs of chat message records due to the nature of its API.
+
+Slack supports two types of pagination: timeline-based and cursor-based
+
+### Timeline Based
 ```
 const response = await apiClient.conversations.history({
      channel,
@@ -21,3 +25,14 @@ const response = await apiClient.conversations.history({
      latest            // end timestamp
 });
 ```
+
+### Cursor Based
+```
+const response = await apiClient.conversations.history({
+     channel,
+     limit,
+     cursor,           // next_cursor from response_metadata
+});
+```
+
+We use Cusor-based pagination here which is equal to 'pageToken` from Google to provide consistency.
\ No newline at end of file

From a7063e857a329c496e2593bf1ff90fa241f512fa Mon Sep 17 00:00:00 2001
From: chime3 
Date: Wed, 23 Oct 2024 16:33:07 -0700
Subject: [PATCH 162/182] fix: update db syncdata

---
 src/providers/google/calendar-event.ts | 112 ++++++++++++-------------
 1 file changed, 55 insertions(+), 57 deletions(-)

diff --git a/src/providers/google/calendar-event.ts b/src/providers/google/calendar-event.ts
index 31f47d08..d0a43495 100644
--- a/src/providers/google/calendar-event.ts
+++ b/src/providers/google/calendar-event.ts
@@ -175,23 +175,23 @@ export default class CalendarEventHandler extends GoogleHandler {
       let eventHistory: SchemaEvent[] = [];
 
       // Iterate over each calendar
-      for (const calendar of calendarList) {
+      for (let i = 0; i < calendarList.length; i++) {
         // Use a separate ItemsRangeTracker for each calendar
-        let rangeTracker = new ItemsRangeTracker(calendar.syncData);
+        let rangeTracker = new ItemsRangeTracker(calendarList[i].syncData);
 
         const fetchedEvents = await this.fetchAndTrackEvents(
-          calendar,
+          calendarList[i],
           rangeTracker,
           apiClient
         );
 
         // Concatenate the fetched events to the total event history
         eventHistory = eventHistory.concat(fetchedEvents);
+       
         totalEvents += fetchedEvents.length;
 
         // Update the calendar's sync data with the latest rangeTracker state
-        calendar.syncData = rangeTracker.export();
-
+        calendarList[i].syncData = rangeTracker.export();
       }
 
       // Finalize sync position and status based on event count
@@ -214,60 +214,60 @@ export default class CalendarEventHandler extends GoogleHandler {
   public buildResult(calendarId: string, event: calendar_v3.Schema$Event): SchemaEvent {
     const eventId = event.id || "";
 
-      const start: DateTimeInfo = {
-        dateTime: event.start?.dateTime || `${event.start?.date}T00:00:00.000Z`
-      };
-      const end: DateTimeInfo = {
-        dateTime: event.end?.dateTime || `${event.end?.date}T00:00:00.000Z`
-      };
+    const start: DateTimeInfo = {
+      dateTime: event.start?.dateTime || `${event.start?.date}T00:00:00.000Z`
+    };
+    const end: DateTimeInfo = {
+      dateTime: event.end?.dateTime || `${event.end?.date}T00:00:00.000Z`
+    };
 
-      // Check for a break based on timestamp
-      const updatedTime = event.updated ? new Date(event.updated).toISOString() : new Date().toISOString();
+    // Check for a break based on timestamp
+    const updatedTime = event.updated ? new Date(event.updated).toISOString() : new Date().toISOString();
 
-      start.timeZone = CalendarHelpers.getUTCOffsetTimezone(event.start?.timeZone);
-      end.timeZone = CalendarHelpers.getUTCOffsetTimezone(event.end?.timeZone);
+    start.timeZone = CalendarHelpers.getUTCOffsetTimezone(event.start?.timeZone);
+    end.timeZone = CalendarHelpers.getUTCOffsetTimezone(event.end?.timeZone);
 
-      const creator: Person = {
-        email: event.creator?.email,
-        displayName: event.creator?.displayName
-      };
+    const creator: Person = {
+      email: event.creator?.email,
+      displayName: event.creator?.displayName
+    };
 
-      const organizer: Person = {
-        email: event.organizer?.email,
-        displayName: event.organizer?.displayName
-      };
+    const organizer: Person = {
+      email: event.organizer?.email,
+      displayName: event.organizer?.displayName
+    };
 
-      let attendees: Person[] = [];
-      if (event.attendees) {
-        attendees = event.attendees.filter(attendee => attendee.email) as Person[];
-      }
+    let attendees: Person[] = [];
+    if (event.attendees) {
+      attendees = event.attendees.filter(attendee => attendee.email) as Person[];
+    }
 
-      const attachments: CalendarAttachment[] = event.attachments as CalendarAttachment[];
-
-      const eventRecord: SchemaEvent = {
-        _id: this.buildItemId(eventId),
-        name: event.summary ?? "Unknown",
-        sourceAccountId: this.provider.getAccountId(),
-        sourceData: event,
-        sourceApplication: this.getProviderApplicationUrl(),
-        sourceId: eventId,
-        schema: CONFIG.verida.schemas.EVENT,
-        uri: event.htmlLink,
-        calendarId: calendarId,
-        start,
-        end,
-        creator,
-        organizer,
-        location: event.location,
-        description: event.description,
-        status: event.status,
-        conferenceData: event.conferenceData,
-        attendees,
-        attachments,
-        insertedAt: updatedTime
-      }
+    const attachments: CalendarAttachment[] = event.attachments as CalendarAttachment[];
+
+    const eventRecord: SchemaEvent = {
+      _id: this.buildItemId(eventId),
+      name: event.summary ?? "Unknown",
+      sourceAccountId: this.provider.getAccountId(),
+      sourceData: event,
+      sourceApplication: this.getProviderApplicationUrl(),
+      sourceId: eventId,
+      schema: CONFIG.verida.schemas.EVENT,
+      uri: event.htmlLink,
+      calendarId: calendarId,
+      start,
+      end,
+      creator,
+      organizer,
+      location: event.location,
+      description: event.description,
+      status: event.status,
+      conferenceData: event.conferenceData,
+      attendees,
+      attachments,
+      insertedAt: updatedTime
+    }
 
-      return eventRecord
+    return eventRecord
   }
 
   private async buildResults(
@@ -316,8 +316,7 @@ export default class CalendarEventHandler extends GoogleHandler {
     let query: calendar_v3.Params$Resource$Events$List = {
       calendarId: calendar.sourceId,
       maxResults: this.config.eventBatchSize,
-      singleEvents: true,
-      orderBy: "updated"
+      singleEvents: true
     };
 
     if (currentRange.startId) {
@@ -342,7 +341,7 @@ export default class CalendarEventHandler extends GoogleHandler {
           startId: items[0].sourceId,
           endId: response.data?.nextPageToken
         },
-        latestResult.breakHit === SyncItemsBreak.ID
+        latestResult.breakHit == SyncItemsBreak.ID
       );
     } else {
       rangeTracker.completedRange({ startId: undefined, endId: undefined }, false);
@@ -356,7 +355,6 @@ export default class CalendarEventHandler extends GoogleHandler {
         maxResults: this.config.eventBatchSize - items.length,
         pageToken: currentRange.startId,
         singleEvents: true,
-        orderBy: "updated"
       };
 
       const backfillResponse = await apiClient.events.list(query);
@@ -372,7 +370,7 @@ export default class CalendarEventHandler extends GoogleHandler {
       if (backfillResult.items.length) {
         rangeTracker.completedRange({
           startId: backfillResult.items[0].sourceId,
-          endId: response.data?.nextPageToken
+          endId: backfillResponse.data?.nextPageToken
         }, backfillResult.breakHit == SyncItemsBreak.ID)
       } else {
         rangeTracker.completedRange({

From 40dd6a1aedd9a1f70c35536cf930ebd59fb49bc5 Mon Sep 17 00:00:00 2001
From: chime3 
Date: Wed, 23 Oct 2024 16:59:26 -0700
Subject: [PATCH 163/182] fix: removed grouplimit

---
 src/providers/slack/chat-message.ts | 4 ++--
 src/serverconfig.example.json       | 1 -
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/providers/slack/chat-message.ts b/src/providers/slack/chat-message.ts
index 4b2b5bb8..a6c9d55e 100644
--- a/src/providers/slack/chat-message.ts
+++ b/src/providers/slack/chat-message.ts
@@ -198,7 +198,7 @@ export default class SlackChatMessageHandler extends BaseSyncHandler {
           startId: items[0].sourceId,
           endId: response.response_metadata?.next_cursor,
         },
-        latestResult.breakHit === SyncItemsBreak.ID
+        latestResult.breakHit == SyncItemsBreak.ID
       );
     } else {
       rangeTracker.completedRange(
@@ -233,7 +233,7 @@ export default class SlackChatMessageHandler extends BaseSyncHandler {
       if (backfillResult.items.length) {
         rangeTracker.completedRange({
           startId: backfillResult.items[0].sourceId,
-          endId: response.response_metadata?.next_cursor
+          endId: backfillResponse.response_metadata?.next_cursor
         }, backfillResult.breakHit == SyncItemsBreak.ID)
       } else {
         rangeTracker.completedRange({
diff --git a/src/serverconfig.example.json b/src/serverconfig.example.json
index 66ffa679..c88e5741 100644
--- a/src/serverconfig.example.json
+++ b/src/serverconfig.example.json
@@ -121,7 +121,6 @@
             "clientSecret": "",
             "stateSecret": "",
             "maxSyncLoops": 1,
-            "groupLimit": 20,
             "messagesPerGroupLimit": 50            
         }
     },

From 061487ff71c1300d3d818e75e77db8609a4a2cbe Mon Sep 17 00:00:00 2001
From: chime3 
Date: Wed, 23 Oct 2024 19:15:41 -0700
Subject: [PATCH 164/182] fix: removed archived channels

---
 src/providers/slack/chat-message.ts | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/src/providers/slack/chat-message.ts b/src/providers/slack/chat-message.ts
index a6c9d55e..1c7ac63c 100644
--- a/src/providers/slack/chat-message.ts
+++ b/src/providers/slack/chat-message.ts
@@ -77,11 +77,14 @@ export default class SlackChatMessageHandler extends BaseSyncHandler {
     const client = this.getSlackClient();
     let channelList: SchemaSocialChatGroup[] = [];
     const types = ["im", "private_channel", "public_channel"];
-
+    
     // Loop through each type of channel (DM, private, public)
     for (const type of types) {
       const conversations = await client.conversations.list({ types: type });
       for (const channel of conversations.channels || []) {
+        // Skip archived channels
+        if(channel?.is_archived) continue;
+        
         const group: SchemaSocialChatGroup = this.buildChatGroup(channel);
         channelList.push(group);
       }
@@ -254,6 +257,9 @@ export default class SlackChatMessageHandler extends BaseSyncHandler {
     let breakHit: SyncItemsBreak;
 
     for (const message of response.messages || []) {
+      // skip if bot message
+      if (message.subtype === 'bot_message') continue;
+
       const messageId = message.ts || "";
 
       // Break if the message ID matches breakId
@@ -281,6 +287,7 @@ export default class SlackChatMessageHandler extends BaseSyncHandler {
     groupId: string,
     message: MessageElement
   ): Promise {
+
     const user = await SlackHelpers.getUserInfo(
       this.connection.accessToken,
       message.user
@@ -289,7 +296,6 @@ export default class SlackChatMessageHandler extends BaseSyncHandler {
     return {
       _id: this.buildItemId(message.ts),
       groupId: groupId,
-      groupName: groupId,
       messageText: message.text,
       fromHandle: user.profile.email ?? "Unknown",
       sourceAccountId: this.provider.getAccountId(),

From 51b7a41ac5e2566e093612465948275fa721cde2 Mon Sep 17 00:00:00 2001
From: chime3 
Date: Wed, 23 Oct 2024 19:51:10 -0700
Subject: [PATCH 165/182] feat: updated slack unit tests

---
 tests/providers/slack/chat-message.ts | 99 +++++++++++++++++++--------
 1 file changed, 70 insertions(+), 29 deletions(-)

diff --git a/tests/providers/slack/chat-message.ts b/tests/providers/slack/chat-message.ts
index d0ac41bf..7794e82f 100644
--- a/tests/providers/slack/chat-message.ts
+++ b/tests/providers/slack/chat-message.ts
@@ -1,5 +1,7 @@
 const assert = require("assert");
+import CONFIG from "../../../src/config";
 import {
+  BaseProviderConfig,
   Connection,
   SyncHandlerPosition,
   SyncHandlerStatus
@@ -10,34 +12,35 @@ import CommonUtils, { NetworkInstance } from "../../common.utils";
 import SlackChatMessageHandler from "../../../src/providers/slack/chat-message";
 import BaseProvider from "../../../src/providers/BaseProvider";
 import { CommonTests, GenericTestConfig } from "../../common.tests";
+import { SchemaSocialChatGroup, SchemaSocialChatMessage, SchemaRecord } from "../../../src/schemas";
 import { SlackHandlerConfig } from "../../../src/providers/slack/interfaces";
-import { SchemaSocialChatGroup, SchemaSocialChatMessage } from "../../../src/schemas";
+import { SlackHelpers } from "../../../src/providers/slack/helpers";
 
+// Define the provider ID
 const providerId = "slack";
 let network: NetworkInstance;
 let connection: Connection;
 let provider: BaseProvider;
 let handlerName = "chat-message";
 let testConfig: GenericTestConfig;
-let providerConfig: Omit = {
-  maxSyncLoops: 1,
-  groupLimit: 2,
-  messageMaxAgeDays: 7,
-  messageBatchSize: 20,
-  messagesPerGroupLimit: 10,
-  maxGroupSize: 100,
-  useDbPos: false
+
+// Configure provider and handler without certain attributes
+let providerConfig: Omit = {};
+let handlerConfig: SlackHandlerConfig = {
+  messagesPerGroupLimit: 3
 };
 
-// Check if it sync channels and conversation
-describe(`${providerId} chat tests`, function () {
+// Test suite for Slack Chat Message syncing
+describe(`${providerId} chat message tests`, function () {
   this.timeout(100000);
 
+  // Before all tests, set up the network, connection, and provider
   this.beforeAll(async function () {
     network = await CommonUtils.getNetwork();
     connection = await CommonUtils.getConnection(providerId);
     provider = Providers(providerId, network.context, connection);
 
+    // Configure test settings
     testConfig = {
       idPrefix: `${provider.getProviderId()}-${connection.profile.id}`,
       timeOrderAttribute: "insertedAt",
@@ -45,9 +48,11 @@ describe(`${providerId} chat tests`, function () {
     };
   });
 
+  // Test fetching data for Slack Chat
   describe(`Fetch ${providerId} data`, () => {
 
     it(`Can pass basic tests: ${handlerName}`, async () => {
+      // Build the necessary test objects
       const { api, handler, provider } = await CommonTests.buildTestObjects(
         providerId,
         SlackChatMessageHandler,
@@ -55,7 +60,11 @@ describe(`${providerId} chat tests`, function () {
         connection
       );
 
+      // Set the handler configuration
+      handler.setConfig(handlerConfig);
+
       try {
+        // Set up initial sync position
         const syncPosition: SyncHandlerPosition = {
           _id: `${providerId}-${handlerName}`,
           providerId,
@@ -64,36 +73,68 @@ describe(`${providerId} chat tests`, function () {
           status: SyncHandlerStatus.ENABLED,
         };
 
-        // Batch 1
+        // Start the sync process
         const response = await handler._sync(api, syncPosition);
+        const results = response.results;
+
+        // Extract chat groups and messages from the results
+        const chatGroups = results.filter(result => result.schema === CONFIG.verida.schemas.CHAT_GROUP);
+        const chatMessages = results.filter(result => result.schema === CONFIG.verida.schemas.CHAT_MESSAGE);
+
+        // Ensure results are returned
+        assert.ok(results && results.length, "Have results returned");
+
+        // Check IDs in the returned items
+        CommonTests.checkItem(results[0], handler, provider);
+
+        // Verify sync status is active
+        assert.equal(
+          SyncHandlerStatus.SYNCING,
+          response.position.status,
+          "Sync is active"
+        );
 
-        // Make sure group and message limit were respected
-        let groupMessages: Record = {};
-        let groups: SchemaSocialChatGroup[] = [];
-        for (const result of (response.results)) {
-          if (result.groupId) {
-            if (!groupMessages[result.groupId]) {
-              groupMessages[result.groupId] = [];
-            }
+        // Ensure the message batch per group works
+        const firstGroupId = chatMessages[0].groupId;
+        const firstGroupMessages = chatMessages.filter(msg => msg.groupId === firstGroupId);
+        assert.equal(firstGroupMessages.length, handlerConfig.messagesPerGroupLimit, "Processed correct number of messages per group");
 
-            groupMessages[result.groupId].push(result);
-          } else {
-            groups.push(result);
-          }
-        }
+         /**
+          * Start the second sync batch process
+          */
+        const secondBatchResponse = await handler._sync(api, response.position);
+        const secondBatchResults = secondBatchResponse.results;
 
-        // Ensure results are returned before performing assertions
-        assert(response.results.length > 0, "Results are returned");
+        // Extract chat groups and messages from the second batch results
+        const secondBatchChatGroups = secondBatchResults.filter(result => result.schema === CONFIG.verida.schemas.CHAT_GROUP);
+        const secondBatchChatMessages = secondBatchResults.filter(result => result.schema === CONFIG.verida.schemas.CHAT_MESSAGE);
+
+        // Ensure second batch results are returned
+        assert.ok(secondBatchResults && secondBatchResults.length, "Have second batch results returned");
+
+        // Check IDs in the returned items for the second batch
+        CommonTests.checkItem(secondBatchResults[0], handler, provider);
+
+        // Verify sync status is still active for the second batch
+        assert.equal(
+          SyncHandlerStatus.SYNCING,
+          secondBatchResponse.position.status,
+          "Sync is still active after second batch"
+        );
+
+        // Check if synced every chat group correctly
+        const syncedGroup = (secondBatchChatGroups.filter(group => group.sourceId === secondBatchChatMessages[0].groupId))[0];
+        assert.ok(syncedGroup.syncData, "Have a sync range per chat group.");
 
       } catch (err) {
-        // ensure provider closes even if there's an error
+        // Ensure provider closes even if an error occurs
         await provider.close();
-
         throw err;
       }
     });
   });
 
+  // After all tests, close the network context
   this.afterAll(async function () {
     const { context } = await CommonUtils.getNetwork();
     await context.close();

From ed7a9374d87b18ce69e1435b67a52f9ef890d69e Mon Sep 17 00:00:00 2001
From: chime3 
Date: Wed, 23 Oct 2024 19:54:06 -0700
Subject: [PATCH 166/182] fix: removed total batch size config

---
 src/providers/slack/interfaces.ts | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/providers/slack/interfaces.ts b/src/providers/slack/interfaces.ts
index b7ebce86..2910ba14 100644
--- a/src/providers/slack/interfaces.ts
+++ b/src/providers/slack/interfaces.ts
@@ -1,8 +1,6 @@
 import { BaseHandlerConfig, BaseProviderConfig } from "../../interfaces";
 
 export interface SlackHandlerConfig extends BaseHandlerConfig {
-  // Maximum number of messages to process in a given batch
-  messageBatchSize: number
   // Maximum number of messages to process in a group
   messagesPerGroupLimit: number
 }

From bc76c4d00a188640bc346cbdc110e8b83ad5346f Mon Sep 17 00:00:00 2001
From: chime3 
Date: Wed, 23 Oct 2024 20:23:19 -0700
Subject: [PATCH 167/182] fix: added chat message schem to slack record

---
 src/providers/slack/chat-message.ts | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/providers/slack/chat-message.ts b/src/providers/slack/chat-message.ts
index 1c7ac63c..509e5d22 100644
--- a/src/providers/slack/chat-message.ts
+++ b/src/providers/slack/chat-message.ts
@@ -301,6 +301,7 @@ export default class SlackChatMessageHandler extends BaseSyncHandler {
       sourceAccountId: this.provider.getAccountId(),
       sourceApplication: this.getProviderApplicationUrl(),
       sourceId: message.ts,
+      schema: CONFIG.verida.schemas.CHAT_MESSAGE,
       sourceData: message,
       insertedAt: new Date(parseFloat(message.ts) * 1000).toISOString(),
       sentAt: new Date(parseFloat(message.ts) * 1000).toISOString(),

From 5e086480e716859213cf06f5731b94580a77e0e9 Mon Sep 17 00:00:00 2001
From: chime3 
Date: Wed, 23 Oct 2024 20:40:47 -0700
Subject: [PATCH 168/182] feat: added bulk user info to improve performance

---
 src/providers/slack/chat-message.ts | 84 ++++++++++++-----------------
 src/providers/slack/helpers.ts      | 16 ++++++
 2 files changed, 50 insertions(+), 50 deletions(-)

diff --git a/src/providers/slack/chat-message.ts b/src/providers/slack/chat-message.ts
index 509e5d22..459322a8 100644
--- a/src/providers/slack/chat-message.ts
+++ b/src/providers/slack/chat-message.ts
@@ -1,4 +1,8 @@
-import { ConversationsHistoryArguments, ConversationsHistoryResponse, WebClient } from "@slack/web-api";
+import {
+  ConversationsHistoryArguments,
+  ConversationsHistoryResponse,
+  WebClient,
+} from "@slack/web-api";
 import CONFIG from "../../config";
 import {
   SyncResponse,
@@ -77,14 +81,11 @@ export default class SlackChatMessageHandler extends BaseSyncHandler {
     const client = this.getSlackClient();
     let channelList: SchemaSocialChatGroup[] = [];
     const types = ["im", "private_channel", "public_channel"];
-    
-    // Loop through each type of channel (DM, private, public)
+
     for (const type of types) {
       const conversations = await client.conversations.list({ types: type });
       for (const channel of conversations.channels || []) {
-        // Skip archived channels
-        if(channel?.is_archived) continue;
-        
+        if (channel?.is_archived) continue;
         const group: SchemaSocialChatGroup = this.buildChatGroup(channel);
         channelList.push(group);
       }
@@ -112,8 +113,7 @@ export default class SlackChatMessageHandler extends BaseSyncHandler {
   ): Promise {
     try {
       const apiClient = this.getSlackClient();
-      const groupList = await this.buildChatGroupList(); // Fetch chat groups
-
+      const groupList = await this.buildChatGroupList();
       const groupDs = await this.provider.getDatastore(
         CONFIG.verida.schemas.CHAT_GROUP
       );
@@ -121,15 +121,13 @@ export default class SlackChatMessageHandler extends BaseSyncHandler {
         sourceAccountId: this.provider.getAccountId(),
       });
 
-      const mergedGroupList = this.mergeGroupLists(groupList, groupDbItems); // Merge new and existing groups
-
+      const mergedGroupList = this.mergeGroupLists(groupList, groupDbItems);
       let totalMessages = 0;
       let chatHistory: SchemaSocialChatMessage[] = [];
 
-      // Iterate over each group
       for (let i = 0; i < mergedGroupList.length; i++) {
         const group = mergedGroupList[i];
-        let rangeTracker = new ItemsRangeTracker(group.syncData); // Track items for each group
+        let rangeTracker = new ItemsRangeTracker(group.syncData);
 
         const fetchedMessages = await this.fetchAndTrackMessages(
           group,
@@ -137,22 +135,14 @@ export default class SlackChatMessageHandler extends BaseSyncHandler {
           apiClient
         );
 
-        // Concatenate fetched messages
         chatHistory = chatHistory.concat(fetchedMessages);
         totalMessages += fetchedMessages.length;
 
-        // Update the group sync data in the mergedGroupList at the current index
         mergedGroupList[i].syncData = rangeTracker.export();
       }
 
-      // Update sync position and status
-      this.updateSyncPosition(
-        syncPosition,
-        totalMessages,
-        mergedGroupList.length
-      );
+      this.updateSyncPosition(syncPosition, totalMessages, mergedGroupList.length);
 
-      // Return the sync response
       return {
         results: mergedGroupList.concat(chatHistory),
         position: syncPosition,
@@ -184,9 +174,7 @@ export default class SlackChatMessageHandler extends BaseSyncHandler {
       query.cursor = currentRange.startId;
     }
 
-    // Fetch messages from Slack API
     const response = await apiClient.conversations.history(query);
-
     const latestResult = await this.buildResults(
       group.sourceId!,
       response,
@@ -210,14 +198,9 @@ export default class SlackChatMessageHandler extends BaseSyncHandler {
       );
     }
 
-    // Back fill
-
     currentRange = rangeTracker.nextRange();
 
-    if (
-      items.length != this.config.messagesPerGroupLimit &&
-      currentRange.startId
-    ) {
+    if (items.length != this.config.messagesPerGroupLimit && currentRange.startId) {
       const query: ConversationsHistoryArguments = {
         channel: group.sourceId!,
         limit: this.config.messagesPerGroupLimit - items.length,
@@ -225,7 +208,6 @@ export default class SlackChatMessageHandler extends BaseSyncHandler {
       };
 
       const backfillResponse = await apiClient.conversations.history(query);
-
       const backfillResult = await this.buildResults(
         group.sourceId!,
         backfillResponse,
@@ -236,13 +218,13 @@ export default class SlackChatMessageHandler extends BaseSyncHandler {
       if (backfillResult.items.length) {
         rangeTracker.completedRange({
           startId: backfillResult.items[0].sourceId,
-          endId: backfillResponse.response_metadata?.next_cursor
-        }, backfillResult.breakHit == SyncItemsBreak.ID)
+          endId: backfillResponse.response_metadata?.next_cursor,
+        }, backfillResult.breakHit == SyncItemsBreak.ID);
       } else {
         rangeTracker.completedRange({
           startId: undefined,
-          endId: undefined
-        }, backfillResult.breakHit == SyncItemsBreak.ID)
+          endId: undefined,
+        }, backfillResult.breakHit == SyncItemsBreak.ID);
       }
     }
     return items;
@@ -254,26 +236,33 @@ export default class SlackChatMessageHandler extends BaseSyncHandler {
     breakId: string
   ): Promise {
     const results: SchemaSocialChatMessage[] = [];
+    const userIds = new Set();
     let breakHit: SyncItemsBreak;
 
+    // Collect unique user IDs from messages
     for (const message of response.messages || []) {
-      // skip if bot message
       if (message.subtype === 'bot_message') continue;
+      userIds.add(message.user);
+    }
 
-      const messageId = message.ts || "";
+    // Fetch user info for all unique user IDs in parallel
+    const userInfoMap = await SlackHelpers.fetchUserInfoBulk(this.connection.accessToken, Array.from(userIds));
+
+    for (const message of response.messages || []) {
+      if (message.subtype === 'bot_message') continue;
 
-      // Break if the message ID matches breakId
+      const messageId = message.ts || "";
       if (messageId === breakId) {
-        const logEvent: SyncProviderLogEvent = {
+        this.emit("log", {
           level: SyncProviderLogLevel.DEBUG,
           message: `Break ID hit (${breakId}) in group (${groupId})`,
-        };
-        this.emit("log", logEvent);
+        });
         breakHit = SyncItemsBreak.ID;
         break;
       }
 
-      const messageRecord = await this.buildResult(groupId, message);
+      const user = userInfoMap[message.user];
+      const messageRecord = await this.buildResult(groupId, message, user);
       results.push(messageRecord);
     }
 
@@ -285,19 +274,14 @@ export default class SlackChatMessageHandler extends BaseSyncHandler {
 
   private async buildResult(
     groupId: string,
-    message: MessageElement
+    message: MessageElement,
+    user: any
   ): Promise {
-
-    const user = await SlackHelpers.getUserInfo(
-      this.connection.accessToken,
-      message.user
-    );
-
     return {
       _id: this.buildItemId(message.ts),
       groupId: groupId,
       messageText: message.text,
-      fromHandle: user.profile.email ?? "Unknown",
+      fromHandle: user.profile.email,
       sourceAccountId: this.provider.getAccountId(),
       sourceApplication: this.getProviderApplicationUrl(),
       sourceId: message.ts,
@@ -309,7 +293,7 @@ export default class SlackChatMessageHandler extends BaseSyncHandler {
         message.user === this.connection.profile.id
           ? SchemaChatMessageType.SEND
           : SchemaChatMessageType.RECEIVE,
-      fromId: message.user ?? "Unknown",
+      fromId: message.user,
       name: message.text.substring(0, 30),
     };
   }
diff --git a/src/providers/slack/helpers.ts b/src/providers/slack/helpers.ts
index 3d74ecf4..90dd4baa 100644
--- a/src/providers/slack/helpers.ts
+++ b/src/providers/slack/helpers.ts
@@ -24,6 +24,22 @@ export class SlackHelpers {
     }
   }
 
+  // Helper function to fetch user info for all unique user IDs in parallel
+  static async fetchUserInfoBulk(accessToken: string, userIds: string[]): Promise<{ [key: string]: any }> {
+    const promises = userIds.map((userId) =>
+      this.getUserInfo(accessToken, userId)
+    );
+    const results = await Promise.all(promises);
+
+    // Map user info by userId
+    const userInfoMap: { [key: string]: any } = {};
+    results.forEach((userInfo, index) => {
+      userInfoMap[userIds[index]] = userInfo;
+    });
+
+    return userInfoMap;
+  }
+
   static getGroupPositionIndex(
     groupList: SchemaSocialChatGroup[],
     groupId: string|undefined

From f7f75920396927ea987e7730f1089b5ae649eed3 Mon Sep 17 00:00:00 2001
From: chime3 
Date: Wed, 23 Oct 2024 20:57:54 -0700
Subject: [PATCH 169/182] feat: added a slack unit test case to check user info
 correct

---
 tests/providers/slack/chat-message.ts | 53 +++++++++++++++++++++++++++
 1 file changed, 53 insertions(+)

diff --git a/tests/providers/slack/chat-message.ts b/tests/providers/slack/chat-message.ts
index 7794e82f..cc0c2d00 100644
--- a/tests/providers/slack/chat-message.ts
+++ b/tests/providers/slack/chat-message.ts
@@ -132,6 +132,59 @@ describe(`${providerId} chat message tests`, function () {
         throw err;
       }
     });
+
+    it(`Should match email from User Info with message's fromHandle`, async () => {
+      try {
+        // Build the necessary test objects
+        const { api, handler, provider } = await CommonTests.buildTestObjects(
+          providerId,
+          SlackChatMessageHandler,
+          providerConfig,
+          connection
+        );
+
+        // Set the handler configuration
+        handler.setConfig(handlerConfig);
+
+        // Set up initial sync position
+        const syncPosition: SyncHandlerPosition = {
+          _id: `${providerId}-${handlerName}`,
+          providerId,
+          handlerId: handler.getId(),
+          accountId: provider.getAccountId(),
+          status: SyncHandlerStatus.ENABLED,
+        };
+
+        // Start the sync process
+        const response = await handler._sync(api, syncPosition);
+        const secondBatchResults = response.results;
+
+        // Extract chat messages from the second batch results
+        const chatMessages = (
+          secondBatchResults.filter(
+            (result) => result.schema === CONFIG.verida.schemas.CHAT_MESSAGE
+          )
+        );
+
+        // Check email comparison for the first message
+        const firstMessage = chatMessages[0];
+        const userInfo = await SlackHelpers.getUserInfo(
+          connection.accessToken,
+          firstMessage.fromId
+        );
+
+        // Compare email from fromHandle and userInfo
+        assert.equal(
+          firstMessage.fromHandle,
+          userInfo.profile.email,
+          "fromHandle email matches userInfo email"
+        );
+
+      } catch (err) {
+        await provider.close();
+        throw err;
+      }
+    });
   });
 
   // After all tests, close the network context

From cbc181861a65f4bc49c9636bcef92a67d9c3e876 Mon Sep 17 00:00:00 2001
From: Chris 
Date: Fri, 25 Oct 2024 10:49:51 +1030
Subject: [PATCH 170/182] Remvoe unused include

---
 src/providers/slack/chat-message.ts | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/providers/slack/chat-message.ts b/src/providers/slack/chat-message.ts
index 459322a8..680f4c51 100644
--- a/src/providers/slack/chat-message.ts
+++ b/src/providers/slack/chat-message.ts
@@ -12,7 +12,6 @@ import {
   SyncHandlerPosition,
   SyncItemsResult,
   SyncItemsBreak,
-  SyncProviderLogEvent,
   SyncProviderLogLevel,
 } from "../../interfaces";
 import {

From a14ce9168db8f3fed3f653c6973dd0009239b623 Mon Sep 17 00:00:00 2001
From: chime3 
Date: Sun, 27 Oct 2024 20:20:06 -0700
Subject: [PATCH 171/182] fix: used channelTypes config

---
 src/providers/slack/interfaces.ts | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/providers/slack/interfaces.ts b/src/providers/slack/interfaces.ts
index 2910ba14..5ec12d94 100644
--- a/src/providers/slack/interfaces.ts
+++ b/src/providers/slack/interfaces.ts
@@ -13,8 +13,8 @@ export interface SlackProviderConfig extends BaseProviderConfig {
 }
 
 export enum SlackChatGroupType {
-  CHANNEL = "channel", // Public channel
-  GROUP = "group", // Private channel
+  PUBLIC_CHANNEL = "public_channel", // Public channel
+  PRIVATE_CHANNEL = "private_channel", // Private channel
   IM = "im", // DM
   MPIM = "mpim" // Multi-person DM
 }
\ No newline at end of file

From 70a037e0c096a89e4819a2060ea8038de4bdc274 Mon Sep 17 00:00:00 2001
From: chime3 
Date: Sun, 27 Oct 2024 20:21:19 -0700
Subject: [PATCH 172/182] fix: use MAX_BATCH_SIZE

---
 src/providers/slack/chat-message.ts | 43 +++++++++++++++++------------
 1 file changed, 26 insertions(+), 17 deletions(-)

diff --git a/src/providers/slack/chat-message.ts b/src/providers/slack/chat-message.ts
index 680f4c51..8391657c 100644
--- a/src/providers/slack/chat-message.ts
+++ b/src/providers/slack/chat-message.ts
@@ -28,6 +28,9 @@ import { MessageElement } from "@slack/web-api/dist/types/response/Conversations
 
 const _ = require("lodash");
 
+// Slack recommends no more than 200, although max value 1000
+// See slack documentation: https://api.slack.com/methods/conversations.list
+const MAX_BATCH_SIZE = 200;
 export interface SyncChatItemsResult extends SyncItemsResult {
   items: SchemaSocialChatMessage[];
 }
@@ -58,14 +61,14 @@ export default class SlackChatMessageHandler extends BaseSyncHandler {
         label: "Channel types",
         type: ConnectionOptionType.ENUM_MULTI,
         enumOptions: [
-          { label: "Public Channel", value: SlackChatGroupType.CHANNEL },
-          { label: "Private Channel", value: SlackChatGroupType.GROUP },
+          { label: "Public Channel", value: SlackChatGroupType.PUBLIC_CHANNEL },
+          { label: "Private Channel", value: SlackChatGroupType.PRIVATE_CHANNEL },
           { label: "Direct Messages", value: SlackChatGroupType.IM },
         ],
         defaultValue: [
-          SlackChatGroupType.CHANNEL,
-          SlackChatGroupType.GROUP,
-          SlackChatGroupType.IM,
+          SlackChatGroupType.IM, // DM first
+          SlackChatGroupType.PRIVATE_CHANNEL,
+          SlackChatGroupType.PUBLIC_CHANNEL,
         ].join(","),
       },
     ];
@@ -79,21 +82,20 @@ export default class SlackChatMessageHandler extends BaseSyncHandler {
   protected async buildChatGroupList(): Promise {
     const client = this.getSlackClient();
     let channelList: SchemaSocialChatGroup[] = [];
-    const types = ["im", "private_channel", "public_channel"];
-
-    for (const type of types) {
-      const conversations = await client.conversations.list({ types: type });
-      for (const channel of conversations.channels || []) {
-        if (channel?.is_archived) continue;
-        const group: SchemaSocialChatGroup = this.buildChatGroup(channel);
-        channelList.push(group);
-      }
+
+    const conversations = await client.conversations.list({ types: this.config["channelTypes"].toString(), limit: MAX_BATCH_SIZE });
+
+    for (const channel of conversations.channels || []) {
+      if (channel?.is_archived) continue;
+      const group: SchemaSocialChatGroup = this.buildChatGroup(channel);
+      channelList.push(group);
     }
 
     return channelList;
   }
 
   private buildChatGroup(channel: any): SchemaSocialChatGroup {
+
     return {
       _id: this.buildItemId(channel.id),
       name: channel.name || channel.user,
@@ -102,7 +104,7 @@ export default class SlackChatMessageHandler extends BaseSyncHandler {
       sourceId: channel.id,
       schema: CONFIG.verida.schemas.CHAT_GROUP,
       sourceData: channel,
-      insertedAt: new Date().toISOString(),
+      insertedAt: new Date(channel.updated).toISOString(),
     };
   }
 
@@ -118,6 +120,8 @@ export default class SlackChatMessageHandler extends BaseSyncHandler {
       );
       const groupDbItems = await groupDs.getMany({
         sourceAccountId: this.provider.getAccountId(),
+      }, {
+        limit: MAX_BATCH_SIZE
       });
 
       const mergedGroupList = this.mergeGroupLists(groupList, groupDbItems);
@@ -302,8 +306,13 @@ export default class SlackChatMessageHandler extends BaseSyncHandler {
     totalMessages: number,
     groupCount: number
   ) {
-    syncPosition.status = SyncHandlerStatus.SYNCING;
-    syncPosition.syncMessage = `Batch complete (${totalMessages}) across (${groupCount} groups)`;
+    if (totalMessages) {
+      syncPosition.status = SyncHandlerStatus.SYNCING;
+      syncPosition.syncMessage = `Batch complete (${totalMessages}) across (${groupCount} groups)`;
+    } else {
+      syncPosition.status = SyncHandlerStatus.ENABLED
+      syncPosition.syncMessage = `Stopping. No results found.`
+    }
   }
 
   private mergeGroupLists(

From b5ca5ce3e9f61b30a2e31fa53c9e5b918d2405ac Mon Sep 17 00:00:00 2001
From: chime3 
Date: Sun, 27 Oct 2024 20:23:24 -0700
Subject: [PATCH 173/182] fix: removed schema record type field

---
 src/schemas.ts | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/schemas.ts b/src/schemas.ts
index 18106b27..a1f9e1d6 100644
--- a/src/schemas.ts
+++ b/src/schemas.ts
@@ -93,7 +93,6 @@ export interface SchemaFavourite extends SchemaRecord {
 export interface SchemaSocialChatGroup extends SchemaRecord {
     newestId?: string
     syncData?: string
-    type?: string
 }
 
 export enum SchemaChatMessageType {

From 45b16d65f877c79e4068aa2d15d2400ab7b219c9 Mon Sep 17 00:00:00 2001
From: Chris 
Date: Fri, 1 Nov 2024 15:05:13 +1030
Subject: [PATCH 174/182] Fix tests not working

---
 tests/providers/slack/chat-message.ts | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/tests/providers/slack/chat-message.ts b/tests/providers/slack/chat-message.ts
index cc0c2d00..686bc94a 100644
--- a/tests/providers/slack/chat-message.ts
+++ b/tests/providers/slack/chat-message.ts
@@ -13,7 +13,7 @@ import SlackChatMessageHandler from "../../../src/providers/slack/chat-message";
 import BaseProvider from "../../../src/providers/BaseProvider";
 import { CommonTests, GenericTestConfig } from "../../common.tests";
 import { SchemaSocialChatGroup, SchemaSocialChatMessage, SchemaRecord } from "../../../src/schemas";
-import { SlackHandlerConfig } from "../../../src/providers/slack/interfaces";
+import { SlackHandlerConfig, SlackChatGroupType } from "../../../src/providers/slack/interfaces";
 import { SlackHelpers } from "../../../src/providers/slack/helpers";
 
 // Define the provider ID
@@ -27,7 +27,12 @@ let testConfig: GenericTestConfig;
 // Configure provider and handler without certain attributes
 let providerConfig: Omit = {};
 let handlerConfig: SlackHandlerConfig = {
-  messagesPerGroupLimit: 3
+  messagesPerGroupLimit: 3,
+  channelTypes: [
+    SlackChatGroupType.IM.toString(), // DM first
+    SlackChatGroupType.PRIVATE_CHANNEL.toString(),
+    SlackChatGroupType.PUBLIC_CHANNEL.toString(),
+  ].join(',')
 };
 
 // Test suite for Slack Chat Message syncing

From 38daba7e29fa326926068cbe0a9675bbed2100b9 Mon Sep 17 00:00:00 2001
From: Chris 
Date: Fri, 1 Nov 2024 15:05:34 +1030
Subject: [PATCH 175/182] Add slack public notes

---
 src/providers/slack/NOTES.md | 12 ++++++++++++
 1 file changed, 12 insertions(+)
 create mode 100644 src/providers/slack/NOTES.md

diff --git a/src/providers/slack/NOTES.md b/src/providers/slack/NOTES.md
new file mode 100644
index 00000000..4b87e736
--- /dev/null
+++ b/src/providers/slack/NOTES.md
@@ -0,0 +1,12 @@
+
+# Setup
+
+Slack only permits users to extract their data by installing the Verida Data Connector application into the Slack workspace. You will require an administrator to approve this application installation before you can start to sync your data.
+
+# Sync
+
+The following data is syncronized:
+
+- DMs
+- Public channels you can access
+- Private channels you can access
\ No newline at end of file

From 13486f14fa475fc1787c91ad6695b8ba6a543873 Mon Sep 17 00:00:00 2001
From: chime3 
Date: Sun, 3 Nov 2024 19:57:59 -0700
Subject: [PATCH 176/182] feat: added max batch size config for slack

---
 src/providers/slack/interfaces.ts | 4 ++++
 src/serverconfig.example.json     | 1 +
 2 files changed, 5 insertions(+)

diff --git a/src/providers/slack/interfaces.ts b/src/providers/slack/interfaces.ts
index 5ec12d94..b5cbd958 100644
--- a/src/providers/slack/interfaces.ts
+++ b/src/providers/slack/interfaces.ts
@@ -1,6 +1,10 @@
 import { BaseHandlerConfig, BaseProviderConfig } from "../../interfaces";
 
 export interface SlackHandlerConfig extends BaseHandlerConfig {
+  // Max batch size to cover all chat groups
+  // Slack recommends no more than 200, although max value 1000
+  // See slack documentation: https://api.slack.com/methods/conversations.list
+  maxBatchSize: number  
   // Maximum number of messages to process in a group
   messagesPerGroupLimit: number
 }
diff --git a/src/serverconfig.example.json b/src/serverconfig.example.json
index c88e5741..084dad9a 100644
--- a/src/serverconfig.example.json
+++ b/src/serverconfig.example.json
@@ -121,6 +121,7 @@
             "clientSecret": "",
             "stateSecret": "",
             "maxSyncLoops": 1,
+            "maxBatchSize": 200,
             "messagesPerGroupLimit": 50            
         }
     },

From 4672ca80e89e71a4f5e01d05ca74ec26c19c73fd Mon Sep 17 00:00:00 2001
From: chime3 
Date: Sun, 3 Nov 2024 19:58:25 -0700
Subject: [PATCH 177/182] fix: rewrite for loop

---
 src/providers/slack/chat-message.ts | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/src/providers/slack/chat-message.ts b/src/providers/slack/chat-message.ts
index 8391657c..eeae0054 100644
--- a/src/providers/slack/chat-message.ts
+++ b/src/providers/slack/chat-message.ts
@@ -28,9 +28,6 @@ import { MessageElement } from "@slack/web-api/dist/types/response/Conversations
 
 const _ = require("lodash");
 
-// Slack recommends no more than 200, although max value 1000
-// See slack documentation: https://api.slack.com/methods/conversations.list
-const MAX_BATCH_SIZE = 200;
 export interface SyncChatItemsResult extends SyncItemsResult {
   items: SchemaSocialChatMessage[];
 }
@@ -83,7 +80,7 @@ export default class SlackChatMessageHandler extends BaseSyncHandler {
     const client = this.getSlackClient();
     let channelList: SchemaSocialChatGroup[] = [];
 
-    const conversations = await client.conversations.list({ types: this.config["channelTypes"].toString(), limit: MAX_BATCH_SIZE });
+    const conversations = await client.conversations.list({ types: this.config["channelTypes"].toString(), limit: this.config.maxBatchSize });
 
     for (const channel of conversations.channels || []) {
       if (channel?.is_archived) continue;
@@ -121,15 +118,15 @@ export default class SlackChatMessageHandler extends BaseSyncHandler {
       const groupDbItems = await groupDs.getMany({
         sourceAccountId: this.provider.getAccountId(),
       }, {
-        limit: MAX_BATCH_SIZE
+        limit: this.config.maxBatchSize
       });
 
       const mergedGroupList = this.mergeGroupLists(groupList, groupDbItems);
       let totalMessages = 0;
       let chatHistory: SchemaSocialChatMessage[] = [];
 
-      for (let i = 0; i < mergedGroupList.length; i++) {
-        const group = mergedGroupList[i];
+      for (const group of mergedGroupList) {
+        
         let rangeTracker = new ItemsRangeTracker(group.syncData);
 
         const fetchedMessages = await this.fetchAndTrackMessages(
@@ -141,7 +138,7 @@ export default class SlackChatMessageHandler extends BaseSyncHandler {
         chatHistory = chatHistory.concat(fetchedMessages);
         totalMessages += fetchedMessages.length;
 
-        mergedGroupList[i].syncData = rangeTracker.export();
+        group.syncData = rangeTracker.export();
       }
 
       this.updateSyncPosition(syncPosition, totalMessages, mergedGroupList.length);
@@ -168,6 +165,7 @@ export default class SlackChatMessageHandler extends BaseSyncHandler {
     let items: SchemaSocialChatMessage[] = [];
     let currentRange = rangeTracker.nextRange();
 
+    // Slack has limit of 1000, and if this.config.messagesPerGroupLimit is higher than 1000, will return maximum 1000 items
     let query: ConversationsHistoryArguments = {
       channel: group.sourceId!,
       limit: this.config.messagesPerGroupLimit,

From 24359e755b0061158909bfa3e815b5d871d2dbc0 Mon Sep 17 00:00:00 2001
From: chime3 
Date: Sun, 17 Nov 2024 19:45:38 -0700
Subject: [PATCH 178/182] feat: added channel types

---
 src/providers/slack/interfaces.ts | 1 +
 src/serverconfig.example.json     | 3 ++-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/providers/slack/interfaces.ts b/src/providers/slack/interfaces.ts
index b5cbd958..905b2624 100644
--- a/src/providers/slack/interfaces.ts
+++ b/src/providers/slack/interfaces.ts
@@ -7,6 +7,7 @@ export interface SlackHandlerConfig extends BaseHandlerConfig {
   maxBatchSize: number  
   // Maximum number of messages to process in a group
   messagesPerGroupLimit: number
+  channelTypes?: string
 }
 
 export interface SlackProviderConfig extends BaseProviderConfig {
diff --git a/src/serverconfig.example.json b/src/serverconfig.example.json
index 084dad9a..bb040fec 100644
--- a/src/serverconfig.example.json
+++ b/src/serverconfig.example.json
@@ -122,7 +122,8 @@
             "stateSecret": "",
             "maxSyncLoops": 1,
             "maxBatchSize": 200,
-            "messagesPerGroupLimit": 50            
+            "messagesPerGroupLimit": 50,
+            "channelTypes": ["im", "private_channel", "public_channel"]            
         }
     },
     "providerDefaults": {

From 34ad2d7f67836f1b16d0d7be84a3ab08c1487f25 Mon Sep 17 00:00:00 2001
From: chime3 
Date: Sun, 17 Nov 2024 19:46:13 -0700
Subject: [PATCH 179/182] feat: added test cases for timestamps comparison

---
 tests/providers/slack/chat-message.ts | 149 ++++++++++++++++++++++++--
 1 file changed, 142 insertions(+), 7 deletions(-)

diff --git a/tests/providers/slack/chat-message.ts b/tests/providers/slack/chat-message.ts
index 686bc94a..bda93222 100644
--- a/tests/providers/slack/chat-message.ts
+++ b/tests/providers/slack/chat-message.ts
@@ -32,7 +32,8 @@ let handlerConfig: SlackHandlerConfig = {
     SlackChatGroupType.IM.toString(), // DM first
     SlackChatGroupType.PRIVATE_CHANNEL.toString(),
     SlackChatGroupType.PUBLIC_CHANNEL.toString(),
-  ].join(',')
+  ].join(','),
+  maxBatchSize: 50
 };
 
 // Test suite for Slack Chat Message syncing
@@ -104,9 +105,9 @@ describe(`${providerId} chat message tests`, function () {
         const firstGroupMessages = chatMessages.filter(msg => msg.groupId === firstGroupId);
         assert.equal(firstGroupMessages.length, handlerConfig.messagesPerGroupLimit, "Processed correct number of messages per group");
 
-         /**
-          * Start the second sync batch process
-          */
+        /**
+         * Start the second sync batch process
+         */
         const secondBatchResponse = await handler._sync(api, response.position);
         const secondBatchResults = secondBatchResponse.results;
 
@@ -127,9 +128,15 @@ describe(`${providerId} chat message tests`, function () {
           "Sync is still active after second batch"
         );
 
-        // Check if synced every chat group correctly
-        const syncedGroup = (secondBatchChatGroups.filter(group => group.sourceId === secondBatchChatMessages[0].groupId))[0];
-        assert.ok(syncedGroup.syncData, "Have a sync range per chat group.");
+        // Check if every chat group with messages is synced correctly
+        const groupIdsWithMessages = new Set(secondBatchChatMessages.map((msg) => msg.groupId));
+
+        groupIdsWithMessages.forEach((groupId) => {
+          const syncedGroup = secondBatchChatGroups.find((group) => group.sourceId === groupId);
+          assert.ok(syncedGroup, `Chat group with sourceId ${groupId} exists in the batch`);
+          assert.ok(syncedGroup!.syncData, `Chat group with sourceId ${groupId} has a valid sync range`);
+        });
+
 
       } catch (err) {
         // Ensure provider closes even if an error occurs
@@ -190,6 +197,134 @@ describe(`${providerId} chat message tests`, function () {
         throw err;
       }
     });
+
+    it(`Should process most recent messages first`, async () => {
+      const { api, handler, provider } = await CommonTests.buildTestObjects(
+        providerId,
+        SlackChatMessageHandler,
+        providerConfig,
+        connection
+      );
+      handler.setConfig(handlerConfig);
+
+      const syncPosition: SyncHandlerPosition = {
+        _id: `${providerId}-${handlerName}`,
+        providerId,
+        handlerId: handler.getId(),
+        accountId: provider.getAccountId(),
+        status: SyncHandlerStatus.ENABLED,
+      };
+
+      const response = await handler._sync(api, syncPosition);
+      const results = (response.results).filter(
+        (result) => result.schema === CONFIG.verida.schemas.CHAT_MESSAGE
+      );
+
+      const timestamps = results.map((msg) => new Date(msg.insertedAt!).getTime());
+      const isSortedDescending = timestamps.every(
+        (val, i, arr) => i === 0 || arr[i - 1] >= val
+      );
+
+      assert.ok(isSortedDescending, "Messages are processed from most recent to oldest");
+    });
+
+    it(`Should ensure second batch items aren't in the first batch`, async () => {
+      const { api, handler, provider } = await CommonTests.buildTestObjects(
+        providerId,
+        SlackChatMessageHandler,
+        providerConfig,
+        connection
+      );
+      handler.setConfig(handlerConfig);
+
+      const firstBatchResponse = await handler._sync(api, {
+        _id: `${providerId}-${handlerName}`,
+        providerId,
+        handlerId: handler.getId(),
+        accountId: provider.getAccountId(),
+        status: SyncHandlerStatus.ENABLED,
+      });
+
+      const firstBatchMessages = (firstBatchResponse.results).filter(
+        (result) => result.schema === CONFIG.verida.schemas.CHAT_MESSAGE
+      );
+
+      setTimeout(() => {
+        console.log("Wait for saving sync data");
+      }, 3000);
+      const secondBatchResponse = await handler._sync(api, firstBatchResponse.position);
+      const secondBatchMessages = (secondBatchResponse.results).filter(
+        (result) => result.schema === CONFIG.verida.schemas.CHAT_MESSAGE
+      );
+
+      const firstBatchIds = firstBatchMessages.map((msg) => msg.sourceId);
+      const secondBatchIds = secondBatchMessages.map((msg) => msg.sourceId);
+
+      const intersection = firstBatchIds.filter((id) => secondBatchIds.includes(id));
+      assert.equal(intersection.length, 0, "No overlapping messages between batches");
+    });
+
+    it(`Should process each type of chat group correctly`, async () => {
+      const { api, handler, provider } = await CommonTests.buildTestObjects(
+        providerId,
+        SlackChatMessageHandler,
+        providerConfig,
+        connection
+      );
+      handler.setConfig(handlerConfig);
+
+      const syncPosition: SyncHandlerPosition = {
+        _id: `${providerId}-${handlerName}`,
+        providerId,
+        handlerId: handler.getId(),
+        accountId: provider.getAccountId(),
+        status: SyncHandlerStatus.ENABLED,
+      };
+
+      const response = await handler._sync(api, syncPosition);
+      const results = (response.results).filter(
+        (result) => result.schema === CONFIG.verida.schemas.CHAT_MESSAGE
+      );
+
+      // Check if each type of group has at least one message
+      const groupTypes = {
+        [SlackChatGroupType.IM]: true,
+        [SlackChatGroupType.PRIVATE_CHANNEL]: true,
+        [SlackChatGroupType.PUBLIC_CHANNEL]: true,
+      };
+
+      results.forEach((message) => {
+        const group = (response.results).find(
+          (result) =>
+            result.schema === CONFIG.verida.schemas.CHAT_GROUP &&
+            result.sourceId === message.groupId
+        );
+
+        if (group) {
+          if (group.sourceData!.hasOwnProperty('is_im') && (group.sourceData! as any).is_im) {
+            groupTypes[SlackChatGroupType.IM] = false;
+          }
+
+          if (group.sourceData!.hasOwnProperty('is_private') && (group.sourceData! as any).is_private) {
+            groupTypes[SlackChatGroupType.PRIVATE_CHANNEL] = true;
+          }
+
+          if (group.sourceData!.hasOwnProperty('is_channel') && (group.sourceData! as any).is_channel) {
+            groupTypes[SlackChatGroupType.PUBLIC_CHANNEL] = true;
+          }
+        }
+      });
+
+      // Assert that all group types are represented
+      Object.entries(groupTypes).forEach(([type, isPresent]) => {
+        assert.ok(
+          isPresent,
+          `Chat group type ${type} should have messages processed`
+        );
+      });
+    });
+
+
   });
 
   // After all tests, close the network context

From 8de8425b84f3e9e958ffdbcc7cfeb003f09e02da Mon Sep 17 00:00:00 2001
From: chime3 
Date: Sun, 17 Nov 2024 19:56:35 -0700
Subject: [PATCH 180/182] Merge branch 'develop' into
 feature/98-slack-connector

---
 tests/providers/slack/chat-message.ts | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/providers/slack/chat-message.ts b/tests/providers/slack/chat-message.ts
index bda93222..43099846 100644
--- a/tests/providers/slack/chat-message.ts
+++ b/tests/providers/slack/chat-message.ts
@@ -288,7 +288,7 @@ describe(`${providerId} chat message tests`, function () {
 
       // Check if each type of group has at least one message
       const groupTypes = {
-        [SlackChatGroupType.IM]: true,
+        [SlackChatGroupType.IM]: false,
         [SlackChatGroupType.PRIVATE_CHANNEL]: true,
         [SlackChatGroupType.PUBLIC_CHANNEL]: true,
       };
@@ -302,7 +302,7 @@ describe(`${providerId} chat message tests`, function () {
 
         if (group) {
           if (group.sourceData!.hasOwnProperty('is_im') && (group.sourceData! as any).is_im) {
-            groupTypes[SlackChatGroupType.IM] = false;
+            groupTypes[SlackChatGroupType.IM] = true;
           }
 
           if (group.sourceData!.hasOwnProperty('is_private') && (group.sourceData! as any).is_private) {

From 347ebdab1100fa868c9f9458978b9bc86c2e2d71 Mon Sep 17 00:00:00 2001
From: chime3 
Date: Sun, 22 Dec 2024 19:53:08 -0700
Subject: [PATCH 181/182] Chore: slack deps

---
 yarn.lock | 99 +++++++++++++++++++++++++++++--------------------------
 1 file changed, 52 insertions(+), 47 deletions(-)

diff --git a/yarn.lock b/yarn.lock
index c5f81da2..4f43d7df 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -1545,6 +1545,48 @@
   resolved "https://registry.yarnpkg.com/@sindresorhus/is/-/is-0.14.0.tgz#9fb3a3cf3132328151f353de4632e01e52102bea"
   integrity sha512-9NET910DNaIPngYnLLPeg+Ogzqsi9uM4mSboU5y6p8S5DzMTVEsJZrawi+BoDNUVBa2DhJqQYUFvMDfgU062LQ==
 
+"@slack/logger@^4", "@slack/logger@^4.0.0":
+  version "4.0.0"
+  resolved "https://registry.yarnpkg.com/@slack/logger/-/logger-4.0.0.tgz#788303ff1840be91bdad7711ef66ca0cbc7073d2"
+  integrity sha512-Wz7QYfPAlG/DR+DfABddUZeNgoeY7d1J39OCR2jR+v7VBsB8ezulDK5szTnDDPDwLH5IWhLvXIHlCFZV7MSKgA==
+  dependencies:
+    "@types/node" ">=18.0.0"
+
+"@slack/oauth@^3.0.1":
+  version "3.0.1"
+  resolved "https://registry.yarnpkg.com/@slack/oauth/-/oauth-3.0.1.tgz#079cde13f998be2d458c20dfcae288067464536e"
+  integrity sha512-TuR9PI6bYKX6qHC7FQI4keMnhj45TNfSNQtTU3mtnHUX4XLM2dYLvRkUNADyiLTle2qu2rsOQtCIsZJw6H0sDA==
+  dependencies:
+    "@slack/logger" "^4"
+    "@slack/web-api" "^7.3.4"
+    "@types/jsonwebtoken" "^9"
+    "@types/node" ">=18"
+    jsonwebtoken "^9"
+    lodash.isstring "^4"
+
+"@slack/types@^2.9.0":
+  version "2.13.1"
+  resolved "https://registry.yarnpkg.com/@slack/types/-/types-2.13.1.tgz#d1af332103ce96e22b10692bef9b34483f4c002c"
+  integrity sha512-YVtJCVtDcjOPKsvOedIThb7YmKNCcSoZN0mUSQqD2fc2ZyI59gOLCF4rYGfw/0C0agzFxAmb7hV5tbMGrgK0Tg==
+
+"@slack/web-api@^7.3.4", "@slack/web-api@^7.4.0":
+  version "7.4.0"
+  resolved "https://registry.yarnpkg.com/@slack/web-api/-/web-api-7.4.0.tgz#56f8376203b4172c02812f0fc092b65c351a74ba"
+  integrity sha512-U+0pui9VcddRt3yTSkLAArVB6G8EyOb0g+/vL1E6u8k46IYc1H4UKDuULF734A2ynVyxYKHIXK8OHilX6YhwjQ==
+  dependencies:
+    "@slack/logger" "^4.0.0"
+    "@slack/types" "^2.9.0"
+    "@types/node" ">=18.0.0"
+    "@types/retry" "0.12.0"
+    axios "^1.7.4"
+    eventemitter3 "^5.0.1"
+    form-data "^4.0.0"
+    is-electron "2.2.2"
+    is-stream "^2"
+    p-queue "^6"
+    p-retry "^4"
+    retry "^0.13.1"
+
 "@smithy/abort-controller@^3.1.8":
   version "3.1.8"
   resolved "https://registry.yarnpkg.com/@smithy/abort-controller/-/abort-controller-3.1.8.tgz#ce0c10ddb2b39107d70b06bbb8e4f6e368bc551d"
@@ -2063,48 +2105,6 @@
     "@smithy/util-buffer-from" "^3.0.0"
     tslib "^2.6.2"
 
-"@slack/logger@^4", "@slack/logger@^4.0.0":
-  version "4.0.0"
-  resolved "https://registry.yarnpkg.com/@slack/logger/-/logger-4.0.0.tgz#788303ff1840be91bdad7711ef66ca0cbc7073d2"
-  integrity sha512-Wz7QYfPAlG/DR+DfABddUZeNgoeY7d1J39OCR2jR+v7VBsB8ezulDK5szTnDDPDwLH5IWhLvXIHlCFZV7MSKgA==
-  dependencies:
-    "@types/node" ">=18.0.0"
-
-"@slack/oauth@^3.0.1":
-  version "3.0.1"
-  resolved "https://registry.yarnpkg.com/@slack/oauth/-/oauth-3.0.1.tgz#079cde13f998be2d458c20dfcae288067464536e"
-  integrity sha512-TuR9PI6bYKX6qHC7FQI4keMnhj45TNfSNQtTU3mtnHUX4XLM2dYLvRkUNADyiLTle2qu2rsOQtCIsZJw6H0sDA==
-  dependencies:
-    "@slack/logger" "^4"
-    "@slack/web-api" "^7.3.4"
-    "@types/jsonwebtoken" "^9"
-    "@types/node" ">=18"
-    jsonwebtoken "^9"
-    lodash.isstring "^4"
-
-"@slack/types@^2.9.0":
-  version "2.13.1"
-  resolved "https://registry.yarnpkg.com/@slack/types/-/types-2.13.1.tgz#d1af332103ce96e22b10692bef9b34483f4c002c"
-  integrity sha512-YVtJCVtDcjOPKsvOedIThb7YmKNCcSoZN0mUSQqD2fc2ZyI59gOLCF4rYGfw/0C0agzFxAmb7hV5tbMGrgK0Tg==
-
-"@slack/web-api@^7.3.4", "@slack/web-api@^7.4.0":
-  version "7.4.0"
-  resolved "https://registry.yarnpkg.com/@slack/web-api/-/web-api-7.4.0.tgz#56f8376203b4172c02812f0fc092b65c351a74ba"
-  integrity sha512-U+0pui9VcddRt3yTSkLAArVB6G8EyOb0g+/vL1E6u8k46IYc1H4UKDuULF734A2ynVyxYKHIXK8OHilX6YhwjQ==
-  dependencies:
-    "@slack/logger" "^4.0.0"
-    "@slack/types" "^2.9.0"
-    "@types/node" ">=18.0.0"
-    "@types/retry" "0.12.0"
-    axios "^1.7.4"
-    eventemitter3 "^5.0.1"
-    form-data "^4.0.0"
-    is-electron "2.2.2"
-    is-stream "^2"
-    p-queue "^6"
-    p-retry "^4"
-    retry "^0.13.1"
-
 "@stablelib/aead@^1.0.1":
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/@stablelib/aead/-/aead-1.0.1.tgz#c4b1106df9c23d1b867eb9b276d8f42d5fc4c0c3"
@@ -3420,7 +3420,7 @@ axios@^1.2.3, axios@^1.6.2:
     form-data "^4.0.0"
     proxy-from-env "^1.1.0"
 
-axios@^1.3.3:
+axios@^1.3.3, axios@^1.7.4, axios@^1.7.7:
   version "1.7.9"
   resolved "https://registry.yarnpkg.com/axios/-/axios-1.7.9.tgz#d7d071380c132a24accda1b2cfc1535b79ec650a"
   integrity sha512-LhLcE7Hbiryz8oMDdDptSrWowmB4Bl6RCt6sIJKpRB4XtVf0iEgewX3au/pJqm+Py1kCASkb/FFKjxQaLtxJvw==
@@ -4721,6 +4721,11 @@ eventemitter3@^4.0.4:
   resolved "https://registry.yarnpkg.com/eventemitter3/-/eventemitter3-4.0.7.tgz#2de9b68f6528d5644ef5c59526a1b4a07306169f"
   integrity sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==
 
+eventemitter3@^5.0.1:
+  version "5.0.1"
+  resolved "https://registry.yarnpkg.com/eventemitter3/-/eventemitter3-5.0.1.tgz#53f5ffd0a492ac800721bb42c66b841de96423c4"
+  integrity sha512-GWkBvjiSZK87ELrYOSESUYeVIc9mvLLf/nXalMOS5dYrgZq9o5OVkbZAVM06CVxYsCwH9BDZFPlQTlPA1j4ahA==
+
 events@1.1.1:
   version "1.1.1"
   resolved "https://registry.yarnpkg.com/events/-/events-1.1.1.tgz#9ebdb7635ad099c70dcc4c2a1f5004288e8bd924"
@@ -5787,7 +5792,7 @@ jsonpointer@^5.0.1:
   resolved "https://registry.yarnpkg.com/jsonpointer/-/jsonpointer-5.0.1.tgz#2110e0af0900fd37467b5907ecd13a7884a1b559"
   integrity sha512-p/nXbhSEcu3pZRdkW1OfJhpsVtW1gd4Wa1fnQc9YLiTfAjn0312eMKimbdIQzuZl9aa9xUGaRlP9T/CJE/ditQ==
 
-jsonwebtoken@^9.0.0:
+jsonwebtoken@^9, jsonwebtoken@^9.0.0:
   version "9.0.2"
   resolved "https://registry.yarnpkg.com/jsonwebtoken/-/jsonwebtoken-9.0.2.tgz#65ff91f4abef1784697d40952bb1998c504caaf3"
   integrity sha512-PRp66vJ865SSqOlgqS8hujT5U4AOgMfhrwYIuIhfKaoSCZcirrmASQr8CX7cUg+RMih+hgznrjp99o+W4pJLHQ==
@@ -6129,7 +6134,7 @@ lodash.isplainobject@^4.0.6:
   resolved "https://registry.yarnpkg.com/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz#7c526a52d89b45c45cc690b88163be0497f550cb"
   integrity sha1-fFJqUtibRcRcxpC4gWO+BJf1UMs=
 
-lodash.isstring@^4.0.1:
+lodash.isstring@^4, lodash.isstring@^4.0.1:
   version "4.0.1"
   resolved "https://registry.yarnpkg.com/lodash.isstring/-/lodash.isstring-4.0.1.tgz#d527dfb5456eca7cc9bb95d5daeaf88ba54a5451"
   integrity sha512-0wJxfxH1wgO3GrbuP+dTTk7op+6L41QCXbGINEmD+ny/G/eCqGzxyCsh7159S+mgDDcoarnBw6PC1PS5+wUGgw==
@@ -6730,7 +6735,7 @@ p-locate@^5.0.0:
   dependencies:
     p-limit "^3.0.2"
 
-p-queue@^6.6.2:
+p-queue@^6, p-queue@^6.6.2:
   version "6.6.2"
   resolved "https://registry.yarnpkg.com/p-queue/-/p-queue-6.6.2.tgz#2068a9dcf8e67dd0ec3e7a2bcb76810faa85e426"
   integrity sha512-RwFpb72c/BhQLEXIZ5K2e+AhgNVmIejGlTgiB9MzZ0e93GRvqZ7uSi0dvRF7/XIXDeNkra2fNHBxTyPDGySpjQ==
@@ -6738,7 +6743,7 @@ p-queue@^6.6.2:
     eventemitter3 "^4.0.4"
     p-timeout "^3.2.0"
 
-p-retry@4:
+p-retry@4, p-retry@^4:
   version "4.6.2"
   resolved "https://registry.yarnpkg.com/p-retry/-/p-retry-4.6.2.tgz#9baae7184057edd4e17231cee04264106e092a16"
   integrity sha512-312Id396EbJdvRONlngUx0NydfrIQ5lsYu0znKVUzVvArzEIt08V1qhtyESbGVd1FGX7UKtiFp5uwKZdM8wIuQ==

From a2ece977dbed7054941b8afd4f547844d48ea6c1 Mon Sep 17 00:00:00 2001
From: chime3 
Date: Sun, 22 Dec 2024 20:02:29 -0700
Subject: [PATCH 182/182] feat: set slack status as active

---
 src/serverconfig.example.json | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/serverconfig.example.json b/src/serverconfig.example.json
index 60b6b847..0d15182c 100644
--- a/src/serverconfig.example.json
+++ b/src/serverconfig.example.json
@@ -131,6 +131,7 @@
             "useDbPos": true
         },
         "slack": {
+            "status": "active",
             "label": "Slack",
             "clientId": "",
             "clientSecret": "",


             
                 
ProviderSource
                     Profile
                     Sync Details
                     Handlers