From 3d44391bd1642f1bd70208bf97916b9305ec73f1 Mon Sep 17 00:00:00 2001 From: chime3 Date: Wed, 25 Sep 2024 22:30:26 -0700 Subject: [PATCH 01/12] fix: wrong function use --- src/providers/discord/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/providers/discord/index.ts b/src/providers/discord/index.ts index c52347ec..f96c5cdc 100644 --- a/src/providers/discord/index.ts +++ b/src/providers/discord/index.ts @@ -49,7 +49,7 @@ export default class DiscordProvider extends Base { public async connect(req: Request, res: Response, next: any): Promise { this.init() - const auth = await passport.authenticate(this.getAccountId()) + const auth = await passport.authenticate(this.getProviderId()) return auth(req, res, next) } @@ -65,7 +65,7 @@ export default class DiscordProvider extends Base { this.init() const promise = new Promise((resolve, rejects) => { - const auth = passport.authenticate(this.getAccountId(), { + const auth = passport.authenticate(this.getProviderId(), { scope: SCOPE, failureRedirect: '/failure/discord', failureMessage: true From f566e434098ea3b3bc6d3cc8f050ede77bda659e Mon Sep 17 00:00:00 2001 From: chime3 Date: Wed, 25 Sep 2024 23:10:23 -0700 Subject: [PATCH 02/12] fix: discord profile --- src/providers/discord/index.ts | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/src/providers/discord/index.ts b/src/providers/discord/index.ts index f96c5cdc..6be63d7f 100644 --- a/src/providers/discord/index.ts +++ b/src/providers/discord/index.ts @@ -1,6 +1,6 @@ import { Request, Response } from 'express' import Base from "../BaseProvider" -import { BaseProviderConfig, ConnectionCallbackResponse } from '../../interfaces' +import { BaseProviderConfig, ConnectionCallbackResponse, PassportProfile } from '../../interfaces' const passport = require("passport") import { Strategy as DiscordStrategy, Scope } from '@oauth-everything/passport-discord'; @@ -73,11 +73,32 @@ export default class DiscordProvider extends Base { if (err) { rejects(err) } else { + console.log("=======") + + const profile: PassportProfile = { + id: data.profile.id, // Discord user ID + provider: data.profile.provider, // discord + displayName: data.profile.displayName, + name: { + familyName: '', // Discord does not provide family name + givenName: data.profile._json.global_name, // Global name as the given name + }, + emails: data.profile.emails, + photos: data.profile.photos, // Photos array from Discord profile + connectionProfile: { + username: data.profile.username, // Discord username + email: data.profile._json.email, // Email from profile + readableId: data.profile.displayName, + verified: data.profile._json.verified // Verified status from Discord profile + } + }; + + console.log(profile); const connectionToken: ConnectionCallbackResponse = { id: data.profile.id, accessToken: data.accessToken, refreshToken: data.refreshToken, - profile: data.profile + profile: profile } resolve(connectionToken) From 5d47d14d8096a691a45e8851f6248f8bb3bab4af Mon Sep 17 00:00:00 2001 From: chime3 Date: Wed, 25 Sep 2024 23:11:34 -0700 Subject: [PATCH 03/12] chore: remove console --- src/providers/discord/index.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/providers/discord/index.ts b/src/providers/discord/index.ts index 6be63d7f..a2e78650 100644 --- a/src/providers/discord/index.ts +++ b/src/providers/discord/index.ts @@ -73,8 +73,7 @@ export default class DiscordProvider extends Base { if (err) { rejects(err) } else { - console.log("=======") - + const profile: PassportProfile = { id: data.profile.id, // Discord user ID provider: data.profile.provider, // discord @@ -92,8 +91,7 @@ export default class DiscordProvider extends Base { verified: data.profile._json.verified // Verified status from Discord profile } }; - - console.log(profile); + const connectionToken: ConnectionCallbackResponse = { id: data.profile.id, accessToken: data.accessToken, From b837bd0f6527c49495a82842065d9585bb3aa9ee Mon Sep 17 00:00:00 2001 From: chime3 Date: Wed, 25 Sep 2024 23:33:40 -0700 Subject: [PATCH 04/12] feat: define interfaces --- src/providers/discord/interfaces.ts | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 src/providers/discord/interfaces.ts diff --git a/src/providers/discord/interfaces.ts b/src/providers/discord/interfaces.ts new file mode 100644 index 00000000..746de5f8 --- /dev/null +++ b/src/providers/discord/interfaces.ts @@ -0,0 +1,27 @@ +import { BaseHandlerConfig, BaseProviderConfig } from "../../interfaces"; + +export interface DiscordHandlerConfig extends BaseHandlerConfig { + // Maximum number of channels to process + channelLimit: number; + // Maximum number of messages to process in a given batch + messageBatchSize: number; + // Maximum number of messages to process in a channel + messagesPerChannelLimit: number; +} + +export interface DiscordProviderConfig extends BaseProviderConfig { + clientId: string; + clientSecret: string; + token: string; + callbackUrl: string; +} + +export enum DiscordChatGroupType { + GUILD_TEXT = "guild_text", // Text channel in a server + DM = "dm", // Direct message + GUILD_VOICE = "guild_voice", // Voice channel in a server + GROUP_DM = "group_dm", // Group direct message + GUILD_CATEGORY = "guild_category", // Category that contains channels + GUILD_NEWS = "guild_news", // News channels (for announcements) + GUILD_STORE = "guild_store", // Store channels (for selling items) +} From c74b76d6b587951a2891bdc4325cd7234dfb87d5 Mon Sep 17 00:00:00 2001 From: chime3 Date: Wed, 25 Sep 2024 23:33:57 -0700 Subject: [PATCH 05/12] feat: added chat message handler --- src/providers/discord/chat-message.ts | 287 ++++++++++++++++++++++++++ 1 file changed, 287 insertions(+) create mode 100644 src/providers/discord/chat-message.ts diff --git a/src/providers/discord/chat-message.ts b/src/providers/discord/chat-message.ts new file mode 100644 index 00000000..d07e74bb --- /dev/null +++ b/src/providers/discord/chat-message.ts @@ -0,0 +1,287 @@ +import { Client, GatewayIntentBits, TextChannel, DMChannel } from 'discord.js'; +import CONFIG from '../../config'; +import { + SyncItemsResult, + SyncResponse, + SyncHandlerStatus, + ProviderHandlerOption, + ConnectionOptionType, + SyncHandlerPosition, +} from '../../interfaces'; +import { + SchemaChatMessageType, + SchemaSocialChatGroup, + SchemaSocialChatMessage, +} from '../../schemas'; +import { DiscordChatGroupType, DiscordHandlerConfig } from './interfaces'; +import BaseSyncHandler from '../BaseSyncHandler'; +import { ItemsRangeTracker } from '../../helpers/itemsRangeTracker'; +import { ItemsRange } from '../../helpers/interfaces'; + +export default class DiscordChatMessageHandler extends BaseSyncHandler { + protected config: DiscordHandlerConfig; + + public getName(): string { + return 'chat-message'; + } + + public getLabel(): string { + return 'Chat Messages'; + } + + public getSchemaUri(): string { + return CONFIG.verida.schemas.CHAT_MESSAGE; + } + + public getProviderApplicationUrl(): string { + return 'https://discord.com/'; + } + + public getOptions(): ProviderHandlerOption[] { + return [ + { + id: 'channelTypes', + label: 'Channel types', + type: ConnectionOptionType.ENUM_MULTI, + enumOptions: [ + { label: 'Text Channels', value: DiscordChatGroupType.GUILD_TEXT }, + { label: 'Direct Messages', value: DiscordChatGroupType.DM }, + ], + defaultValue: [ + DiscordChatGroupType.GUILD_TEXT, + DiscordChatGroupType.DM, + ].join(','), + }, + ]; + } + + public getDiscordClient(): Client { + const token = this.connection.accessToken; + return new Client({ + intents: [ + GatewayIntentBits.Guilds, + GatewayIntentBits.GuildMessages, + GatewayIntentBits.DirectMessages, + ], + }).login(token); + } + + protected async buildChatGroupList(): Promise { + const client = await this.getDiscordClient(); + + let channelList: SchemaSocialChatGroup[] = []; + + // Fetch all types of conversations: DMs and Text Channels + const channels = client.channels.cache; + + for (const [id, channel] of channels) { + if (channel.isText()) { + const group: SchemaSocialChatGroup = { + _id: this.buildItemId(channel.id), + name: channel.name, + sourceAccountId: this.provider.getAccountId(), + sourceApplication: this.getProviderApplicationUrl(), + sourceId: channel.id, + schema: CONFIG.verida.schemas.CHAT_GROUP, + sourceData: channel, + insertedAt: new Date().toISOString(), + }; + channelList.push(group); + } else if (channel.isDM()) { + const group: SchemaSocialChatGroup = { + _id: this.buildItemId(channel.id), + name: `DM with ${channel.recipient.username}`, + sourceAccountId: this.provider.getAccountId(), + sourceApplication: this.getProviderApplicationUrl(), + sourceId: channel.id, + schema: CONFIG.verida.schemas.CHAT_GROUP, + sourceData: channel, + insertedAt: new Date().toISOString(), + }; + channelList.push(group); + } + } + return channelList; + } + + protected async fetchMessageRange( + chatGroup: SchemaSocialChatGroup, + range: ItemsRange, + apiClient: Client + ): Promise { + const messages: SchemaSocialChatMessage[] = []; + const channel = apiClient.channels.cache.get(chatGroup.sourceId!) as TextChannel | DMChannel; + + if (!channel) return messages; + + const fetchedMessages = await channel.messages.fetch({ + after: range.startId, + before: range.endId, + }); + + for (const message of fetchedMessages.values()) { + const chatMessage: SchemaSocialChatMessage = { + _id: this.buildItemId(message.id), + groupId: chatGroup._id, + groupName: chatGroup.name, + messageText: message.content, + fromHandle: message.author.username, + sourceAccountId: this.provider.getAccountId(), + sourceApplication: this.getProviderApplicationUrl(), + sourceId: message.id, + sourceData: message, + insertedAt: new Date(message.createdTimestamp).toISOString(), + sentAt: new Date(message.createdTimestamp).toISOString(), + type: + message.author.id === this.connection.profile.id + ? SchemaChatMessageType.SEND + : SchemaChatMessageType.RECEIVE, + fromId: message.author.id, + name: message.content.substring(0, 30), + }; + messages.push(chatMessage); + } + + return messages; + } + + public async _sync( + api: any, + syncPosition: SyncHandlerPosition + ): Promise { + try { + const apiClient = await this.getDiscordClient(); + const groupList = await this.buildChatGroupList(); // Fetch all Text Channels and DM groups + + let totalMessages = 0; + let chatHistory: SchemaSocialChatMessage[] = []; + + // Determine the current group position + let groupPosition = this.getGroupPositionIndex(groupList, syncPosition); + + const groupCount = groupList.length; + + // Iterate over each group + for (let i = 0; i < Math.min(groupCount, this.config.channelLimit); i++) { + const groupIndex = (groupPosition + i) % groupCount; // Rotate through groups + const group = groupList[groupIndex]; + + // Use a separate ItemsRangeTracker for each group + let rangeTracker = new ItemsRangeTracker(group.syncData); + + const fetchedMessages = await this.fetchAndTrackMessages( + group, + rangeTracker, + apiClient + ); + + // Concatenate the fetched messages to the total chat history + chatHistory = chatHistory.concat(fetchedMessages); + totalMessages += fetchedMessages.length; + + // Update the group's sync data with the latest rangeTracker state + group.syncData = rangeTracker.export(); + + // Stop if the total messages fetched reach the batch size + if (totalMessages >= this.config.messageBatchSize) { + syncPosition.thisRef = groupList[(groupIndex + 1) % groupCount].sourceId; // Continue from the next group in the next sync + break; + } + } + + // Finalize sync position and status based on message count + this.updateSyncPosition( + syncPosition, + totalMessages, + groupCount, + chatHistory + ); + + // Concatenate only items after syncPosition.thisRef and chatHistory + const remainingGroups = groupList.slice(groupPosition + 1); + + return { + results: remainingGroups.concat(chatHistory), + position: syncPosition, + }; + } catch (err: any) { + console.error(err); + throw err; + } + } + + private getGroupPositionIndex( + groupList: SchemaSocialChatGroup[], + syncPosition: SyncHandlerPosition + ): number { + const groupPosition = groupList.findIndex( + (group) => group.sourceId === syncPosition.thisRef + ); + + // If not found, return 0 to start from the beginning + return groupPosition === -1 ? 0 : groupPosition; + } + + private async fetchAndTrackMessages( + group: SchemaSocialChatGroup, + rangeTracker: ItemsRangeTracker, + apiClient: Client + ): Promise { + // Validate group and group.id + if (!group || !group.sourceId) { + throw new Error('Invalid group or missing group sourceId'); + } + + // Initialize range from tracker + let currentRange = rangeTracker.nextRange(); + let items: SchemaSocialChatMessage[] = []; + + while (true) { + // Fetch messages for the current range using fetchMessageRange + const messages = await this.fetchMessageRange(group, currentRange, apiClient); + + if (!messages.length) break; + + // Add fetched messages to the main list + items = items.concat(messages); + + // Break loop if messages reached group limit + if (items.length > this.config.messagesPerChannelLimit) { + // Mark the current range as complete and stop + rangeTracker.completedRange({ + startId: messages[0].sourceId, + endId: messages[messages.length - 1].sourceId, + }, false); + break; + } else { + // Update rangeTracker and continue fetching + rangeTracker.completedRange({ + startId: messages[0].sourceId, + endId: messages[messages.length - 1].sourceId, + }, false); + + // Move to the next range + currentRange = rangeTracker.nextRange(); + } + } + + return items; + } + + private updateSyncPosition( + syncPosition: SyncHandlerPosition, + totalMessages: number, + groupCount: number, + chatHistory: SchemaSocialChatMessage[] + ) { + if (totalMessages === 0) { + syncPosition.status = SyncHandlerStatus.ENABLED; + syncPosition.syncMessage = 'No new messages found.'; + } else if (totalMessages < this.config.messageBatchSize) { + syncPosition.syncMessage = `Processed ${totalMessages} messages across ${groupCount} groups. Sync complete.`; + syncPosition.status = SyncHandlerStatus.ENABLED; + } else { + syncPosition.syncMessage = `Batch complete (${this.config.messageBatchSize}). More results pending.`; + } + } +} From 091456ba783809b12ef9996e9f1c483fff9d2c85 Mon Sep 17 00:00:00 2001 From: chime3 Date: Thu, 26 Sep 2024 22:36:58 -0700 Subject: [PATCH 06/12] fix: fetch guilds and channels --- src/providers/discord/chat-message.ts | 91 ++++++++++++++++----------- src/providers/discord/index.ts | 4 +- 2 files changed, 57 insertions(+), 38 deletions(-) diff --git a/src/providers/discord/chat-message.ts b/src/providers/discord/chat-message.ts index d07e74bb..d6e4c872 100644 --- a/src/providers/discord/chat-message.ts +++ b/src/providers/discord/chat-message.ts @@ -56,53 +56,69 @@ export default class DiscordChatMessageHandler extends BaseSyncHandler { } public getDiscordClient(): Client { + const token = this.connection.accessToken; - return new Client({ + + const client = new Client({ intents: [ GatewayIntentBits.Guilds, GatewayIntentBits.GuildMessages, GatewayIntentBits.DirectMessages, + GatewayIntentBits.MessageContent ], - }).login(token); - } - - protected async buildChatGroupList(): Promise { - const client = await this.getDiscordClient(); + }); + + client.login(token); + return client; + } + protected async buildChatGroupList(api: any): Promise { let channelList: SchemaSocialChatGroup[] = []; + let channels = [] + + const guilds: any = await api.get('/users/@me/guilds') + + for (const guild of guilds) { + channels = await api.get(`/guilds/${guild.id}/channels`) + + console.log(`Channels in guild ${guild.name}:`, channels); + } - // Fetch all types of conversations: DMs and Text Channels - const channels = client.channels.cache; - + for (const [id, channel] of channels) { - if (channel.isText()) { - const group: SchemaSocialChatGroup = { - _id: this.buildItemId(channel.id), - name: channel.name, - sourceAccountId: this.provider.getAccountId(), - sourceApplication: this.getProviderApplicationUrl(), - sourceId: channel.id, - schema: CONFIG.verida.schemas.CHAT_GROUP, - sourceData: channel, - insertedAt: new Date().toISOString(), - }; - channelList.push(group); - } else if (channel.isDM()) { - const group: SchemaSocialChatGroup = { - _id: this.buildItemId(channel.id), - name: `DM with ${channel.recipient.username}`, - sourceAccountId: this.provider.getAccountId(), - sourceApplication: this.getProviderApplicationUrl(), - sourceId: channel.id, - schema: CONFIG.verida.schemas.CHAT_GROUP, - sourceData: channel, - insertedAt: new Date().toISOString(), - }; - channelList.push(group); + if (channel.isTextBased()) { + if (channel instanceof TextChannel) { + const textChannel = channel as TextChannel; + const group: SchemaSocialChatGroup = { + _id: this.buildItemId(textChannel.id), + name: textChannel.name, + sourceAccountId: this.provider.getAccountId(), + sourceApplication: this.getProviderApplicationUrl(), + sourceId: textChannel.id, + schema: CONFIG.verida.schemas.CHAT_GROUP, + sourceData: textChannel, + insertedAt: new Date().toISOString(), + }; + channelList.push(group); + } else if (channel.isDMBased()) { + const dmChannel = channel as DMChannel; + const group: SchemaSocialChatGroup = { + _id: this.buildItemId(dmChannel.id), + name: `DM with ${dmChannel.recipient?.username}`, + sourceAccountId: this.provider.getAccountId(), + sourceApplication: this.getProviderApplicationUrl(), + sourceId: dmChannel.id, + schema: CONFIG.verida.schemas.CHAT_GROUP, + sourceData: dmChannel, + insertedAt: new Date().toISOString(), + }; + channelList.push(group); + } } } return channelList; } + protected async fetchMessageRange( chatGroup: SchemaSocialChatGroup, @@ -149,9 +165,10 @@ export default class DiscordChatMessageHandler extends BaseSyncHandler { api: any, syncPosition: SyncHandlerPosition ): Promise { + try { - const apiClient = await this.getDiscordClient(); - const groupList = await this.buildChatGroupList(); // Fetch all Text Channels and DM groups + //const apiClient = await this.getDiscordClient(); + const groupList = await this.buildChatGroupList(api); // Fetch all Text Channels and DM groups let totalMessages = 0; let chatHistory: SchemaSocialChatMessage[] = []; @@ -172,7 +189,7 @@ export default class DiscordChatMessageHandler extends BaseSyncHandler { const fetchedMessages = await this.fetchAndTrackMessages( group, rangeTracker, - apiClient + api ); // Concatenate the fetched messages to the total chat history @@ -225,7 +242,7 @@ export default class DiscordChatMessageHandler extends BaseSyncHandler { private async fetchAndTrackMessages( group: SchemaSocialChatGroup, rangeTracker: ItemsRangeTracker, - apiClient: Client + apiClient: any ): Promise { // Validate group and group.id if (!group || !group.sourceId) { diff --git a/src/providers/discord/index.ts b/src/providers/discord/index.ts index a2e78650..f7a239f5 100644 --- a/src/providers/discord/index.ts +++ b/src/providers/discord/index.ts @@ -12,6 +12,7 @@ import axios from 'axios' //import SBTs from './sbts' import Following from './following' import TokenExpiredError from '../TokenExpiredError' +import DiscordChatMessageHandler from './chat-message'; export interface DiscordProviderConfig extends BaseProviderConfig { clientID: string @@ -42,7 +43,8 @@ export default class DiscordProvider extends Base { public syncHandlers(): any[] { return [ //SBTs - Following + //Following, + DiscordChatMessageHandler ] return [] } From e10c91b14475af467d5f9d5ba90b0c194da618c2 Mon Sep 17 00:00:00 2001 From: chime3 Date: Sun, 29 Sep 2024 21:05:00 -0700 Subject: [PATCH 07/12] feat: added discord routes --- package.json | 2 + src/providers/discord/chat-message.ts | 19 +++++--- src/providers/discord/index.ts | 2 +- yarn.lock | 65 +++++++++++++++++++++++++++ 4 files changed, 81 insertions(+), 7 deletions(-) diff --git a/package.json b/package.json index d02079bd..5a21766f 100644 --- a/package.json +++ b/package.json @@ -32,6 +32,7 @@ }, "homepage": "https://github.com/verida/server-template#readme", "dependencies": { + "@discordjs/rest": "^2.4.0", "@oauth-everything/passport-discord": "^1.0.2", "@sapphire/snowflake": "^3.4.2", "@superfaceai/passport-twitter-oauth2": "^1.2.3", @@ -52,6 +53,7 @@ "cors": "^2.8.5", "crypto-js": "^4.2.0", "dayjs": "^1.11.1", + "discord-api-types": "^0.37.101", "discord.js": "^14.9.0", "dotenv": "^8.2.0", "express": "^4.17.1", diff --git a/src/providers/discord/chat-message.ts b/src/providers/discord/chat-message.ts index d6e4c872..fd02d277 100644 --- a/src/providers/discord/chat-message.ts +++ b/src/providers/discord/chat-message.ts @@ -1,4 +1,5 @@ import { Client, GatewayIntentBits, TextChannel, DMChannel } from 'discord.js'; +import { Routes } from 'discord-api-types/v10'; import CONFIG from '../../config'; import { SyncItemsResult, @@ -76,14 +77,20 @@ export default class DiscordChatMessageHandler extends BaseSyncHandler { let channelList: SchemaSocialChatGroup[] = []; let channels = [] - const guilds: any = await api.get('/users/@me/guilds') + try { + const guilds: any = await api.get(Routes.userGuilds()); - for (const guild of guilds) { - channels = await api.get(`/guilds/${guild.id}/channels`) - - console.log(`Channels in guild ${guild.name}:`, channels); + for (const guild of guilds) { + channels = await api.get(Routes.guildChannels(guild.id)) + + console.log(`Channels in guild ${guild.name}:`, channels); + } + + } catch (error) { + console.error('Error fetching user guilds & channels:', error); + return []; } - + for (const [id, channel] of channels) { if (channel.isTextBased()) { diff --git a/src/providers/discord/index.ts b/src/providers/discord/index.ts index f7a239f5..1716fb24 100644 --- a/src/providers/discord/index.ts +++ b/src/providers/discord/index.ts @@ -4,7 +4,7 @@ import { BaseProviderConfig, ConnectionCallbackResponse, PassportProfile } from const passport = require("passport") import { Strategy as DiscordStrategy, Scope } from '@oauth-everything/passport-discord'; -import { REST, Client, GatewayIntentBits } from 'discord.js' +import { REST } from '@discordjs/rest' import { DiscordSnowflake } from '@sapphire/snowflake' import dayjs from 'dayjs' import axios from 'axios' diff --git a/yarn.lock b/yarn.lock index 21e0a1db..fdc72eb7 100644 --- a/yarn.lock +++ b/yarn.lock @@ -35,6 +35,11 @@ resolved "https://registry.yarnpkg.com/@discordjs/collection/-/collection-1.5.0.tgz#478acd5d510cb5996c5101f47b24959ac7499cc2" integrity sha512-suyVndkEAAWrGxyw/CPGdtXoRRU6AUNkibtnbJevQzpelkJh3Q1gQqWDpqf5i39CnAn5+LrN0YS+cULeEjq2Yw== +"@discordjs/collection@^2.1.1": + version "2.1.1" + resolved "https://registry.yarnpkg.com/@discordjs/collection/-/collection-2.1.1.tgz#901917bc538c12b9c3613036d317847baee08cae" + integrity sha512-LiSusze9Tc7qF03sLCujF5iZp7K+vRNEDBZ86FT9aQAv3vxMLihUvKvpsCWiQ2DJq1tVckopKm1rxomgNUc9hg== + "@discordjs/formatters@^0.3.0": version "0.3.0" resolved "https://registry.yarnpkg.com/@discordjs/formatters/-/formatters-0.3.0.tgz#8313d158c5e974597eec43b1f381d870a507d133" @@ -56,11 +61,31 @@ tslib "^2.5.0" undici "^5.21.0" +"@discordjs/rest@^2.4.0": + version "2.4.0" + resolved "https://registry.yarnpkg.com/@discordjs/rest/-/rest-2.4.0.tgz#63bfc816af58af844914e3589d7eae609cd199b5" + integrity sha512-Xb2irDqNcq+O8F0/k/NaDp7+t091p+acb51iA4bCKfIn+WFWd6HrNvcsSbMMxIR9NjcMZS6NReTKygqiQN+ntw== + dependencies: + "@discordjs/collection" "^2.1.1" + "@discordjs/util" "^1.1.1" + "@sapphire/async-queue" "^1.5.3" + "@sapphire/snowflake" "^3.5.3" + "@vladfrangu/async_event_emitter" "^2.4.6" + discord-api-types "0.37.97" + magic-bytes.js "^1.10.0" + tslib "^2.6.3" + undici "6.19.8" + "@discordjs/util@^0.2.0": version "0.2.0" resolved "https://registry.yarnpkg.com/@discordjs/util/-/util-0.2.0.tgz#91b590dae3934ffa5fe34530afc5212c569d6751" integrity sha512-/8qNbebFzLWKOOg+UV+RB8itp4SmU5jw0tBUD3ifElW6rYNOj1Ku5JaSW7lLl/WgjjxF01l/1uQPCzkwr110vg== +"@discordjs/util@^1.1.1": + version "1.1.1" + resolved "https://registry.yarnpkg.com/@discordjs/util/-/util-1.1.1.tgz#bafcde0faa116c834da1258d78ec237080bbab29" + integrity sha512-eddz6UnOBEB1oITPinyrB2Pttej49M9FZQY8NxgEvc3tq6ZICZ19m70RsmzRdDHk80O9NoYN/25AqJl8vPVf/g== + "@ethersproject/abi@5.7.0", "@ethersproject/abi@^5.7.0": version "5.7.0" resolved "https://registry.yarnpkg.com/@ethersproject/abi/-/abi-5.7.0.tgz#b3f3e045bbbeed1af3947335c247ad625a44e449" @@ -468,6 +493,11 @@ resolved "https://registry.yarnpkg.com/@sapphire/async-queue/-/async-queue-1.5.0.tgz#2f255a3f186635c4fb5a2381e375d3dfbc5312d8" integrity sha512-JkLdIsP8fPAdh9ZZjrbHWR/+mZj0wvKS5ICibcLrRI1j84UmLMshx5n9QmL8b95d4onJ2xxiyugTgSAX7AalmA== +"@sapphire/async-queue@^1.5.3": + version "1.5.3" + resolved "https://registry.yarnpkg.com/@sapphire/async-queue/-/async-queue-1.5.3.tgz#03cd2a2f3665068f314736bdc56eee2025352422" + integrity sha512-x7zadcfJGxFka1Q3f8gCts1F0xMwCKbZweM85xECGI0hBTeIZJGGCrHgLggihBoprlQ/hBmDR5LKfIPqnmHM3w== + "@sapphire/shapeshift@^3.8.1": version "3.8.2" resolved "https://registry.yarnpkg.com/@sapphire/shapeshift/-/shapeshift-3.8.2.tgz#f9f25cba74c710b56f8790de76a9642a9635e7db" @@ -481,6 +511,11 @@ resolved "https://registry.yarnpkg.com/@sapphire/snowflake/-/snowflake-3.4.2.tgz#365af8e7b57ada924ec8e85383b921280f81d128" integrity sha512-KJwlv5gkGjs1uFV7/xx81n3tqgBwBJvH94n1xDyH3q+JSmtsMeSleJffarEBfG2yAFeJiFA4BnGOK6FFPHc19g== +"@sapphire/snowflake@^3.5.3": + version "3.5.3" + resolved "https://registry.yarnpkg.com/@sapphire/snowflake/-/snowflake-3.5.3.tgz#0c102aa2ec5b34f806e9bc8625fc6a5e1d0a0c6a" + integrity sha512-jjmJywLAFoWeBi1W7994zZyiNWPIiqRRNAmSERxyg93xRGzNYvGjlZ0gR6x0F4gPRi2+0O6S71kOZYyr3cxaIQ== + "@selderee/plugin-htmlparser2@^0.11.0": version "0.11.0" resolved "https://registry.yarnpkg.com/@selderee/plugin-htmlparser2/-/plugin-htmlparser2-0.11.0.tgz#d5b5e29a7ba6d3958a1972c7be16f4b2c188c517" @@ -1191,6 +1226,11 @@ axios "^1.2.3" ethers "^5.7.2" +"@vladfrangu/async_event_emitter@^2.4.6": + version "2.4.6" + resolved "https://registry.yarnpkg.com/@vladfrangu/async_event_emitter/-/async_event_emitter-2.4.6.tgz#508b6c45b03f917112a9008180b308ba0e4d1805" + integrity sha512-RaI5qZo6D2CVS6sTHFKg1v5Ohq/+Bo2LZ5gzUEwZ/WkHhwtGTCB/sVLw8ijOkAUxasZ+WshN/Rzj4ywsABJ5ZA== + "@xmldom/xmldom@^0.8.10", "@xmldom/xmldom@^0.8.6": version "0.8.10" resolved "https://registry.yarnpkg.com/@xmldom/xmldom/-/xmldom-0.8.10.tgz#a1337ca426aa61cef9fe15b5b28e340a72f6fa99" @@ -2415,6 +2455,16 @@ dingbat-to-unicode@^1.0.1: resolved "https://registry.yarnpkg.com/dingbat-to-unicode/-/dingbat-to-unicode-1.0.1.tgz#5091dd673241453e6b5865e26e5a4452cdef5c83" integrity sha512-98l0sW87ZT58pU4i61wa2OHwxbiYSbuxsCBozaVnYX2iCnr3bLM3fIes1/ej7h1YdOKuKt/MLs706TVnALA65w== +discord-api-types@0.37.97: + version "0.37.97" + resolved "https://registry.yarnpkg.com/discord-api-types/-/discord-api-types-0.37.97.tgz#d658573f726ad179261d538dbad4e7e8eca48d11" + integrity sha512-No1BXPcVkyVD4ZVmbNgDKaBoqgeQ+FJpzZ8wqHkfmBnTZig1FcH3iPPersiK1TUIAzgClh2IvOuVUYfcWLQAOA== + +discord-api-types@^0.37.101: + version "0.37.101" + resolved "https://registry.yarnpkg.com/discord-api-types/-/discord-api-types-0.37.101.tgz#1512a26e35b700b9dfb903a5747741a097f9c8bb" + integrity sha512-2wizd94t7G3A8U5Phr3AiuL4gSvhqistDwWnlk1VLTit8BI1jWUncFqFQNdPbHqS3661+Nx/iEyIwtVjPuBP3w== + discord-api-types@^0.37.37: version "0.37.38" resolved "https://registry.yarnpkg.com/discord-api-types/-/discord-api-types-0.37.38.tgz#0599937f64aff63a2b534376563021f17537d166" @@ -4021,6 +4071,11 @@ ltgt@2.2.1, ltgt@^2.1.2: resolved "https://registry.yarnpkg.com/ltgt/-/ltgt-2.2.1.tgz#f35ca91c493f7b73da0e07495304f17b31f87ee5" integrity sha512-AI2r85+4MquTw9ZYqabu4nMwy9Oftlfa/e/52t9IjtfG+mGBbTNdAoZ3RQKLHR6r0wQnwZnPIEh/Ya6XTWAKNA== +magic-bytes.js@^1.10.0: + version "1.10.0" + resolved "https://registry.yarnpkg.com/magic-bytes.js/-/magic-bytes.js-1.10.0.tgz#c41cf4bc2f802992b05e64962411c9dd44fdef92" + integrity sha512-/k20Lg2q8LE5xiaaSkMXk4sfvI+9EGEykFS4b0CHHGWqDYU0bGUFSwchNOMA56D7TCs9GwVTkqe9als1/ns8UQ== + mailparser@^3.7.1: version "3.7.1" resolved "https://registry.yarnpkg.com/mailparser/-/mailparser-3.7.1.tgz#4d0ea2eeb50a73dd10854a71ef1d4553bdce01cb" @@ -5691,6 +5746,11 @@ tslib@^2.5.0: resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.5.0.tgz#42bfed86f5787aeb41d031866c8f402429e0fddf" integrity sha512-336iVw3rtn2BUK7ORdIAHTyxHGRIHVReokCR3XjbckJMK7ms8FysBfhLR8IXnAgy7T0PTPNBWKiH514FOW/WSg== +tslib@^2.6.3: + version "2.7.0" + resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.7.0.tgz#d9b40c5c40ab59e8738f297df3087bf1a2690c01" + integrity sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA== + tunnel-agent@^0.6.0: version "0.6.0" resolved "https://registry.yarnpkg.com/tunnel-agent/-/tunnel-agent-0.6.0.tgz#27a5dea06b36b04a0a9966774b290868f0fc40fd" @@ -5805,6 +5865,11 @@ undici-types@~5.26.4: resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-5.26.5.tgz#bcd539893d00b56e964fd2657a4866b221a65617" integrity sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA== +undici@6.19.8: + version "6.19.8" + resolved "https://registry.yarnpkg.com/undici/-/undici-6.19.8.tgz#002d7c8a28f8cc3a44ff33c3d4be4d85e15d40e1" + integrity sha512-U8uCCl2x9TK3WANvmBavymRzxbfFYG+tAu+fgx3zxQy3qdagQqBLwJVrdyO1TBfUXvfKveMKJZhpvUYoOjM+4g== + undici@^5.21.0: version "5.21.2" resolved "https://registry.yarnpkg.com/undici/-/undici-5.21.2.tgz#329f628aaea3f1539a28b9325dccc72097d29acd" From a6a7276d89d7ed9ff9b2da26dec12917b6f245ba Mon Sep 17 00:00:00 2001 From: chime3 Date: Wed, 30 Oct 2024 15:30:17 -0700 Subject: [PATCH 08/12] chore: removed invalid handlers --- src/providers/discord/following.ts | 67 --------------------- src/providers/discord/index.ts | 5 +- src/providers/discord/sbts.ts | 96 ------------------------------ yarn.lock | 51 ++++------------ 4 files changed, 12 insertions(+), 207 deletions(-) delete mode 100644 src/providers/discord/following.ts delete mode 100644 src/providers/discord/sbts.ts diff --git a/src/providers/discord/following.ts b/src/providers/discord/following.ts deleted file mode 100644 index 890e45f3..00000000 --- a/src/providers/discord/following.ts +++ /dev/null @@ -1,67 +0,0 @@ -import { SyncHandlerPosition } from "../../interfaces" -import BaseSyncHandler from "../BaseSyncHandler" -const _ = require('lodash') - -const log4js = require("log4js") -const logger = log4js.getLogger() - -//import { Client } from 'discord.js' - -export default class Following extends BaseSyncHandler { - - protected static schemaUri: string = 'https://common.schemas.verida.io/social/following/v0.1.0/schema.json' - - public getLabel(): string { - return "Joined Servers" - } - - public getName(): string { - return 'following' - } - - /** - * @todo: Support paging through all results - * @todo: Correctly support `this.config.limitResults` - * - * @param api - */ - public async sync(api: any, syncPosition: SyncHandlerPosition): Promise { - const guildResponse: any = await api.get('/users/@me/guilds') - console.log(syncPosition) - - const now = (new Date()).toISOString() - const guilds: any = [] - for (let i in guildResponse) { - try { - const guildItem = guildResponse[i] - console.log('Processing', guildItem.name) - /* - Note: It's not possible to fetch followedTimestamp. It appears Discord rate limiting is - really extreme and only allows 5 requests per minute. - - const guildMemberInfo = await api.get(`/users/@me/guilds/${guildItem.id}/member`) - const followedTimestamp = guildMemberInfo.joined_at - */ - - const guildIcon = api.cdn.icon(guildItem.id, guildItem.icon) - const guildEntry: any = { - _id: `discord-${guildItem.id}`, - name: guildItem.name, - icon: guildIcon, - summary: `Discord guild: ${guildItem.name}`, - //uri: Discord doesn't support URL's for servers - sourceApplication: 'https://discord.com/', - sourceId: guildItem.id, - //followedTimestamp: now, - insertedAt: now - } - - guilds.push(guildEntry) - } catch (err) { - console.log(err) - } - } - - return guilds - } -} \ No newline at end of file diff --git a/src/providers/discord/index.ts b/src/providers/discord/index.ts index 84a88bff..9d6b784a 100644 --- a/src/providers/discord/index.ts +++ b/src/providers/discord/index.ts @@ -9,9 +9,8 @@ import { DiscordSnowflake } from '@sapphire/snowflake' import dayjs from 'dayjs' import axios from 'axios' -//import SBTs from './sbts' -import Following from './following' import InvalidTokenError from '../InvalidTokenError'; +import DiscordChatMessageHandler from './chat-message'; export interface DiscordProviderConfig extends BaseProviderConfig { clientID: string @@ -41,8 +40,6 @@ export default class DiscordProvider extends Base { public syncHandlers(): any[] { return [ - //SBTs - //Following, DiscordChatMessageHandler ] return [] diff --git a/src/providers/discord/sbts.ts b/src/providers/discord/sbts.ts deleted file mode 100644 index c43e598e..00000000 --- a/src/providers/discord/sbts.ts +++ /dev/null @@ -1,96 +0,0 @@ -import BaseSyncHandler from "../BaseSyncHandler" -import { SyncHandlerPosition } from "../../interfaces" -import { REST } from 'discord.js' -import DiscordProvider from "." -const _ = require('lodash') - -// import dayjs from 'dayjs' -//const log4js = require("log4js") -//const logger = log4js.getLogger() - -export default class SBTs extends BaseSyncHandler { - protected static schemaUri: string = 'https://common.schemas.verida.io/credential/base/v0.2.0/schema.json' - - public getName(): string { - return 'sbts' - } - - /** - * @todo: Support paging through all results - * @todo: Correctly support `this.config.limitResults` - * - * @param api - */ - public async sync(api: REST, syncPosition: SyncHandlerPosition): Promise { - console.log('fetching sbt credentials to sync') - - const guildResponse: any = await api.get('/users/@me/guilds') - - //const genericApi = new REST({ version: '10', authPrefix: 'Bearer' }).setToken(provider.getConfig()); - - const guilds = {} - return guilds - for (let i in guildResponse) { - try { - const guildItem = guildResponse[i] - console.log('fetching member guild info for ', guildItem.name) - const guildMemberInfo = await api.get(`/users/@me/guilds/${guildItem.id}/member`) - console.log('fetching guild info for ', guildItem.name) - const guildInfo = await api.get(`/guilds/${guildItem.id}`) - console.log(guildItem, guildMemberInfo, guildInfo) - } catch (err) { - console.log(err) - } - } - - return guilds - /*console.log('following.sync()') - console.log(syncConfig) - const me = await api.v2.me() - const limit = syncConfig.limit ? syncConfig.limit : this.config.followingLimit - const sinceId = syncConfig.sinceId ? syncConfig.sinceId.substring(8) : undefined - console.log('limit', limit) - console.log('sinceId', sinceId) - - const followingResult = await api.v2.following(me.data.id, { - 'user.fields': ['profile_image_url', 'description'], - asPaginator: true - }) - - const results = [] - const now = (new Date()).toISOString() - for await (const user of followingResult) { - if (sinceId && user.id == sinceId) { - console.log('latest user id found, exiting') - break - } - - // Iterate until rate limit is hit - // or API calls returns no more results - - //console.log(user) - - results.push({ - _id: `twitter-${user.id}`, - name: user.name, - icon: user.profile_image_url, - summary: user.description.substring(0,256), - uri: `https://twitter.com/${user.username}`, - sourceApplication: 'https://twitter.com/', - sourceId: user.id, - // twitter doesn't support a timestamp on when the user - // was followed, so set to current timestamp - followedTimestamp: now, - insertedAt: now - }) - - if (results.length >= limit) { - break - } - } - - console.log('returning following results:', results.length) - return results*/ - } - -} \ No newline at end of file diff --git a/yarn.lock b/yarn.lock index 51fa34ed..06033ce6 100644 --- a/yarn.lock +++ b/yarn.lock @@ -62,25 +62,6 @@ tslib "^2.6.3" undici "6.19.8" -"@discordjs/rest@^2.4.0": - version "2.4.0" - resolved "https://registry.yarnpkg.com/@discordjs/rest/-/rest-2.4.0.tgz#63bfc816af58af844914e3589d7eae609cd199b5" - integrity sha512-Xb2irDqNcq+O8F0/k/NaDp7+t091p+acb51iA4bCKfIn+WFWd6HrNvcsSbMMxIR9NjcMZS6NReTKygqiQN+ntw== - dependencies: - "@discordjs/collection" "^2.1.1" - "@discordjs/util" "^1.1.1" - "@sapphire/async-queue" "^1.5.3" - "@sapphire/snowflake" "^3.5.3" - "@vladfrangu/async_event_emitter" "^2.4.6" - discord-api-types "0.37.97" - magic-bytes.js "^1.10.0" - tslib "^2.6.3" - undici "6.19.8" - -"@discordjs/util@^0.2.0": - version "0.2.0" - resolved "https://registry.yarnpkg.com/@discordjs/util/-/util-0.2.0.tgz#91b590dae3934ffa5fe34530afc5212c569d6751" - integrity sha512-/8qNbebFzLWKOOg+UV+RB8itp4SmU5jw0tBUD3ifElW6rYNOj1Ku5JaSW7lLl/WgjjxF01l/1uQPCzkwr110vg== "@discordjs/util@^1.1.0", "@discordjs/util@^1.1.1": version "1.1.1" resolved "https://registry.yarnpkg.com/@discordjs/util/-/util-1.1.1.tgz#bafcde0faa116c834da1258d78ec237080bbab29" @@ -101,11 +82,6 @@ tslib "^2.6.2" ws "^8.16.0" -"@discordjs/util@^1.1.1": - version "1.1.1" - resolved "https://registry.yarnpkg.com/@discordjs/util/-/util-1.1.1.tgz#bafcde0faa116c834da1258d78ec237080bbab29" - integrity sha512-eddz6UnOBEB1oITPinyrB2Pttej49M9FZQY8NxgEvc3tq6ZICZ19m70RsmzRdDHk80O9NoYN/25AqJl8vPVf/g== - "@ethersproject/abi@5.7.0", "@ethersproject/abi@^5.7.0": version "5.7.0" resolved "https://registry.yarnpkg.com/@ethersproject/abi/-/abi-5.7.0.tgz#b3f3e045bbbeed1af3947335c247ad625a44e449" @@ -519,11 +495,6 @@ resolved "https://registry.yarnpkg.com/@sapphire/snowflake/-/snowflake-3.4.2.tgz#365af8e7b57ada924ec8e85383b921280f81d128" integrity sha512-KJwlv5gkGjs1uFV7/xx81n3tqgBwBJvH94n1xDyH3q+JSmtsMeSleJffarEBfG2yAFeJiFA4BnGOK6FFPHc19g== -"@sapphire/snowflake@^3.5.3": - version "3.5.3" - resolved "https://registry.yarnpkg.com/@sapphire/snowflake/-/snowflake-3.5.3.tgz#0c102aa2ec5b34f806e9bc8625fc6a5e1d0a0c6a" - integrity sha512-jjmJywLAFoWeBi1W7994zZyiNWPIiqRRNAmSERxyg93xRGzNYvGjlZ0gR6x0F4gPRi2+0O6S71kOZYyr3cxaIQ== - "@selderee/plugin-htmlparser2@^0.11.0": version "0.11.0" resolved "https://registry.yarnpkg.com/@selderee/plugin-htmlparser2/-/plugin-htmlparser2-0.11.0.tgz#d5b5e29a7ba6d3958a1972c7be16f4b2c188c517" @@ -2699,21 +2670,21 @@ dingbat-to-unicode@^1.0.1: resolved "https://registry.yarnpkg.com/dingbat-to-unicode/-/dingbat-to-unicode-1.0.1.tgz#5091dd673241453e6b5865e26e5a4452cdef5c83" integrity sha512-98l0sW87ZT58pU4i61wa2OHwxbiYSbuxsCBozaVnYX2iCnr3bLM3fIes1/ej7h1YdOKuKt/MLs706TVnALA65w== +discord-api-types@0.37.100: + version "0.37.100" + resolved "https://registry.yarnpkg.com/discord-api-types/-/discord-api-types-0.37.100.tgz#5979892d39511bc7f1dbb9660d2d2cad698b3de7" + integrity sha512-a8zvUI0GYYwDtScfRd/TtaNBDTXwP5DiDVX7K5OmE+DRT57gBqKnwtOC5Ol8z0mRW8KQfETIgiB8U0YZ9NXiCA== + +discord-api-types@0.37.83: + version "0.37.83" + resolved "https://registry.yarnpkg.com/discord-api-types/-/discord-api-types-0.37.83.tgz#a22a799729ceded8176ea747157837ddf4708b1f" + integrity sha512-urGGYeWtWNYMKnYlZnOnDHm8fVRffQs3U0SpE8RHeiuLKb/u92APS8HoQnPTFbnXmY1vVnXjXO4dOxcAn3J+DA== + discord-api-types@0.37.97: version "0.37.97" resolved "https://registry.yarnpkg.com/discord-api-types/-/discord-api-types-0.37.97.tgz#d658573f726ad179261d538dbad4e7e8eca48d11" integrity sha512-No1BXPcVkyVD4ZVmbNgDKaBoqgeQ+FJpzZ8wqHkfmBnTZig1FcH3iPPersiK1TUIAzgClh2IvOuVUYfcWLQAOA== -discord-api-types@^0.37.101: - version "0.37.101" - resolved "https://registry.yarnpkg.com/discord-api-types/-/discord-api-types-0.37.101.tgz#1512a26e35b700b9dfb903a5747741a097f9c8bb" - integrity sha512-2wizd94t7G3A8U5Phr3AiuL4gSvhqistDwWnlk1VLTit8BI1jWUncFqFQNdPbHqS3661+Nx/iEyIwtVjPuBP3w== - -discord-api-types@^0.37.37: - version "0.37.38" - resolved "https://registry.yarnpkg.com/discord-api-types/-/discord-api-types-0.37.38.tgz#0599937f64aff63a2b534376563021f17537d166" - integrity sha512-p9dibYycLXY1FUM9bIuohYeHHwa/634394QE90n6Tlb6ID5HtCtXBPYY68oaOlFnlFLBJAOYATMwX5oiV+VqiA== - discord.js@^14.16.3: version "14.16.3" resolved "https://registry.yarnpkg.com/discord.js/-/discord.js-14.16.3.tgz#9553366953c992469f47a55af2a11c2054a9babe" @@ -5933,7 +5904,7 @@ tslib@2.4.0: resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.4.0.tgz#7cecaa7f073ce680a05847aa77be941098f36dc3" integrity sha512-d6xOpEDfsi2CZVlPQzGeux8XMwLT9hssAsaPYExaQMuYskwb+x1x7J371tWlbBdWHroy99KnVB6qIkUbs5X3UQ== -tslib@^2.6.2, tslib@^2.6.3: +tslib@^2.6.2: version "2.8.0" resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.8.0.tgz#d124c86c3c05a40a91e6fdea4021bd31d377971b" integrity sha512-jWVzBLplnCmoaTr13V9dYbiQ99wvZRd0vNWaDRg+aVYRcjDF3nDksxFDE/+fkXnKhpnUUkmx5pK/v8mCtLVqZA== From c1ea7405f6537f99d667c5b2bdcc221ac7fe29eb Mon Sep 17 00:00:00 2001 From: chime3 Date: Wed, 30 Oct 2024 15:30:51 -0700 Subject: [PATCH 09/12] fix: refactored discord chat message handler --- src/providers/discord/chat-message.ts | 160 +++++++++----------------- src/providers/discord/interfaces.ts | 4 - 2 files changed, 53 insertions(+), 111 deletions(-) diff --git a/src/providers/discord/chat-message.ts b/src/providers/discord/chat-message.ts index fd02d277..6c695a53 100644 --- a/src/providers/discord/chat-message.ts +++ b/src/providers/discord/chat-message.ts @@ -1,8 +1,8 @@ -import { Client, GatewayIntentBits, TextChannel, DMChannel } from 'discord.js'; +import { Client, GatewayIntentBits, DMChannel } from 'discord.js'; +import { REST } from '@discordjs/rest'; import { Routes } from 'discord-api-types/v10'; import CONFIG from '../../config'; import { - SyncItemsResult, SyncResponse, SyncHandlerStatus, ProviderHandlerOption, @@ -14,7 +14,7 @@ import { SchemaSocialChatGroup, SchemaSocialChatMessage, } from '../../schemas'; -import { DiscordChatGroupType, DiscordHandlerConfig } from './interfaces'; +import { DiscordHandlerConfig } from './interfaces'; import BaseSyncHandler from '../BaseSyncHandler'; import { ItemsRangeTracker } from '../../helpers/itemsRangeTracker'; import { ItemsRange } from '../../helpers/interfaces'; @@ -45,87 +45,72 @@ export default class DiscordChatMessageHandler extends BaseSyncHandler { label: 'Channel types', type: ConnectionOptionType.ENUM_MULTI, enumOptions: [ - { label: 'Text Channels', value: DiscordChatGroupType.GUILD_TEXT }, - { label: 'Direct Messages', value: DiscordChatGroupType.DM }, + { label: 'Direct Messages', value: 'DM' }, ], - defaultValue: [ - DiscordChatGroupType.GUILD_TEXT, - DiscordChatGroupType.DM, - ].join(','), + defaultValue: 'DM', }, ]; } public getDiscordClient(): Client { - const token = this.connection.accessToken; - + const client = new Client({ intents: [ - GatewayIntentBits.Guilds, - GatewayIntentBits.GuildMessages, GatewayIntentBits.DirectMessages, - GatewayIntentBits.MessageContent + GatewayIntentBits.Guilds, + GatewayIntentBits.MessageContent, ], }); - + client.login(token); return client; - } + } protected async buildChatGroupList(api: any): Promise { let channelList: SchemaSocialChatGroup[] = []; - let channels = [] - +//let dmChannels: any[] = []; + try { - const guilds: any = await api.get(Routes.userGuilds()); - - for (const guild of guilds) { - channels = await api.get(Routes.guildChannels(guild.id)) - - console.log(`Channels in guild ${guild.name}:`, channels); - } - + // const client = new REST({ version: '10', authPrefix: 'Bearer' }).setToken(this.connection.accessToken); + // const channels = await client.get('/users/@me/guilds'); + // Fetch DM channels only + // dmChannels = await api.post(Routes.userChannels()); + const client = this.getDiscordClient(); + + // Wait until the client is ready to ensure all channels are accessible + await new Promise(resolve => client.once('ready', resolve)); + + const dmChannels = client.channels.cache.filter( + channel => channel.isDMBased() + ) as Map; + console.log('DM Channels===========:'); + console.log(dmChannels) + } catch (error) { - console.error('Error fetching user guilds & channels:', error); + console.error('Error fetching DM channels:', error); return []; } - - - for (const [id, channel] of channels) { - if (channel.isTextBased()) { - if (channel instanceof TextChannel) { - const textChannel = channel as TextChannel; - const group: SchemaSocialChatGroup = { - _id: this.buildItemId(textChannel.id), - name: textChannel.name, - sourceAccountId: this.provider.getAccountId(), - sourceApplication: this.getProviderApplicationUrl(), - sourceId: textChannel.id, - schema: CONFIG.verida.schemas.CHAT_GROUP, - sourceData: textChannel, - insertedAt: new Date().toISOString(), - }; - channelList.push(group); - } else if (channel.isDMBased()) { - const dmChannel = channel as DMChannel; - const group: SchemaSocialChatGroup = { - _id: this.buildItemId(dmChannel.id), - name: `DM with ${dmChannel.recipient?.username}`, - sourceAccountId: this.provider.getAccountId(), - sourceApplication: this.getProviderApplicationUrl(), - sourceId: dmChannel.id, - schema: CONFIG.verida.schemas.CHAT_GROUP, - sourceData: dmChannel, - insertedAt: new Date().toISOString(), - }; - channelList.push(group); - } +/* + for (const channel of dmChannels) { + if (channel.isDMBased()) { + const dmChannel = channel as DMChannel; + const group: SchemaSocialChatGroup = { + _id: this.buildItemId(dmChannel.id), + name: `DM with ${dmChannel.recipient?.username}`, + sourceAccountId: this.provider.getAccountId(), + sourceApplication: this.getProviderApplicationUrl(), + sourceId: dmChannel.id, + schema: CONFIG.verida.schemas.CHAT_GROUP, + sourceData: dmChannel, + insertedAt: new Date().toISOString(), + }; + channelList.push(group); } - } + }*/ + return channelList; } - protected async fetchMessageRange( chatGroup: SchemaSocialChatGroup, @@ -133,7 +118,7 @@ export default class DiscordChatMessageHandler extends BaseSyncHandler { apiClient: Client ): Promise { const messages: SchemaSocialChatMessage[] = []; - const channel = apiClient.channels.cache.get(chatGroup.sourceId!) as TextChannel | DMChannel; + const channel = apiClient.channels.cache.get(chatGroup.sourceId!) as DMChannel; if (!channel) return messages; @@ -172,25 +157,16 @@ export default class DiscordChatMessageHandler extends BaseSyncHandler { api: any, syncPosition: SyncHandlerPosition ): Promise { - try { - //const apiClient = await this.getDiscordClient(); - const groupList = await this.buildChatGroupList(api); // Fetch all Text Channels and DM groups + const groupList = await this.buildChatGroupList(api); let totalMessages = 0; let chatHistory: SchemaSocialChatMessage[] = []; - // Determine the current group position - let groupPosition = this.getGroupPositionIndex(groupList, syncPosition); - const groupCount = groupList.length; - // Iterate over each group - for (let i = 0; i < Math.min(groupCount, this.config.channelLimit); i++) { - const groupIndex = (groupPosition + i) % groupCount; // Rotate through groups - const group = groupList[groupIndex]; + for (const group of groupList) { - // Use a separate ItemsRangeTracker for each group let rangeTracker = new ItemsRangeTracker(group.syncData); const fetchedMessages = await this.fetchAndTrackMessages( @@ -199,33 +175,19 @@ export default class DiscordChatMessageHandler extends BaseSyncHandler { api ); - // Concatenate the fetched messages to the total chat history chatHistory = chatHistory.concat(fetchedMessages); totalMessages += fetchedMessages.length; - // Update the group's sync data with the latest rangeTracker state group.syncData = rangeTracker.export(); - - // Stop if the total messages fetched reach the batch size - if (totalMessages >= this.config.messageBatchSize) { - syncPosition.thisRef = groupList[(groupIndex + 1) % groupCount].sourceId; // Continue from the next group in the next sync - break; - } } - // Finalize sync position and status based on message count this.updateSyncPosition( syncPosition, - totalMessages, - groupCount, - chatHistory + totalMessages ); - // Concatenate only items after syncPosition.thisRef and chatHistory - const remainingGroups = groupList.slice(groupPosition + 1); - return { - results: remainingGroups.concat(chatHistory), + results: groupList.concat(chatHistory), position: syncPosition, }; } catch (err: any) { @@ -234,18 +196,6 @@ export default class DiscordChatMessageHandler extends BaseSyncHandler { } } - private getGroupPositionIndex( - groupList: SchemaSocialChatGroup[], - syncPosition: SyncHandlerPosition - ): number { - const groupPosition = groupList.findIndex( - (group) => group.sourceId === syncPosition.thisRef - ); - - // If not found, return 0 to start from the beginning - return groupPosition === -1 ? 0 : groupPosition; - } - private async fetchAndTrackMessages( group: SchemaSocialChatGroup, rangeTracker: ItemsRangeTracker, @@ -294,18 +244,14 @@ export default class DiscordChatMessageHandler extends BaseSyncHandler { private updateSyncPosition( syncPosition: SyncHandlerPosition, - totalMessages: number, - groupCount: number, - chatHistory: SchemaSocialChatMessage[] + totalMessages: number ) { if (totalMessages === 0) { syncPosition.status = SyncHandlerStatus.ENABLED; syncPosition.syncMessage = 'No new messages found.'; - } else if (totalMessages < this.config.messageBatchSize) { - syncPosition.syncMessage = `Processed ${totalMessages} messages across ${groupCount} groups. Sync complete.`; - syncPosition.status = SyncHandlerStatus.ENABLED; } else { - syncPosition.syncMessage = `Batch complete (${this.config.messageBatchSize}). More results pending.`; + syncPosition.syncMessage = `Batch complete (${totalMessages}). More results pending.`; } } + } diff --git a/src/providers/discord/interfaces.ts b/src/providers/discord/interfaces.ts index 746de5f8..81d8b6f4 100644 --- a/src/providers/discord/interfaces.ts +++ b/src/providers/discord/interfaces.ts @@ -1,10 +1,6 @@ import { BaseHandlerConfig, BaseProviderConfig } from "../../interfaces"; export interface DiscordHandlerConfig extends BaseHandlerConfig { - // Maximum number of channels to process - channelLimit: number; - // Maximum number of messages to process in a given batch - messageBatchSize: number; // Maximum number of messages to process in a channel messagesPerChannelLimit: number; } From 55ba963f8c86df3ed3af4951f515d71dbaf27b62 Mon Sep 17 00:00:00 2001 From: chime3 Date: Thu, 31 Oct 2024 21:10:00 -0700 Subject: [PATCH 10/12] fix: revert changes --- src/providers/discord/chat-message.ts | 22 ++++++---------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/src/providers/discord/chat-message.ts b/src/providers/discord/chat-message.ts index 6c695a53..4b941f9b 100644 --- a/src/providers/discord/chat-message.ts +++ b/src/providers/discord/chat-message.ts @@ -69,29 +69,19 @@ export default class DiscordChatMessageHandler extends BaseSyncHandler { protected async buildChatGroupList(api: any): Promise { let channelList: SchemaSocialChatGroup[] = []; -//let dmChannels: any[] = []; + let dmChannels: any[] = []; try { - // const client = new REST({ version: '10', authPrefix: 'Bearer' }).setToken(this.connection.accessToken); - // const channels = await client.get('/users/@me/guilds'); + const client = new REST({ version: '10', authPrefix: 'Bearer' }).setToken(this.connection.accessToken); + const channels = await client.get('/users/@me/guilds'); // Fetch DM channels only - // dmChannels = await api.post(Routes.userChannels()); - const client = this.getDiscordClient(); - - // Wait until the client is ready to ensure all channels are accessible - await new Promise(resolve => client.once('ready', resolve)); - - const dmChannels = client.channels.cache.filter( - channel => channel.isDMBased() - ) as Map; - console.log('DM Channels===========:'); - console.log(dmChannels) + dmChannels = await api.post(Routes.userChannels()); } catch (error) { console.error('Error fetching DM channels:', error); return []; } -/* + for (const channel of dmChannels) { if (channel.isDMBased()) { const dmChannel = channel as DMChannel; @@ -107,7 +97,7 @@ export default class DiscordChatMessageHandler extends BaseSyncHandler { }; channelList.push(group); } - }*/ + } return channelList; } From fd5b87e61f887da062f66be2aa4eda9d5c4e2544 Mon Sep 17 00:00:00 2001 From: chime3 Date: Thu, 31 Oct 2024 21:37:37 -0700 Subject: [PATCH 11/12] feat:added discord read me --- src/providers/discord/README.md | 46 +++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 src/providers/discord/README.md diff --git a/src/providers/discord/README.md b/src/providers/discord/README.md new file mode 100644 index 00000000..9e531197 --- /dev/null +++ b/src/providers/discord/README.md @@ -0,0 +1,46 @@ +# Discord Connector Configuration + +This guide provides instructions for configuring a Discord connector. + +## Steps for Setting up a Discord App + +1. **Go to the [Discord Developer Portal](https://discord.com/developers/applications)** + Access the Discord Developer Portal to create a new application and manage its settings. + +2. **Create a New Application** + - Click **New Application**. + - Enter an **App Name** and select the associated workspace or team for development. + +3. **Retrieve Client ID and Client Secret** + - Under the **OAuth2** tab, locate the `Client ID` and `Client Secret`, which are required for authentication. + +4. **Configure Redirect URL and Permissions Scopes** + - Navigate to the **OAuth2** section to set up redirect URLs and required scopes. + - **Redirect URL**: `https://127.0.0.1:5021/callback/discord` + - Add the following scopes: + - `identify` + - `guilds` + - `guilds.members.read` + - `messages.read` + - `email` + - `dm_channels.read` + - `dm_channels.messages.read` + +### Notes + +Discord servers contain numerous public channels and general messages. To ensure relevant data access, We process only **Direct Messages (DMs)** here. + +To use DM-related scopes, the App should be approved by Discord team due to security reasons. +## Pagination in Discord + +Discord uses cursor-based pagination, which allows fetching messages in relation to specific message IDs (`before` or `after` parameters). + + +``` + const response = await apiClient.channels.messages.list({ + channel_id, + limit, + before, // Message ID to retrieve messages sent before this ID + after // Message ID to retrieve messages sent after this ID + }); +``` \ No newline at end of file From 43baa89af0a42306df78256555007df0c2241789 Mon Sep 17 00:00:00 2001 From: chime3 Date: Tue, 14 Jan 2025 21:30:44 -0700 Subject: [PATCH 12/12] fix: update discord oAuth2 scopes --- src/providers/discord/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/providers/discord/index.ts b/src/providers/discord/index.ts index 9d6b784a..cfc2fad3 100644 --- a/src/providers/discord/index.ts +++ b/src/providers/discord/index.ts @@ -20,7 +20,7 @@ export interface DiscordProviderConfig extends BaseProviderConfig { } // Note: If scopes change a user needs to disconnect and reconnect the app -const SCOPE = [Scope.IDENTIFY, Scope.EMAIL, Scope.GUILDS, 'guilds.members.read'] +const SCOPE = [Scope.IDENTIFY, Scope.EMAIL, Scope.GUILDS, 'guilds.members.read', Scope.MESSAGES_READ, 'dm_channels.read'] export default class DiscordProvider extends Base {