diff --git a/package.json b/package.json index 0be4c9f0..7a490da6 100644 --- a/package.json +++ b/package.json @@ -33,6 +33,7 @@ "homepage": "https://github.com/verida/server-template#readme", "dependencies": { "@notionhq/client": "^2.2.15", + "@discordjs/rest": "^2.4.0", "@oauth-everything/passport-discord": "^1.0.2", "@sapphire/snowflake": "^3.4.2", "@superfaceai/passport-twitter-oauth2": "^1.2.3", diff --git a/src/providers/discord/README.md b/src/providers/discord/README.md new file mode 100644 index 00000000..9e531197 --- /dev/null +++ b/src/providers/discord/README.md @@ -0,0 +1,46 @@ +# Discord Connector Configuration + +This guide provides instructions for configuring a Discord connector. + +## Steps for Setting up a Discord App + +1. **Go to the [Discord Developer Portal](https://discord.com/developers/applications)** + Access the Discord Developer Portal to create a new application and manage its settings. + +2. **Create a New Application** + - Click **New Application**. + - Enter an **App Name** and select the associated workspace or team for development. + +3. **Retrieve Client ID and Client Secret** + - Under the **OAuth2** tab, locate the `Client ID` and `Client Secret`, which are required for authentication. + +4. **Configure Redirect URL and Permissions Scopes** + - Navigate to the **OAuth2** section to set up redirect URLs and required scopes. + - **Redirect URL**: `https://127.0.0.1:5021/callback/discord` + - Add the following scopes: + - `identify` + - `guilds` + - `guilds.members.read` + - `messages.read` + - `email` + - `dm_channels.read` + - `dm_channels.messages.read` + +### Notes + +Discord servers contain numerous public channels and general messages. To ensure relevant data access, We process only **Direct Messages (DMs)** here. + +To use DM-related scopes, the App should be approved by Discord team due to security reasons. +## Pagination in Discord + +Discord uses cursor-based pagination, which allows fetching messages in relation to specific message IDs (`before` or `after` parameters). + + +``` + const response = await apiClient.channels.messages.list({ + channel_id, + limit, + before, // Message ID to retrieve messages sent before this ID + after // Message ID to retrieve messages sent after this ID + }); +``` \ No newline at end of file diff --git a/src/providers/discord/chat-message.ts b/src/providers/discord/chat-message.ts new file mode 100644 index 00000000..4b941f9b --- /dev/null +++ b/src/providers/discord/chat-message.ts @@ -0,0 +1,247 @@ +import { Client, GatewayIntentBits, DMChannel } from 'discord.js'; +import { REST } from '@discordjs/rest'; +import { Routes } from 'discord-api-types/v10'; +import CONFIG from '../../config'; +import { + SyncResponse, + SyncHandlerStatus, + ProviderHandlerOption, + ConnectionOptionType, + SyncHandlerPosition, +} from '../../interfaces'; +import { + SchemaChatMessageType, + SchemaSocialChatGroup, + SchemaSocialChatMessage, +} from '../../schemas'; +import { DiscordHandlerConfig } from './interfaces'; +import BaseSyncHandler from '../BaseSyncHandler'; +import { ItemsRangeTracker } from '../../helpers/itemsRangeTracker'; +import { ItemsRange } from '../../helpers/interfaces'; + +export default class DiscordChatMessageHandler extends BaseSyncHandler { + protected config: DiscordHandlerConfig; + + public getName(): string { + return 'chat-message'; + } + + public getLabel(): string { + return 'Chat Messages'; + } + + public getSchemaUri(): string { + return CONFIG.verida.schemas.CHAT_MESSAGE; + } + + public getProviderApplicationUrl(): string { + return 'https://discord.com/'; + } + + public getOptions(): ProviderHandlerOption[] { + return [ + { + id: 'channelTypes', + label: 'Channel types', + type: ConnectionOptionType.ENUM_MULTI, + enumOptions: [ + { label: 'Direct Messages', value: 'DM' }, + ], + defaultValue: 'DM', + }, + ]; + } + + public getDiscordClient(): Client { + const token = this.connection.accessToken; + + const client = new Client({ + intents: [ + GatewayIntentBits.DirectMessages, + GatewayIntentBits.Guilds, + GatewayIntentBits.MessageContent, + ], + }); + + client.login(token); + return client; + } + + protected async buildChatGroupList(api: any): Promise { + let channelList: SchemaSocialChatGroup[] = []; + let dmChannels: any[] = []; + + try { + const client = new REST({ version: '10', authPrefix: 'Bearer' }).setToken(this.connection.accessToken); + const channels = await client.get('/users/@me/guilds'); + // Fetch DM channels only + dmChannels = await api.post(Routes.userChannels()); + + } catch (error) { + console.error('Error fetching DM channels:', error); + return []; + } + + for (const channel of dmChannels) { + if (channel.isDMBased()) { + const dmChannel = channel as DMChannel; + const group: SchemaSocialChatGroup = { + _id: this.buildItemId(dmChannel.id), + name: `DM with ${dmChannel.recipient?.username}`, + sourceAccountId: this.provider.getAccountId(), + sourceApplication: this.getProviderApplicationUrl(), + sourceId: dmChannel.id, + schema: CONFIG.verida.schemas.CHAT_GROUP, + sourceData: dmChannel, + insertedAt: new Date().toISOString(), + }; + channelList.push(group); + } + } + + return channelList; + } + + protected async fetchMessageRange( + chatGroup: SchemaSocialChatGroup, + range: ItemsRange, + apiClient: Client + ): Promise { + const messages: SchemaSocialChatMessage[] = []; + const channel = apiClient.channels.cache.get(chatGroup.sourceId!) as DMChannel; + + if (!channel) return messages; + + const fetchedMessages = await channel.messages.fetch({ + after: range.startId, + before: range.endId, + }); + + for (const message of fetchedMessages.values()) { + const chatMessage: SchemaSocialChatMessage = { + _id: this.buildItemId(message.id), + groupId: chatGroup._id, + groupName: chatGroup.name, + messageText: message.content, + fromHandle: message.author.username, + sourceAccountId: this.provider.getAccountId(), + sourceApplication: this.getProviderApplicationUrl(), + sourceId: message.id, + sourceData: message, + insertedAt: new Date(message.createdTimestamp).toISOString(), + sentAt: new Date(message.createdTimestamp).toISOString(), + type: + message.author.id === this.connection.profile.id + ? SchemaChatMessageType.SEND + : SchemaChatMessageType.RECEIVE, + fromId: message.author.id, + name: message.content.substring(0, 30), + }; + messages.push(chatMessage); + } + + return messages; + } + + public async _sync( + api: any, + syncPosition: SyncHandlerPosition + ): Promise { + try { + const groupList = await this.buildChatGroupList(api); + + let totalMessages = 0; + let chatHistory: SchemaSocialChatMessage[] = []; + + const groupCount = groupList.length; + + for (const group of groupList) { + + let rangeTracker = new ItemsRangeTracker(group.syncData); + + const fetchedMessages = await this.fetchAndTrackMessages( + group, + rangeTracker, + api + ); + + chatHistory = chatHistory.concat(fetchedMessages); + totalMessages += fetchedMessages.length; + + group.syncData = rangeTracker.export(); + } + + this.updateSyncPosition( + syncPosition, + totalMessages + ); + + return { + results: groupList.concat(chatHistory), + position: syncPosition, + }; + } catch (err: any) { + console.error(err); + throw err; + } + } + + private async fetchAndTrackMessages( + group: SchemaSocialChatGroup, + rangeTracker: ItemsRangeTracker, + apiClient: any + ): Promise { + // Validate group and group.id + if (!group || !group.sourceId) { + throw new Error('Invalid group or missing group sourceId'); + } + + // Initialize range from tracker + let currentRange = rangeTracker.nextRange(); + let items: SchemaSocialChatMessage[] = []; + + while (true) { + // Fetch messages for the current range using fetchMessageRange + const messages = await this.fetchMessageRange(group, currentRange, apiClient); + + if (!messages.length) break; + + // Add fetched messages to the main list + items = items.concat(messages); + + // Break loop if messages reached group limit + if (items.length > this.config.messagesPerChannelLimit) { + // Mark the current range as complete and stop + rangeTracker.completedRange({ + startId: messages[0].sourceId, + endId: messages[messages.length - 1].sourceId, + }, false); + break; + } else { + // Update rangeTracker and continue fetching + rangeTracker.completedRange({ + startId: messages[0].sourceId, + endId: messages[messages.length - 1].sourceId, + }, false); + + // Move to the next range + currentRange = rangeTracker.nextRange(); + } + } + + return items; + } + + private updateSyncPosition( + syncPosition: SyncHandlerPosition, + totalMessages: number + ) { + if (totalMessages === 0) { + syncPosition.status = SyncHandlerStatus.ENABLED; + syncPosition.syncMessage = 'No new messages found.'; + } else { + syncPosition.syncMessage = `Batch complete (${totalMessages}). More results pending.`; + } + } + +} diff --git a/src/providers/discord/following.ts b/src/providers/discord/following.ts deleted file mode 100644 index 890e45f3..00000000 --- a/src/providers/discord/following.ts +++ /dev/null @@ -1,67 +0,0 @@ -import { SyncHandlerPosition } from "../../interfaces" -import BaseSyncHandler from "../BaseSyncHandler" -const _ = require('lodash') - -const log4js = require("log4js") -const logger = log4js.getLogger() - -//import { Client } from 'discord.js' - -export default class Following extends BaseSyncHandler { - - protected static schemaUri: string = 'https://common.schemas.verida.io/social/following/v0.1.0/schema.json' - - public getLabel(): string { - return "Joined Servers" - } - - public getName(): string { - return 'following' - } - - /** - * @todo: Support paging through all results - * @todo: Correctly support `this.config.limitResults` - * - * @param api - */ - public async sync(api: any, syncPosition: SyncHandlerPosition): Promise { - const guildResponse: any = await api.get('/users/@me/guilds') - console.log(syncPosition) - - const now = (new Date()).toISOString() - const guilds: any = [] - for (let i in guildResponse) { - try { - const guildItem = guildResponse[i] - console.log('Processing', guildItem.name) - /* - Note: It's not possible to fetch followedTimestamp. It appears Discord rate limiting is - really extreme and only allows 5 requests per minute. - - const guildMemberInfo = await api.get(`/users/@me/guilds/${guildItem.id}/member`) - const followedTimestamp = guildMemberInfo.joined_at - */ - - const guildIcon = api.cdn.icon(guildItem.id, guildItem.icon) - const guildEntry: any = { - _id: `discord-${guildItem.id}`, - name: guildItem.name, - icon: guildIcon, - summary: `Discord guild: ${guildItem.name}`, - //uri: Discord doesn't support URL's for servers - sourceApplication: 'https://discord.com/', - sourceId: guildItem.id, - //followedTimestamp: now, - insertedAt: now - } - - guilds.push(guildEntry) - } catch (err) { - console.log(err) - } - } - - return guilds - } -} \ No newline at end of file diff --git a/src/providers/discord/index.ts b/src/providers/discord/index.ts index d7a74040..cfc2fad3 100644 --- a/src/providers/discord/index.ts +++ b/src/providers/discord/index.ts @@ -1,17 +1,16 @@ import { Request, Response } from 'express' import Base from "../BaseProvider" -import { BaseProviderConfig, ConnectionCallbackResponse } from '../../interfaces' +import { BaseProviderConfig, ConnectionCallbackResponse, PassportProfile } from '../../interfaces' const passport = require("passport") import { Strategy as DiscordStrategy, Scope } from '@oauth-everything/passport-discord'; -import { REST, Client, GatewayIntentBits } from 'discord.js' +import { REST } from '@discordjs/rest' import { DiscordSnowflake } from '@sapphire/snowflake' import dayjs from 'dayjs' import axios from 'axios' -//import SBTs from './sbts' -import Following from './following' import InvalidTokenError from '../InvalidTokenError'; +import DiscordChatMessageHandler from './chat-message'; export interface DiscordProviderConfig extends BaseProviderConfig { clientID: string @@ -21,7 +20,7 @@ export interface DiscordProviderConfig extends BaseProviderConfig { } // Note: If scopes change a user needs to disconnect and reconnect the app -const SCOPE = [Scope.IDENTIFY, Scope.EMAIL, Scope.GUILDS, 'guilds.members.read'] +const SCOPE = [Scope.IDENTIFY, Scope.EMAIL, Scope.GUILDS, 'guilds.members.read', Scope.MESSAGES_READ, 'dm_channels.read'] export default class DiscordProvider extends Base { @@ -41,15 +40,14 @@ export default class DiscordProvider extends Base { public syncHandlers(): any[] { return [ - //SBTs - Following + DiscordChatMessageHandler ] return [] } public async connect(req: Request, res: Response, next: any): Promise { this.init() - const auth = await passport.authenticate(this.getAccountId()) + const auth = await passport.authenticate(this.getProviderId()) return auth(req, res, next) } @@ -65,7 +63,7 @@ export default class DiscordProvider extends Base { this.init() const promise = new Promise((resolve, rejects) => { - const auth = passport.authenticate(this.getAccountId(), { + const auth = passport.authenticate(this.getProviderId(), { scope: SCOPE, failureRedirect: '/failure/discord', failureMessage: true @@ -73,11 +71,30 @@ export default class DiscordProvider extends Base { if (err) { rejects(err) } else { + + const profile: PassportProfile = { + id: data.profile.id, // Discord user ID + provider: data.profile.provider, // discord + displayName: data.profile.displayName, + name: { + familyName: '', // Discord does not provide family name + givenName: data.profile._json.global_name, // Global name as the given name + }, + emails: data.profile.emails, + photos: data.profile.photos, // Photos array from Discord profile + connectionProfile: { + username: data.profile.username, // Discord username + email: data.profile._json.email, // Email from profile + readableId: data.profile.displayName, + verified: data.profile._json.verified // Verified status from Discord profile + } + }; + const connectionToken: ConnectionCallbackResponse = { id: data.profile.id, accessToken: data.accessToken, refreshToken: data.refreshToken, - profile: data.profile + profile: profile } resolve(connectionToken) diff --git a/src/providers/discord/interfaces.ts b/src/providers/discord/interfaces.ts new file mode 100644 index 00000000..81d8b6f4 --- /dev/null +++ b/src/providers/discord/interfaces.ts @@ -0,0 +1,23 @@ +import { BaseHandlerConfig, BaseProviderConfig } from "../../interfaces"; + +export interface DiscordHandlerConfig extends BaseHandlerConfig { + // Maximum number of messages to process in a channel + messagesPerChannelLimit: number; +} + +export interface DiscordProviderConfig extends BaseProviderConfig { + clientId: string; + clientSecret: string; + token: string; + callbackUrl: string; +} + +export enum DiscordChatGroupType { + GUILD_TEXT = "guild_text", // Text channel in a server + DM = "dm", // Direct message + GUILD_VOICE = "guild_voice", // Voice channel in a server + GROUP_DM = "group_dm", // Group direct message + GUILD_CATEGORY = "guild_category", // Category that contains channels + GUILD_NEWS = "guild_news", // News channels (for announcements) + GUILD_STORE = "guild_store", // Store channels (for selling items) +} diff --git a/src/providers/discord/sbts.ts b/src/providers/discord/sbts.ts deleted file mode 100644 index c43e598e..00000000 --- a/src/providers/discord/sbts.ts +++ /dev/null @@ -1,96 +0,0 @@ -import BaseSyncHandler from "../BaseSyncHandler" -import { SyncHandlerPosition } from "../../interfaces" -import { REST } from 'discord.js' -import DiscordProvider from "." -const _ = require('lodash') - -// import dayjs from 'dayjs' -//const log4js = require("log4js") -//const logger = log4js.getLogger() - -export default class SBTs extends BaseSyncHandler { - protected static schemaUri: string = 'https://common.schemas.verida.io/credential/base/v0.2.0/schema.json' - - public getName(): string { - return 'sbts' - } - - /** - * @todo: Support paging through all results - * @todo: Correctly support `this.config.limitResults` - * - * @param api - */ - public async sync(api: REST, syncPosition: SyncHandlerPosition): Promise { - console.log('fetching sbt credentials to sync') - - const guildResponse: any = await api.get('/users/@me/guilds') - - //const genericApi = new REST({ version: '10', authPrefix: 'Bearer' }).setToken(provider.getConfig()); - - const guilds = {} - return guilds - for (let i in guildResponse) { - try { - const guildItem = guildResponse[i] - console.log('fetching member guild info for ', guildItem.name) - const guildMemberInfo = await api.get(`/users/@me/guilds/${guildItem.id}/member`) - console.log('fetching guild info for ', guildItem.name) - const guildInfo = await api.get(`/guilds/${guildItem.id}`) - console.log(guildItem, guildMemberInfo, guildInfo) - } catch (err) { - console.log(err) - } - } - - return guilds - /*console.log('following.sync()') - console.log(syncConfig) - const me = await api.v2.me() - const limit = syncConfig.limit ? syncConfig.limit : this.config.followingLimit - const sinceId = syncConfig.sinceId ? syncConfig.sinceId.substring(8) : undefined - console.log('limit', limit) - console.log('sinceId', sinceId) - - const followingResult = await api.v2.following(me.data.id, { - 'user.fields': ['profile_image_url', 'description'], - asPaginator: true - }) - - const results = [] - const now = (new Date()).toISOString() - for await (const user of followingResult) { - if (sinceId && user.id == sinceId) { - console.log('latest user id found, exiting') - break - } - - // Iterate until rate limit is hit - // or API calls returns no more results - - //console.log(user) - - results.push({ - _id: `twitter-${user.id}`, - name: user.name, - icon: user.profile_image_url, - summary: user.description.substring(0,256), - uri: `https://twitter.com/${user.username}`, - sourceApplication: 'https://twitter.com/', - sourceId: user.id, - // twitter doesn't support a timestamp on when the user - // was followed, so set to current timestamp - followedTimestamp: now, - insertedAt: now - }) - - if (results.length >= limit) { - break - } - } - - console.log('returning following results:', results.length) - return results*/ - } - -} \ No newline at end of file diff --git a/yarn.lock b/yarn.lock index 90a72609..06033ce6 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5904,11 +5904,16 @@ tslib@2.4.0: resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.4.0.tgz#7cecaa7f073ce680a05847aa77be941098f36dc3" integrity sha512-d6xOpEDfsi2CZVlPQzGeux8XMwLT9hssAsaPYExaQMuYskwb+x1x7J371tWlbBdWHroy99KnVB6qIkUbs5X3UQ== -tslib@^2.6.2, tslib@^2.6.3: +tslib@^2.6.2: version "2.8.0" resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.8.0.tgz#d124c86c3c05a40a91e6fdea4021bd31d377971b" integrity sha512-jWVzBLplnCmoaTr13V9dYbiQ99wvZRd0vNWaDRg+aVYRcjDF3nDksxFDE/+fkXnKhpnUUkmx5pK/v8mCtLVqZA== +tslib@^2.6.3: + version "2.7.0" + resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.7.0.tgz#d9b40c5c40ab59e8738f297df3087bf1a2690c01" + integrity sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA== + tunnel-agent@^0.6.0: version "0.6.0" resolved "https://registry.yarnpkg.com/tunnel-agent/-/tunnel-agent-0.6.0.tgz#27a5dea06b36b04a0a9966774b290868f0fc40fd"