diff --git a/assets/fireflies/icon.png b/assets/fireflies/icon.png new file mode 100644 index 00000000..a0f492af Binary files /dev/null and b/assets/fireflies/icon.png differ diff --git a/src/api/rest/v1/fireflies/controller.ts b/src/api/rest/v1/fireflies/controller.ts new file mode 100644 index 00000000..fe086515 --- /dev/null +++ b/src/api/rest/v1/fireflies/controller.ts @@ -0,0 +1,21 @@ +import { Request, Response, NextFunction } from "express"; + +export default class Controller { + public static async apiKeySubmit(req: Request, res: Response, next: NextFunction) { + try { + const apiKey = req.body.apiKey; + + // @todo Validate API key if any (pending from FireFlies) + + // Redirect to the callback endpoint with the apiKey + res.status(200).send({ + redirect: `/callback/fireflies?apiKey=${encodeURIComponent(apiKey)}` + }); + + } catch (error) { + res.status(400).send({ + error: error.message + }); + } + } +} diff --git a/src/api/rest/v1/fireflies/routes.ts b/src/api/rest/v1/fireflies/routes.ts new file mode 100644 index 00000000..920148be --- /dev/null +++ b/src/api/rest/v1/fireflies/routes.ts @@ -0,0 +1,8 @@ +import express from 'express' +import Controller from './controller' + +const router = express.Router() + +router.post('/apiKeySubmit', Controller.apiKeySubmit) + +export default router \ No newline at end of file diff --git a/src/api/rest/v1/routes.ts b/src/api/rest/v1/routes.ts index 1987890c..ba5c3dad 100644 --- a/src/api/rest/v1/routes.ts +++ b/src/api/rest/v1/routes.ts @@ -9,6 +9,7 @@ import AdminRoutes from './admin/routes' import InfoRoutes from './info/routes' import LLMRoutes from './llm/routes' import TelegramRoutes from './telegram/routes' +import FireFliesRoutes from './fireflies/routes' import SearchRoutes from "./search/routes" import AccountRoutes from './account/routes' import AuthRoutes from "./auth/routes" @@ -30,5 +31,6 @@ router.use('/account', AccountRoutes) router.use('/app', AppRoutes) router.use('/telegram', TelegramRoutes) +router.use('/fireflies', FireFliesRoutes) export default router diff --git a/src/providers/fireflies/README.md b/src/providers/fireflies/README.md new file mode 100644 index 00000000..053b827f --- /dev/null +++ b/src/providers/fireflies/README.md @@ -0,0 +1,10 @@ +## Authentication + +Fireflies uses API key authentication and GraphQL for all API interactions. + +#### Getting an API Key + +1. Log in to your Fireflies.ai account +2. Navigate to Settings > Developer Settings +3. Use the existing one or reset another +4. Copy and securely store your API key diff --git a/src/providers/fireflies/api.ts b/src/providers/fireflies/api.ts new file mode 100644 index 00000000..dd2aca6a --- /dev/null +++ b/src/providers/fireflies/api.ts @@ -0,0 +1,90 @@ +import axios, { AxiosResponse } from 'axios'; + +export interface FireFliesConfig { + apiKey: string; + baseUrl?: string; +} + +export interface User { + user_id: string; + name: string; + email: string; + is_admin?: string; + num_transcripts?: string; + integrations?: string[]; + +} + +export interface GraphQLResponse { + data: T; + errors?: { message: string }[]; +} + +export class FireFliesClient { + private apiKey: string; + private baseUrl: string; + + constructor(config: FireFliesConfig) { + this.apiKey = config.apiKey; + this.baseUrl = config.baseUrl || 'https://api.fireflies.ai/graphql'; + } + + private get headers() { + return { + 'Content-Type': 'application/json', + Authorization: `Bearer ${this.apiKey}` + }; + } + + public async executeQuery(query: string, variables?: Record): Promise> { + const payload = { query, variables }; + + try { + const response: AxiosResponse> = await axios.post(this.baseUrl, payload, { headers: this.headers }); + if (response.data.errors) { + throw new Error(`GraphQL error: ${response.data.errors.map(e => e.message).join(', ')}`); + } + return response.data; + } catch (error) { + console.error('Error executing GraphQL query:', error); + throw error; + } + } + + /** + * + * @param userId Optional + * + * @returns Owner by default + */ + public async getUser(userId?: string): Promise { + const query = userId + ? ` + query User($userId: String!) { + user(id: $userId) { + user_id + name + email + } + } + ` + : ` + query { + user { + user_id + name + email + } + } + `; + + // Pass variables only if userId is defined + const response = userId + ? await this.executeQuery<{ user: User }>(query, { userId }) + : await this.executeQuery<{ user: User }>(query); + + return response.data.user; + } + + // You can add more methods to handle other queries or mutations here +} diff --git a/src/providers/fireflies/index.ts b/src/providers/fireflies/index.ts new file mode 100644 index 00000000..ba08ca97 --- /dev/null +++ b/src/providers/fireflies/index.ts @@ -0,0 +1,85 @@ +import { Request, Response } from 'express' +import Base from "../BaseProvider" + +import { BaseProviderConfig, ConnectionCallbackResponse, PassportProfile } from '../../interfaces' +import { FireFliesClient, FireFliesConfig } from './api' +import MeetingTranscriptHandler from './meeting-transcript' + +export default class FireFliesProvider extends Base { + + protected config: BaseProviderConfig + + + public getProviderName() { + return 'fireflies' + } + + public getProviderLabel() { + return 'FireFlies' + } + + public getProviderApplicationUrl() { + return 'https://fireflies.ai/' + } + + public setConfig(config: BaseProviderConfig) { + this.config = config + } + + public syncHandlers(): any[] { + return [ + MeetingTranscriptHandler + ] + } + + public async connect(req: Request, res: Response, next: any): Promise { + return res.redirect('/provider/fireflies') + } + + public async callback(req: Request, res: Response, next: any): Promise { + const apiKey = req.query.apiKey!.toString(); + + // Initialize Fireflies client configuration + const config: FireFliesConfig = { + apiKey: apiKey + }; + + const client = new FireFliesClient(config); + + // Fetch user profile from Fireflies + const ffProfile = await client.getUser(); + + // Set up display name + const displayName = ffProfile.name.trim(); + + // Construct the profile structure similar to the Telegram format + const profile: PassportProfile = { + id: ffProfile.user_id.toString(), + provider: this.getProviderId(), // Assuming getProviderId() returns 'fireflies' or similar identifier + displayName: displayName, + name: { + familyName: ffProfile.name.split(" ").slice(-1)[0], // Last word as family name + givenName: ffProfile.name.split(" ").slice(0, -1).join(" ") // First part as given name + }, + connectionProfile: { + username: ffProfile.email.split('@')[0], // Username from email prefix + readableId: ffProfile.user_id, + email: ffProfile.email, + verified: true // Assuming profile is verified + } + }; + + return { + id: profile.id, + accessToken: apiKey, + refreshToken: apiKey, + profile + }; + } + + public async getApi( + accessToken?: string, + refreshToken?: string + ): Promise { } +} + diff --git a/src/providers/fireflies/meeting-transcript.ts b/src/providers/fireflies/meeting-transcript.ts new file mode 100644 index 00000000..ed1054b8 --- /dev/null +++ b/src/providers/fireflies/meeting-transcript.ts @@ -0,0 +1,223 @@ +import axios from 'axios'; +import CONFIG from "../../config"; +import { BaseHandlerConfig, SyncHandlerPosition, SyncItemsBreak, SyncItemsResult, SyncProviderLogLevel } from '../../interfaces'; +import { ItemsRangeTracker } from "../../helpers/itemsRangeTracker"; + +import { + SyncResponse, + SyncHandlerStatus, + ProviderHandlerOption, + ConnectionOptionType, +} from "../../interfaces"; +import { SchemaMeetingTranscript } from "../../schemas"; +import AccessDeniedError from "../AccessDeniedError"; +import InvalidTokenError from "../InvalidTokenError"; +import BaseSyncHandler from '../BaseSyncHandler'; +import { FireFliesClient } from './api'; + +const MAX_BATCH_SIZE = 50; // Maximum limit for Fireflies API queries + +export interface SyncTranscriptItemsResult extends SyncItemsResult { + items: SchemaMeetingTranscript[]; +} + +export default class MeetingTranscriptHandler extends BaseSyncHandler { + + protected config: BaseHandlerConfig; + + public getLabel(): string { + return "Meeting Transcript"; + } + + public getName(): string { + return 'meeting-transcript'; + } + + public getSchemaUri(): string { + return CONFIG.verida.schemas.MEETING_TRANSCRIPT; + } + + public getProviderApplicationUrl() { + return 'https://app.fireflies.ai/'; + } + + public getOptions(): ProviderHandlerOption[] { + return [{ + id: 'dateRange', + label: 'Transcript Date Range', + type: ConnectionOptionType.ENUM, + enumOptions: [{ + value: '1-month', + label: '1 month' + }, { + value: '3-months', + label: '3 months' + }, { + value: '6-months', + label: '6 months' + }], + defaultValue: '3-months' + }]; + } + + public async _sync( + api: any, + syncPosition: SyncHandlerPosition + ): Promise { + + try { + if (this.config.batchSize > MAX_BATCH_SIZE) { + throw new Error(`Batch size (${this.config.batchSize}) is larger than permitted (${MAX_BATCH_SIZE})`); + } + + const client = new FireFliesClient({ + apiKey: this.connection.accessToken + }); + + const rangeTracker = new ItemsRangeTracker(syncPosition.thisRef); + let items: SchemaMeetingTranscript[] = []; + + let currentRange = rangeTracker.nextRange(); + + const query = ` + query Transcripts($limit: Int, $skip: Int) { + transcripts(limit: $limit, skip: $skip) { + id + title + organizer_email + user { + email + name + } + date + speakers { + id + name + } + sentences { + speaker_name + raw_text + } + meeting_attendees { + displayName + email + phoneNumber + name + } + duration + summary { + short_summary + } + cal_id + } + } + `; + + const variables = { + limit: this.config.batchSize, + skip: currentRange.startId || 0 + }; + + const response = await client.executeQuery(query, variables) + + const resultData = await this.buildResults(response.data.transcripts, currentRange.endId); + + items = resultData.items; + + if (!items.length) { + syncPosition.syncMessage = "No transcripts found within specified range."; + syncPosition.status = SyncHandlerStatus.ENABLED; + } else { + syncPosition.syncMessage = `Fetched ${items.length} transcripts.`; + } + + syncPosition.thisRef = rangeTracker.export(); + + return { + results: items, + position: syncPosition, + }; + } catch (err: any) { + if (err.response && err.response.status == 403) { + throw new AccessDeniedError(err.message); + } else if (err.response && err.response.status == 401) { + throw new InvalidTokenError(err.message); + } + throw err; + } + } + + protected async buildResults( + transcripts: any[], + breakId: string + ): Promise { + const results: SchemaMeetingTranscript[] = []; + let breakHit: SyncItemsBreak | undefined; + + for (const transcript of transcripts) { + const transcriptId = transcript.id; + + // Check for the break ID to stop processing + if (transcriptId === breakId) { + this.emit('log', { + level: SyncProviderLogLevel.DEBUG, + message: `Break ID hit (${breakId})`, + }); + breakHit = SyncItemsBreak.ID; + break; + } + + // Map transcript fields to SchemaMeetingTranscript + results.push({ + _id: this.buildItemId(transcriptId), // Unique ID for each transcript + name: transcript.title || 'Untitled meeting', + organizerEmail: transcript.organizer_email, + user: transcript.user + ? { + email: transcript.user.email, + displayName: transcript.user.name || '', + name: transcript.user.name || undefined, + } + : undefined, + speakers: transcript.speakers + ? transcript.speakers.map((speaker: any) => ({ + displayName: speaker.name, + email: speaker?.email, + })) + : [], + meetingAttendees: transcript.meeting_attendees + ? transcript.meeting_attendees.map((attendee: any) => ({ + displayName: typeof attendee.displayName === 'string' && attendee.displayName.trim() + ? attendee.displayName.trim() + : '', + email: typeof attendee.email === 'string' && attendee.email.trim() + ? attendee.email.trim() + : '', + phoneNumber: typeof attendee.phoneNumber === 'string' && attendee.phoneNumber.trim() + ? attendee.phoneNumber.trim() + : '', + name: typeof attendee.name === 'string' && attendee.name.trim() + ? attendee.name.trim() + : '', + })) + : [], + + duration: transcript.duration, + dateTime: new Date(transcript.date).toISOString(), + sentence: transcript.sentences + ? transcript.sentences.map((sentence: any) => ({ + rawText: sentence.raw_text, + speakerName: sentence.speaker_name, + })) + : [], + calendarEventId: transcript.cal_id || undefined, + insertedAt: new Date().toISOString(), // Add the current timestamp + }); + } + + return { + items: results, + breakHit, // Indicates if a break ID was encountered + }; + } +} diff --git a/src/schemas.ts b/src/schemas.ts index a1f9e1d6..af11096b 100644 --- a/src/schemas.ts +++ b/src/schemas.ts @@ -156,3 +156,26 @@ export interface SchemaEvent extends SchemaRecord { attachments?: CalendarAttachment[] } + +export interface SchemaMeetingTranscript extends SchemaRecord{ + organizerEmail: string; + user?: SchemaPerson; + speakers?: SchemaPerson[]; + meetingAttendees?: SchemaPerson[]; + duration?: number; + dateTime?: string; + sentence?: SchemaMeetingTranscriptSentence[]; + calendarEventId?: string; +} + +export interface SchemaPerson { + email?: string; + displayName: string; + name?: string; + phoneNumber?: string; +} + +export interface SchemaMeetingTranscriptSentence { + rawText: string; + speakerName: string; +} diff --git a/src/serverconfig.example.json b/src/serverconfig.example.json index a94fd73e..ca7cd7fc 100644 --- a/src/serverconfig.example.json +++ b/src/serverconfig.example.json @@ -33,7 +33,8 @@ "CHAT_GROUP": "https://common.schemas.verida.io/social/chat/group/v0.1.0/schema.json", "CHAT_MESSAGE": "https://common.schemas.verida.io/social/chat/message/v0.1.0/schema.json", "CALENDAR": "https://common.schemas.verida.io/social/calendar/v0.1.0/schema.json", - "EVENT": "https://common.schemas.verida.io/social/event/v0.1.0/schema.json" + "EVENT": "https://common.schemas.verida.io/social/event/v0.1.0/schema.json", + "MEETING_TRANSCRIPT": "https://common.schemas.verida.io/social/meeting-transcript/v0.1.0/schema.json" }, "llms": { "bedrockEndpoint": "", @@ -158,6 +159,12 @@ "messagesPerGroupLimit": 100, "maxGroupSize": 50, "useDbPos": true + }, + "fireflies": { + "status": "active", + "label": "FireFlies", + "maxSyncLoops": 1, + "batchSize": 50 } }, "providerDefaults": { diff --git a/src/web/developer/data/data.js b/src/web/developer/data/data.js index ec9202a0..f64a3762 100644 --- a/src/web/developer/data/data.js +++ b/src/web/developer/data/data.js @@ -22,7 +22,8 @@ $(document).ready(function() { "Chat Message": "https://common.schemas.verida.io/social/chat/message/v0.1.0/schema.json", "Files": "https://common.schemas.verida.io/file/v0.1.0/schema.json", "Calendar": "https://common.schemas.verida.io/social/calendar/v0.1.0/schema.json", - "Event": "https://common.schemas.verida.io/social/event/v0.1.0/schema.json" + "Event": "https://common.schemas.verida.io/social/event/v0.1.0/schema.json", + "Meeting Transcript": "https://common.schemas.verida.io/social/meeting-transcript/v0.1.0/schema.json" }; // Load Verida Key and Schema from local storage diff --git a/src/web/provider/fireflies/index.html b/src/web/provider/fireflies/index.html new file mode 100644 index 00000000..53cbe25f --- /dev/null +++ b/src/web/provider/fireflies/index.html @@ -0,0 +1,53 @@ + + + + + + Fireflies.ai Login + + + + + + +
+

Login with Fireflies.ai

+ + + + + +
+ + + +
+ + + +
+ + + diff --git a/src/web/provider/fireflies/script.js b/src/web/provider/fireflies/script.js new file mode 100644 index 00000000..1f666179 --- /dev/null +++ b/src/web/provider/fireflies/script.js @@ -0,0 +1,39 @@ +function setError(message) { + $("#errorContainer").show(); + $("#errorMessage").text(message); + setTimeout(() => { + $("#errorContainer").hide(); + }, 5000); +} + +function disableButton() { + $('#submitButton').prop('disabled', true); +} + +function enableButton() { + $('#submitButton').prop('disabled', false); +} + +function submitApiKey() { + disableButton(); + const apiKey = $('.form-control').val().trim(); + + if (!apiKey) { + setError("API Key is required."); + enableButton(); + return; + } + + $.post('/api/rest/v1/fireflies/apiKeySubmit', { apiKey: apiKey }, null, 'json') + .done((response) => { + window.location.href = response.redirect; + }) + .fail(() => { + setError("Failed to connect to Fireflies API. Please try again later."); + enableButton(); + }); +} + +$(document).ready(function() { + $("#errorContainer").hide(); +}); diff --git a/tests/providers/fireflies/meeting-transcript.tests.ts b/tests/providers/fireflies/meeting-transcript.tests.ts new file mode 100644 index 00000000..28a09935 --- /dev/null +++ b/tests/providers/fireflies/meeting-transcript.tests.ts @@ -0,0 +1,215 @@ +const assert = require("assert"); +import CONFIG from "../../../src/config"; +import { + BaseHandlerConfig, + BaseProviderConfig, + Connection, + SyncHandlerPosition, + SyncHandlerStatus +} from "../../../src/interfaces"; +import Providers from "../../../src/providers"; +import CommonUtils, { NetworkInstance } from "../../common.utils"; + +import FirefliesMeetingTranscriptHandler from "../../../src/providers/fireflies/meeting-transcript"; +import BaseProvider from "../../../src/providers/BaseProvider"; +import { CommonTests, GenericTestConfig } from "../../common.tests"; +import { SchemaMeetingTranscript, SchemaRecord } from "../../../src/schemas"; + +const providerId = "fireflies"; +let network: NetworkInstance; +let connection: Connection; +let provider: BaseProvider; +let handlerName = "meeting-transcript"; +let testConfig: GenericTestConfig; + +let providerConfig: Omit = {}; +let handlerConfig: BaseHandlerConfig = { + batchSize: 20 +}; + +describe(`${providerId} meeting transcript tests`, function () { + this.timeout(100000); + + this.beforeAll(async function () { + network = await CommonUtils.getNetwork(); + connection = await CommonUtils.getConnection(providerId); + provider = Providers(providerId, network.context, connection); + + testConfig = { + idPrefix: `${provider.getProviderId()}-${connection.profile.id}`, + timeOrderAttribute: "insertedAt", + batchSizeLimitAttribute: "batchSize", + }; + }); + + describe(`Fetch ${providerId} data`, () => { + it(`Can pass basic tests: ${handlerName}`, async () => { + const { api, handler, provider } = await CommonTests.buildTestObjects( + providerId, + FirefliesMeetingTranscriptHandler, + providerConfig, + connection + ); + + handler.setConfig(handlerConfig); + + try { + const syncPosition: SyncHandlerPosition = { + _id: `${providerId}-${handlerName}`, + providerId, + handlerId: handler.getId(), + accountId: provider.getAccountId(), + status: SyncHandlerStatus.ENABLED, + }; + + // First batch + const response = await handler._sync(api, syncPosition); + const results = response.results; + + // Basic assertions + assert.ok(results && results.length, "Have results returned"); + assert.ok(results.length > 0, "Have meeting-transcripts returned"); + + // Check first item structure + CommonTests.checkItem(results[0], handler, provider); + + // Verify sync status + assert.equal( + SyncHandlerStatus.SYNCING, + response.position.status, + "Sync is active" + ); + + // Second batch + const secondBatchResponse = await handler._sync(api, response.position); + const secondBatchResults = secondBatchResponse.results; + + // Verify second batch + assert.ok(secondBatchResults && secondBatchResults.length, "Have second batch results"); + } catch (err) { + await provider.close(); + throw err; + } + }); + + it(`Should have valid meeting-transcript data structure`, async () => { + const { api, handler, provider } = await CommonTests.buildTestObjects( + providerId, + FirefliesMeetingTranscriptHandler, + providerConfig, + connection + ); + handler.setConfig(handlerConfig); + + const syncPosition: SyncHandlerPosition = { + _id: `${providerId}-${handlerName}`, + providerId, + handlerId: handler.getId(), + accountId: provider.getAccountId(), + status: SyncHandlerStatus.ENABLED, + }; + + const response = await handler._sync(api, syncPosition); + const results = response.results; + + // Check meeting-transcript structure + const firstItem = results[0]; + assert.ok(firstItem.organizerEmail, "Item has organizer"); + assert.ok(firstItem.duration, "Item has duration"); + assert.ok(firstItem.sourceId, "Item has sourceId"); + assert.ok(firstItem.summary, "Item has summary"); + }); + + it(`Should process meeting-transcripts in chronological order`, async () => { + const { api, handler, provider } = await CommonTests.buildTestObjects( + providerId, + FirefliesMeetingTranscriptHandler, + providerConfig, + connection + ); + handler.setConfig(handlerConfig); + + const syncPosition: SyncHandlerPosition = { + _id: `${providerId}-${handlerName}`, + providerId, + handlerId: handler.getId(), + accountId: provider.getAccountId(), + status: SyncHandlerStatus.ENABLED, + }; + + const response = await handler._sync(api, syncPosition); + const results = response.results; + + const timestamps = results.map(t => new Date(t.dateTime!).getTime()); + const isSortedAscending = timestamps.every( + (val, i, arr) => i === 0 || val >= arr[i - 1] + ); + + assert.ok(isSortedAscending, "Meeting-transcripts are processed in chronological order"); + }); + + it(`Should ensure second batch items aren't in the first batch`, async () => { + const { api, handler, provider } = await CommonTests.buildTestObjects( + providerId, + FirefliesMeetingTranscriptHandler, + providerConfig, + connection + ); + handler.setConfig(handlerConfig); + + const firstBatchResponse = await handler._sync(api, { + _id: `${providerId}-${handlerName}`, + providerId, + handlerId: handler.getId(), + accountId: provider.getAccountId(), + status: SyncHandlerStatus.ENABLED, + }); + + const firstBatchItems = firstBatchResponse.results; + + const secondBatchResponse = await handler._sync(api, firstBatchResponse.position); + const secondBatchItems = secondBatchResponse.results; + + const firstBatchIds = firstBatchItems.map(item => item.sourceId); + const secondBatchIds = secondBatchItems.map(item => item.sourceId); + + const intersection = firstBatchIds.filter(id => secondBatchIds.includes(id)); + assert.equal(intersection.length, 0, "No overlapping items between batches"); + }); + + it(`Should handle AI-generated metadata correctly`, async () => { + const { api, handler, provider } = await CommonTests.buildTestObjects( + providerId, + FirefliesMeetingTranscriptHandler, + providerConfig, + connection + ); + handler.setConfig(handlerConfig); + + const syncPosition: SyncHandlerPosition = { + _id: `${providerId}-${handlerName}`, + providerId, + handlerId: handler.getId(), + accountId: provider.getAccountId(), + status: SyncHandlerStatus.ENABLED, + }; + + const response = await handler._sync(api, syncPosition); + const results = response.results; + + results.forEach(item => { + // Check AI-generated metadata(summary, topics, score etc), summary only + assert.ok(item.summary, "Item has AI summary"); + + // Check meeting metadata(start, end time, participants etc), duration only + assert.ok(typeof item.duration === 'number', "Item has valid duration"); + }); + }); + + }); + + this.afterAll(async function () { + const { context } = await CommonUtils.getNetwork(); + await context.close(); + }); +});