diff --git a/assets/notion/icon.png b/assets/notion/icon.png new file mode 100644 index 00000000..833be386 Binary files /dev/null and b/assets/notion/icon.png differ diff --git a/package.json b/package.json index fe27a62d..ddac771f 100644 --- a/package.json +++ b/package.json @@ -43,6 +43,7 @@ "@sapphire/snowflake": "^3.4.2", "@superfaceai/passport-twitter-oauth2": "^1.2.3", "@tensorflow-models/universal-sentence-encoder": "^1.3.3", + "@types/passport-strategy": "^0.2.38", "@verida/account-node": "^4.4.1", "@verida/client-ts": "^4.4.1", "@verida/did-client": "^4.4.1", @@ -87,6 +88,7 @@ "passport": "^0.5.2", "passport-facebook": "^3.0.0", "passport-google-oauth20": "^2.0.0", + "passport-strategy": "^1.0.0", "pdf-parse": "^1.1.1", "sanitize-html": "^2.13.1", "string-strip-html": "8.5.0", diff --git a/src/providers/notion/NotionStrategy.ts b/src/providers/notion/NotionStrategy.ts new file mode 100644 index 00000000..1bbb0e65 --- /dev/null +++ b/src/providers/notion/NotionStrategy.ts @@ -0,0 +1,93 @@ +import { Request } from "express"; +import https from "https"; +import { URL } from "url"; +import passport from "passport"; + +interface NotionStrategyOptions { + clientID: string; + clientSecret: string; + callbackURL: string; +} + +export default class NotionStrategy extends passport.Strategy { + name = "notion"; + private _clientID: string; + private _clientSecret: string; + private _callbackURL: string; + private _authorizationURL: string; + private _tokenURL: string; + + constructor({ clientID, clientSecret, callbackURL }: NotionStrategyOptions) { + super(); + if (!clientID || !clientSecret || !callbackURL) { + throw new TypeError("Missing required options for NotionStrategy"); + } + this._clientID = clientID; + this._clientSecret = clientSecret; + this._callbackURL = callbackURL; + this._authorizationURL = "https://api.notion.com/v1/oauth/authorize"; + this._tokenURL = "https://api.notion.com/v1/oauth/token"; + } + + async authenticate(req: Request, options?: any) { + options = options || {}; + if (req.query?.code) { + try { + const oauthData = await this.getOAuthAccessToken(req.query.code as string); + + if (oauthData.owner.type !== "user") { + return this.fail(`Notion API token not owned by user, instead: ${oauthData.owner.type}`); + } + + return this.success(oauthData); + } catch (error) { + return this.error(error); + } + } else { + const authUrl = new URL(this._authorizationURL); + authUrl.searchParams.set("client_id", this._clientID); + authUrl.searchParams.set("redirect_uri", this._callbackURL); + authUrl.searchParams.set("response_type", "code"); + return this.redirect(authUrl.toString()); + } + } + + private async getOAuthAccessToken(code: string): Promise { + const accessTokenBody = { + grant_type: "authorization_code", + code, + redirect_uri: this._callbackURL, + }; + const encodedCredential = Buffer.from(`${this._clientID}:${this._clientSecret}`).toString("base64"); + + const requestOptions = { + hostname: new URL(this._tokenURL).hostname, + path: new URL(this._tokenURL).pathname, + headers: { + Authorization: `Basic ${encodedCredential}`, + "Content-Type": "application/json", + }, + method: "POST", + }; + + return new Promise((resolve, reject) => { + const accessTokenRequest = https.request(requestOptions, (res) => { + let data = ""; + res.on("data", (d) => { + data += d; + }); + res.on("end", () => { + try { + resolve(JSON.parse(data)); + } catch (error) { + reject(error); + } + }); + }); + + accessTokenRequest.on("error", reject); + accessTokenRequest.write(JSON.stringify(accessTokenBody)); + accessTokenRequest.end(); + }); + } +} diff --git a/src/providers/notion/README.md b/src/providers/notion/README.md new file mode 100644 index 00000000..30b427d2 --- /dev/null +++ b/src/providers/notion/README.md @@ -0,0 +1,55 @@ +# Notion Provider Configuration + +## Notion Integration Setup + +1. Go to [Notion Integrations](https://www.notion.so/my-integrations) +2. Create a new integration: + - Click **New Integration** + - Add an integration name and redirect Urls + - Choose the capabilities needed (Read Content, Read Comments, Read Users) +3. Copy the `Secret Key` and `Public Key` - store it securely + +## Authentication + +The Notion provider uses Integration Token authentication: +- Use the Integration Token as the `accessToken` +- No `refreshToken` is required + +## Data Access + +Notion API provides access to: +- Pages +- Databases +- Blocks (content elements) +- Users +- Comments + +### Pagination + +Notion uses cursor-based pagination: +- `start_cursor` and `has_more` for pagination control +- Default page size of 100 items +- Maximum page size of 100 items + +## Rate Limits + +- Rate limits vary by tier +- Standard tier: ~3 requests per second +- See [Notion API Limits](https://developers.notion.com/reference/request-limits) for current limits + +## Notes + +- Database queries support filtering and sorting +- Block content is retrieved recursively for nested structures +- Rich text content includes formatting metadata +- User permissions are respected based on integration access +- Some features may require specific Notion plan types + +#### Example – Cursor-Based Pagination + +```javascript +const response = await notion.databases.query({ + database_id: databaseId, + page_size: 10, + start_cursor: nextCursor, // optional, for pagination +}); diff --git a/src/providers/notion/block.ts b/src/providers/notion/block.ts new file mode 100644 index 00000000..b2920d76 --- /dev/null +++ b/src/providers/notion/block.ts @@ -0,0 +1,163 @@ +import CONFIG from "../../config"; +import { BaseHandlerConfig, SyncItemsBreak, SyncItemsResult, SyncProviderLogEvent, SyncProviderLogLevel } from "../../interfaces"; +import { Client } from "@notionhq/client"; +import { ItemsRangeTracker } from "../../helpers/itemsRangeTracker"; +import { + SyncResponse, + SyncHandlerStatus, + ProviderHandlerOption, + ConnectionOptionType, +} from "../../interfaces"; +import { SchemaRecord } from "../../schemas"; +import AccessDeniedError from "../AccessDeniedError"; +import InvalidTokenError from "../InvalidTokenError"; +import BaseSyncHandler from "../BaseSyncHandler"; + +const MAX_BATCH_SIZE = 500; + +export interface SyncBlockItemsResult extends SyncItemsResult { + items: any[]; +} + +export default class NotionBlockHandler extends BaseSyncHandler { + protected config: BaseHandlerConfig; + + public getLabel(): string { + return "Notion Blocks"; + } + + public getName(): string { + return "notion_blocks"; + } + + public getSchemaUri(): string { + return CONFIG.verida.schemas.BLOCK; + } + + public getProviderApplicationUrl(): string { + return "https://notion.so/"; + } + + public getNotionClient(): Client { + return new Client({ auth: this.connection.accessToken }); + } + + public getOptions(): ProviderHandlerOption[] { + return [ + { + id: "syncDepth", + label: "Sync Depth", + type: ConnectionOptionType.ENUM, + enumOptions: [ + { value: "1-level", label: "1 Level" }, + { value: "2-levels", label: "2 Levels" }, + { value: "all", label: "All Levels" }, + ], + defaultValue: "1-level", + }, + ]; + } + + public async _sync(api: any, syncPosition: any): Promise { + try { + if (this.config.batchSize > MAX_BATCH_SIZE) { + throw new Error(`Batch size (${this.config.batchSize}) exceeds max limit (${MAX_BATCH_SIZE})`); + } + + const notion = this.getNotionClient(); + const rangeTracker = new ItemsRangeTracker(syncPosition.thisRef); + let items: any[] = []; + + let currentRange = rangeTracker.nextRange(); + + const pages = await this.getPageList(); + + const page = await notion.pages.retrieve({ + page_id: pages[0].id + }) + + let query = { block_id: pages[0].id, start_cursor: currentRange.startId }; + + const latestResponse = await notion.blocks.children.list(query); + + + const latestResult = await this.buildResults(notion, latestResponse, currentRange.endId); + + items = latestResult.items; + let nextPageCursor = latestResponse.next_cursor; + + if (items.length) { + rangeTracker.completedRange({ + startId: items[0].id, + endId: nextPageCursor, + }, latestResult.breakHit === SyncItemsBreak.ID); + } else { + rangeTracker.completedRange({ startId: undefined, endId: undefined }, false); + } + + if (!items.length) { + syncPosition.syncMessage = "Stopping. No results found."; + syncPosition.status = SyncHandlerStatus.ENABLED; + } else { + syncPosition.syncMessage = items.length !== this.config.batchSize && !nextPageCursor + ? `Processed ${items.length} items. Stopping. No more results.` + : `Batch complete (${this.config.batchSize}). More results pending.`; + } + + syncPosition.thisRef = rangeTracker.export(); + + return { results: items, position: syncPosition }; + } catch (err: any) { + if (err.status === 403) throw new AccessDeniedError(err.message); + if (err.status === 401) throw new InvalidTokenError(err.message); + throw err; + } + + + } + + protected async buildResults( + notion: Client, + serverResponse: any, + breakId: string + ): Promise { + const results: any[] = []; + let breakHit: SyncItemsBreak; + + for (const block of serverResponse.results) { + if (block.id === breakId) { + this.emit("log", { level: SyncProviderLogLevel.DEBUG, message: `Break ID hit (${breakId})` }); + breakHit = SyncItemsBreak.ID; + break; + } + + results.push({ + _id: this.buildItemId(block.id), + type: block.type, + sourceId: block.id, + sourceApplication: this.getProviderApplicationUrl(), + content: JSON.stringify(block), + insertedAt: new Date().toISOString(), + }); + } + + return { items: results, breakHit }; + } + + public async getPageList(): Promise { + try { + const notion = await this.getNotionClient(); + const response = await notion.search({ + filter: { property: "object", value: "page" }, + sort: { direction: "ascending", timestamp: "last_edited_time" }, + page_size: 50 // Max is 100 + }); + + return response.results; + + } catch (error) { + console.error("Error fetching Notion pages:", error); + } + } + +} diff --git a/src/providers/notion/index.ts b/src/providers/notion/index.ts new file mode 100644 index 00000000..204ab78f --- /dev/null +++ b/src/providers/notion/index.ts @@ -0,0 +1,111 @@ +import { Request, Response } from "express"; +import { Client } from "@notionhq/client"; +import Base from "../BaseProvider"; +import { NotionProviderConfig } from "./interfaces"; +import { ConnectionCallbackResponse, PassportProfile } from "../../interfaces"; +import passport from "passport"; +import NotionStrategy from "./NotionStrategy"; +import NotionBlockHandler from "./block"; + +export default class NotionProvider extends Base { + protected config: NotionProviderConfig; + + public getProviderName() { + return "notion"; + } + + public getProviderLabel() { + return "Notion"; + } + + public getProviderApplicationUrl() { + return "https://www.notion.so/"; + } + + public syncHandlers(): any[] { + return [ + NotionBlockHandler + ]; + } + + public getScopes(): string[] { + return ["read_content", "read_comment"]; + } + + public async connect(req: Request, res: Response, next: any): Promise { + this.init(); + const auth = passport.authenticate("notion", { + scope: this.getScopes().join(" "), + }); + return auth(req, res, next); + } + + public async callback(req: Request, res: Response, next: any): Promise { + this.init(); + return new Promise((resolve, reject) => { + passport.authenticate( + "notion", + { failureRedirect: "/failure/notion", failureMessage: true }, + (err: any, user: any) => { + if (err) { + return reject(err); + } + if (!user) { + return reject(new Error("No user data returned from Notion")); + } + + const profile = this.formatProfile(user); + + resolve({ + id: profile.id, + accessToken: user.access_token, + refreshToken: user.access_token, // Notion does not provide refresh tokens currently + profile: { + username: profile.connectionProfile.username, + ...profile, + }, + }); + } + )(req, res, next); + }); + } + + public async getApi(accessToken?: string): Promise { + if (!accessToken) { + throw new Error("Access token is required"); + } + return new Client({ auth: accessToken }); + } + + public init() { + passport.use( + new NotionStrategy({ + clientID: this.config.clientId, + clientSecret: this.config.clientSecret, + callbackURL: this.config.callbackUrl, + }) + ); + } + + private formatProfile(notionData: any): PassportProfile { + const owner = notionData.owner?.user; + const email = owner?.person?.email || null; + + return { + id: owner?.id || "", + provider: this.getProviderName(), + displayName: owner?.name || email || owner?.id || "Unknown", + name: { + familyName: "", + givenName: owner?.name || "", + }, + photos: owner?.avatar_url ? [{ value: owner.avatar_url }] : [], + connectionProfile: { + username: email ? email.split("@")[0] : owner?.id || "unknown", + readableId: email || owner?.id || "unknown", + email: email, + verified: true, + }, + }; + } +} diff --git a/src/providers/notion/interfaces.ts b/src/providers/notion/interfaces.ts new file mode 100644 index 00000000..1aafa243 --- /dev/null +++ b/src/providers/notion/interfaces.ts @@ -0,0 +1,11 @@ +import { BaseHandlerConfig, BaseProviderConfig } from "../../../src/interfaces"; + +export interface NotionProviderConfig extends BaseProviderConfig { + clientId: string; + clientSecret: string; + callbackUrl: string; +} + +export interface NotionHandlerConfig extends BaseHandlerConfig { + batchSize: number +} \ No newline at end of file diff --git a/src/serverconfig.example.json b/src/serverconfig.example.json index 75bec705..863e8d77 100644 --- a/src/serverconfig.example.json +++ b/src/serverconfig.example.json @@ -180,6 +180,14 @@ "messagesPerGroupLimit": 100, "maxGroupSize": 50, "useDbPos": true + }, + "notion": { + "status": "active", + "label": "Notion", + "clientId": "", + "clientSecret": "", + "batchSize": 50, + "maxSyncLoops": 1 } }, "providerDefaults": { diff --git a/yarn.lock b/yarn.lock index d9338981..46e3cf7f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2489,6 +2489,14 @@ "@types/oauth" "*" "@types/passport" "*" +"@types/passport-strategy@^0.2.38": + version "0.2.38" + resolved "https://registry.yarnpkg.com/@types/passport-strategy/-/passport-strategy-0.2.38.tgz#482abba0b165cd4553ec8b748f30b022bd6c04d3" + integrity sha512-GC6eMqqojOooq993Tmnmp7AUTbbQSgilyvpCYQjT+H6JfG/g6RGc7nXEniZlp0zyKJ0WUdOiZWLBZft9Yug1uA== + dependencies: + "@types/express" "*" + "@types/passport" "*" + "@types/passport@*", "@types/passport@1.x": version "1.0.12" resolved "https://registry.yarnpkg.com/@types/passport/-/passport-1.0.12.tgz#7dc8ab96a5e895ec13688d9e3a96920a7f42e73e" @@ -6988,7 +6996,7 @@ passport-oauth2@^1.5.0, passport-oauth2@^1.6.1: uid2 "0.0.x" utils-merge "1.x.x" -passport-strategy@1.x.x: +passport-strategy@1.x.x, passport-strategy@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/passport-strategy/-/passport-strategy-1.0.0.tgz#b5539aa8fc225a3d1ad179476ddf236b440f52e4" integrity sha1-tVOaqPwiWj0a0XlHbd8ja0QPUuQ=