Add Notion Connector

package.json

@@ -43,6 +43,7 @@
     "@sapphire/snowflake": "^3.4.2",
     "@superfaceai/passport-twitter-oauth2": "^1.2.3",
     "@tensorflow-models/universal-sentence-encoder": "^1.3.3",
+    "@types/passport-strategy": "^0.2.38",
     "@verida/account-node": "^4.4.1",
     "@verida/client-ts": "^4.4.1",
     "@verida/did-client": "^4.4.1",
@@ -87,6 +88,7 @@
     "passport": "^0.5.2",
     "passport-facebook": "^3.0.0",
     "passport-google-oauth20": "^2.0.0",
+    "passport-strategy": "^1.0.0",
     "pdf-parse": "^1.1.1",
     "sanitize-html": "^2.13.1",
     "string-strip-html": "8.5.0",

src/providers/notion/NotionStrategy.ts

@@ -0,0 +1,93 @@
+import { Request } from "express";
+import https from "https";
+import { URL } from "url";
+import passport from "passport";
+
+interface NotionStrategyOptions {
+    clientID: string;
+    clientSecret: string;
+    callbackURL: string;
+}
+
+export default class NotionStrategy extends passport.Strategy {
+    name = "notion";
+    private _clientID: string;
+    private _clientSecret: string;
+    private _callbackURL: string;
+    private _authorizationURL: string;
+    private _tokenURL: string;
+
+    constructor({ clientID, clientSecret, callbackURL }: NotionStrategyOptions) {
+        super();
+        if (!clientID || !clientSecret || !callbackURL) {
+            throw new TypeError("Missing required options for NotionStrategy");
+        }
+        this._clientID = clientID;
+        this._clientSecret = clientSecret;
+        this._callbackURL = callbackURL;
+        this._authorizationURL = "https://api.notion.com/v1/oauth/authorize";
+        this._tokenURL = "https://api.notion.com/v1/oauth/token";
+    }
+
+    async authenticate(req: Request, options?: any) {
+        options = options || {};
+        if (req.query?.code) {
+            try {
+                const oauthData = await this.getOAuthAccessToken(req.query.code as string);
+
+                if (oauthData.owner.type !== "user") {
+                    return this.fail(`Notion API token not owned by user, instead: ${oauthData.owner.type}`);
+                }
+
+                return this.success(oauthData);
+            } catch (error) {
+                return this.error(error);
+            }
+        } else {
+            const authUrl = new URL(this._authorizationURL);
+            authUrl.searchParams.set("client_id", this._clientID);
+            authUrl.searchParams.set("redirect_uri", this._callbackURL);
+            authUrl.searchParams.set("response_type", "code");
+            return this.redirect(authUrl.toString());
+        }
+    }
+
+    private async getOAuthAccessToken(code: string): Promise<any> {
+        const accessTokenBody = {
+            grant_type: "authorization_code",
+            code,
+            redirect_uri: this._callbackURL,
+        };
+        const encodedCredential = Buffer.from(`${this._clientID}:${this._clientSecret}`).toString("base64");
+
+        const requestOptions = {
+            hostname: new URL(this._tokenURL).hostname,
+            path: new URL(this._tokenURL).pathname,
+            headers: {
+                Authorization: `Basic ${encodedCredential}`,
+                "Content-Type": "application/json",
+            },
+            method: "POST",
+        };
+
+        return new Promise((resolve, reject) => {
+            const accessTokenRequest = https.request(requestOptions, (res) => {
+                let data = "";
+                res.on("data", (d) => {
+                    data += d;
+                });
+                res.on("end", () => {
+                    try {
+                        resolve(JSON.parse(data));
+                    } catch (error) {
+                        reject(error);
+                    }
+                });
+            });
+
+            accessTokenRequest.on("error", reject);
+            accessTokenRequest.write(JSON.stringify(accessTokenBody));
+            accessTokenRequest.end();
+        });
+    }
+}

src/providers/notion/README.md

@@ -0,0 +1,55 @@
+# Notion Provider Configuration  
+
+## Notion Integration Setup  
+
+1. Go to [Notion Integrations](https://www.notion.so/my-integrations)  
+2. Create a new integration:  
+    - Click **New Integration**  
+    - Add an integration name and redirect Urls    
+    - Choose the capabilities needed (Read Content, Read Comments, Read Users)  
+3. Copy the `Secret Key` and `Public Key` - store it securely  
+
+## Authentication  
+
+The Notion provider uses Integration Token authentication:  
+- Use the Integration Token as the `accessToken`  
+- No `refreshToken` is required  
+
+## Data Access  
+
+Notion API provides access to:  
+- Pages  
+- Databases  
+- Blocks (content elements)  
+- Users  
+- Comments  
+
+### Pagination  
+
+Notion uses cursor-based pagination:  
+- `start_cursor` and `has_more` for pagination control  
+- Default page size of 100 items  
+- Maximum page size of 100 items  
+
+## Rate Limits  
+
+- Rate limits vary by tier  
+- Standard tier: ~3 requests per second  
+- See [Notion API Limits](https://developers.notion.com/reference/request-limits) for current limits  
+
+## Notes  
+
+- Database queries support filtering and sorting  
+- Block content is retrieved recursively for nested structures  
+- Rich text content includes formatting metadata  
+- User permissions are respected based on integration access  
+- Some features may require specific Notion plan types  
+
+#### Example – Cursor-Based Pagination  
+
+```javascript
+const response = await notion.databases.query({
+    database_id: databaseId,
+    page_size: 10,
+    start_cursor: nextCursor,  // optional, for pagination
+});

src/providers/notion/block.ts

@@ -0,0 +1,163 @@
+import CONFIG from "../../config";
+import { BaseHandlerConfig, SyncItemsBreak, SyncItemsResult, SyncProviderLogEvent, SyncProviderLogLevel } from "../../interfaces";
+import { Client } from "@notionhq/client";
+import { ItemsRangeTracker } from "../../helpers/itemsRangeTracker";
+import {
+    SyncResponse,
+    SyncHandlerStatus,
+    ProviderHandlerOption,
+    ConnectionOptionType,
+} from "../../interfaces";
+import { SchemaRecord } from "../../schemas";
+import AccessDeniedError from "../AccessDeniedError";
+import InvalidTokenError from "../InvalidTokenError";
+import BaseSyncHandler from "../BaseSyncHandler";
+
+const MAX_BATCH_SIZE = 500;
+
+export interface SyncBlockItemsResult extends SyncItemsResult {
+    items: any[];
+}
+
+export default class NotionBlockHandler extends BaseSyncHandler {
+    protected config: BaseHandlerConfig;
+
+    public getLabel(): string {
+        return "Notion Blocks";
+    }
+
+    public getName(): string {
+        return "notion_blocks";
+    }
+
+    public getSchemaUri(): string {
+        return CONFIG.verida.schemas.BLOCK;
+    }
+
+    public getProviderApplicationUrl(): string {
+        return "https://notion.so/";
+    }
+
+    public getNotionClient(): Client {
+        return new Client({ auth: this.connection.accessToken });
+    }
+
+    public getOptions(): ProviderHandlerOption[] {
+        return [
+            {
+                id: "syncDepth",
+                label: "Sync Depth",
+                type: ConnectionOptionType.ENUM,
+                enumOptions: [
+                    { value: "1-level", label: "1 Level" },
+                    { value: "2-levels", label: "2 Levels" },
+                    { value: "all", label: "All Levels" },
+                ],
+                defaultValue: "1-level",
+            },
+        ];
+    }
+
+    public async _sync(api: any, syncPosition: any): Promise<SyncResponse> {
+        try {
+            if (this.config.batchSize > MAX_BATCH_SIZE) {
+                throw new Error(`Batch size (${this.config.batchSize}) exceeds max limit (${MAX_BATCH_SIZE})`);
+            }
+
+            const notion = this.getNotionClient();
+            const rangeTracker = new ItemsRangeTracker(syncPosition.thisRef);
+            let items: any[] = [];
+
+            let currentRange = rangeTracker.nextRange();
+
+            const pages = await this.getPageList();
+
+            const page = await notion.pages.retrieve({
+                page_id: pages[0].id
+            })
+
+            let query = { block_id: pages[0].id, start_cursor: currentRange.startId };
+
+            const latestResponse = await notion.blocks.children.list(query);
+
+
+            const latestResult = await this.buildResults(notion, latestResponse, currentRange.endId);
+
+            items = latestResult.items;
+            let nextPageCursor = latestResponse.next_cursor;
+
+            if (items.length) {
+                rangeTracker.completedRange({
+                    startId: items[0].id,
+                    endId: nextPageCursor,
+                }, latestResult.breakHit === SyncItemsBreak.ID);
+            } else {
+                rangeTracker.completedRange({ startId: undefined, endId: undefined }, false);
+            }
+
+            if (!items.length) {
+                syncPosition.syncMessage = "Stopping. No results found.";
+                syncPosition.status = SyncHandlerStatus.ENABLED;
+            } else {
+                syncPosition.syncMessage = items.length !== this.config.batchSize && !nextPageCursor
+                    ? `Processed ${items.length} items. Stopping. No more results.`
+                    : `Batch complete (${this.config.batchSize}). More results pending.`;
+            }
+
+            syncPosition.thisRef = rangeTracker.export();
+
+            return { results: items, position: syncPosition };
+        } catch (err: any) {
+            if (err.status === 403) throw new AccessDeniedError(err.message);
+            if (err.status === 401) throw new InvalidTokenError(err.message);
+            throw err;
+        }
+
+
+    }
+
+    protected async buildResults(
+        notion: Client,
+        serverResponse: any,
+        breakId: string
+    ): Promise<SyncBlockItemsResult> {
+        const results: any[] = [];
+        let breakHit: SyncItemsBreak;
+
+        for (const block of serverResponse.results) {
+            if (block.id === breakId) {
+                this.emit("log", { level: SyncProviderLogLevel.DEBUG, message: `Break ID hit (${breakId})` });
+                breakHit = SyncItemsBreak.ID;
+                break;
+            }
+
+            results.push({
+                _id: this.buildItemId(block.id),
+                type: block.type,
+                sourceId: block.id,
+                sourceApplication: this.getProviderApplicationUrl(),
+                content: JSON.stringify(block),
+                insertedAt: new Date().toISOString(),
+            });
+        }
+
+        return { items: results, breakHit };
+    }
+
+    public async getPageList(): Promise<any[]> {
+        try {
+            const notion = await this.getNotionClient();
+            const response = await notion.search({
+                filter: { property: "object", value: "page" },
+                sort: { direction: "ascending", timestamp: "last_edited_time" },
+                page_size: 50 // Max is 100
+            });
+
+            return response.results;
+
+        } catch (error) {
+            console.error("Error fetching Notion pages:", error);
+        }
+    }
+
+}