From 74645df044c14091747d527f0bf5dfe2c3677529 Mon Sep 17 00:00:00 2001 From: Ashar Irfan Date: Thu, 30 Jan 2025 21:44:52 +0500 Subject: [PATCH 1/3] =?UTF-8?q?=F0=9F=93=A6=20NEW:=20Crawl=20function=20in?= =?UTF-8?q?=20SDK?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/langbase/src/langbase/langbase.ts | 46 ++++++++++++++++++++-- 1 file changed, 43 insertions(+), 3 deletions(-) diff --git a/packages/langbase/src/langbase/langbase.ts b/packages/langbase/src/langbase/langbase.ts index 8954f69..8ba29d4 100644 --- a/packages/langbase/src/langbase/langbase.ts +++ b/packages/langbase/src/langbase/langbase.ts @@ -252,7 +252,7 @@ export interface ToolWebSearchOptions { query: string; total_results?: number; domains?: string[]; - apiKey?: string; + api_key?: string; } export interface ToolWebSearchResponse { @@ -260,6 +260,20 @@ export interface ToolWebSearchResponse { content: string; } +export interface ToolCrawlOptions { + url: string[]; + max_pages?: number; + api_key?: string; +} + +export interface ToolCrawlResponse { + success: boolean; + data: { + url: string; + content: string; + }[]; +} + export class Langbase { private request: Request; private apiKey: string; @@ -296,6 +310,7 @@ export class Langbase { }; public tool: { + crawl: (options: ToolCrawlOptions) => Promise; webSearch: ( options: ToolWebSearchOptions, ) => Promise; @@ -331,6 +346,7 @@ export class Langbase { }; this.tool = { + crawl: this.webCrawl.bind(this), webSearch: this.webSearch.bind(this), }; } @@ -559,12 +575,36 @@ export class Langbase { private async webSearch( options: ToolWebSearchOptions, ): Promise { + const apiKey = options.api_key ? options.api_key : null; + apiKey && delete options.api_key; return this.request.post({ endpoint: '/v1/tools/web-search', body: options, headers: { - ...(options.apiKey && { - 'LB-WEB-SEARCH-KEY': options.apiKey, + ...(apiKey && { + 'LB-WEB-SEARCH-KEY': apiKey, + }), + }, + }); + } + + /** + * Performs a web crawls on target websites using the Langbase API. + * + * @param options - Crawl configuration options + * @returns An array of responses containing data from the crawl operation. + */ + private async webCrawl( + options: ToolCrawlOptions, + ): Promise { + const apiKey = options.api_key ? options.api_key : null; + apiKey && delete options.api_key; + return this.request.post({ + endpoint: '/v1/tools/crawl', + body: options, + headers: { + ...(apiKey && { + 'LB-CRAWL-KEY': apiKey, }), }, }); From 5e9da3e2375021a46781ba6ca8aa88bcb4b6311c Mon Sep 17 00:00:00 2001 From: Ashar Irfan Date: Thu, 30 Jan 2025 21:45:09 +0500 Subject: [PATCH 2/3] =?UTF-8?q?=F0=9F=93=A6=20NEW:=20Crawl=20example?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- examples/nodejs/.env.example | 1 + examples/nodejs/examples/tools/crawl.ts | 18 ++++++++++++++++++ examples/nodejs/examples/tools/web-search.ts | 2 +- examples/nodejs/package.json | 3 ++- examples/nodejs/readme.md | 1 + 5 files changed, 23 insertions(+), 2 deletions(-) create mode 100644 examples/nodejs/examples/tools/crawl.ts diff --git a/examples/nodejs/.env.example b/examples/nodejs/.env.example index dfb1f50..12bd349 100644 --- a/examples/nodejs/.env.example +++ b/examples/nodejs/.env.example @@ -4,3 +4,4 @@ LANGBASE_SDK_GENERATE_PIPE="" LANGBASE_SDK_CHAT_PIPE="" LANGBASE_API_KEY="" WEB_SEARCH_KEY="" +CRAWL_KEY="" diff --git a/examples/nodejs/examples/tools/crawl.ts b/examples/nodejs/examples/tools/crawl.ts new file mode 100644 index 0000000..80f6bed --- /dev/null +++ b/examples/nodejs/examples/tools/crawl.ts @@ -0,0 +1,18 @@ +import 'dotenv/config'; +import {Langbase} from 'langbase'; + +const langbase = new Langbase({ + apiKey: process.env.LANGBASE_API_KEY!, +}); + +async function main() { + const results = await langbase.tool.crawl({ + url: ['https://langbase.com', 'https://langbase.com/about'], + max_pages: 1, + api_key: process.env.CRAWL_KEY, + }); + + console.log(results); +} + +main(); diff --git a/examples/nodejs/examples/tools/web-search.ts b/examples/nodejs/examples/tools/web-search.ts index 5d90673..9f3444c 100644 --- a/examples/nodejs/examples/tools/web-search.ts +++ b/examples/nodejs/examples/tools/web-search.ts @@ -8,7 +8,7 @@ const langbase = new Langbase({ async function main() { const results = await langbase.tool.webSearch({ query: 'AI Engineer', - apiKey: process.env.WEB_SEARCH_KEY, + api_key: process.env.WEB_SEARCH_KEY, }); console.log(results); diff --git a/examples/nodejs/package.json b/examples/nodejs/package.json index 3fb78f0..98f08cb 100644 --- a/examples/nodejs/package.json +++ b/examples/nodejs/package.json @@ -28,7 +28,8 @@ "stream-text": "npx tsx ./examples/pipes/stream-text.ts", "stream-text-generate-pipe": "npx tsx ./examples/pipes/stream-text-generate-pipe.ts", "stream-text-chat-pipe": "npx tsx ./examples/pipes/stream-text-chat-pipe.ts", - "tools.web-search": "npx tsx ./examples/tools/web-search.ts" + "tools.web-search": "npx tsx ./examples/tools/web-search.ts", + "tools.crawl": "npx tsx ./examples/tools/crawl.ts" }, "keywords": [], "author": "Ahmad Awais (https://twitter.com/MrAhmadAwais)", diff --git a/examples/nodejs/readme.md b/examples/nodejs/readme.md index a2db50d..c9ffb56 100644 --- a/examples/nodejs/readme.md +++ b/examples/nodejs/readme.md @@ -32,4 +32,5 @@ npm run memory.docs.retry-embed # tools npm run tools.web-search +npm run tools.crawl ``` From 7a2751bca18bcd6bd6e994b9c49fee2e70f6338e Mon Sep 17 00:00:00 2001 From: Ashar Irfan Date: Thu, 30 Jan 2025 22:05:49 +0500 Subject: [PATCH 3/3] =?UTF-8?q?=F0=9F=91=8C=20IMPROVE:=20Crawl=20response?= =?UTF-8?q?=20type?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/langbase/src/langbase/langbase.ts | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/packages/langbase/src/langbase/langbase.ts b/packages/langbase/src/langbase/langbase.ts index 8ba29d4..7946c21 100644 --- a/packages/langbase/src/langbase/langbase.ts +++ b/packages/langbase/src/langbase/langbase.ts @@ -267,11 +267,8 @@ export interface ToolCrawlOptions { } export interface ToolCrawlResponse { - success: boolean; - data: { - url: string; - content: string; - }[]; + url: string; + content: string; } export class Langbase {