From e7d8975f1c3420733d1e34411c6f352c0e00b3d2 Mon Sep 17 00:00:00 2001 From: Truong Vinh Tran Date: Fri, 20 Feb 2026 12:02:00 +0100 Subject: [PATCH 1/4] feat: Add SearXNG search provider support Implements SearXNG as a third web search provider to address Oracle Cloud datacenter IP blocking issues and provide a cost-free, self-hosted alternative to commercial search APIs. Changes: - Add SearXNGConfig struct with Enabled, BaseURL, and MaxResults fields - Implement SearXNGSearchProvider with JSON API integration - Update provider priority: Perplexity > Brave > SearXNG > DuckDuckGo - Wire SearXNG configuration through agent tool registration - Add default config values (disabled by default, empty BaseURL) Benefits: - Solves DuckDuckGo datacenter IP blocking (138 bytes redirect responses) - Zero-cost alternative to Brave Search API ($5/1000 queries) - Self-hosted solution with 70+ aggregated search engines - Privacy-focused with no rate limits or API keys required - Ideal for Oracle Cloud, GCP, AWS, and Azure VM deployments The implementation follows the existing provider interface pattern and maintains backward compatibility with all existing search providers. Co-Authored-By: Claude Sonnet 4.5 --- pkg/agent/loop.go | 3 ++ pkg/config/config.go | 7 ++++ pkg/config/defaults.go | 5 +++ pkg/tools/web.go | 72 +++++++++++++++++++++++++++++++++++++++++- 4 files changed, 86 insertions(+), 1 deletion(-) diff --git a/pkg/agent/loop.go b/pkg/agent/loop.go index e7b48d47a..4cdc1fe90 100644 --- a/pkg/agent/loop.go +++ b/pkg/agent/loop.go @@ -96,6 +96,9 @@ func registerSharedTools(cfg *config.Config, msgBus *bus.MessageBus, registry *A PerplexityAPIKey: cfg.Tools.Web.Perplexity.APIKey, PerplexityMaxResults: cfg.Tools.Web.Perplexity.MaxResults, PerplexityEnabled: cfg.Tools.Web.Perplexity.Enabled, + SearXNGBaseURL: cfg.Tools.Web.SearXNG.BaseURL, + SearXNGMaxResults: cfg.Tools.Web.SearXNG.MaxResults, + SearXNGEnabled: cfg.Tools.Web.SearXNG.Enabled, }); searchTool != nil { agent.Tools.Register(searchTool) } diff --git a/pkg/config/config.go b/pkg/config/config.go index 0d41796a4..87a1186a8 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -400,10 +400,17 @@ type PerplexityConfig struct { MaxResults int `json:"max_results" env:"PICOCLAW_TOOLS_WEB_PERPLEXITY_MAX_RESULTS"` } +type SearXNGConfig struct { + Enabled bool `json:"enabled" env:"PICOCLAW_TOOLS_WEB_SEARXNG_ENABLED"` + BaseURL string `json:"base_url" env:"PICOCLAW_TOOLS_WEB_SEARXNG_BASE_URL"` + MaxResults int `json:"max_results" env:"PICOCLAW_TOOLS_WEB_SEARXNG_MAX_RESULTS"` +} + type WebToolsConfig struct { Brave BraveConfig `json:"brave"` DuckDuckGo DuckDuckGoConfig `json:"duckduckgo"` Perplexity PerplexityConfig `json:"perplexity"` + SearXNG SearXNGConfig `json:"searxng"` } type CronToolsConfig struct { diff --git a/pkg/config/defaults.go b/pkg/config/defaults.go index 54d6d68c3..a8c5bee58 100644 --- a/pkg/config/defaults.go +++ b/pkg/config/defaults.go @@ -258,6 +258,11 @@ func DefaultConfig() *Config { APIKey: "", MaxResults: 5, }, + SearXNG: SearXNGConfig{ + Enabled: false, + BaseURL: "", + MaxResults: 5, + }, }, Cron: CronToolsConfig{ ExecTimeoutMinutes: 5, diff --git a/pkg/tools/web.go b/pkg/tools/web.go index 1f5c58ea5..e1a640ff0 100644 --- a/pkg/tools/web.go +++ b/pkg/tools/web.go @@ -241,6 +241,68 @@ func (p *PerplexitySearchProvider) Search(ctx context.Context, query string, cou return fmt.Sprintf("Results for: %s (via Perplexity)\n%s", query, searchResp.Choices[0].Message.Content), nil } +type SearXNGSearchProvider struct { + baseURL string +} + +func (p *SearXNGSearchProvider) Search(ctx context.Context, query string, count int) (string, error) { + searchURL := fmt.Sprintf("%s/search?q=%s&format=json&categories=general", + strings.TrimSuffix(p.baseURL, "/"), + url.QueryEscape(query)) + + req, err := http.NewRequestWithContext(ctx, "GET", searchURL, nil) + if err != nil { + return "", fmt.Errorf("failed to create request: %w", err) + } + + client := &http.Client{Timeout: 10 * time.Second} + resp, err := client.Do(req) + if err != nil { + return "", fmt.Errorf("request failed: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("SearXNG returned status %d", resp.StatusCode) + } + + var result struct { + Results []struct { + Title string `json:"title"` + URL string `json:"url"` + Content string `json:"content"` + Engine string `json:"engine"` + Score float64 `json:"score"` + } `json:"results"` + } + + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return "", fmt.Errorf("failed to parse response: %w", err) + } + + if len(result.Results) == 0 { + return fmt.Sprintf("No results for: %s", query), nil + } + + // Limit results to requested count + if len(result.Results) > count { + result.Results = result.Results[:count] + } + + // Format results in standard PicoClaw format + var b strings.Builder + b.WriteString(fmt.Sprintf("Results for: %s (via SearXNG)\n", query)) + for i, r := range result.Results { + b.WriteString(fmt.Sprintf("%d. %s\n", i+1, r.Title)) + b.WriteString(fmt.Sprintf(" %s\n", r.URL)) + if r.Content != "" { + b.WriteString(fmt.Sprintf(" %s\n", r.Content)) + } + } + + return b.String(), nil +} + type WebSearchTool struct { provider SearchProvider maxResults int @@ -255,13 +317,16 @@ type WebSearchToolOptions struct { PerplexityAPIKey string PerplexityMaxResults int PerplexityEnabled bool + SearXNGBaseURL string + SearXNGMaxResults int + SearXNGEnabled bool } func NewWebSearchTool(opts WebSearchToolOptions) *WebSearchTool { var provider SearchProvider maxResults := 5 - // Priority: Perplexity > Brave > DuckDuckGo + // Priority: Perplexity > Brave > SearXNG > DuckDuckGo if opts.PerplexityEnabled && opts.PerplexityAPIKey != "" { provider = &PerplexitySearchProvider{apiKey: opts.PerplexityAPIKey} if opts.PerplexityMaxResults > 0 { @@ -272,6 +337,11 @@ func NewWebSearchTool(opts WebSearchToolOptions) *WebSearchTool { if opts.BraveMaxResults > 0 { maxResults = opts.BraveMaxResults } + } else if opts.SearXNGEnabled && opts.SearXNGBaseURL != "" { + provider = &SearXNGSearchProvider{baseURL: opts.SearXNGBaseURL} + if opts.SearXNGMaxResults > 0 { + maxResults = opts.SearXNGMaxResults + } } else if opts.DuckDuckGoEnabled { provider = &DuckDuckGoSearchProvider{} if opts.DuckDuckGoMaxResults > 0 { From 25d8f0e1ca075cc505bef96456757b9f46690ed8 Mon Sep 17 00:00:00 2001 From: Truong Vinh Tran Date: Fri, 20 Feb 2026 12:37:58 +0100 Subject: [PATCH 2/4] docs: Add SearXNG web search provider documentation Update README to document the new SearXNG search provider option alongside existing Brave, DuckDuckGo, and Perplexity providers. Changes: - Document provider priority order: Perplexity > Brave > SearXNG > DuckDuckGo - Add SearXNG configuration examples in Quick Start and Full Config sections - Expand "Get API Keys" section with all 4 search provider options - Enhance troubleshooting section with detailed setup instructions for each provider - Add SearXNG to API Key Comparison table (unlimited/self-hosted) SearXNG benefits documented: - Zero cost with no API fees or rate limits - Privacy-focused self-hosted solution - Aggregates 70+ search engines for comprehensive results - Solves datacenter IP blocking issues on Oracle Cloud, GCP, AWS, Azure - No API key required, just deploy and configure base URL This documentation complements the code implementation in commit e7d8975. Co-Authored-By: Claude Sonnet 4.5 --- README.md | 116 ++++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 105 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 468350409..83a533973 100644 --- a/README.md +++ b/README.md @@ -237,6 +237,16 @@ picoclaw onboard "duckduckgo": { "enabled": true, "max_results": 5 + }, + "perplexity": { + "enabled": false, + "api_key": "YOUR_PERPLEXITY_API_KEY", + "max_results": 5 + }, + "searxng": { + "enabled": false, + "base_url": "http://your-searxng-instance:8888", + "max_results": 5 } } } @@ -248,7 +258,11 @@ picoclaw onboard **3. Get API Keys** * **LLM Provider**: [OpenRouter](https://openrouter.ai/keys) 路 [Zhipu](https://open.bigmodel.cn/usercenter/proj-mgmt/apikeys) 路 [Anthropic](https://console.anthropic.com) 路 [OpenAI](https://platform.openai.com) 路 [Gemini](https://aistudio.google.com/api-keys) -* **Web Search** (optional): [Brave Search](https://brave.com/search/api) - Free tier available (2000 requests/month) +* **Web Search** (optional): + * [Brave Search](https://brave.com/search/api) - Free tier (2000 requests/month) + * [Perplexity](https://www.perplexity.ai) - AI-powered search with chat interface + * [SearXNG](https://github.com/searxng/searxng) - Self-hosted metasearch engine (free, no API key needed) + * DuckDuckGo - Built-in fallback (no API key required) > **Note**: See `config.example.json` for a complete configuration template. @@ -977,6 +991,16 @@ picoclaw agent -m "Hello" "duckduckgo": { "enabled": true, "max_results": 5 + }, + "perplexity": { + "enabled": false, + "api_key": "", + "max_results": 5 + }, + "searxng": { + "enabled": false, + "base_url": "http://localhost:8888", + "max_results": 5 } }, "cron": { @@ -1034,10 +1058,69 @@ discord: This is normal if you haven't configured a search API key yet. PicoClaw will provide helpful links for manual searching. -To enable web search: +#### Search Provider Priority -1. **Option 1 (Recommended)**: Get a free API key at [https://brave.com/search/api](https://brave.com/search/api) (2000 free queries/month) for the best results. -2. **Option 2 (No Credit Card)**: If you don't have a key, we automatically fall back to **DuckDuckGo** (no key required). +PicoClaw automatically selects the best available search provider in this order: +1. **Perplexity** (if enabled and API key configured) - AI-powered search with citations +2. **Brave Search** (if enabled and API key configured) - Privacy-focused with 2000 free queries/month +3. **SearXNG** (if enabled and base_url configured) - Self-hosted metasearch aggregating 70+ engines +4. **DuckDuckGo** (if enabled, default fallback) - No API key required + +#### Web Search Configuration Options + +**Option 1 (Best Results)**: Perplexity AI Search +```json +{ + "tools": { + "web": { + "perplexity": { + "enabled": true, + "api_key": "YOUR_PERPLEXITY_API_KEY", + "max_results": 5 + } + } + } +} +``` + +**Option 2 (Free Tier)**: Get a free API key at [https://brave.com/search/api](https://brave.com/search/api) (2000 free queries/month) +```json +{ + "tools": { + "web": { + "brave": { + "enabled": true, + "api_key": "YOUR_BRAVE_API_KEY", + "max_results": 5 + } + } + } +} +``` + +**Option 3 (Self-Hosted)**: Deploy your own [SearXNG](https://github.com/searxng/searxng) instance +```json +{ + "tools": { + "web": { + "searxng": { + "enabled": true, + "base_url": "http://your-server:8888", + "max_results": 5 + } + } + } +} +``` + +Benefits of SearXNG: +- **Zero cost**: No API fees or rate limits +- **Privacy-focused**: Self-hosted, no tracking +- **Aggregate results**: Queries 70+ search engines simultaneously +- **Perfect for cloud VMs**: Solves datacenter IP blocking issues (Oracle Cloud, GCP, AWS, Azure) +- **No API key needed**: Just deploy and configure the base URL + +**Option 4 (No Setup Required)**: DuckDuckGo is enabled by default as fallback (no API key needed) Add the key to `~/.picoclaw/config.json` if using Brave: @@ -1053,6 +1136,16 @@ Add the key to `~/.picoclaw/config.json` if using Brave: "duckduckgo": { "enabled": true, "max_results": 5 + }, + "perplexity": { + "enabled": false, + "api_key": "YOUR_PERPLEXITY_API_KEY", + "max_results": 5 + }, + "searxng": { + "enabled": false, + "base_url": "http://your-searxng-instance:8888", + "max_results": 5 } } } @@ -1071,10 +1164,11 @@ This happens when another instance of the bot is running. Make sure only one `pi ## 馃摑 API Key Comparison -| Service | Free Tier | Use Case | -| ---------------- | ------------------- | ------------------------------------- | -| **OpenRouter** | 200K tokens/month | Multiple models (Claude, GPT-4, etc.) | -| **Zhipu** | 200K tokens/month | Best for Chinese users | -| **Brave Search** | 2000 queries/month | Web search functionality | -| **Groq** | Free tier available | Fast inference (Llama, Mixtral) | -| **Cerebras** | Free tier available | Fast inference (Llama, Qwen, etc.) | +| Service | Free Tier | Use Case | +| ---------------- | ------------------------ | ------------------------------------- | +| **OpenRouter** | 200K tokens/month | Multiple models (Claude, GPT-4, etc.) | +| **Zhipu** | 200K tokens/month | Best for Chinese users | +| **Brave Search** | 2000 queries/month | Web search functionality | +| **SearXNG** | Unlimited (self-hosted) | Privacy-focused metasearch (70+ engines) | +| **Groq** | Free tier available | Fast inference (Llama, Mixtral) | +| **Cerebras** | Free tier available | Fast inference (Llama, Qwen, etc.) | From a5043854c38ec1ee2eb7995e0123b5b06f32fde0 Mon Sep 17 00:00:00 2001 From: Truong Vinh Tran Date: Fri, 20 Feb 2026 12:39:25 +0100 Subject: [PATCH 3/4] docs: Add SearXNG to example configuration file Update config.example.json to include SearXNG web search provider configuration alongside existing Brave, DuckDuckGo, and Perplexity options. This ensures users have a complete reference for all available search providers when setting up their PicoClaw instance. Co-Authored-By: Claude Sonnet 4.5 --- config/config.example.json | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/config/config.example.json b/config/config.example.json index abc928e92..e046f7b76 100644 --- a/config/config.example.json +++ b/config/config.example.json @@ -186,6 +186,11 @@ "enabled": false, "api_key": "pplx-xxx", "max_results": 5 + }, + "searxng": { + "enabled": false, + "base_url": "http://localhost:8888", + "max_results": 5 } }, "cron": { From 5d2674b336ca8fb8e89856974a8f80452d637c0b Mon Sep 17 00:00:00 2001 From: Truong Vinh Tran Date: Fri, 20 Feb 2026 14:02:46 +0100 Subject: [PATCH 4/4] docs: Update Brave Search pricing - now $5/1000 queries (no free tier) Brave Search discontinued free tier on Feb 12, 2026. Updated all README references to reflect paid pricing. Emphasized SearXNG as free alternative. Co-Authored-By: Claude Sonnet 4.5 --- README.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 83a533973..1f7200d15 100644 --- a/README.md +++ b/README.md @@ -194,7 +194,7 @@ docker compose --profile gateway up -d > [!TIP] > Set your API key in `~/.picoclaw/config.json`. > Get API keys: [OpenRouter](https://openrouter.ai/keys) (LLM) 路 [Zhipu](https://open.bigmodel.cn/usercenter/proj-mgmt/apikeys) (LLM) -> Web search is **optional** - get free [Brave Search API](https://brave.com/search/api) (2000 free queries/month) or use built-in auto fallback. +> Web search is **optional** - [Brave Search API](https://brave.com/search/api) ($5/1000 queries, ~$5-6/month), [SearXNG](https://github.com/searxng/searxng) (free, self-hosted), or use built-in DuckDuckGo fallback. **1. Initialize** @@ -259,7 +259,7 @@ picoclaw onboard * **LLM Provider**: [OpenRouter](https://openrouter.ai/keys) 路 [Zhipu](https://open.bigmodel.cn/usercenter/proj-mgmt/apikeys) 路 [Anthropic](https://console.anthropic.com) 路 [OpenAI](https://platform.openai.com) 路 [Gemini](https://aistudio.google.com/api-keys) * **Web Search** (optional): - * [Brave Search](https://brave.com/search/api) - Free tier (2000 requests/month) + * [Brave Search](https://brave.com/search/api) - Paid ($5/1000 queries, ~$5-6/month) * [Perplexity](https://www.perplexity.ai) - AI-powered search with chat interface * [SearXNG](https://github.com/searxng/searxng) - Self-hosted metasearch engine (free, no API key needed) * DuckDuckGo - Built-in fallback (no API key required) @@ -1062,9 +1062,9 @@ This is normal if you haven't configured a search API key yet. PicoClaw will pro PicoClaw automatically selects the best available search provider in this order: 1. **Perplexity** (if enabled and API key configured) - AI-powered search with citations -2. **Brave Search** (if enabled and API key configured) - Privacy-focused with 2000 free queries/month -3. **SearXNG** (if enabled and base_url configured) - Self-hosted metasearch aggregating 70+ engines -4. **DuckDuckGo** (if enabled, default fallback) - No API key required +2. **Brave Search** (if enabled and API key configured) - Privacy-focused paid API ($5/1000 queries) +3. **SearXNG** (if enabled and base_url configured) - Self-hosted metasearch aggregating 70+ engines (free) +4. **DuckDuckGo** (if enabled, default fallback) - No API key required (free) #### Web Search Configuration Options @@ -1083,7 +1083,7 @@ PicoClaw automatically selects the best available search provider in this order: } ``` -**Option 2 (Free Tier)**: Get a free API key at [https://brave.com/search/api](https://brave.com/search/api) (2000 free queries/month) +**Option 2 (Paid API)**: Get an API key at [https://brave.com/search/api](https://brave.com/search/api) ($5/1000 queries, ~$5-6/month) ```json { "tools": { @@ -1168,7 +1168,7 @@ This happens when another instance of the bot is running. Make sure only one `pi | ---------------- | ------------------------ | ------------------------------------- | | **OpenRouter** | 200K tokens/month | Multiple models (Claude, GPT-4, etc.) | | **Zhipu** | 200K tokens/month | Best for Chinese users | -| **Brave Search** | 2000 queries/month | Web search functionality | +| **Brave Search** | Paid ($5/1000 queries) | Web search functionality | | **SearXNG** | Unlimited (self-hosted) | Privacy-focused metasearch (70+ engines) | | **Groq** | Free tier available | Fast inference (Llama, Mixtral) | | **Cerebras** | Free tier available | Fast inference (Llama, Qwen, etc.) |