diff --git a/src/Promptable.php b/src/Promptable.php index 79851888..acfe3666 100644 --- a/src/Promptable.php +++ b/src/Promptable.php @@ -6,6 +6,7 @@ use Illuminate\Broadcasting\Channel; use Illuminate\Container\Container; use Illuminate\Queue\SerializesModels; +use Illuminate\Support\Str; use Laravel\Ai\Attributes\Model as ModelAttribute; use Laravel\Ai\Attributes\Provider as ProviderAttribute; use Laravel\Ai\Attributes\Timeout as TimeoutAttribute; @@ -20,6 +21,7 @@ use Laravel\Ai\Prompts\AgentPrompt; use Laravel\Ai\Providers\Provider; use Laravel\Ai\Responses\AgentResponse; +use Laravel\Ai\Responses\Data\Meta; use Laravel\Ai\Responses\QueuedAgentResponse; use Laravel\Ai\Responses\StreamableAgentResponse; use Laravel\Ai\Streaming\Events\StreamEvent; @@ -71,12 +73,57 @@ public function stream( ?string $model = null, ?int $timeout = null): StreamableAgentResponse { - return $this->withModelFailover( - fn (Provider $provider, string $model) => $provider->stream( - new AgentPrompt($this, $prompt, $attachments, $provider, $model, $this->getTimeout($timeout)) - ), - $provider, - $model, + $providers = $this->getProvidersAndModels($provider, $model); + $resolvedTimeout = $this->getTimeout($timeout); + + $resolvedProviders = []; + + foreach ($providers as $p => $m) { + $resolved = Ai::textProviderFor($this, $p); + $resolvedProviders[] = [$resolved, $m ?? $this->getDefaultModelFor($resolved)]; + } + + if (empty($resolvedProviders)) { + throw new RuntimeException('No AI providers were configured.'); + } + + // Single provider: direct path (no failover needed) + if (count($resolvedProviders) === 1) { + [$singleProvider, $singleModel] = $resolvedProviders[0]; + + return $singleProvider->stream( + new AgentPrompt($this, $prompt, $attachments, $singleProvider, $singleModel, $resolvedTimeout) + ); + } + + // Multiple providers: failover-aware streaming + $agent = $this; + [$firstProvider, $firstModel] = $resolvedProviders[0]; + + return new StreamableAgentResponse( + (string) Str::uuid7(), + function () use ($resolvedProviders, $agent, $prompt, $attachments, $resolvedTimeout) { + $lastException = null; + + foreach ($resolvedProviders as [$provider, $model]) { + try { + yield from $provider->stream( + new AgentPrompt($agent, $prompt, $attachments, $provider, $model, $resolvedTimeout) + ); + + return; + } catch (FailoverableException $e) { + $lastException = $e; + + event(new AgentFailedOver($agent, $provider, $model, $e)); + + continue; + } + } + + throw $lastException; + }, + new Meta($firstProvider->name(), $firstModel), ); } diff --git a/tests/Feature/AgentStreamFailoverTest.php b/tests/Feature/AgentStreamFailoverTest.php new file mode 100644 index 00000000..a26205f9 --- /dev/null +++ b/tests/Feature/AgentStreamFailoverTest.php @@ -0,0 +1,89 @@ + ['driver' => 'groq', 'key' => 'test-key'], + 'ai.providers.backup' => ['driver' => 'groq', 'key' => 'test-key'], + ]); + + Http::preventStrayRequests(); + + Http::fakeSequence() + ->push(status: 429) + ->push($this->fakeGroqStreamBody(), 200); + + $response = (new AssistantAgent)->stream( + 'Hello', + provider: ['primary', 'backup'], + ); + + $events = []; + + foreach ($response as $event) { + $events[] = $event; + } + + $this->assertTrue( + collect($events)->whereInstanceOf(TextDelta::class)->isNotEmpty() + ); + + Event::assertDispatched(AgentFailedOver::class); + } + + public function test_stream_throws_last_exception_when_all_providers_fail(): void + { + config([ + 'ai.providers.primary' => ['driver' => 'groq', 'key' => 'test-key'], + 'ai.providers.backup' => ['driver' => 'groq', 'key' => 'test-key'], + ]); + + Http::preventStrayRequests(); + + Http::fakeSequence() + ->push(status: 429) + ->push(status: 429); + + $this->expectException(RateLimitedException::class); + + $response = (new AssistantAgent)->stream( + 'Hello', + provider: ['primary', 'backup'], + ); + + foreach ($response as $event) { + // + } + } + + private function fakeGroqStreamBody(): string + { + $chunks = [ + '{"id":"chatcmpl-1","object":"chat.completion.chunk","created":1,"model":"test","choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":null}]}', + '{"id":"chatcmpl-1","object":"chat.completion.chunk","created":1,"model":"test","choices":[{"index":0,"delta":{"content":"Hello"},"finish_reason":null}]}', + '{"id":"chatcmpl-1","object":"chat.completion.chunk","created":1,"model":"test","choices":[{"index":0,"delta":{},"finish_reason":"stop"}],"usage":{"prompt_tokens":5,"completion_tokens":1,"total_tokens":6}}', + ]; + + $body = ''; + + foreach ($chunks as $chunk) { + $body .= "data: {$chunk}\n\n"; + } + + return $body . "data: [DONE]\n\n"; + } +}