Skip to content

Commit

Permalink
fix: stream response buffer issue (#1703)
Browse files Browse the repository at this point in the history
  • Loading branch information
cr7258 authored Jan 22, 2025
1 parent ce94c6e commit 133a30b
Showing 1 changed file with 7 additions and 5 deletions.
12 changes: 7 additions & 5 deletions plugins/wasm-go/extensions/ai-proxy/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -95,10 +95,9 @@ func onHttpRequestHeader(ctx wrapper.HttpContext, pluginConfig config.PluginConf
// Disable the route re-calculation since the plugin may modify some headers related to the chosen route.
ctx.DisableReroute()

_, needHandleStreamingBody := activeProvider.(provider.StreamingResponseBodyHandler)
if needHandleStreamingBody {
proxywasm.RemoveHttpRequestHeader("Accept-Encoding")
}
// Always remove the Accept-Encoding header to prevent the LLM from sending compressed responses,
// allowing plugins to inspect or modify the response correctly
proxywasm.RemoveHttpRequestHeader("Accept-Encoding")

if handler, ok := activeProvider.(provider.RequestHeadersHandler); ok {
// Set the apiToken for the current request.
Expand Down Expand Up @@ -200,8 +199,11 @@ func onHttpResponseHeaders(ctx wrapper.HttpContext, pluginConfig config.PluginCo
util.ReplaceResponseHeaders(headers)

checkStream(ctx, log)
_, needHandleBody := activeProvider.(provider.TransformResponseBodyHandler)
_, needHandleStreamingBody := activeProvider.(provider.StreamingResponseBodyHandler)
if !needHandleStreamingBody {
if !needHandleBody && !needHandleStreamingBody {
ctx.DontReadResponseBody()
} else if !needHandleStreamingBody {
ctx.BufferResponseBody()
}

Expand Down

0 comments on commit 133a30b

Please sign in to comment.