refactor: lock only one slot for embedding at pooling

Signed-off-by: thxCode <thxcode0824@gmail.com>
gpustack · Aug 6, 2024 · 67215b9 · 67215b9
1 parent 5e41179
commit 67215b9
Showing 1 changed file with 2 additions and 0 deletions.
diff --git a/llama-box/main.cpp b/llama-box/main.cpp
@@ -2342,6 +2342,8 @@ struct server_context {
                         batch_type = slot_type;
                     } else if (batch_type != slot_type) {
                         continue;
+                    } else if (batch_type == 1 && llama_pooling_type(ctx) > 0) {
+                        continue;
                     }
 
                     // keep only the common part