Concurrent request error

[Eric-chy]

Traceback (most recent call last):
File "/usr/local/lib/python3.8/site-packages/flask/app.py", line 1463, in wsgi_app
response = self.full_dispatch_request()
File "/usr/local/lib/python3.8/site-packages/flask/app.py", line 872, in full_dispatch_request
rv = self.handle_user_exception(e)
File "/usr/local/lib/python3.8/site-packages/flask/app.py", line 870, in full_dispatch_request
rv = self.dispatch_request()
File "/usr/local/lib/python3.8/site-packages/flask/app.py", line 855, in dispatch_request
return self.ensure_sync(self.view_functions[rule.endpoint])(**view_args) # type: ignore[no-any-return]
File "/app/nllb_serve/app.py", line 141, in translate
inputs = tokenizer(sources, return_tensors="pt", padding=True)
File "/usr/local/lib/python3.8/site-packages/transformers/tokenization_utils_base.py", line 2882, in call
self._switch_to_input_mode()
File "/usr/local/lib/python3.8/site-packages/transformers/models/nllb/tokenization_nllb_fast.py", line 273, in _switch_to_input_mode
return self.set_src_lang_special_tokens(self.src_lang)
File "/usr/local/lib/python3.8/site-packages/transformers/models/nllb/tokenization_nllb_fast.py", line 295, in set_src_lang_special_tokens
self._tokenizer.post_processor = processors.TemplateProcessing(
RuntimeError: Already borrowed

[Eric-chy]

It works fine when I make a single request, but when I make concurrent requests, an error occurs.

[sgowdaks]

Hi @Eric-chy I am looking into this, could you please provide a sample code of how are you making concurrent requests?

[tickscn]

@sgowdaks I've encountered the same issue. Below is a simple PHP code snippet to reproduce the problem

<?php
require "./vendor/autoload.php"; 
use GuzzleHttp\Client;
use GuzzleHttp\Pool;

$client = new Client([
    'base_uri' => getenv('TRANSLATE_HOST'),
    'time_out' => 20,
]);

$content = json_decode(file_get_contents('content.json'), true);
$request = function ($content) use ($client) {
    $chunks = array_chunk($content, 10);
    foreach ($chunks as $chunk) {
        yield function () use ($chunk, $client) {
            return $client->postAsync('/translate', [
                'json' => [
                    'source' => $chunk,
                    'src_lang' => 'eng_Latn',
                    'tgt_lang' => 'deu_Latn',
                ]
            ]);
        };
    }
};
$res = [];
$pool = new Pool($client, $request($content), [
    'concurrency' => 10,
    'fulfilled' => function ($response, $index) use (&$res) {
        $resp = json_decode($response->getBody()->getContents(), true);
        $res[$index] = $resp['translation'] ?? [];

    },
    'rejected' => function ($reason, $index) {
        if ($reason instanceof \GuzzleHttp\Exception\RequestException) {
            if ($reason->hasResponse()) {
                $response = $reason->getResponse();
                echo $response->getBody() . PHP_EOL;
            }
        }
        throw $reason;
    },
]);
$promise = $pool->promise();
$promise->wait();

file_put_contents('res.json', json_decode($res));

content.json