Use multiprocessing queue

nesis/api/core/document_loaders/minio.py

@@ -1,6 +1,6 @@
 import logging
+import multiprocessing
 import os
-import queue
 import tempfile
 from typing import Dict, Any
 
@@ -38,7 +38,7 @@ def __init__(
         self._http_client = http_client
         self._cache_client = cache_client
         self._datasource = datasource
-        self._task_queue = queue.Queue(5)
+        self._task_queue = multiprocessing.Queue(maxsize=3)
 
         _extract_runner = None
         _ingest_runner = IngestRunner(config=config, http_client=http_client)

nesis/api/core/document_loaders/stores.py

@@ -1,9 +1,7 @@
 import json
 import logging
-from datetime import datetime
 
 import sqlalchemy as sa
-from sqlalchemy import DateTime
 from sqlalchemy.orm import registry, Session
 
 from nesis.api.core.models.entities import Document, DocumentObject

nesis/api/tests/core/document_loaders/test_minio.py

@@ -1,17 +1,15 @@
 import datetime
 import json
 import os
-import random
 import unittest as ut
 import unittest.mock as mock
 import uuid
 
 import pytest
-from sqlalchemy import create_engine
 from sqlalchemy.orm.session import Session
 
-import nesis.api.core.services as services
 import nesis.api.core.document_loaders.minio as minio
+import nesis.api.core.services as services
 from nesis.api import tests
 from nesis.api.core.document_loaders.stores import SqlDocumentStore
 from nesis.api.core.models import DBSession
@@ -20,12 +18,10 @@
     Datasource,
     Document,
 )
-
 from nesis.api.core.models.objects import (
     DatasourceType,
     DatasourceStatus,
 )
-from nesis.api.core.util import http
 
 
 @pytest.fixture