Fix download tests (#76)

* Fix download tests

* Remove unfunctional mockup

---------

Co-authored-by: thomas.grandjean <thomas.grandjean@developpement-durable.gouv.fr>

Author: tgrandje

cartiflette/download/download.py

@@ -11,6 +11,7 @@
 from pebble import ThreadPool
 import s3fs
 import shutil
+import traceback
 from typing import Union
 
 from cartiflette import BUCKET, PATH_WITHIN_BUCKET, FS, THREADS_DOWNLOAD
@@ -280,7 +281,8 @@ def func(args):
                 else:
                     paths = {}
                     # cleanup temp files
-                    shutil.rmtree(result["root_cleanup"])
+                    if result["root_cleanup"]:
+                        shutil.rmtree(result["root_cleanup"])
 
                 del result["hash"], result["root_cleanup"], result["layers"]
                 result["paths"] = paths
@@ -303,6 +305,7 @@ def func(args):
                         break
                     except Exception as e:
                         logger.error(e)
+                        logger.error(traceback.format_exc())
         else:
             for args in combinations:
                 files = deep_dict_update(files, func(args))

tests/mockups.py

@@ -2,6 +2,7 @@
 import pytest
 import requests
 from requests_cache import CachedSession
+import s3fs
 import logging
 
 from tests.conftest import (
@@ -10,7 +11,9 @@
     CONTENT_DUMMY,
 )
 
+from cartiflette import FS
 from cartiflette.download.dataset import Dataset
+from cartiflette.download.scraper import MasterScraper
 
 
 logging.basicConfig(level=logging.INFO)
@@ -19,31 +22,28 @@
 @pytest.fixture
 def mock_Dataset_without_s3(monkeypatch):
     monkeypatch.setattr(Dataset, "_get_last_md5", lambda x: None)
-    monkeypatch.setattr("FS")
+    # monkeypatch.setattr("FS")
 
 
 @pytest.fixture
 def total_mock_s3(monkeypatch):
     monkeypatch.setattr(Dataset, "_get_last_md5", lambda x: None)
 
-    def mock_unpack(x):
+    def mock_unpack(self, x):
         return {
-            x.provider: {
-                x.dataset_family: {
-                    x.source: {
-                        x.territory: {
-                            x.year: {
-                                "downloaded": False,
-                                "paths": None,
-                                "hash": None,
-                            }
-                        }
-                    }
-                }
-            }
+            "downloaded": False,
+            "layers": None,
+            "hash": None,
+            "root_cleanup": None,
         }
 
-    monkeypatch.setattr(Dataset, "download_unpack", lambda x: mock_unpack)
+    monkeypatch.setattr(MasterScraper, "download_unpack", mock_unpack)
+    # monkeypatch.setattr("cartiflette.THREADS_DOWNLOAD", 1)
+
+    def mock_ls(folder):
+        return [f"{folder}/md5.json"]
+
+    monkeypatch.setattr(FS, "ls", mock_ls)
 
 
 class MockResponse:

tests/test_download.py

@@ -4,16 +4,18 @@
 import os
 import requests_cache
 import logging
+
 from cartiflette.download.dataset import Dataset
 from cartiflette.download.scraper import MasterScraper
-
 from cartiflette.download.download import _download_sources
+from cartiflette.download import download_all
 from cartiflette.utils import import_yaml_config
 from tests.conftest import (
     DUMMY_FILE_1,
     DUMMY_FILE_2,
     HASH_DUMMY,
 )
+
 from tests.mockups import (
     mock_httpscraper_download_success,
     mock_httpscraper_download_success_corrupt_hash,
@@ -285,5 +287,6 @@ def test_sources_yaml(mock_Dataset_without_s3):
 
 
 def test_download_all(total_mock_s3):
-    ret = _download_sources()
+    ret = download_all()
     assert isinstance(ret, dict)
+    assert len(ret) > 0