EVA-3322 - refactor (#48)

* Proposal for refactor of pyutils
* update to 0.6

.github/workflows/python-package.yml

@@ -44,7 +44,7 @@ jobs:
         python -m pip install --upgrade pip
         pip install flake8 pytest
         if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
-        python setup.py install
+        pip install .[eva-internal]
     - name: Lint with flake8
       run: |
         # stop the build if there are Python syntax errors or undefined names

.gitignore

@@ -130,3 +130,5 @@ dmypy.json
 
 # Pycharm project
 .idea/
+
+venv/

ebi_eva_common_pyutils/variation/assembly_utils.py → ebi_eva_common_pyutils/assembly_utils.py

@@ -12,9 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from ftplib import FTP
 import http
-import re
 import requests
 
 from ebi_eva_common_pyutils.assembly import NCBIAssembly

ebi_eva_common_pyutils/config_utils.py → ebi_eva_internal_pyutils/config_utils.py

@@ -18,7 +18,6 @@
 import yaml
 import urllib.request
 
-from pymongo.uri_parser import split_hosts
 from retry import retry
 
 

ebi_eva_common_pyutils/metadata_utils.py → ebi_eva_internal_pyutils/metadata_utils.py

@@ -17,11 +17,11 @@
 
 import psycopg2
 
-from ebi_eva_common_pyutils.config_utils import get_metadata_creds_for_profile
+from ebi_eva_internal_pyutils.config_utils import get_metadata_creds_for_profile
 from ebi_eva_common_pyutils.ena_utils import get_scientific_name_and_common_name
 from ebi_eva_common_pyutils.logger import logging_config
 from ebi_eva_common_pyutils.ncbi_utils import get_ncbi_assembly_name_from_term
-from ebi_eva_common_pyutils.pg_utils import get_result_cursor, get_all_results_for_query, execute_query
+from ebi_eva_internal_pyutils.pg_utils import get_result_cursor, get_all_results_for_query, execute_query
 from ebi_eva_common_pyutils.taxonomy.taxonomy import get_scientific_name_from_ensembl
 
 logger = logging_config.get_logger(__name__)

ebi_eva_common_pyutils/mongo_utils.py → ebi_eva_internal_pyutils/mongo_utils.py

@@ -19,7 +19,7 @@
 
 from ebi_eva_common_pyutils.command_utils import run_command_with_output
 from ebi_eva_common_pyutils.common_utils import merge_two_dicts
-from ebi_eva_common_pyutils.config_utils import get_mongo_uri_for_eva_profile, get_mongo_creds_for_profile
+from ebi_eva_internal_pyutils.config_utils import get_mongo_uri_for_eva_profile, get_mongo_creds_for_profile
 
 
 class MongoConfig:

ebi_eva_internal_pyutils/mongodb/__init__.py

@@ -0,0 +1,3 @@
+from ebi_eva_internal_pyutils.mongodb.mongo_database import MongoDatabase
+
+

ebi_eva_internal_pyutils/nextflow/__init__.py

@@ -0,0 +1 @@
+from ebi_eva_internal_pyutils.nextflow.nextflow_pipeline import LinearNextFlowPipeline, NextFlowPipeline, NextFlowProcess

ebi_eva_common_pyutils/spring_properties.py → ebi_eva_internal_pyutils/spring_properties.py

@@ -13,7 +13,7 @@
 # limitations under the License.
 from collections import defaultdict
 
-from ebi_eva_common_pyutils.config_utils import get_mongo_creds_for_profile, get_accession_pg_creds_for_profile, \
+from ebi_eva_internal_pyutils.config_utils import get_mongo_creds_for_profile, get_accession_pg_creds_for_profile, \
     get_count_service_creds_for_profile, get_properties_from_xml_file, get_variant_load_job_tracker_creds_for_profile
 
 

setup.py

@@ -5,15 +5,15 @@
 
 setup(
     name='ebi_eva_common_pyutils',
-    scripts=[os.path.join(os.path.dirname(__file__), 'ebi_eva_common_pyutils', 'archive_directory.py')],
+    scripts=[os.path.join(os.path.dirname(__file__), 'ebi_eva_internal_pyutils', 'archive_directory.py')],
     packages=find_packages(),
-    version='0.5.8.dev1',
+    version='0.6.0.dev0',
     license='Apache',
     description='EBI EVA - Common Python Utilities',
     url='https://github.com/EBIVariation/eva-common-pyutils',
     keywords=['EBI', 'EVA', 'PYTHON', 'UTILITIES'],
-    install_requires=['psycopg2-binary', 'requests', 'pymongo', 'lxml', 'pyyaml', 'cached-property', 'retry',
-                      'networkx<=2.5'],
+    install_requires=['requests', 'lxml', 'pyyaml', 'cached-property', 'retry'],
+    extras_require={'eva-internal': ['psycopg2-binary', 'pymongo', 'networkx<=2.5']},
     classifiers=[
         'Development Status :: 5 - Production/Stable',
         'Intended Audience :: Developers',

tests/common/test_variation.py

@@ -0,0 +1,9 @@
+import unittest
+
+from ebi_eva_common_pyutils.assembly_utils import retrieve_genbank_assembly_accessions_from_ncbi
+
+
+class TestAssemblyUtils(unittest.TestCase):
+    def test_retrieve_genbank_assembly_accessions_from_ncbi(self):
+        info = retrieve_genbank_assembly_accessions_from_ncbi('GCA_002263795.2')
+        print(info)

tests/test_archive.py → tests/internal/test_archive.py

@@ -3,12 +3,12 @@
 import tarfile
 from unittest import TestCase
 
-from ebi_eva_common_pyutils.archive_directory import archive_directory
+from ebi_eva_internal_pyutils.archive_directory import archive_directory
 
 
 class TestArchive(TestCase):
 
-    archive = os.path.join(os.path.dirname(__file__), 'resources', 'archive')
+    archive = os.path.join(os.path.dirname(__file__), '../resources', 'archive')
 
     def setUp(self) -> None:
         self.src_dir = os.path.join(self.archive, 'src')

tests/test_config_utils.py → tests/internal/test_config_utils.py

@@ -14,7 +14,7 @@
 import os
 from lxml.etree import XPathEvalError
 
-from ebi_eva_common_pyutils.config_utils import EVAPrivateSettingsXMLConfig, get_pg_metadata_uri_for_eva_profile, \
+from ebi_eva_internal_pyutils.config_utils import EVAPrivateSettingsXMLConfig, get_pg_metadata_uri_for_eva_profile, \
     get_mongo_uri_for_eva_profile, get_mongo_creds_for_profile
 from tests.test_common import TestCommon
 
@@ -47,7 +47,7 @@ def test_get_value_with_xpath(self):
 class TestDatabaseConfig(TestCommon):
 
     def setUp(self) -> None:
-        self.config_file = os.path.join(os.path.dirname(__file__), 'resources', 'test_config_file.xml')
+        self.config_file = os.path.join(os.path.dirname(__file__), '../resources', 'test_config_file.xml')
 
     def test_get_pg_metadata_uri_for_eva_profile(self):
         self.assertEqual(

tests/test_metadata_utils.py → tests/internal/test_metadata_utils.py

@@ -1,7 +1,7 @@
 from unittest import TestCase
-from unittest.mock import Mock, MagicMock, patch
+from unittest.mock import MagicMock, patch
 
-from ebi_eva_common_pyutils.metadata_utils import resolve_variant_warehouse_db_name, get_taxonomy_code_from_metadata, \
+from ebi_eva_internal_pyutils.metadata_utils import resolve_variant_warehouse_db_name, get_taxonomy_code_from_metadata, \
     get_assembly_code_from_metadata, insert_new_assembly_and_taxonomy, build_taxonomy_code, \
     ensure_taxonomy_is_in_evapro, insert_assembly_in_evapro, update_accessioning_status
 
@@ -17,13 +17,13 @@ def test_build_taxonomy_code(self):
 
     def test_get_taxonomy_code_from_metadata(self):
         db_handle = MagicMock()
-        with patch('ebi_eva_common_pyutils.metadata_utils.get_all_results_for_query', return_value=[('hsapiens',)]):
+        with patch('ebi_eva_internal_pyutils.metadata_utils.get_all_results_for_query', return_value=[('hsapiens',)]):
             taxcode = get_taxonomy_code_from_metadata(db_handle, 9096)
             assert taxcode == 'hsapiens'
 
     def test_get_assembly_code_from_metadata(self):
         db_handle = MagicMock()
-        with patch('ebi_eva_common_pyutils.metadata_utils.get_all_results_for_query', return_value=[('grch38',)]):
+        with patch('ebi_eva_internal_pyutils.metadata_utils.get_all_results_for_query', return_value=[('grch38',)]):
             taxcode = get_assembly_code_from_metadata(db_handle, 'GCA_000001405.15')
             assert taxcode == 'grch38'
 
@@ -49,9 +49,9 @@ def test_resolve_variant_warehouse_db_name_from_database(self):
             ('GCA_900491625.1', 3329, 'eva_pabies_a541150contigsfastagz')
         ]
         # Patch the lookup functions
-        ptaxonomy = patch('ebi_eva_common_pyutils.metadata_utils.get_taxonomy_code_from_metadata',
+        ptaxonomy = patch('ebi_eva_internal_pyutils.metadata_utils.get_taxonomy_code_from_metadata',
                           side_effect=['maize',  None,   'pabies'])
-        passembly = patch('ebi_eva_common_pyutils.metadata_utils.get_assembly_code_from_metadata',
+        passembly = patch('ebi_eva_internal_pyutils.metadata_utils.get_assembly_code_from_metadata',
                           side_effect=['agpv2', 'umd311', None])
         with ptaxonomy, passembly:
             for assembly, taxonomy, expected_db_name in expected_results:
@@ -89,13 +89,13 @@ def test_ensure_taxonomy_is_in_evapro(self):
     def test_insert_assembly_in_evapro_no_insert(self):
         db_handle = MagicMock()
         db_handle.cursor().__enter__().fetchall.return_value = (9606,)
-        with patch('ebi_eva_common_pyutils.metadata_utils.insert_taxonomy') as mock_insert:
+        with patch('ebi_eva_internal_pyutils.metadata_utils.insert_taxonomy') as mock_insert:
             ensure_taxonomy_is_in_evapro(db_handle, 9606)
         mock_insert.assert_not_called()
 
     def test_insert_assembly_in_evapro(self):
         db_handle = MagicMock()
-        with patch('ebi_eva_common_pyutils.metadata_utils.get_all_results_for_query') as mock_results:
+        with patch('ebi_eva_internal_pyutils.metadata_utils.get_all_results_for_query') as mock_results:
             # This will be used when retrieving the assembly set id after it's been inserted
             assembly_set_id = 27
             mock_results.return_value = ((assembly_set_id,),)
@@ -122,21 +122,21 @@ def test_update_accessioning_status(self):
 
     def test_insert_new_assembly_and_taxonomy(self):
         db_handle = MagicMock()
-        with patch('ebi_eva_common_pyutils.metadata_utils.ensure_taxonomy_is_in_evapro') as mock_tax_in_evapro, \
-                patch('ebi_eva_common_pyutils.metadata_utils.insert_assembly_in_evapro') as mock_insert_assembly, \
-                patch('ebi_eva_common_pyutils.metadata_utils.update_accessioning_status') as mock_update_status, \
-                patch('ebi_eva_common_pyutils.metadata_utils.get_all_results_for_query') as mock_get_results:
+        with patch('ebi_eva_internal_pyutils.metadata_utils.ensure_taxonomy_is_in_evapro') as mock_tax_in_evapro, \
+                patch('ebi_eva_internal_pyutils.metadata_utils.insert_assembly_in_evapro') as mock_insert_assembly, \
+                patch('ebi_eva_internal_pyutils.metadata_utils.update_accessioning_status') as mock_update_status, \
+                patch('ebi_eva_internal_pyutils.metadata_utils.get_all_results_for_query') as mock_get_results:
             insert_new_assembly_and_taxonomy(db_handle, 'GCA_000001405.15', 9606)
         mock_tax_in_evapro.assert_called_once_with(db_handle, 9606, None)
         mock_insert_assembly.assert_called_once_with(db_handle, 9606, 'GCA_000001405.15', 'GRCh38', 'grch38')
         mock_update_status.assert_called_once_with(db_handle, 'GCA_000001405.15', True)
 
     def test_no_insert_new_assembly_and_taxonomy(self):
         db_handle = MagicMock()
-        with patch('ebi_eva_common_pyutils.metadata_utils.ensure_taxonomy_is_in_evapro') as mock_tax_in_evapro, \
-                patch('ebi_eva_common_pyutils.metadata_utils.insert_assembly_in_evapro') as mock_insert_assembly, \
-                patch('ebi_eva_common_pyutils.metadata_utils.update_accessioning_status') as mock_update_status, \
-                patch('ebi_eva_common_pyutils.metadata_utils.get_all_results_for_query') as mock_get_results:
+        with patch('ebi_eva_internal_pyutils.metadata_utils.ensure_taxonomy_is_in_evapro') as mock_tax_in_evapro, \
+                patch('ebi_eva_internal_pyutils.metadata_utils.insert_assembly_in_evapro') as mock_insert_assembly, \
+                patch('ebi_eva_internal_pyutils.metadata_utils.update_accessioning_status') as mock_update_status, \
+                patch('ebi_eva_internal_pyutils.metadata_utils.get_all_results_for_query') as mock_get_results:
             mock_get_results.return_value = ((1,),)
             insert_new_assembly_and_taxonomy(db_handle, 'GCA_000001405.15', 9606)
         mock_tax_in_evapro.assert_not_called()

tests/test_mongodb.py → tests/internal/test_mongodb.py

@@ -6,16 +6,16 @@
 from pymongo.read_concern import ReadConcern
 
 from ebi_eva_common_pyutils.command_utils import run_command_with_output
-from ebi_eva_common_pyutils.mongo_utils import get_mongo_connection_handle
-from ebi_eva_common_pyutils.mongodb import MongoDatabase
+from ebi_eva_internal_pyutils.mongo_utils import get_mongo_connection_handle
+from ebi_eva_internal_pyutils.mongodb import MongoDatabase
 from tests.test_common import TestCommon
 
 
 class TestMongoDatabase(TestCommon):
     dump_db_name = "test_mongo_db"
     uri = "mongodb://localhost:27017/admin"
     local_mongo_handle = pymongo.MongoClient()
-    config_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'resources/test_config_file.xml')
+    config_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), '../resources/test_config_file.xml')
 
     # Tests expect a local sharded Mongo instance
     def setUp(self) -> None:

tests/test_nextflow.py → tests/internal/test_nextflow.py

@@ -1,6 +1,6 @@
 import os
 import shutil
-from ebi_eva_common_pyutils.nextflow import LinearNextFlowPipeline, NextFlowPipeline, NextFlowProcess
+from ebi_eva_internal_pyutils.nextflow import LinearNextFlowPipeline, NextFlowPipeline, NextFlowProcess
 from tests.test_common import TestCommon
 
 

tests/test_spring_prop.py → tests/internal/test_spring_prop.py

@@ -1,7 +1,6 @@
 import os
-from unittest import TestCase
 
-from ebi_eva_common_pyutils.spring_properties import SpringPropertiesGenerator
+from ebi_eva_internal_pyutils.spring_properties import SpringPropertiesGenerator
 from tests.test_common import TestCommon