Fix downloading of ISIN file

Author: Jay Vala

requirements/ci.txt

@@ -1,18 +1,17 @@
-black>=19.10b0 ; python_version >= "3.6"
-isort~=4.3.21
-flake8~=3.7.9
-flake8-absolute-import~=1.0 ; python_version >= "3.6"
-flake8-black~=0.1.1 ; python_version >= "3.6"
-flake8-blind-except~=0.1.1 ; python_version >= "3.6"
-flake8-builtins~=1.5.2 ; python_version >= "3.6"
-flake8-comprehensions~=3.2.2 ; python_version >= "3.6"
-flake8-docstrings~=1.5.0 ; python_version >= "3.6"
-flake8-mutable~=1.2.0 ; python_version >= "3.6"
-flake8-print~=3.1.4 ; python_version >= "3.6"
-flake8-quotes~=3.0.0 ; python_version >= "3.6"
-flake8-tuple~=0.4.1 ; python_version >= "3.6"
-pytest~=4.6  # pytest 5 requires py3
-pytest-cov~=2.8.1
-pytest-env~=0.6.2
-pytest-sugar~=0.9.2
-testfixtures~=6.14.0
+black
+isort
+flake8
+flake8-absolute-import
+flake8-black
+flake8-blind-except
+flake8-comprehensions
+flake8-docstrings
+flake8-mutable
+flake8-print
+flake8-quotes
+flake8-tuple
+pytest
+pytest-cov
+pytest-env
+pytest-sugar
+testfixtures

requirements/prod.txt

@@ -1,3 +1,4 @@
 pandas
 texttable
-requests
+requests
+bs4

src/python_lei/isin_lei.py

@@ -1,5 +1,3 @@
-import pandas as pd
-import requests
 from python_lei.exceptions import InvalidISIN, InvalidLEI
 from python_lei.utils import load_data
 

src/python_lei/utils.py

@@ -8,6 +8,7 @@
 
 import pandas as pd
 import requests
+from bs4 import BeautifulSoup
 
 logger = logging.getLogger(__name__)
 logging.basicConfig(level=logging.INFO)
@@ -30,8 +31,7 @@ def __init__(self, _is_actions=False):
         Args:
             _is_actions (bool): For setting path of downloaded resources on Github Actions
         """
-        self.data_url = f"https://isinmapping.gleif.org/file-by-date/{TODAY}"
-
+        self.data_url = "https://www.gleif.org/en/lei-data/lei-mapping/download-isin-to-lei-relationship-files"        
         self._download(_is_actions)
 
     def _download(self, _is_actions):
@@ -42,8 +42,13 @@ def _download(self, _is_actions):
             logger.info(f"No resources directory found, creating resources directory.")
             os.mkdir(RESOURCE_DIR)
 
+        download_link = self._scrape_isin_file()
+
+        if not download_link:
+            raise ValueError("Downloading of isin file not available.")
+
         try:
-            response = requests.get(self.data_url)
+            response = requests.get(download_link)
         except requests.exceptions as err:
             logger.error(
                 "Connection Error, Unable to download data at this time. Please check you have working internet connection or try again later."
@@ -52,19 +57,31 @@ def _download(self, _is_actions):
             logger.error("No response from GLEIF server.")
 
         logger.info("The file could be over 50 Mb.")
-        # TODO: Add progress bar
         zipped_content = zipfile.ZipFile(io.BytesIO(response.content))
-        # TODO: Remove this
         if _is_actions:
             zipped_content.extractall(
                 "/home/runner/work/python-lei/python-lei/resources"
             )
         else:
             zipped_content.extractall(RESOURCE_DIR)
             logger.info(f"Extraction complete in {RESOURCE_DIR}")
+    
+    def _scrape_isin_file(self):
+        """
+        Scrape the data.
+        """
+        try:
+            response = requests.get(self.data_url)
+            response.raise_for_status()
+            soup = BeautifulSoup(response.text)
+        
+            # find all the tr and td and get to the href
+            download_link = soup.find_all("tr")[1].find_all("td")[1].find("a")["href"]
+            return download_link
 
-    # TODO: Covert the dataframe to parquet and use it.
-
+        except requests.ConnectionError:
+            logger.error(f"Error connecting to {self.data_url}")
+       
 
 class Update:
     """
@@ -83,12 +100,12 @@ def __init__(self):
             logger.info(
                 "Resource directory not found or LEI ISIN mappings not found. Downloading now."
             )
-            download = Download()
+            Download()
 
         if os.listdir(RESOURCE_DIR) != []:
             shutil.rmtree(RESOURCE_DIR)
             logger.info(f"Downloading Data in {RESOURCE_DIR}")
-            download = Download()
+            Download()
 
 
 def load_data():

tests/test_isin_lei.py

@@ -1,9 +1,6 @@
-from pathlib import Path
-from unittest.mock import Mock, patch
+from unittest.mock import patch
 
-import pandas as pd
 import pytest
-from python_lei.exceptions import NotFound
 from python_lei.lei_search import SearchLEI
 
 

tests/test_lei_search.py

@@ -1,9 +1,8 @@
-from pathlib import Path
-from unittest.mock import Mock, patch
+from unittest.mock import patch
 
 import pandas as pd
 import pytest
-from python_lei.exceptions import InvalidISIN, InvalidLEI
+from python_lei.exceptions import InvalidLEI
 from python_lei.pylei import pyLEI
 from python_lei.utils import PROJECT_ROOT