diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c53af54..4c28522 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -8,18 +8,24 @@ on: tags: ["*"] jobs: CI: - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 + strategy: + matrix: + python-version: ["3.6", "3.7", "3.8", "3.9", "3.10", "3.11"] # Specify Python versions here steps: - - uses: actions/checkout@v2 - - name: Set up Python 3.7 - uses: actions/setup-python@v2 + - uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 with: - python-version: 3.7 + python-version: ${{ matrix.python-version }} + - name: Install dependencies run: | python -m pip install --upgrade pip pip install -r requirements/ci.txt pip install -e . + - name: Download Data run: python -c "from python_lei.utils import Download; Download(_is_actions=True)" @@ -38,12 +44,12 @@ jobs: if: startsWith(github.ref, 'refs/tags/') runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - - name: Set up Python 3.7 - uses: actions/setup-python@v2 + - name: Set up Python 3.10 + uses: actions/setup-python@v5 with: - python-version: 3.7 + python-version: 3.10 - name: PyPi Deploy preparation run: | @@ -53,4 +59,4 @@ jobs: uses: pypa/gh-action-pypi-publish@v1.0.0a0 with: user: ${{ secrets.PYPI_USER }} - password: ${{ secrets.PYPI_PASSWORD }} \ No newline at end of file + password: ${{ secrets.PYPI_PASSWORD }} diff --git a/requirements/ci.txt b/requirements/ci.txt index f36b481..8e70901 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -1,18 +1,17 @@ -black>=19.10b0 ; python_version >= "3.6" -isort~=4.3.21 -flake8~=3.7.9 -flake8-absolute-import~=1.0 ; python_version >= "3.6" -flake8-black~=0.1.1 ; python_version >= "3.6" -flake8-blind-except~=0.1.1 ; python_version >= "3.6" -flake8-builtins~=1.5.2 ; python_version >= "3.6" -flake8-comprehensions~=3.2.2 ; python_version >= "3.6" -flake8-docstrings~=1.5.0 ; python_version >= "3.6" -flake8-mutable~=1.2.0 ; python_version >= "3.6" -flake8-print~=3.1.4 ; python_version >= "3.6" -flake8-quotes~=3.0.0 ; python_version >= "3.6" -flake8-tuple~=0.4.1 ; python_version >= "3.6" -pytest~=4.6 # pytest 5 requires py3 -pytest-cov~=2.8.1 -pytest-env~=0.6.2 -pytest-sugar~=0.9.2 -testfixtures~=6.14.0 \ No newline at end of file +black +isort +flake8 +flake8-absolute-import +flake8-black +flake8-blind-except +flake8-comprehensions +flake8-docstrings +flake8-mutable +flake8-print +flake8-quotes +flake8-tuple +pytest +pytest-cov +pytest-env +pytest-sugar +testfixtures diff --git a/requirements/prod.txt b/requirements/prod.txt index 8b36a96..41c3f4f 100644 --- a/requirements/prod.txt +++ b/requirements/prod.txt @@ -1,3 +1,4 @@ pandas texttable -requests \ No newline at end of file +requests +bs4 diff --git a/src/python_lei/isin_lei.py b/src/python_lei/isin_lei.py index 86f3ea5..94fe15e 100644 --- a/src/python_lei/isin_lei.py +++ b/src/python_lei/isin_lei.py @@ -1,5 +1,3 @@ -import pandas as pd -import requests from python_lei.exceptions import InvalidISIN, InvalidLEI from python_lei.utils import load_data diff --git a/src/python_lei/utils.py b/src/python_lei/utils.py index 0924ac1..576f7ba 100644 --- a/src/python_lei/utils.py +++ b/src/python_lei/utils.py @@ -8,6 +8,7 @@ import pandas as pd import requests +from bs4 import BeautifulSoup logger = logging.getLogger(__name__) logging.basicConfig(level=logging.INFO) @@ -30,8 +31,7 @@ def __init__(self, _is_actions=False): Args: _is_actions (bool): For setting path of downloaded resources on Github Actions """ - self.data_url = f"https://isinmapping.gleif.org/file-by-date/{TODAY}" - + self.data_url = "https://www.gleif.org/en/lei-data/lei-mapping/download-isin-to-lei-relationship-files" self._download(_is_actions) def _download(self, _is_actions): @@ -42,8 +42,13 @@ def _download(self, _is_actions): logger.info(f"No resources directory found, creating resources directory.") os.mkdir(RESOURCE_DIR) + download_link = self._scrape_isin_file() + + if not download_link: + raise ValueError("Downloading of isin file not available.") + try: - response = requests.get(self.data_url) + response = requests.get(download_link) except requests.exceptions as err: logger.error( "Connection Error, Unable to download data at this time. Please check you have working internet connection or try again later." @@ -52,9 +57,7 @@ def _download(self, _is_actions): logger.error("No response from GLEIF server.") logger.info("The file could be over 50 Mb.") - # TODO: Add progress bar zipped_content = zipfile.ZipFile(io.BytesIO(response.content)) - # TODO: Remove this if _is_actions: zipped_content.extractall( "/home/runner/work/python-lei/python-lei/resources" @@ -62,9 +65,23 @@ def _download(self, _is_actions): else: zipped_content.extractall(RESOURCE_DIR) logger.info(f"Extraction complete in {RESOURCE_DIR}") + + def _scrape_isin_file(self): + """ + Scrape the data. + """ + try: + response = requests.get(self.data_url) + response.raise_for_status() + soup = BeautifulSoup(response.text) + + # find all the tr and td and get to the href + download_link = soup.find_all("tr")[1].find_all("td")[1].find("a")["href"] + return download_link - # TODO: Covert the dataframe to parquet and use it. - + except requests.ConnectionError: + logger.error(f"Error connecting to {self.data_url}") + class Update: """ @@ -83,12 +100,12 @@ def __init__(self): logger.info( "Resource directory not found or LEI ISIN mappings not found. Downloading now." ) - download = Download() + Download() if os.listdir(RESOURCE_DIR) != []: shutil.rmtree(RESOURCE_DIR) logger.info(f"Downloading Data in {RESOURCE_DIR}") - download = Download() + Download() def load_data(): diff --git a/tests/test_isin_lei.py b/tests/test_isin_lei.py deleted file mode 100644 index 5d56bf5..0000000 --- a/tests/test_isin_lei.py +++ /dev/null @@ -1,55 +0,0 @@ -from pathlib import Path - -import pandas as pd -import pytest -from python_lei.exceptions import InvalidISIN, InvalidLEI -from python_lei.isin_lei import ISINtoLEI, LEItoISIN -from python_lei.utils import PROJECT_ROOT - - -@pytest.fixture(scope="module") -def get_isin_response(): - return [ - "SE0000382335", - "US052800AB59", - "US0528002084", - "US0528003074", - "US0528001094", - "US0528001177", - ] - - -def test_get_isin(get_isin_response): - leitoisin = LEItoISIN() - isin_list, isin_dataframe = leitoisin.get_isin( - "A23RUXWKASG834LTMK28", return_dataframe=True - ) - assert isin_list == get_isin_response - - df_isin = pd.read_csv(f"{PROJECT_ROOT}/tests/assets/isin_response.csv") - # TODO: Use pandas to test the dataframes - assert isin_dataframe["ISIN"].tolist() == df_isin["ISIN"].tolist() - assert isin_dataframe["LEI"].tolist() == df_isin["LEI"].tolist() - - isin_list = leitoisin.get_isin("A23RUXWKASG834LTMK28") - assert isin_list == get_isin_response - - -def test_get_isin_incorrect_lei(): - leitoisin = LEItoISIN() - with pytest.raises(InvalidLEI) as exc_info: - isin_list = leitoisin.get_isin("A23RUXWKASG834LTMK2") - assert str(exc_info.value) == "Invalid LEI number" - - -def test_get_lei(): - isintolei = ISINtoLEI() - lei_list = isintolei.get_lei("SE0000382335") - assert lei_list == ["A23RUXWKASG834LTMK28"] - - -def test_get_lei_incorrect_isin(): - isintolei = ISINtoLEI() - with pytest.raises(InvalidISIN) as exc_info: - lei_list = isintolei.get_lei("SE000038233") - assert str(exc_info.value) == "Invalid ISIN number" diff --git a/tests/test_lei_search.py b/tests/test_lei_search.py index 033d28e..ffe6bd3 100644 --- a/tests/test_lei_search.py +++ b/tests/test_lei_search.py @@ -1,9 +1,6 @@ -from pathlib import Path -from unittest.mock import Mock, patch +from unittest.mock import patch -import pandas as pd import pytest -from python_lei.exceptions import NotFound from python_lei.lei_search import SearchLEI diff --git a/tests/test_pylei.py b/tests/test_pylei.py index 704719a..d9c367b 100644 --- a/tests/test_pylei.py +++ b/tests/test_pylei.py @@ -1,9 +1,8 @@ -from pathlib import Path -from unittest.mock import Mock, patch +from unittest.mock import patch import pandas as pd import pytest -from python_lei.exceptions import InvalidISIN, InvalidLEI +from python_lei.exceptions import InvalidLEI from python_lei.pylei import pyLEI from python_lei.utils import PROJECT_ROOT