From 30c29b28d7b75cc19c87c471aace2bc27075e61d Mon Sep 17 00:00:00 2001 From: Jose Tiago Macara Coutinho Date: Thu, 5 Jun 2025 16:13:32 +0200 Subject: [PATCH 1/9] Efficient dataset variables for cube - lazy fetch variables - use cached variables when need to access dataset variables --- src/pycrunch/cubes.py | 30 ++++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/src/pycrunch/cubes.py b/src/pycrunch/cubes.py index cf3cb0e..972c916 100644 --- a/src/pycrunch/cubes.py +++ b/src/pycrunch/cubes.py @@ -74,8 +74,27 @@ def __init__(self, dataset): if not hasattr(dataset, "catalogs"): dataset.refresh() self._dataset = dataset - self._variables_by_alias = dataset.variables.by('alias') - self._variables_by_name = dataset.variables.by('name') + self._variables_cache = None + self._variables_by_alias_cache = None + self._variables_by_name_cache = None + + @property + def _variables(self): + if self._variables_cache is None: + self._variables_cache = self._dataset.variables + return self._variables_cache + + @property + def _variables_by_alias(self): + if self._variables_by_alias_cache is None: + self._variables_by_alias_cache = self._variables.by("alias") + return self._variables_by_alias_cache + + @property + def _variables_by_name(self): + if self._variables_by_name_cache is None: + self._variables_by_name_cache = self._variables.by("name") + return self._variables_by_name_cache def prepare_dimensions(self, dimensions): """Return list of crunch expressions for each cube dimension. @@ -107,17 +126,16 @@ def get_dimension_by_string(self, dim_str): :param dim_str: String representing URL, Name, or Alias of a variable """ - - if dim_str in self._dataset.variables.index: + if dim_str in self._variables.index: # When URL is provided, fetch variable from index - return self._dataset.variables.index[dim_str] + return self._variables.index[dim_str] elif dim_str in self._variables_by_alias: return self._variables_by_alias[dim_str] elif dim_str in self._variables_by_name: return self._variables_by_name[dim_str] elif 'subvariables/' in dim_str: var_url = dim_str.split('subvariables/')[0] - variable = self._dataset.variables.index[var_url] + variable = self._variables.index[var_url] return variable.entity.subvariables.index[dim_str] raise ValueError("Can't find variable {} in dataset {}".format( From de562919e2323c3d58bd4acb32987a3cb2978a35 Mon Sep 17 00:00:00 2001 From: Jose Tiago Macara Coutinho Date: Thu, 5 Jun 2025 16:18:00 +0200 Subject: [PATCH 2/9] Update github actions (deprecation error) --- .github/workflows/build-and-test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-test.yaml b/.github/workflows/build-and-test.yaml index e88a379..9a523b2 100644 --- a/.github/workflows/build-and-test.yaml +++ b/.github/workflows/build-and-test.yaml @@ -12,7 +12,7 @@ on: jobs: test: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 strategy: max-parallel: 4 matrix: From 4101ebadebe859c5dbbd39486d40ca607aab9527 Mon Sep 17 00:00:00 2001 From: Jose Tiago Macara Coutinho Date: Thu, 5 Jun 2025 16:22:16 +0200 Subject: [PATCH 3/9] Remove python 2.7 github actions test (unsupported in ubuntu-22.04) --- .github/workflows/build-and-test.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/build-and-test.yaml b/.github/workflows/build-and-test.yaml index 9a523b2..5f9b81a 100644 --- a/.github/workflows/build-and-test.yaml +++ b/.github/workflows/build-and-test.yaml @@ -17,7 +17,6 @@ jobs: max-parallel: 4 matrix: python-version: - - 2.7 - 3.6 - 3.7 - 3.8 @@ -25,7 +24,6 @@ jobs: steps: - uses: actions/checkout@v1 - name: Set up Python ${{ matrix.python-version }} - if: matrix.python-version != '2.7' uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} @@ -38,7 +36,7 @@ jobs: - name: Test with tox ${{env.TOXENV}} if: matrix.python-version == '3.6' env: - TOXENV: py27,py36,coverage + TOXENV: py36,coverage run: tox - name: Test with tox Python 3.7 From 419fcc3e17b17beda81c84a285e64a4b79ddd89f Mon Sep 17 00:00:00 2001 From: Jose Tiago Macara Coutinho Date: Thu, 5 Jun 2025 16:24:45 +0200 Subject: [PATCH 4/9] Remove python 3.6 github actions test (unsupported in ubuntu-22.04) --- .github/workflows/build-and-test.yaml | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build-and-test.yaml b/.github/workflows/build-and-test.yaml index 5f9b81a..982945d 100644 --- a/.github/workflows/build-and-test.yaml +++ b/.github/workflows/build-and-test.yaml @@ -17,7 +17,6 @@ jobs: max-parallel: 4 matrix: python-version: - - 3.6 - 3.7 - 3.8 - 3.9 @@ -33,12 +32,6 @@ jobs: pip install --upgrade pip pip install tox tox-gh-actions - - name: Test with tox ${{env.TOXENV}} - if: matrix.python-version == '3.6' - env: - TOXENV: py36,coverage - run: tox - - name: Test with tox Python 3.7 if: matrix.python-version == '3.7' env: @@ -57,8 +50,8 @@ jobs: TOXENV: py39 run: tox - - name: Lint Test with tox Python 3.6 - if: matrix.python-version == '3.6' + - name: Lint Test with tox Python 3.9 + if: matrix.python-version == '3.9' env: TOXENV: lint run: tox From 3517fd5fae0e3fb1c6becce92ddc7f1ab06d518c Mon Sep 17 00:00:00 2001 From: Jose Tiago Macara Coutinho Date: Thu, 5 Jun 2025 16:31:13 +0200 Subject: [PATCH 5/9] Update tox & github actions --- .github/workflows/build-and-test.yaml | 38 +++++++++++++++++++++++---- tox.ini | 21 +++++++++++---- 2 files changed, 49 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build-and-test.yaml b/.github/workflows/build-and-test.yaml index 982945d..66ca0a7 100644 --- a/.github/workflows/build-and-test.yaml +++ b/.github/workflows/build-and-test.yaml @@ -17,9 +17,13 @@ jobs: max-parallel: 4 matrix: python-version: - - 3.7 - - 3.8 - - 3.9 + - "3.7" + - "3.8" + - "3.9" + - "3.10" + - "3.11" + - "3.12" + - "3.13" steps: - uses: actions/checkout@v1 - name: Set up Python ${{ matrix.python-version }} @@ -50,8 +54,32 @@ jobs: TOXENV: py39 run: tox - - name: Lint Test with tox Python 3.9 - if: matrix.python-version == '3.9' + - name: Test with tox Python 3.10 + if: matrix.python-version == '3.10' + env: + TOXENV: py310 + run: tox + + - name: Test with tox Python 3.11 + if: matrix.python-version == '3.11' + env: + TOXENV: py311 + run: tox + + - name: Test with tox Python 3.12 + if: matrix.python-version == '3.12' + env: + TOXENV: py312 + run: tox + + - name: Test with tox Python 3.13 + if: matrix.python-version == '3.13' + env: + TOXENV: py313 + run: tox + + - name: Lint Test with tox Python 3.11 + if: matrix.python-version == '3.11' env: TOXENV: lint run: tox diff --git a/tox.ini b/tox.ini index f824e4b..66d3f80 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] envlist = - py27,py34,py35,py36,py37,py38,py39 + py27,py34,py35,py36,py37,py38,py39,py310,py311,py312,py313 coverage,lint [gh-actions] @@ -8,7 +8,14 @@ python = 2.7: py27 3.4: py34 3.5: py35 - 3.6: py27, py36, coverage, lint + 3.6: py36 + 3.7: py37 + 3.8: py38 + 3.9: py39 + 3.10: py310 + 3.11: py311, coverage, lint + 3.12: py312 + 3.13: py313 [testenv] basepython = @@ -19,8 +26,12 @@ basepython = py37: python3.7 py38: python3.8 py39: python3.9 + py310: python3.10 + py311: python3.11 + py312: python3.12 + py313: python3.13 py2: python2.7 - py3: python3.6 + py3: python3.11 commands = pip install pycrunch[testing] @@ -31,7 +42,7 @@ setenv = [testenv:coverage] skip_install = True -basepython = python3.6 +basepython = python3.11 commands = coverage combine coverage report @@ -42,7 +53,7 @@ setenv = [testenv:lint] skip_install = True -basepython = python3.6 +basepython = python3.11 commands = python setup.py check -r -s -m check-manifest From 997eeb42ec5d1d53a69b1554db4b6f2158cab54e Mon Sep 17 00:00:00 2001 From: Jose Tiago Macara Coutinho Date: Thu, 5 Jun 2025 16:57:03 +0200 Subject: [PATCH 6/9] Fix tests --- tests/test_cube.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_cube.py b/tests/test_cube.py index 86e0ab0..a879669 100644 --- a/tests/test_cube.py +++ b/tests/test_cube.py @@ -42,8 +42,8 @@ def dimension_string_fixture(request): dim_str, str_type, dim, subvar = request.param preparer = DimensionsPreparer(Mock()) preparer._dataset.variables.index = dict() - preparer._variables_by_alias = dict() - preparer._variables_by_name = dict() + preparer._variables_by_alias_cache = dict() + preparer._variables_by_name_cache = dict() if str_type == 'URL': preparer._dataset.variables.index[dim_str] = dim elif str_type == 'URL_SUBVAR': From bc16897748d85c76f74e0e3f05dd93e887b26e09 Mon Sep 17 00:00:00 2001 From: Jose Tiago Macara Coutinho Date: Thu, 5 Jun 2025 23:11:11 +0200 Subject: [PATCH 7/9] Remove dependency on httpbin.org serving requests --- tests/test_http.py | 73 +++++++++++++++++++++++++++++++++------------- 1 file changed, 52 insertions(+), 21 deletions(-) diff --git a/tests/test_http.py b/tests/test_http.py index b31fff1..cc5d7b6 100644 --- a/tests/test_http.py +++ b/tests/test_http.py @@ -1,3 +1,6 @@ +import json +import threading +import sys from unittest import TestCase import pytest @@ -16,27 +19,55 @@ import mock -class TestHTTPRequests(TestCase): - - @classmethod - def setUpClass(cls): - cls.s = Session("not an email", "not a password", site_url="https://app.crunch.io/api/") - cls.r = cls.s.get("http://httpbin.org/headers") - - def test_request_sends_user_agent(self): - pycrunch_ua = 'pycrunch/%s' % __version__ - req_headers_sent = self.r.request.headers - req_headers_received = self.r.json()['headers'] - self.assertTrue('user-agent' in req_headers_sent) - self.assertTrue('User-Agent' in req_headers_received) - self.assertTrue(pycrunch_ua in req_headers_sent.get('user-agent', '')) - self.assertTrue(pycrunch_ua in req_headers_received.get('User-Agent', '')) - - def test_request_sends_gzip(self): - req_headers_sent = self.r.request.headers - req_headers_received = self.r.json()['headers'] - self.assertIn("gzip", req_headers_sent['Accept-Encoding']) - self.assertIn("gzip", req_headers_received['Accept-Encoding']) +@pytest.fixture(scope="module") +def http_server(): + import http.server + class Handler(http.server.BaseHTTPRequestHandler): + def do_GET(self): + if self.path == '/headers': + reply = json.dumps({"headers":dict(self.headers)}).encode() + self.send_response(200) + self.send_header('Content-Type', 'application/json') + else: + self.send_response(404) + self.send_header('Content-Type', 'text/plain') + reply = b"Not found" + self.send_header('Content-Length', len(reply)) + self.end_headers() + self.wfile.write(reply) + + server = http.server.HTTPServer(("127.0.0.1", 0), Handler) + task = threading.Thread(target=server.serve_forever, daemon=True) + task.start() + yield server + server.shutdown() + server.server_close() + task.join(timeout=1) + + +@pytest.mark.skipif(sys.version_info < (3, 4), reason="requires python 3.4 or higher") +def test_request_sends_user_agent(http_server): + session = Session("not an email", "not a password", site_url="https://app.crunch.io/api/") + url = "http://127.0.0.1:{}/headers".format(http_server.server_port) + pycrunch_ua = 'pycrunch/%s' % __version__ + response = session.get(url) + req_headers_sent = response.request.headers + req_headers_received = response.json()['headers'] + assert 'user-agent' in req_headers_sent + assert 'user-agent' in req_headers_received + assert pycrunch_ua in req_headers_sent.get('user-agent', '') + assert pycrunch_ua in req_headers_received.get('user-agent', '') + + +@pytest.mark.skipif(sys.version_info < (3, 4), reason="requires python 3.4 or higher") +def test_request_sends_gzip(http_server): + session = Session("not an email", "not a password", site_url="https://app.crunch.io/api/") + url = "http://127.0.0.1:{}/headers".format(http_server.server_port) + response = session.get(url) + req_headers_sent = response.request.headers + req_headers_received = response.json()['headers'] + assert "gzip" in req_headers_sent['Accept-Encoding'] + assert "gzip" in req_headers_received['Accept-Encoding'] class TestHTTPResponses(TestCase): From 96a6b64f9c7333876ec319b3b35295641e7ee73e Mon Sep 17 00:00:00 2001 From: Jose Tiago Macara Coutinho Date: Thu, 5 Jun 2025 23:21:16 +0200 Subject: [PATCH 8/9] Adapt tests to python >= 3.12 --- tests/test_elements.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/test_elements.py b/tests/test_elements.py index b67da82..7717b8e 100644 --- a/tests/test_elements.py +++ b/tests/test_elements.py @@ -3,6 +3,7 @@ from unittest import TestCase import requests +import six from pycrunch import elements, shoji @@ -38,7 +39,7 @@ def test_attribute_access(self): def test_attribute_error(self): foo = self.Foo(bar=42) msg = 'Foo has no attribute nope' - self.assertRaisesRegexp(AttributeError, msg, getattr, foo, 'nope') + six.assertRaisesRegex(self, AttributeError, msg, getattr, foo, 'nope') def test_copy(self): foo = self.Foo(bar=42) @@ -165,7 +166,7 @@ def test_follow_uri_template(self): def test_follow_no_link(self): person = self.Person(session=None, self='some uri') msg = 'Person has no link foo' - self.assertRaisesRegexp(AttributeError, msg, person.follow, 'foo') + six.assertRaisesRegex(self, AttributeError, msg, person.follow, 'foo') def test_refresh(self): before = { @@ -199,7 +200,7 @@ def test_refresh_no_response(self): person = self.Person(session=session_mock, self='some uri') msg = 'Response could not be parsed.' - self.assertRaisesRegexp(TypeError, msg, person.refresh) + six.assertRaisesRegex(self, TypeError, msg, person.refresh) session_mock.get.assert_called_once_with('some uri') def test_post(self): From 20dc5cd142b9d334a0316c9cb10cba958a26d0e6 Mon Sep 17 00:00:00 2001 From: Jose Tiago Macara Coutinho Date: Fri, 6 Jun 2025 06:58:41 +0200 Subject: [PATCH 9/9] Use requests-mock --- setup.py | 1 + tests/test_http.py | 72 +++++++++++++--------------------------------- 2 files changed, 21 insertions(+), 52 deletions(-) diff --git a/setup.py b/setup.py index 2dbf983..2cadf4a 100644 --- a/setup.py +++ b/setup.py @@ -27,6 +27,7 @@ def get_long_desc(): tests_requires = [ 'mock', + 'requests-mock', 'pytest', 'pytest-cov', ] diff --git a/tests/test_http.py b/tests/test_http.py index cc5d7b6..f1a7235 100644 --- a/tests/test_http.py +++ b/tests/test_http.py @@ -1,10 +1,8 @@ -import json -import threading -import sys from unittest import TestCase import pytest import requests +import requests_mock from pycrunch import connect, connect_with_token, Session, __version__ from pycrunch.lemonpy import ServerError @@ -19,55 +17,25 @@ import mock -@pytest.fixture(scope="module") -def http_server(): - import http.server - class Handler(http.server.BaseHTTPRequestHandler): - def do_GET(self): - if self.path == '/headers': - reply = json.dumps({"headers":dict(self.headers)}).encode() - self.send_response(200) - self.send_header('Content-Type', 'application/json') - else: - self.send_response(404) - self.send_header('Content-Type', 'text/plain') - reply = b"Not found" - self.send_header('Content-Length', len(reply)) - self.end_headers() - self.wfile.write(reply) - - server = http.server.HTTPServer(("127.0.0.1", 0), Handler) - task = threading.Thread(target=server.serve_forever, daemon=True) - task.start() - yield server - server.shutdown() - server.server_close() - task.join(timeout=1) - - -@pytest.mark.skipif(sys.version_info < (3, 4), reason="requires python 3.4 or higher") -def test_request_sends_user_agent(http_server): - session = Session("not an email", "not a password", site_url="https://app.crunch.io/api/") - url = "http://127.0.0.1:{}/headers".format(http_server.server_port) - pycrunch_ua = 'pycrunch/%s' % __version__ - response = session.get(url) - req_headers_sent = response.request.headers - req_headers_received = response.json()['headers'] - assert 'user-agent' in req_headers_sent - assert 'user-agent' in req_headers_received - assert pycrunch_ua in req_headers_sent.get('user-agent', '') - assert pycrunch_ua in req_headers_received.get('user-agent', '') - - -@pytest.mark.skipif(sys.version_info < (3, 4), reason="requires python 3.4 or higher") -def test_request_sends_gzip(http_server): - session = Session("not an email", "not a password", site_url="https://app.crunch.io/api/") - url = "http://127.0.0.1:{}/headers".format(http_server.server_port) - response = session.get(url) - req_headers_sent = response.request.headers - req_headers_received = response.json()['headers'] - assert "gzip" in req_headers_sent['Accept-Encoding'] - assert "gzip" in req_headers_received['Accept-Encoding'] +class TestHTTPRequests(TestCase): + + def setUp(self): + self.s = Session("not an email", "not a password", site_url="https://app.crunch.io/api/") + adapter = requests_mock.Adapter() + adapter.register_uri('GET', "http://httpbin.org/headers", text='data') + self.s.mount("mock://", adapter) + + def test_request_sends_user_agent(self): + pycrunch_ua = 'pycrunch/%s' % __version__ + resp = self.s.get('http://httpbin.org/headers') + req_headers_sent = resp.request.headers + assert 'user-agent' in req_headers_sent + assert pycrunch_ua in req_headers_sent.get('user-agent', '') + + def test_request_sends_gzip(self): + resp = self.s.get('http://httpbin.org/headers') + req_headers_sent = resp.request.headers + self.assertIn("gzip", req_headers_sent['Accept-Encoding']) class TestHTTPResponses(TestCase):