Add support for Pyrate Limiter v3 and drop support for Python 3.6 and…

… 3.7 (#6)

* Drop old python versions and fix failing rate limiter

* Remove .idea

* fix issues

* fix makefile

* update tox file to have 3.8+

.github/workflows/continuous_integration.yml

@@ -9,13 +9,13 @@ on:
     - cron: "0 0 1 * *"
 
 jobs:
-  test:
-    name: ${{ matrix.os }} (${{ matrix.python-version }})
+  unit-tests:
+    name: Unit tests | Python ${{ matrix.python-version }} on ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
     strategy:
-      fail-fast: true
+      fail-fast: false
       matrix:
-        python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"]
+        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12-dev"]
         os: [ubuntu-latest, macos-latest, windows-latest]
         include:
           - os: ubuntu-latest
@@ -25,14 +25,14 @@ jobs:
           - os: windows-latest
             path: ~\AppData\Local\pip\Cache
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/cache@v2
+      - uses: actions/checkout@v3
+      - uses: actions/cache@v3
         with:
           path: ${{ matrix.path }}
           key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
           restore-keys: ${{ runner.os }}-pip-
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install dependencies
@@ -42,28 +42,29 @@ jobs:
       - name: Test with tox
         run: tox
       - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v2
+        uses: codecov/codecov-action@v3
         with:
           file: coverage.xml
 
-  lint:
+  linting:
+    name: Linting
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/cache@v2
+      - uses: actions/checkout@v3
+      - uses: actions/cache@v3
         with:
           path: ~/.cache/pip
           key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
           restore-keys: ${{ runner.os }}-pip-
-      - uses: actions/cache@v2
+      - uses: actions/cache@v3
         with:
           path: ~/.cache/pre-commit
           key: ${{ runner.os }}-pre-commit-${{ hashFiles('**/.pre-commit-config.yaml') }}
           restore-keys: ${{ runner.os }}-pre-commit-
-      - name: Set up Python 3.10
-        uses: actions/setup-python@v2
+      - name: Set up Python ${{ runner.python-version }}
+        uses: actions/setup-python@v4
         with:
-          python-version: "3.10"
+          python-version: "3.11"
       - name: Install dependencies
         run: |
           pip install -U pip setuptools wheel

.github/workflows/release.yml

@@ -10,15 +10,15 @@ jobs:
   release-to-pypi:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/cache@v2
+      - uses: actions/checkout@v3
+      - uses: actions/cache@v3
         with:
           path: ~/.cache/pip
           key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
           restore-keys: ${{ runner.os }}-pip-
-      - uses: actions/setup-python@v2
+      - uses: actions/setup-python@v4
         with:
-          python-version: "3.10"
+          python-version: "3.11"
       - name: Install dependencies
         run: |
           pip install -U pip setuptools wheel

.gitignore

@@ -1,5 +1,6 @@
 test.py
 .vscode/
+.idea/
 
 # Byte-compiled / optimized / DLL files
 __pycache__/

.pre-commit-config.yaml

@@ -2,12 +2,12 @@ default_language_version:
   python: python3
 repos:
   - repo: https://github.com/PyCQA/bandit
-    rev: 1.7.4
+    rev: 1.7.5
     hooks:
       - id: bandit
         args: [-lll, --quiet, --exclude=tests/**]
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.1.0
+    rev: v4.4.0
     hooks:
       - id: check-case-conflict
       - id: check-docstring-first
@@ -26,16 +26,16 @@ repos:
         args: [requirements/requirements.txt, requirements/requirements-docs.txt, requirements/requirements-tests.txt]
       - id: trailing-whitespace
   - repo: https://github.com/timothycrosley/isort
-    rev: 5.10.1
+    rev: 5.12.0
     hooks:
       - id: isort
   - repo: https://github.com/asottile/pyupgrade
-    rev: v2.31.1
+    rev: v3.13.0
     hooks:
       - id: pyupgrade
         args: [--py36-plus]
   - repo: https://github.com/pre-commit/pygrep-hooks
-    rev: v1.9.0
+    rev: v1.10.0
     hooks:
       - id: python-no-eval
       - id: python-check-blanket-noqa
@@ -46,17 +46,17 @@ repos:
       - id: rst-inline-touching-normal
       - id: text-unicode-replacement-char
   - repo: https://github.com/psf/black
-    rev: 22.3.0
+    rev: 23.9.1
     hooks:
       - id: black
         args: [--quiet, --target-version=py36]
-  - repo: https://gitlab.com/pycqa/flake8
-    rev: 3.9.2
+  - repo: https://github.com/pycqa/flake8
+    rev: 6.1.0
     hooks:
       - id: flake8
-        additional_dependencies: [flake8-bugbear]
+        additional_dependencies: [flake8-bugbear, flake8-comprehensions]
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: 'v0.942'
+    rev: 'v1.5.1'
     hooks:
       - id: mypy
         additional_dependencies: [pandas-stubs, types-requests]

.readthedocs.yaml

@@ -0,0 +1,29 @@
+# Read the Docs configuration file for Sphinx projects
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+version: 2
+
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.11"
+
+# Build documentation in the "docs/" directory with Sphinx
+sphinx:
+  configuration: docs/conf.py
+  # You can configure Sphinx to use a different builder, for instance use the dirhtml builder for simpler URLs
+  # builder: "dirhtml"
+  # Fail on all warnings to avoid broken references
+  # fail_on_warning: true
+
+# Optionally build your docs in additional formats such as PDF and ePub
+# formats:
+#    - pdf
+#    - epub
+
+# Optional but recommended, declare the Python requirements required
+# to build your documentation
+# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html
+python:
+  install:
+  - requirements: requirements/requirements-docs.txt

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2022 Jad Chaar
+Copyright (c) 2023 Jad Chaar
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

Makefile

@@ -1,32 +1,27 @@
 .PHONY: auto test docs clean
 
-auto: build310
+auto: build311
 
-build36: PYTHON_VER = python3.6
-build37: PYTHON_VER = python3.7
 build38: PYTHON_VER = python3.8
 build39: PYTHON_VER = python3.9
 build310: PYTHON_VER = python3.10
+build311: PYTHON_VER = python3.11
+build312: PYTHON_VER = python3.12
 
-build36 build37 build38 build39 build310: clean
+build38 build39 build310 build311 build312: clean
 	$(PYTHON_VER) -m venv venv
 	. venv/bin/activate; \
 	pip install -U pip setuptools wheel; \
-	pip install -r requirements/requirements-tests.txt; \
 	pip install -r requirements/requirements-docs.txt; \
+	pip install -r requirements/requirements-tests.txt; \
 	pre-commit install
 
 test:
 	rm -f .coverage coverage.xml
-	. venv/bin/activate; \
-	pytest
+	. venv/bin/activate; pytest
 
 lint:
-	. venv/bin/activate; \
-	pre-commit run --all-files
-
-clean-docs:
-	rm -rf docs/_build
+	. venv/bin/activate; pre-commit run --all-files --show-diff-on-failure
 
 docs: clean-docs
 	. venv/bin/activate; \
@@ -37,6 +32,11 @@ live-docs: clean-docs
 	. venv/bin/activate; \
 	sphinx-autobuild docs docs/_build/html
 
+build-dist: clean-dist
+	. venv/bin/activate; \
+	pip install -U flit; \
+	flit build
+
 deep-clean-dry-run:
 	git clean -xdn
 
@@ -46,19 +46,12 @@ deep-clean:
 clean-env:
 	rm -rf venv .tox
 
-clean: clean-env clean-dist clean-docs
-	rm -rf .pytest_cache ./**/__pycache__ .mypy_cache
-	rm -f .coverage coverage.xml ./**/*.pyc
-
 clean-dist:
 	rm -rf dist build *.egg *.eggs *.egg-info
 
-build-dist: clean-dist
-	. venv/bin/activate; \
-	pip install -U flit; \
-	flit build --setup-py
+clean-docs:
+	rm -rf docs/_build
 
-publish: test clean-dist
-	. venv/bin/activate; \
-	pip install -U flit; \
-	flit publish --setup-py
+clean: clean-env clean-dist clean-docs
+	rm -rf .pytest_cache ./**/__pycache__ .mypy_cache
+	rm -f .coverage coverage.xml ./**/*.pyc

pyproject.toml

@@ -18,19 +18,19 @@ classifiers = [
     "Topic :: Office/Business :: Financial :: Investment",
     "License :: OSI Approved :: MIT License",
     "Programming Language :: Python :: 3",
-    "Programming Language :: Python :: 3.6",
-    "Programming Language :: Python :: 3.7",
     "Programming Language :: Python :: 3.8",
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
     "Programming Language :: Python :: 3 :: Only",
     "Operating System :: OS Independent",
 ]
 dependencies = [
     "requests",
-    "pyrate-limiter",
+    "pyrate-limiter>=3.1.0"
 ]
-requires-python = ">=3.6"
+requires-python = ">=3.8"
 description = "Unofficial SEC EDGAR API wrapper for Python"
 keywords = ["sec", "edgar", "filing", "financial", "finance", "stocks", "mutual funds", "etfs", "cik", "ticker", "api wrapper"]
 dynamic = ["version"]

requirements/requirements.txt

@@ -1,2 +1,2 @@
-pyrate-limiter
+pyrate-limiter>=3.1.0
 requests

sec_edgar_api/_BaseClient.py

@@ -1,24 +1,35 @@
 from weakref import finalize
 
 import requests
-from pyrate_limiter import Duration, Limiter, RequestRate
+from pyrate_limiter import Duration, Limiter, Rate
 from requests.adapters import HTTPAdapter
 from urllib3.util.retry import Retry
 
 from ._constants import BACKOFF_FACTOR, MAX_REQUESTS_PER_SECOND, MAX_RETRIES
 from ._types import JSONType
 from .EdgarAPIError import EdgarAPIError
 
-# Rate limiter
-rate = RequestRate(MAX_REQUESTS_PER_SECOND, Duration.SECOND)
-limiter = Limiter(rate)
+# 10 requests per second rate limit set by SEC:
+# https://www.sec.gov/os/webmaster-faq#developers
+SEC_THROTTLE_LIMIT_RATE = Rate(MAX_REQUESTS_PER_SECOND, Duration.SECOND)
+
+# Wait up to 60 seconds for the rate-limiter bucket to refill.
+# If the bucket does NOT refill, an exception will be raised.
+limiter = Limiter(
+    SEC_THROTTLE_LIMIT_RATE, raise_when_fail=True, max_delay=60_000
+).as_decorator()
+
+
+def limiter_mapping(*args):
+    return "sec_edgar_api_rate_limit", 1
+
 
 # Specify max number of request retries
 # https://stackoverflow.com/a/35504626/3820660
 retries = Retry(
     total=MAX_RETRIES,
     backoff_factor=BACKOFF_FACTOR,
-    status_forcelist=[403, 500, 502, 503, 504],
+    status_forcelist=[408, 425, 429, 500, 502, 503, 504],
 )
 
 
@@ -40,7 +51,7 @@ def __init__(self, user_agent: str):
         # Source: https://stackoverflow.com/a/67312839/3820660
         finalize(self, self._session.close)
 
-    @limiter.ratelimit(delay=True)
+    @limiter(limiter_mapping)
     def _rate_limited_get(self, url: str) -> JSONType:
         """Make a rate-limited GET request.
 

sec_edgar_api/_constants.py

@@ -9,5 +9,5 @@
 BASE_URL_XBRL_FRAMES = f"{_BASE_URL_XBRL}/frames"
 
 MAX_REQUESTS_PER_SECOND = 10
-MAX_RETRIES = 10
+MAX_RETRIES = 3
 BACKOFF_FACTOR = 1 / MAX_REQUESTS_PER_SECOND

sec_edgar_api/_version.py

@@ -1 +1 @@
-__version__ = "1.0.0"
+__version__ = "1.1.0"

tox.ini

@@ -1,16 +1,15 @@
 [tox]
 minversion = 3.18.0
-envlist = py{36,37,38,39,310}
+envlist = py{38,39,310,311,312}
 skip_missing_interpreters = true
-isolated_build = true
 
 [gh-actions]
 python =
-    3.6: py36
-    3.7: py37
     3.8: py38
     3.9: py39
     3.10: py310
+    3.11: py311
+    3.12: py312
 
 [testenv]
 deps = -r requirements/requirements-tests.txt