From 4495de0c04e8d3bf7233c950433538a91fa35337 Mon Sep 17 00:00:00 2001 From: aan Date: Fri, 29 Nov 2024 18:56:08 +0100 Subject: [PATCH] feat: handle API keys --- README.md | 20 +++++++- bearish/exceptions.py | 2 + bearish/main.py | 16 +++++++ bearish/models/api_keys/__init__.py | 0 bearish/models/api_keys/api_keys.py | 18 ++++++++ bearish/models/base.py | 6 ++- bearish/sources/alphavantage.py | 51 +++++++++++++++------ bearish/sources/base.py | 7 +++ bearish/sources/financedatabase.py | 2 +- bearish/sources/financial_modelling_prep.py | 0 bearish/sources/finnhub.py | 0 bearish/sources/investpy.py | 2 +- bearish/sources/tiingo.py | 0 bearish/sources/yfinance.py | 3 +- poetry.lock | 17 ++++++- pyproject.toml | 1 + tests/sources/test_alphavantage.py | 14 ++++++ tests/test_bearish.py | 13 +++++- 18 files changed, 148 insertions(+), 24 deletions(-) create mode 100644 bearish/exceptions.py create mode 100644 bearish/models/api_keys/__init__.py create mode 100644 bearish/models/api_keys/api_keys.py create mode 100644 bearish/sources/financial_modelling_prep.py create mode 100644 bearish/sources/finnhub.py create mode 100644 bearish/sources/tiingo.py diff --git a/README.md b/README.md index b73b587..0fe2bc4 100644 --- a/README.md +++ b/README.md @@ -8,4 +8,22 @@ https://github.com/alvarobartt/investpy https://github.com/fja05680/sp500 https://site.financialmodelingprep.com/developer/docs/pricing -poetry run python ./bearish/main.py ./test3.db --keywords Michellin --keywords MICP --keywords MI \ No newline at end of file +poetry run python ./bearish/main.py ./test3.db --keywords Michellin --keywords MICP --keywords MI + + +| **API Name** | **Description** | **Free Tier Limitations** | **Pricing** | **Key Features** | **Website** | **Python Package** | +|----------------------------|----------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------|------------------------------------------------|-----------------------------------------| +| **Yahoo Finance (yfinance)** | Widely used for stock market data, including historical and real-time data. | No official API; data accessed via unofficial methods like the `yfinance` Python library, which may have limitations. | Free access through unofficial methods; no official pricing. | Historical data, stock quotes, market summaries, supports Python library (`yfinance`). | [yahoo.com](https://finance.yahoo.com/) | [yfinance](https://pypi.org/project/yfinance/) | +| **Alpha Vantage** | Provides real-time and historical data for stocks, forex, and cryptocurrencies. | 5 API requests per minute and 500 requests per day. | Premium plans start at $49.99/month for higher request limits. | JSON and CSV formats, over 50 technical indicators, global market news. | [alphavantage.co](https://www.alphavantage.co/)| [alpha_vantage](https://pypi.org/project/alpha_vantage/) | +| **Polygon.io** | Real-time and historical data for stocks, options, forex, and crypto. | Free tier offers delayed data only. | Paid plans start at $49/month for real-time data and higher request limits. | RESTful APIs, WebSocket support, low latency, extensive market data coverage. | [polygon.io](https://polygon.io/) | [polygon-api-client](https://pypi.org/project/polygon-api-client/) | +| **Twelve Data** | Offers stock, ETF, forex, and crypto data. | 8 API requests per minute and 800 requests per day. | Paid plans start at $29/month for higher request limits and additional features. | Real-time and historical data, over 100 technical indicators, JSON/CSV support. | [twelvedata.com](https://twelvedata.com/) | [twelvedata](https://pypi.org/project/twelvedata/) | +| **Finnhub** | Provides real-time stock, forex, and crypto data, including financial statements and market news. | 60 API requests per minute. | Premium plans available; pricing upon request. | JSON format, 30+ years of historical data, earnings, insider transactions, market sentiment analysis. | [finnhub.io](https://finnhub.io/) | [finnhub-python](https://pypi.org/project/finnhub-python/) | +| **Quandl** | Offers financial, economic, and alternative datasets for research and analysis. | 500 API calls per day. | Premium datasets available; pricing varies by dataset. | Premium and free datasets, robust documentation, supports Python and other integrations. | [quandl.com](https://www.quandl.com/) | [quandl](https://pypi.org/project/Quandl/) | +| **IEX Cloud** | A flexible platform for U.S. and international stock data. | Free tier includes 500,000 messages per month. | Paid plans start at $9/month for increased message limits and additional data access. | Real-time and historical data, earnings, alternative datasets, customizable plans. | [iexcloud.io](https://iexcloud.io/) | [iexfinance](https://pypi.org/project/iexfinance/) | +| **Marketstack** | Provides real-time and historical data for global markets. | 100 requests per month with end-of-day data only. | Paid plans start at $9.99/month for real-time data and higher request limits. | 170,000+ stock tickers, intraday data, JSON format, 70+ global exchanges. | [marketstack.com](https://marketstack.com/) | [marketstack-api](https://pypi.org/project/marketstack-api/) | +| **Financial Modeling Prep** | Offers free stock data, financial statements, and historical prices. | Free tier available; specific limitations not specified. | Paid plans available; pricing upon request. | JSON format, real-time updates, extensive coverage of financial statements. | [fmp.io](https://site.financialmodelingprep.com/)| [financial-modeling-prep](https://github.com/CodeLlama/financialmodelingprep-python) | +| **StockData.org** | Real-time, intraday, and historical stock data APIs. | 100 requests per day. | Paid plans start at $19/month for higher request limits. | Market news data, U.S. stock prices, JSON format, free and premium plans. | [stockdata.org](https://www.stockdata.org/) | Not available | +| **Barchart OnDemand** | Real-time and historical data for stocks, commodities, and forex markets. | No free tier available. | Pricing upon request; tailored to specific data needs. | REST APIs, customizable data feeds, JSON format, comprehensive documentation. | [barchart.com](https://www.barchart.com/) | Not available | +| **Tiingo** | Provides end-of-day stock prices, IEX real-time prices, and news feeds. | 500 unique symbols per month, 50 requests per hour, 1,000 requests per day, 1 GB bandwidth per month. | Power Plan at $10/month for higher limits; Commercial Plan available for businesses. | REST and WebSocket APIs, extensive historical data, news database with over 50 million articles. | [tiingo.com](https://www.tiingo.com/) | [tiingo](https://pypi.org/project/tiingo/) | +| **Xignite** | Offers real-time, delayed, intraday, and historical stock and ETF data. | No free tier; free access for fintech startups available upon application. | Pricing varies based on data needs; contact for a quote. | Cloud-native APIs, extensive global coverage, multiple data types, unlimited usage in paid plans. | [xignite.com](https://www.xignite.com/) | Not available | +| **RealStonks** | A REST API that scrapes MarketWatch to provide real-time stock prices for NASDAQ-listed stocks. | No official free tier; as it's a scraping tool, usage may be subject to limitations based on source website policies. | Open-source project; free to use but may require self-hosting and maintenance. | Provides real-time stock prices, total stock volume, price change, and percentage change since last update.| [GitHub Repository](https://github.com/MarketWatch/RealStonks) | Not available | diff --git a/bearish/exceptions.py b/bearish/exceptions.py new file mode 100644 index 0000000..8348bfb --- /dev/null +++ b/bearish/exceptions.py @@ -0,0 +1,2 @@ +class InvalidApiKeyError(Exception): + pass diff --git a/bearish/main.py b/bearish/main.py index 1591a5d..ae23d5b 100644 --- a/bearish/main.py +++ b/bearish/main.py @@ -1,4 +1,5 @@ import logging +import os from pathlib import Path from typing import Optional, List, Any @@ -11,6 +12,8 @@ ) from bearish.database.crud import BearishDb +from bearish.exceptions import InvalidApiKeyError +from bearish.models.api_keys.api_keys import SourceApiKeys from bearish.models.assets.assets import Assets from bearish.models.financials.base import Financials from bearish.models.price.price import Price @@ -27,6 +30,7 @@ class Bearish(BaseModel): model_config = ConfigDict(extra="forbid") path: Path + api_keys: SourceApiKeys = Field(default_factory=SourceApiKeys) _bearish_db: BearishDb = PrivateAttr() sources: List[AbstractSource] = Field( default_factory=lambda: [ @@ -38,6 +42,18 @@ class Bearish(BaseModel): def model_post_init(self, __context: Any) -> None: # noqa: ANN401 self._bearish_db = BearishDb(database_path=self.path) + for source in self.sources: + try: + source.set_api_key( + self.api_keys.keys.get( + source.__source__, os.environ.get(source.__source__.upper()) # type: ignore + ) + ) + except InvalidApiKeyError as e: # noqa: PERF203 + logger.error( + f"Invalid API key for {source.__source__}: {e}. It will be removed from sources" + ) + self.sources.remove(source) def write_assets(self, query: Optional[AssetQuery] = None) -> None: for source in self.sources: diff --git a/bearish/models/api_keys/__init__.py b/bearish/models/api_keys/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/bearish/models/api_keys/api_keys.py b/bearish/models/api_keys/api_keys.py new file mode 100644 index 0000000..e309da5 --- /dev/null +++ b/bearish/models/api_keys/api_keys.py @@ -0,0 +1,18 @@ +import json +from pathlib import Path +from typing import Dict + +from pydantic import BaseModel, Field + + +class SourceApiKeys(BaseModel): + keys: Dict[str, str] = Field(default_factory=dict) + + @classmethod + def from_file(cls, api_keys_path: Path) -> "SourceApiKeys": + if not api_keys_path.exists(): + raise FileNotFoundError(f"File not found: {api_keys_path}") + try: + return cls(keys=json.loads(api_keys_path.read_text())) + except Exception as e: + raise ValueError(f"Invalid JSON in file {api_keys_path}: {e}") from e diff --git a/bearish/models/base.py b/bearish/models/base.py index 361cc03..f7731a2 100644 --- a/bearish/models/base.py +++ b/bearish/models/base.py @@ -1,6 +1,7 @@ +import abc import datetime from datetime import date -from typing import Dict, Any, ClassVar +from typing import Dict, Any, ClassVar, Optional from pydantic import ( BaseModel, @@ -17,9 +18,10 @@ class BaseAssets(BaseModel): currencies: Any -class SourceBase(BaseModel): +class SourceBase(BaseModel, abc.ABC): __source__: str __alias__: ClassVar[Dict[str, str]] = {} + __api_key__: ClassVar[Optional[str]] = None model_config = ConfigDict(populate_by_name=True, extra="forbid") diff --git a/bearish/sources/alphavantage.py b/bearish/sources/alphavantage.py index 5b8badb..d31e76a 100644 --- a/bearish/sources/alphavantage.py +++ b/bearish/sources/alphavantage.py @@ -1,36 +1,48 @@ +import functools import logging -import os -from typing import List, Optional, ClassVar, cast, Any, Dict +from typing import List, Optional, ClassVar, cast, Any, Dict, Callable, Type, TypeVar import pandas as pd from alpha_vantage.fundamentaldata import FundamentalData # type: ignore from alpha_vantage.timeseries import TimeSeries # type: ignore from pydantic import BaseModel +from bearish.exceptions import InvalidApiKeyError +from bearish.models.assets.assets import Assets from bearish.models.assets.equity import Equity +from bearish.models.base import SourceBase from bearish.models.financials.balance_sheet import BalanceSheet +from bearish.models.financials.base import Financials from bearish.models.financials.cash_flow import CashFlow from bearish.models.financials.metrics import FinancialMetrics from bearish.models.price.price import Price from bearish.models.query.query import AssetQuery - from bearish.sources.base import ( AbstractSource, ) -from bearish.models.financials.base import Financials -from bearish.models.assets.assets import Assets logger = logging.getLogger(__name__) +T = TypeVar("T", bound=Type[Any]) + -class AlphaVantageBase(BaseModel): +def check_api_key(method: Callable[..., Any]) -> Callable[..., Any]: + @functools.wraps(method) + def wrapper(cls: T, *args: Any, **kwargs: Any) -> Any: # noqa: ANN401 + if not hasattr(cls, "fundamentals") or not hasattr(cls, "timeseries"): + raise InvalidApiKeyError(f"API key not set for {cls.__source__}") + return method(cls, *args, **kwargs) + + return wrapper + + +class AlphaVantageSourceBase(BaseModel): __source__: str = "AlphaVantage" - fundamentals: ClassVar[FundamentalData] = FundamentalData( - key=os.environ.get("ALPHAVANTAGE_API_KEY", "DUMMY") - ) - timeseries: ClassVar[TimeSeries] = TimeSeries( - key=os.environ.get("ALPHAVANTAGE_API_KEY", "DUMMY") - ) + + +class AlphaVantageBase(AlphaVantageSourceBase, SourceBase): + fundamentals: ClassVar[FundamentalData] + timeseries: ClassVar[TimeSeries] class AlphaVantageBaseFinancials(AlphaVantageBase): @@ -87,6 +99,7 @@ class AlphaVantageEquity(AlphaVantageBase, Equity): } @classmethod + @check_api_key def from_tickers(cls, tickers: List[str]) -> List["AlphaVantageEquity"]: equities = [] for ticker in tickers: @@ -126,6 +139,7 @@ class AlphaVantageFinancialMetrics(AlphaVantageBaseFinancials, FinancialMetrics) } @classmethod + @check_api_key def from_ticker(cls, ticker: str) -> List["AlphaVantageFinancialMetrics"]: company_overview, _ = cls.fundamentals.get_company_overview(ticker) return AlphaVantageFinancialMetrics.from_json(company_overview) # type: ignore @@ -164,6 +178,7 @@ class AlphaVantageBalanceSheet(AlphaVantageBaseFinancials, BalanceSheet): } @classmethod + @check_api_key def from_ticker(cls, ticker: str) -> List["AlphaVantageBalanceSheet"]: data_annual, _ = cls.fundamentals.get_balance_sheet_annual(ticker) data_quarterly, _ = cls.fundamentals.get_balance_sheet_quarterly(ticker) @@ -196,6 +211,7 @@ class AlphaVantageCashFlow(AlphaVantageBaseFinancials, CashFlow): } @classmethod + @check_api_key def from_ticker(cls, ticker: str) -> List["AlphaVantageCashFlow"]: data, _ = cls.fundamentals.get_cash_flow_annual(ticker) return AlphaVantageCashFlow.from_dataframe(ticker, data) # type: ignore @@ -212,7 +228,8 @@ class AlphaVantagePrice(AlphaVantageBase, Price): } @classmethod - def from_ticker(cls, ticker: str, type: str) -> List["Price"]: + @check_api_key + def from_ticker(cls, ticker: str, type: str) -> List[Price]: type = "full" if type == "full" else "compact" time_series, metadata = cls.timeseries.get_daily(ticker, outputsize=type) @@ -222,7 +239,7 @@ def from_ticker(cls, ticker: str, type: str) -> List["Price"]: ] -class AlphaVantageSource(AbstractSource): +class AlphaVantageSource(AlphaVantageSourceBase, AbstractSource): def _read_assets(self, query: Optional[AssetQuery] = None) -> Assets: if query is None: return Assets() @@ -237,4 +254,8 @@ def _read_financials(self, ticker: str) -> Financials: ) def read_series(self, ticker: str, type: str) -> List[Price]: - return AlphaVantagePrice.from_ticker(ticker, type) + return cast(List[Price], AlphaVantagePrice.from_ticker(ticker, type)) + + def set_api_key(self, api_key: str) -> None: + AlphaVantageBase.fundamentals = FundamentalData(key=api_key) + AlphaVantageBase.timeseries = TimeSeries(key=api_key) diff --git a/bearish/sources/base.py b/bearish/sources/base.py index acdfa73..afc7f37 100644 --- a/bearish/sources/base.py +++ b/bearish/sources/base.py @@ -46,6 +46,12 @@ def _read_assets(self, query: Optional[AssetQuery] = None) -> Assets: ... @abc.abstractmethod def read_series(self, ticker: str, type: str) -> List[Price]: ... + @abc.abstractmethod + def set_api_key(self, api_key: str) -> None: ... + + def __hash__(self) -> int: + return hash(self.__source__) + class UrlSource(BaseModel): url: str @@ -73,6 +79,7 @@ def to_assets(self) -> Assets: class DatabaseCsvSource(AbstractSource): __url_sources__: UrlSources + def set_api_key(self, api_key: str) -> None: ... def _read_assets(self, query: Optional[AssetQuery] = None) -> Assets: sources = self.__url_sources__ for field in sources.model_fields: diff --git a/bearish/sources/financedatabase.py b/bearish/sources/financedatabase.py index 0566fad..162405a 100644 --- a/bearish/sources/financedatabase.py +++ b/bearish/sources/financedatabase.py @@ -37,7 +37,7 @@ class FinanceDatabaseCurrency(FinanceDatabaseBase, Currency): ... class FinanceDatabaseEtf(FinanceDatabaseBase, Etf): ... -class FinanceDatabaseSource(DatabaseCsvSource): +class FinanceDatabaseSource(FinanceDatabaseBase, DatabaseCsvSource): __url_sources__ = UrlSources( equity=UrlSource( url=RAW_EQUITIES_DATA_URL, diff --git a/bearish/sources/financial_modelling_prep.py b/bearish/sources/financial_modelling_prep.py new file mode 100644 index 0000000..e69de29 diff --git a/bearish/sources/finnhub.py b/bearish/sources/finnhub.py new file mode 100644 index 0000000..e69de29 diff --git a/bearish/sources/investpy.py b/bearish/sources/investpy.py index d36a7cb..cb75511 100644 --- a/bearish/sources/investpy.py +++ b/bearish/sources/investpy.py @@ -48,7 +48,7 @@ class InvestPyEtf(InvestPyBase, Etf): } -class InvestPySource(DatabaseCsvSource): +class InvestPySource(InvestPyBase, DatabaseCsvSource): __url_sources__ = UrlSources( equity=UrlSource( url=RAW_EQUITIES_INVESTSPY_DATA_URL, diff --git a/bearish/sources/tiingo.py b/bearish/sources/tiingo.py new file mode 100644 index 0000000..e69de29 diff --git a/bearish/sources/yfinance.py b/bearish/sources/yfinance.py index 5938181..251e41a 100644 --- a/bearish/sources/yfinance.py +++ b/bearish/sources/yfinance.py @@ -287,7 +287,8 @@ class yFinancePrice(YfinanceBase, Price): # noqa: N801 } -class yFinanceSource(AbstractSource): # noqa: N801 +class yFinanceSource(YfinanceBase, AbstractSource): # noqa: N801 + def set_api_key(self, api_key: str) -> None: ... def _read_assets(self, query: Optional[AssetQuery] = None) -> Assets: if query is None: return Assets() diff --git a/poetry.lock b/poetry.lock index 2152638..86b86da 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2033,6 +2033,21 @@ port-for = ">=0.6.0" pymongo = "*" pytest = ">=6.2" +[[package]] +name = "pytest-order" +version = "1.3.0" +description = "pytest plugin to run your tests in a specific order" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pytest_order-1.3.0-py3-none-any.whl", hash = "sha256:2cd562a21380345dd8d5774aa5fd38b7849b6ee7397ca5f6999bbe6e89f07f6e"}, + {file = "pytest_order-1.3.0.tar.gz", hash = "sha256:51608fec3d3ee9c0adaea94daa124a5c4c1d2bb99b00269f098f414307f23dde"}, +] + +[package.dependencies] +pytest = {version = ">=6.2.4", markers = "python_version >= \"3.10\""} + [[package]] name = "python-dateutil" version = "2.9.0.post0" @@ -2690,4 +2705,4 @@ repair = ["scipy (>=1.6.3)"] [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.13" -content-hash = "dfec44f50d4dd2c98ee8dc4eb19e1468f16510efb54eee2e8f972f027ab7d159" +content-hash = "a602621c490be6cc69da7a1d061628941da4e09c234b2f678d270eaa4af7d333" diff --git a/pyproject.toml b/pyproject.toml index a25367a..c1539ce 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,6 +35,7 @@ types-beautifulsoup4 = "^4.12.0.20240229" pytest-mongo = "^3.1.0" requests-mock = "^1.12.1" black = "^24.10.0" +pytest-order = "^1.3.0" [build-system] requires = ["poetry-core"] diff --git a/tests/sources/test_alphavantage.py b/tests/sources/test_alphavantage.py index 7b61400..9ac541d 100644 --- a/tests/sources/test_alphavantage.py +++ b/tests/sources/test_alphavantage.py @@ -7,6 +7,7 @@ import pytest from alpha_vantage.fundamentaldata import FundamentalData +from bearish.exceptions import InvalidApiKeyError from bearish.models.query.query import Symbols, AssetQuery from bearish.sources.alphavantage import ( AlphaVantageFinancialMetrics, @@ -155,3 +156,16 @@ def test_alphavantage_read_series(): AlphaVantageBase.timeseries = FakeTimeSeries() series = AlphaVantageSource().read_series(ticker, "full") assert series + + +@pytest.mark.order(2) +def test_api_key(): + alpha = AlphaVantageSource() + alpha.set_api_key("test") + assert AlphaVantageCashFlow.from_ticker("AAPL") + + +@pytest.mark.order(1) +def test_no_api_key(): + with pytest.raises(InvalidApiKeyError): + AlphaVantageCashFlow.from_ticker("AAPL") diff --git a/tests/test_bearish.py b/tests/test_bearish.py index 9a22fcd..c516250 100644 --- a/tests/test_bearish.py +++ b/tests/test_bearish.py @@ -6,6 +6,7 @@ from bearish.database.crud import BearishDb from bearish.main import Bearish +from bearish.models.api_keys.api_keys import SourceApiKeys from bearish.models.query.query import AssetQuery, Symbols from bearish.sources.alphavantage import AlphaVantageBase, AlphaVantageSource from bearish.sources.financedatabase import ( @@ -147,7 +148,11 @@ def test_update_series_multiple_times(bearish_db: BearishDb): def test_update_financials_alphavantage(bearish_db: BearishDb): AlphaVantageBase.fundamentals = FakeFundamentalData() AlphaVantageBase.timeseries = FakeTimeSeries() - bearish = Bearish(path=bearish_db.database_path, sources=[AlphaVantageSource()]) + bearish = Bearish( + path=bearish_db.database_path, + api_keys=SourceApiKeys(keys={"AlphaVantage": "AlphaVantage"}), + sources=[AlphaVantageSource()], + ) bearish.read_financials_from_many_sources("AAPL") financials = bearish.read_financials(AssetQuery(symbols=Symbols(equities=["AAPL"]))) assert financials @@ -156,7 +161,11 @@ def test_update_financials_alphavantage(bearish_db: BearishDb): def test_update_series_alphavantage(bearish_db: BearishDb): AlphaVantageBase.fundamentals = FakeFundamentalData() AlphaVantageBase.timeseries = FakeTimeSeries() - bearish = Bearish(path=bearish_db.database_path, sources=[AlphaVantageSource()]) + bearish = Bearish( + path=bearish_db.database_path, + api_keys=SourceApiKeys(keys={"AlphaVantage": "AlphaVantage"}), + sources=[AlphaVantageSource()], + ) bearish.write_series("AAPL", "full") series = bearish.read_series(AssetQuery(symbols=Symbols(equities=["AAPL"]))) assert series