diff --git a/.dockerignore b/.dockerignore index 66d49ea0..ab5e8767 100755 --- a/.dockerignore +++ b/.dockerignore @@ -1,15 +1,11 @@ -#https://stackoverflow.com/questions/28097064/dockerignore-ignore-everything-except-a-file-and-the-dockerfile +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ -# Ignore Everything -** - -!pypfopt -!tests -!setup.py -!README.md -!requirements.txt -!binder -!cookbook - -**/__pycache__ -**/*.pyc +*/__pycache__ +*/*.pyc \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/bug.md b/.github/ISSUE_TEMPLATE/bug.md index 8302570d..65f976df 100644 --- a/.github/ISSUE_TEMPLATE/bug.md +++ b/.github/ISSUE_TEMPLATE/bug.md @@ -13,8 +13,8 @@ A clear and concise description of what the bug is. **Expected behavior** A clear and concise description of what you expected to happen. -**Screenshots** -If applicable, add screenshots to help explain your problem. +**Code sample** +Add a minimal reproducible example (see [here](https://stackoverflow.com/help/minimal-reproducible-example)). **Operating system, python version, PyPortfolioOpt version** e.g MacOS 10.146, python 3.7.3, PyPortfolioOpt 1.2.6 diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index a329c877..69d5617b 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -10,8 +10,8 @@ assignees: '' **Is your feature request related to a problem?** A clear and concise description of what the problem is. -**Describe the solution you'd like** -A clear and concise description of what you want to happen. +**Describe the feature you'd like** +A clear description of the feature you want, or a link to the textbook/article describing the feature. **Additional context** Add any other context or screenshots about the feature request here. diff --git a/.github/ISSUE_TEMPLATE/help-needed.md b/.github/ISSUE_TEMPLATE/help-needed.md index 597fd940..d99ff034 100644 --- a/.github/ISSUE_TEMPLATE/help-needed.md +++ b/.github/ISSUE_TEMPLATE/help-needed.md @@ -10,5 +10,7 @@ assignees: '' **What are you trying to do?** Clear description of the problem you are trying to solve with PyPortfolioOpt +**What have you tried?** + **What data are you using?** What asset class, how many assets, how many data points. Preferably provide a sample of the dataset as a csv attachment. diff --git a/.github/ISSUE_TEMPLATE/installation-error.md b/.github/ISSUE_TEMPLATE/installation-error.md index bd796c97..cccaab2b 100644 --- a/.github/ISSUE_TEMPLATE/installation-error.md +++ b/.github/ISSUE_TEMPLATE/installation-error.md @@ -16,5 +16,5 @@ pip install pyportfolioopt **Error message** ``` -Copy paste the terminal message here +Copy paste the terminal message inside the backticks. ``` diff --git a/.gitignore b/.gitignore index 1868dd62..dca20524 100644 --- a/.gitignore +++ b/.gitignore @@ -11,8 +11,20 @@ DEV/ .pytest_cache/ .vscode/ +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + pip-selfcheck.json +.coverage +.coverage* + html-coverage html-report diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9d2fc515..d203e632 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -4,7 +4,7 @@ Please refer to the roadmap for a list of areas that I think PyPortfolioOpt coul from. In addition, the following is always welcome:: - Improve performance of existing code (but not at the cost of readability) – are there any nice numpy tricks I've missed? -- Add new optimisation objective functions. For example, if you think that the best performance metric has not been included, write it into a function (or suggest it in [Issues](https://github.com/robertmartin8/PyPortfolioOpt/issues) and I will have a go). +- Add new optimization objective functions. For example, if you think that the best performance metric has not been included, write it into a function (or suggest it in [Issues](https://github.com/robertmartin8/PyPortfolioOpt/issues) and I will have a go). - Help me write more tests! If you are someone learning about quant finance and/or unit testing in python, what better way to practice than to write some tests on an open-source project! Feel free to check for edge cases, or test performance on a dataset with more stocks. ## Guidelines @@ -37,7 +37,7 @@ If you have questions unrelated to the project, drop me an email – contact det ## Bugs/issues -If you find any bugs or the portfolio optimisation is not working as expected, feel free to [raise an issue](https://github.com/robertmartin8/PyPortfolioOpt/issues). I would ask that you provide the following information in the issue: +If you find any bugs or the portfolio optimization is not working as expected, feel free to [raise an issue](https://github.com/robertmartin8/PyPortfolioOpt/issues). I would ask that you provide the following information in the issue: - Descriptive title so that other users can see the existing issues - Operating system, python version, and python distribution (optional). diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 21286730..00000000 --- a/Dockerfile +++ /dev/null @@ -1,26 +0,0 @@ -FROM python:3.7.7-slim-stretch as builder - -# this will be user root regardless whether home/beakerx is not -COPY . /tmp/pypfopt - -RUN buildDeps='gcc g++' && \ - apt-get update && apt-get install -y $buildDeps --no-install-recommends && \ - pip install --no-cache-dir -r /tmp/pypfopt/requirements.txt && \ - # One could install the pypfopt library directly in the image. We don't and share via docker-compose instead. - # pip install --no-cache-dir /tmp/pyhrp && \ - rm -r /tmp/pypfopt && \ - apt-get purge -y --auto-remove $buildDeps - - -# ---------------------------------------------------------------------------------------------------------------------- -FROM builder as test - -# COPY tools needed for testing into the image -RUN pip install --no-cache-dir pytest pytest-cov pytest-html - -# COPY the tests over -COPY tests /pypfopt/tests - -WORKDIR /pypfopt - -CMD py.test --cov=pypfopt --cov-report html:artifacts/html-coverage --cov-report term --html=artifacts/html-report/report.html tests diff --git a/Makefile b/Makefile deleted file mode 100644 index c26321f5..00000000 --- a/Makefile +++ /dev/null @@ -1,54 +0,0 @@ -#!make -PROJECT_VERSION := $(shell python setup.py --version) - -SHELL := /bin/bash -PACKAGE := pypfopt - -# needed to get the ${PORT} environment variable -include .env -export - -.PHONY: help build test tag pypi - - -.DEFAULT: help - -help: - @echo "make build" - @echo " Build the docker image." - @echo "make test" - @echo " Build the docker image for testing and run them." - @echo "make doc" - @echo " Construct the documentation." - @echo "make tag" - @echo " Make a tag on Github." - -jupyter: - echo "http://localhost:${PORT}" - docker-compose up jupyter - -build: - docker-compose build pypfopt - -test: - mkdir -p artifacts - docker-compose -f docker-compose.test.yml run sut - -tag: test - git tag -a ${PROJECT_VERSION} -m "new tag" - git push --tags - -# push to pypi -#pypi: tag -# python setup.py sdist -# twine check dist/* -# twine upload dist/* - -# push to dockerhub -# you will need a free account on dockerhub and do docker login... -#hub: tag -# docker build --file binder/Dockerfile --tag ${IMAGE}:latest --no-cache . -# docker push ${IMAGE}:latest -# docker tag ${IMAGE}:latest ${IMAGE}:${PROJECT_VERSION} -# docker push ${IMAGE}:${PROJECT_VERSION} -# docker rmi -f ${IMAGE}:${PROJECT_VERSION} \ No newline at end of file diff --git a/README.md b/README.md index 5a070563..dd063a3f 100755 --- a/README.md +++ b/README.md @@ -26,8 +26,8 @@ -PyPortfolioOpt is a library that implements portfolio optimisation methods, including -classical mean-variance optimisation techniques and Black-Litterman allocation, as well as more +PyPortfolioOpt is a library that implements portfolio optimization methods, including +classical mean-variance optimization techniques and Black-Litterman allocation, as well as more recent developments in the field like shrinkage and Hierarchical Risk Parity, along with some novel experimental features like exponentially-weighted covariance matrices. @@ -49,14 +49,14 @@ Head over to the [documentation on ReadTheDocs](https://pyportfolioopt.readthedo - [For development](#for-development) - [A quick example](#a-quick-example) - [What's new](#whats-new) -- [An overview of classical portfolio optimisation methods](#an-overview-of-classical-portfolio-optimisation-methods) +- [An overview of classical portfolio optimization methods](#an-overview-of-classical-portfolio-optimization-methods) - [Features](#features) - [Expected returns](#expected-returns) - [Risk models (covariance)](#risk-models-covariance) - [Objective functions](#objective-functions) - [Adding constraints or different objectives](#adding-constraints-or-different-objectives) - [Black-Litterman allocation](#black-litterman-allocation) - - [Other optimisers](#other-optimisers) + - [Other optimizers](#other-optimizers) - [Advantages over existing implementations](#advantages-over-existing-implementations) - [Project principles and design decisions](#project-principles-and-design-decisions) - [Testing](#testing) @@ -124,7 +124,7 @@ df = pd.read_csv("tests/resources/stock_prices.csv", parse_dates=True, index_col mu = expected_returns.mean_historical_return(df) S = risk_models.sample_cov(df) -# Optimise for maximal Sharpe ratio +# Optimize for maximal Sharpe ratio ef = EfficientFrontier(mu, S) raw_weights = ef.max_sharpe() cleaned_weights = ef.clean_weights() @@ -206,23 +206,23 @@ As of v1.2.0: new plots. All other plotting functions (scattered in different classes) have been retained, but are now deprecated. -## An overview of classical portfolio optimisation methods +## An overview of classical portfolio optimization methods -Harry Markowitz's 1952 paper is the undeniable classic, which turned portfolio optimisation from an art into a science. The key insight is that by combining assets with different expected returns and volatilities, one can decide on a mathematically optimal allocation which minimises the risk for a target return – the set of all such optimal portfolios is referred to as the **efficient frontier**. +Harry Markowitz's 1952 paper is the undeniable classic, which turned portfolio optimization from an art into a science. The key insight is that by combining assets with different expected returns and volatilities, one can decide on a mathematically optimal allocation which minimises the risk for a target return – the set of all such optimal portfolios is referred to as the **efficient frontier**.
Although much development has been made in the subject, more than half a century later, Markowitz's core ideas are still fundamentally important and see daily use in many portfolio management firms. -The main drawback of mean-variance optimisation is that the theoretical treatment requires knowledge of the expected returns and the future risk-characteristics (covariance) of the assets. Obviously, if we knew the expected returns of a stock life would be much easier, but the whole game is that stock returns are notoriously hard to forecast. As a substitute, we can derive estimates of the expected return and covariance based on historical data – though we do lose the theoretical guarantees provided by Markowitz, the closer our estimates are to the real values, the better our portfolio will be. +The main drawback of mean-variance optimization is that the theoretical treatment requires knowledge of the expected returns and the future risk-characteristics (covariance) of the assets. Obviously, if we knew the expected returns of a stock life would be much easier, but the whole game is that stock returns are notoriously hard to forecast. As a substitute, we can derive estimates of the expected return and covariance based on historical data – though we do lose the theoretical guarantees provided by Markowitz, the closer our estimates are to the real values, the better our portfolio will be. Thus this project provides four major sets of functionality (though of course they are intimately related) - Estimates of expected returns - Estimates of risk (i.e covariance of asset returns) -- Objective functions to be optimised -- Optimisers. +- Objective functions to be optimized +- Optimizers. A key design goal of PyPortfolioOpt is **modularity** – the user should be able to swap in their components while still making use of the framework that PyPortfolioOpt provides. @@ -253,7 +253,7 @@ The covariance matrix encodes not just the volatility of an asset, but also how - an unbiased estimate of the covariance matrix - relatively easy to compute - the de facto standard for many years - - however, it has a high estimation error, which is particularly dangerous in mean-variance optimisation because the optimiser is likely to give excess weight to these erroneous estimates. + - however, it has a high estimation error, which is particularly dangerous in mean-variance optimization because the optimizer is likely to give excess weight to these erroneous estimates. - Semicovariance: a measure of risk that focuses on downside variation. - Exponential covariance: an improvement over sample covariance that gives more weight to recent data - Covariance shrinkage: techniques that involve combining the sample covariance matrix with a structured estimator, to reduce the effect of erroneous weights. PyPortfolioOpt provides wrappers around the efficient vectorised implementations provided by `sklearn.covariance`. @@ -280,7 +280,7 @@ The covariance matrix encodes not just the volatility of an asset, but also how ### Adding constraints or different objectives -- Long/short: by default all of the mean-variance optimisation methods in PyPortfolioOpt are long-only, but they can be initialised to allow for short positions by changing the weight bounds: +- Long/short: by default all of the mean-variance optimization methods in PyPortfolioOpt are long-only, but they can be initialised to allow for short positions by changing the weight bounds: ```python ef = EfficientFrontier(mu, S, weight_bounds=(-1, 1)) @@ -299,7 +299,7 @@ ef.efficient_return(target_return=0.2, market_neutral=True) ef = EfficientFrontier(mu, S, weight_bounds=(0, 0.1)) ``` -One issue with mean-variance optimisation is that it leads to many zero-weights. While these are +One issue with mean-variance optimization is that it leads to many zero-weights. While these are "optimal" in-sample, there is a large body of research showing that this characteristic leads mean-variance portfolios to underperform out-of-sample. To that end, I have introduced an objective function that can reduce the number of negligible weights for any of the objective functions. Essentially, it adds a penalty (parameterised by `gamma`) on small weights, with a term that looks just like L2 regularisation in machine learning. It may be necessary to try several `gamma` values to achieve the desired number of non-negligible weights. For the test portfolio of 20 securities, `gamma ~ 1` is sufficient @@ -328,14 +328,16 @@ ef = EfficientFrontier(rets, S) ef.max_sharpe() ``` -### Other optimisers +### Other optimizers -The features above mostly pertain to solving efficient frontier optimisation problems via quadratic programming (though this is taken care of by `cvxpy`). However, we offer different optimisers as well: +The features above mostly pertain to solving mean-variance optimization problems via quadratic programming (though this is taken care of by `cvxpy`). However, we offer different optimizers as well: +- Mean-semivariance optimization +- Mean-CVaR optimization - Hierarchical Risk Parity, using clustering algorithms to choose uncorrelated assets - Markowitz's critical line algorithm (CLA) -Please refer to the [documentation](https://pyportfolioopt.readthedocs.io/en/latest/OtherOptimisers.html) for more. +Please refer to the [documentation](https://pyportfolioopt.readthedocs.io/en/latest/OtherOptimizers.html) for more. ## Advantages over existing implementations @@ -350,10 +352,10 @@ Please refer to the [documentation](https://pyportfolioopt.readthedocs.io/en/lat ## Project principles and design decisions -- It should be easy to swap out individual components of the optimisation process +- It should be easy to swap out individual components of the optimization process with the user's proprietary improvements. - Usability is everything: it is better to be self-explanatory than consistent. -- There is no point in portfolio optimisation unless it can be practically +- There is no point in portfolio optimization unless it can be practically applied to real asset prices. - Everything that has been implemented should be tested. - Inline documentation is good: dedicated (separate) documentation is better. diff --git a/binder/Dockerfile b/binder/Dockerfile deleted file mode 100644 index 0fbf219d..00000000 --- a/binder/Dockerfile +++ /dev/null @@ -1,20 +0,0 @@ -FROM jupyter/base-notebook:python-3.7.6 as jupyter - -# File Author / Maintainer -# MAINTAINER Thomas Schmelzer "thomas.schmelzer@gmail.com" - -# copy the config file -COPY ./binder/jupyter_notebook_config.py /etc/jupyter/jupyter_notebook_config.py - -# copy the package over and install it -COPY --chown=jovyan:users . /tmp/pyportfolioopt - -RUN conda install -y -c conda-forge --file /tmp/pyportfolioopt/requirements.txt && \ - conda clean -y --all && \ - pip install --no-cache-dir /tmp/pyportfolioopt && \ - pip install yfinance && \ - pip install lxml && \ - rm -rf /tmp/pyportfolioopt - -# hardcoded parameters!? see https://github.com/moby/moby/issues/35018 -COPY --chown=jovyan:users ./cookbook $HOME/work diff --git a/binder/jupyter_notebook_config.py b/binder/jupyter_notebook_config.py deleted file mode 100644 index ef619faa..00000000 --- a/binder/jupyter_notebook_config.py +++ /dev/null @@ -1,7 +0,0 @@ -c = get_config() -c.NotebookApp.ip = '0.0.0.0' -c.NotebookApp.port = 8888 -c.NotebookApp.open_browser = False -c.NotebookApp.token = '' -c.NotebookApp.password = '' -c.NotebookApp.notebook_dir = "/home/jovyan/work" \ No newline at end of file diff --git a/cookbook/1-RiskReturnModels.ipynb b/cookbook/1-RiskReturnModels.ipynb index 645f62f8..a403477b 100644 --- a/cookbook/1-RiskReturnModels.ipynb +++ b/cookbook/1-RiskReturnModels.ipynb @@ -240,7 +240,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "The exponential moving average is marginally better than the others, but the improvement is almost unnoticeable. We also note that the mean absolute deviations are above 25%, meaning that if your expected annual returns are 10%, on average the realised annual return could be anywhere from a 15% loss to a 35% gain. This is a massive range, and gives some context to the advice in the docs suggesting that you optimise without providing an estimate of returns." + "The exponential moving average is marginally better than the others, but the improvement is almost unnoticeable. We also note that the mean absolute deviations are above 25%, meaning that if your expected annual returns are 10%, on average the realised annual return could be anywhere from a 15% loss to a 35% gain. This is a massive range, and gives some context to the advice in the docs suggesting that you optimize without providing an estimate of returns." ] }, { diff --git a/cookbook/2-Mean-Variance-Optimisation.ipynb b/cookbook/2-Mean-Variance-Optimisation.ipynb index 72a58df5..fddacac7 100644 --- a/cookbook/2-Mean-Variance-Optimisation.ipynb +++ b/cookbook/2-Mean-Variance-Optimisation.ipynb @@ -4,11 +4,11 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Mean-variance optimisation\n", + "# Mean-variance optimization\n", "\n", "In this cookbook recipe, we work on several examples demonstrating PyPortfolioOpt's mean-variance capabilities. I will discuss what I think should be your \"default\" options, based on my experience in optimising portfolios.\n", "\n", - "To start, you need a list of tickers. Some people just provide the whole universe of stocks, but I don't think this is a good idea - portfolio optimisation is quite different from asset selection. I would suggest anywhere from 10-50 stocks as a starting point.\n", + "To start, you need a list of tickers. Some people just provide the whole universe of stocks, but I don't think this is a good idea - portfolio optimization is quite different from asset selection. I would suggest anywhere from 10-50 stocks as a starting point.\n", "\n", "Some of the things we cover:\n", "\n", @@ -827,7 +827,7 @@ "source": [ "## Long/short min variance\n", "\n", - "In this section, we construct a long/short portfolio with the objective of minimising variance. There is a good deal of research that demonstrates that these global-minimum variance (GMV) portfolios outperform mean-variance optimised portfolios." + "In this section, we construct a long/short portfolio with the objective of minimising variance. There is a good deal of research that demonstrates that these global-minimum variance (GMV) portfolios outperform mean-variance optimized portfolios." ] }, { @@ -1353,7 +1353,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "We then set up the optimiser and add our constraints. We can use `ef.add_objective()` to add other constraints. For example, let's say that in addition to the above sector constraints, I specifically want:\n", + "We then set up the optimizer and add our constraints. We can use `ef.add_objective()` to add other constraints. For example, let's say that in addition to the above sector constraints, I specifically want:\n", "\n", "- 10% of the portfolio in AMZN\n", "- Less than 5% of my portfolio in TSLA" @@ -1564,7 +1564,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "While this portfolio seems like it meets our objectives, we might be worried by the fact that a lot of the tickers have been assigned zero weight. In effect, the optimiser is \"overfitting\" to the data you have provided -- you are much more likely to get better results by enforcing some level of diversification. One way of doing this is to use **L2 regularisation** – essentially, adding a penalty on the number of near-zero weights." + "While this portfolio seems like it meets our objectives, we might be worried by the fact that a lot of the tickers have been assigned zero weight. In effect, the optimizer is \"overfitting\" to the data you have provided -- you are much more likely to get better results by enforcing some level of diversification. One way of doing this is to use **L2 regularisation** – essentially, adding a penalty on the number of near-zero weights." ] }, { @@ -1837,7 +1837,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Efficient semi-variance optimisation\n", + "## Efficient semi-variance optimization\n", "\n", "In this example, we will minimise the portfolio semivariance (i.e downside volatility) subject to a return constraint (target 20%).\n", "\n", @@ -1927,7 +1927,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "However, this solution is not truly optimal in mean-semivariance space. To do the optimisation properly, we must use the `EfficientSemivariance` class. This requires us to first compute the returns and drop NaNs." + "However, this solution is not truly optimal in mean-semivariance space. To do the optimization properly, we must use the `EfficientSemivariance` class. This requires us to first compute the returns and drop NaNs." ] }, { @@ -2001,7 +2001,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Efficient CVaR optimisation\n", + "## Efficient CVaR optimization\n", "\n", "In this example, we will find the portfolio that maximises return subject to a CVaR constraint.\n", "\n", @@ -2423,7 +2423,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "As per the docs, *before* we call any optimisation function, we should pass this to the plotting module:" + "As per the docs, *before* we call any optimization function, we should pass this to the plotting module:" ] }, { diff --git a/cookbook/3-Advanced-Mean-Variance-Optimisation.ipynb b/cookbook/3-Advanced-Mean-Variance-Optimisation.ipynb index c087b548..5089df48 100644 --- a/cookbook/3-Advanced-Mean-Variance-Optimisation.ipynb +++ b/cookbook/3-Advanced-Mean-Variance-Optimisation.ipynb @@ -344,7 +344,7 @@ "source": [ "## Min volatility with a transaction cost objective\n", "\n", - "Let's say that you already have a portfolio, and want to now optimise it. It could be quite expensive to completely reallocate, so you may want to take into account transaction costs. PyPortfolioOpt provides a simple objective to account for this.\n", + "Let's say that you already have a portfolio, and want to now optimize it. It could be quite expensive to completely reallocate, so you may want to take into account transaction costs. PyPortfolioOpt provides a simple objective to account for this.\n", "\n", "Note: this objective will not play nicely with `max_sharpe`." ] @@ -450,7 +450,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "The optimiser seems to really like JD. The reason for this is that it is highly anticorrelated to other assets (notice the dark column in the covariance plot). Hence, historically, it adds a lot of diversification. But it is dangerous to place too much emphasis on what happened in the past, so we may want to limit the asset weights. \n", + "The optimizer seems to really like JD. The reason for this is that it is highly anticorrelated to other assets (notice the dark column in the covariance plot). Hence, historically, it adds a lot of diversification. But it is dangerous to place too much emphasis on what happened in the past, so we may want to limit the asset weights. \n", "\n", "In addition, we notice that 4 stocks have now been allocated zero weight, which may be undesirable. Both of these problems can be fixed by adding an [L2 regularisation objective](https://pyportfolioopt.readthedocs.io/en/latest/EfficientFrontier.html#more-on-l2-regularisation). " ] @@ -601,13 +601,13 @@ "- Quadratic utility\n", "- Transaction cost model (a simple one)\n", "\n", - "However, you may want have a different objective. If this new objective is **convex**, you can optimise a portfolio with the full benefit of PyPortfolioOpt's modular syntax, for example adding other constraints and objectives.\n", + "However, you may want have a different objective. If this new objective is **convex**, you can optimize a portfolio with the full benefit of PyPortfolioOpt's modular syntax, for example adding other constraints and objectives.\n", "\n", - "To demonstrate this, we will minimise the **logarithmic-barrier** function suggested in the paper 60 Years of Portfolio Optimisation, by Kolm et al (2014):\n", + "To demonstrate this, we will minimise the **logarithmic-barrier** function suggested in the paper 60 Years of Portfolio Optimization, by Kolm et al (2014):\n", "\n", "$$f(w, S, k) = w^T S w - k \\sum_{i=1}^N \\ln w$$\n", "\n", - "We must first convert this mathematical objective into the language of cvxpy. Cvxpy is a powerful modelling language for convex optimisation problems. It is clean and easy to use, the only caveat is that objectives must be expressed with `cvxpy` functions, a list of which can be found [here](https://www.cvxpy.org/tutorial/functions/index.html)." + "We must first convert this mathematical objective into the language of cvxpy. Cvxpy is a powerful modelling language for convex optimization problems. It is clean and easy to use, the only caveat is that objectives must be expressed with `cvxpy` functions, a list of which can be found [here](https://www.cvxpy.org/tutorial/functions/index.html)." ] }, { @@ -740,9 +740,9 @@ "source": [ "## Custom nonconvex objectives\n", "\n", - "In some cases, you may be trying to optimise for nonconvex objectives. Optimisation in general is a very hard problem, so please be aware that you may have mixed results in that case. Convex problems, on the other hand, are well understood and can be solved with nice theoretical guarantees.\n", + "In some cases, you may be trying to optimize for nonconvex objectives. Optimization in general is a very hard problem, so please be aware that you may have mixed results in that case. Convex problems, on the other hand, are well understood and can be solved with nice theoretical guarantees.\n", "\n", - "PyPortfolioOpt does offer some functionality for nonconvex optimisation, but it is not really encouraged. In particular, nonconvex optimisation is not compatible with PyPortfolioOpt's modular constraints API.\n", + "PyPortfolioOpt does offer some functionality for nonconvex optimization, but it is not really encouraged. In particular, nonconvex optimization is not compatible with PyPortfolioOpt's modular constraints API.\n", "\n", "As an example, we will use the Deviation Risk Parity objective from Kolm et al (2014). Because we are not using a convex solver, we don't have to define it using `cvxpy` functions." ] diff --git a/cookbook/4-Black-Litterman-Allocation.ipynb b/cookbook/4-Black-Litterman-Allocation.ipynb index f73e1e52..137de7fa 100644 --- a/cookbook/4-Black-Litterman-Allocation.ipynb +++ b/cookbook/4-Black-Litterman-Allocation.ipynb @@ -15,7 +15,7 @@ "- Downloading data for the Black-Litterman method\n", "- Constructing the prior return vector based on market equilibrium\n", "- Two ways of constructing the uncertainty matrix\n", - "- Combining Black-Litterman with mean-variance optimisation\n", + "- Combining Black-Litterman with mean-variance optimization\n", "\n", "## Downloading data\n", "\n", @@ -818,7 +818,7 @@ "source": [ "## Portfolio allocation\n", "\n", - "Now that we have constructed our Black-Litterman posterior estimate, we can proceed to use any of the optimisers discussed in previous recipes." + "Now that we have constructed our Black-Litterman posterior estimate, we can proceed to use any of the optimizers discussed in previous recipes." ] }, { @@ -839,7 +839,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "/Users/Robert/github/PyPortfolioOpt/pypfopt/efficient_frontier.py:196: UserWarning: max_sharpe transforms the optimisation problem so additional objectives may not work as expected.\n", + "/Users/Robert/github/PyPortfolioOpt/pypfopt/efficient_frontier.py:196: UserWarning: max_sharpe transforms the optimization problem so additional objectives may not work as expected.\n", " warnings.warn(\n" ] }, diff --git a/cookbook/5-Hierarchical-Risk-Parity.ipynb b/cookbook/5-Hierarchical-Risk-Parity.ipynb index f9e7303c..8fa53100 100644 --- a/cookbook/5-Hierarchical-Risk-Parity.ipynb +++ b/cookbook/5-Hierarchical-Risk-Parity.ipynb @@ -7,7 +7,7 @@ "source": [ "# Hierarchical Risk Parity\n", "\n", - "HRP is a modern portfolio optimisation method inspired by machine learning.\n", + "HRP is a modern portfolio optimization method inspired by machine learning.\n", "\n", "The idea is that by examining the hierarchical structure of the market, we can better diversify. \n", "\n", @@ -461,7 +461,7 @@ "id": "answering-tamil", "metadata": {}, "source": [ - "## HRP optimisation\n", + "## HRP optimization\n", "\n", "HRP uses a completely different backend, so it is currently not possible to pass constraints or specify an objective function." ] diff --git a/docker-compose.test.yml b/docker-compose.test.yml deleted file mode 100755 index e44a5448..00000000 --- a/docker-compose.test.yml +++ /dev/null @@ -1,12 +0,0 @@ -version: '3.6' -services: - sut: - build: - context: . - dockerfile: Dockerfile - target: test - - volumes: - - ./artifacts:/artifacts - - ./tests:/pypfopt/tests - - ./pypfopt:/pypfopt/pypfopt:ro diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index 5afa6d4b..00000000 --- a/docker-compose.yml +++ /dev/null @@ -1,16 +0,0 @@ -version: '3.6' -services: - pypfopt: - build: - context: . - dockerfile: Dockerfile - target: builder - - jupyter: - build: - context: . - dockerfile: ./binder/Dockerfile - volumes: - - ./cookbook:${WORK} - ports: - - ${PORT}:8888 \ No newline at end of file diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 00000000..8b4f9f3e --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,30 @@ +FROM python:3.8-slim-buster + +WORKDIR pypfopt +COPY pyproject.toml poetry.lock ./ + +RUN buildDeps='gcc g++' && \ + apt-get update && apt-get install -y $buildDeps --no-install-recommends && \ + pip install --upgrade pip==21.0.1 && \ + pip install "poetry==1.1.4" && \ + poetry install -E optionals --no-root && \ + apt-get purge -y --auto-remove $buildDeps + +COPY . . + +# Usage examples: +# +# Build +# from root of repo: +# docker build -f docker/Dockerfile . -t pypfopt +# +# Run +# iPython interpreter: +# docker run -it pypfopt poetry run ipython +# Jupyter notebook server: +# docker run -it -p 8888:8888 pypfopt poetry run jupyter notebook --allow-root --no-browser --ip 0.0.0.0 +# click on http://127.0.0.1:8888/?token=xxx +# Pytest +# docker run -t pypfopt poetry run pytest +# Bash +# docker run -it pypfopt bash diff --git a/docs/About.rst b/docs/About.rst index b218490a..865158e4 100644 --- a/docs/About.rst +++ b/docs/About.rst @@ -9,10 +9,10 @@ over to my `website `_. I learn fastest when making real projects. In early 2018 I began seriously trying to self-educate on certain topics in quantitative finance, and mean-variance -optimisation is one of the cornerstones of this field. I read quite a few journal +optimization is one of the cornerstones of this field. I read quite a few journal articles and explanations but ultimately felt that a real proof of understanding would lie in the implementation. At the same time, I realised that existing open-source -(python) portfolio optimisation libraries (there are one or two), were unsatisfactory +(python) portfolio optimization libraries (there are one or two), were unsatisfactory for several reasons, and that people 'out there' might benefit from a well-documented and intuitive API. This is what motivated the development of PyPortfolioOpt. diff --git a/docs/BlackLitterman.rst b/docs/BlackLitterman.rst index bba80a81..f7c494a8 100644 --- a/docs/BlackLitterman.rst +++ b/docs/BlackLitterman.rst @@ -45,7 +45,7 @@ Similarly, we can calculate a posterior estimate of the covariance matrix: Though the algorithm is relatively simple, BL proved to be a challenge from a software engineering perspective because it's not quite clear how best to fit it into PyPortfolioOpt's API. The full discussion can be found on a `Github issue thread `_, -but I ultimately decided that though BL is not technically an optimiser, it didn't make sense to +but I ultimately decided that though BL is not technically an optimizer, it didn't make sense to split up its methods into `expected_returns` or `risk_models`. I have thus made it an independent module and owing to the comparatively extensive theory, have given it a dedicated documentation page. I'd like to thank `Felipe Schneider `_ for his multiple @@ -55,7 +55,7 @@ of market cap data for free, please refer to the `cookbook recipe `_ - on top of PyPortfolioOpt, which allows you to visualise BL outputs and compare optimisation objectives. + on top of PyPortfolioOpt, which allows you to visualise BL outputs and compare optimization objectives. Priors ====== @@ -192,7 +192,7 @@ Output of the BL model ====================== The BL model outputs posterior estimates of the returns and covariance matrix. The default suggestion in the literature is to -then input these into an optimiser (see :ref:`efficient-frontier`). A quick alternative, which is quite useful for debugging, is +then input these into an optimizer (see :ref:`efficient-frontier`). A quick alternative, which is quite useful for debugging, is to calculate the weights implied by the returns vector [4]_. It is actually the reverse of the procedure we used to calculate the returns implied by the market weights. diff --git a/docs/Contributing.rst b/docs/Contributing.rst index 2bfbadd7..75effa3c 100644 --- a/docs/Contributing.rst +++ b/docs/Contributing.rst @@ -5,8 +5,8 @@ Contributing Some of the things that I'd love for people to help with: - Improve performance of existing code (but not at the cost of readability) -- Add new optimisation objectives. For example, if you would like to use something other - than the Sharpe ratio, write an optimiser! (or suggest it in +- Add new optimization objectives. For example, if you would like to use something other + than the Sharpe ratio, write an optimizer! (or suggest it in `Issues `_ and I will have a go). - Help me write more tests! If you are someone learning about quant finance and/or unit testing in python, what better way to practice than to write some tests on an @@ -71,7 +71,7 @@ details can be found on my `website `_. Bugs/issues =========== -If you find any bugs or the portfolio optimisation is not working as expected, +If you find any bugs or the portfolio optimization is not working as expected, feel free to `raise an issue `_. I would ask that you provide the following information in the issue: diff --git a/docs/EfficientFrontier.rst b/docs/EfficientFrontier.rst deleted file mode 100644 index f737e7f0..00000000 --- a/docs/EfficientFrontier.rst +++ /dev/null @@ -1,367 +0,0 @@ -.. _efficient-frontier: - -############################### -Efficient Frontier Optimisation -############################### - -Mathematical optimisation is a very difficult problem in general, particularly when we are dealing -with complex objectives and constraints. However, **convex optimisation** problems are a well-understood -class of problems, which happen to be incredibly useful for finance. A convex problem has the following form: - -.. math:: - - \begin{equation*} - \begin{aligned} - & \underset{\mathbf{x}}{\text{minimise}} & & f(\mathbf{x}) \\ - & \text{subject to} & & g_i(\mathbf{x}) \leq 0, i = 1, \ldots, m\\ - &&& A\mathbf{x} = b,\\ - \end{aligned} - \end{equation*} - -where :math:`\mathbf{x} \in \mathbb{R}^n`, and :math:`f(\mathbf{x}), g_i(\mathbf{x})` are convex functions. [1]_ - -Fortunately, portfolio optimisation problems (with standard objectives and constraints) are convex. This -allows us to immediately apply the vast body of theory as well as the refined solving routines -- accordingly, -the main difficulty is inputting our specific problem into a solver. - -PyPortfolioOpt aims to do the hard work for you, allowing for one-liners like ``ef.min_volatility()`` -to generate a portfolio that minimises the volatility, while at the same time allowing for more -complex problems to be built up from modular units. This is all possible thanks to -`cvxpy `_, the *fantastic* python-embedded modelling -language for convex optimisation upon which PyPortfolioOpt's efficient frontier functionality lies. - -As a brief aside, I should note that while "efficient frontier" optimisation is technically a very -specific method, I tend to use it as a blanket term (interchangeably with mean-variance -optimisation) to refer to anything similar, such as minimising variance. - -.. tip:: - - You can find complete examples in the relevant cookbook `recipe `_. - - -Structure -========= - -As shown in the definition of a convex problem, there are essentially two things we need to specify: -the optimisation objective, and the optimisation constraints. For example, the classic portfolio -optimisation problem is to **minimise risk** subject to a **return constraint** (i.e the portfolio -must return more than a certain amount). From an implementation perspective, however, there is -not much difference between an objective and a constraint. Consider a similar problem, which is to -**maximize return** subject to a **risk constraint** -- now, the role of risk and return have swapped. - -To that end, PyPortfolioOpt defines an :py:mod:`objective_functions` module that contains objective functions -(which can also act as constraints, as we have just seen). The actual optimisation occurs in the :py:class:`efficient_frontier.EfficientFrontier` class. -This class provides straightforward methods for optimising different objectives (all documented below). - -However, PyPortfolioOpt was designed so that you can easily add new constraints or objective terms to an existing problem. -For example, adding a regularisation objective (explained below) to a minimum volatility objective is as simple as:: - - ef = EfficientFrontier(expected_returns, cov_matrix) # setup - ef.add_objective(objective_functions.L2_reg) # add a secondary objective - ef.min_volatility() # find the portfolio that minimises volatility and L2_reg - -.. tip:: - - If you would like to plot the efficient frontier, take a look at the :ref:`plotting` module. - -Basic Usage -=========== - -.. automodule:: pypfopt.efficient_frontier - - .. autoclass:: EfficientFrontier - - .. automethod:: __init__ - - .. note:: - - As of v0.5.0, you can pass a collection (list or tuple) of (min, max) pairs - representing different bounds for different assets. - - .. tip:: - - If you want to generate short-only portfolios, there is a quick hack. Multiply - your expected returns by -1, then optimise a long-only portfolio. - - .. automethod:: min_volatility - - .. automethod:: max_sharpe - - .. caution:: - - Because ``max_sharpe()`` makes a variable substitution, additional objectives may - not work as intended. - - - .. automethod:: max_quadratic_utility - - .. note:: - - ``pypfopt.black_litterman`` provides a method for calculating the market-implied - risk-aversion parameter, which gives a useful estimate in the absence of other - information! - - .. automethod:: efficient_risk - - .. caution:: - - If you pass an unreasonable target into :py:meth:`efficient_risk` or - :py:meth:`efficient_return`, the optimiser will fail silently and return - weird weights. *Caveat emptor* applies! - - .. automethod:: efficient_return - - .. automethod:: portfolio_performance - - .. tip:: - - If you would like to use the ``portfolio_performance`` function independently of any - optimiser (e.g for debugging purposes), you can use:: - - from pypfopt import base_optimizer - - base_optimizer.portfolio_performance( - weights, expected_returns, cov_matrix, verbose=True, risk_free_rate=0.02 - ) - -.. note:: - - PyPortfolioOpt defers to cvxpy's default choice of solver. If you would like to explicitly - choose the solver, simply pass the optional ``solver = "ECOS"`` kwarg to the constructor. - You can choose from any of the `supported solvers `_, - and pass in solver params via ``solver_options`` (a ``dict``). - -Adding objectives and constraints -================================= - -EfficientFrontier inherits from the BaseConvexOptimizer class. In particular, the functions to -add constraints and objectives are documented below: - - -.. class:: pypfopt.base_optimizer.BaseConvexOptimizer - - .. automethod:: add_constraint - - .. automethod:: add_sector_constraints - - .. automethod:: add_objective - - -Objective functions -=================== - -.. automodule:: pypfopt.objective_functions - :members: - - - -.. _L2-Regularisation: - -More on L2 Regularisation -========================= - -As has been discussed in the :ref:`user-guide`, efficient frontier optimisation often -results in many weights being negligible, i.e the efficient portfolio does not end up -including most of the assets. This is expected behaviour, but it may be undesirable -if you need a certain number of assets in your portfolio. - -In order to coerce the efficient frontier optimiser to produce more non-negligible -weights, we add what can be thought of as a "small weights penalty" to all -of the objective functions, parameterised by :math:`\gamma` (``gamma``). Considering -the minimum variance objective for instance, we have: - -.. math:: - \underset{w}{\text{minimise}} ~ \left\{w^T \Sigma w \right\} ~~~ \longrightarrow ~~~ - \underset{w}{\text{minimise}} ~ \left\{w^T \Sigma w + \gamma w^T w \right\} - -Note that :math:`w^T w` is the same as the sum of squared weights (I didn't -write this explicitly to reduce confusion caused by :math:`\Sigma` denoting both the -covariance matrix and the summation operator). This term reduces the number of -negligible weights, because it has a minimum value when all weights are -equally distributed, and maximum value in the limiting case where the entire portfolio -is allocated to one asset. I refer to it as **L2 regularisation** because it has -exactly the same form as the L2 regularisation term in machine learning, though -a slightly different purpose (in ML it is used to keep weights small while here it is -used to make them larger). - -.. note:: - - In practice, :math:`\gamma` must be tuned to achieve the level - of regularisation that you want. However, if the universe of assets is small - (less than 20 assets), then ``gamma=1`` is a good starting point. For larger - universes, or if you want more non-negligible weights in the final portfolio, - increase ``gamma``. - - -Efficient Semivariance -====================== - -The mean-variance optimisation methods described above can be used whenever you have a vector -of expected returns and a covariance matrix. - -However, you may want to construct the efficient frontier for an entirely different risk model. -For example, instead of penalising volatility (which is symmetric), you may want to penalise -only the downside deviation (with the idea that upside volatility actually benefits you!) - -There are two approaches to the mean-semivariance optimisation problem. The first is to use a -heuristic (i.e "quick and dirty") solution: pretending that the semicovariance matrix -(implemented in :py:mod:`risk_models`) is a typical covariance matrix and doing standard -mean-variance optimisation. It can be shown that this *does not* yield a portfolio that -is efficient in mean-semivariance space (though it might be a good-enough approximation). - -Fortunately, it is possible to write mean-semivariance optimisation as a convex problem -(albeit one with many variables), that can be solved to give an "exact" solution. -For example, to maximise return for a target semivariance -:math:`s^*` (long-only), we would solve the following problem: - -.. math:: - - \begin{equation*} - \begin{aligned} - & \underset{w}{\text{maximise}} & & w^T \mu \\ - & \text{subject to} & & n^T n \leq s^* \\ - &&& B w - p + n = 0 \\ - &&& w^T \mathbf{1} = 1 \\ - &&& n \geq 0 \\ - &&& p \geq 0. \\ - \end{aligned} - \end{equation*} - -Here, **B** is the :math:`T \times N` (scaled) matrix of excess returns: -``B = (returns - benchmark) / sqrt(T)``. Additional linear equality constraints and -convex inequality constraints can be added. - -PyPortfolioOpt allows users to optimise along the efficient semivariance frontier -via the :py:class:`EfficientSemivariance` class. :py:class:`EfficientSemivariance` inherits from -:py:class:`EfficientFrontier`, so it has the same utility methods -(e.g :py:func:`add_constraint`, :py:func:`portfolio_performance`), but finds portfolios on the mean-semivariance -frontier. Note that some of the parent methods, like :py:func:`max_sharpe` and :py:func:`min_volatility` -are not applicable to mean-semivariance portfolios, so calling them returns an error. - -:py:class:`EfficientSemivariance` has a slightly different API to :py:class:`EfficientFrontier`. Instead of passing -in a covariance matrix, you should past in a dataframe of historical/simulated returns (this can be constructed -from your price dataframe using the helper method :py:func:`expected_returns.returns_from_prices`). Here -is a full example, in which we seek the portfolio that minimises the semivariance for a target -annual return of 20%:: - - from pypfopt import expected_returns, EfficientSemivariance - - df = ... # your dataframe of prices - mu = expected_returns.mean_historical_returns(df) - historical_returns = expected_returns.returns_from_prices(df) - - es = EfficientSemivariance(mu, historical_returns) - es.efficient_return(0.20) - - # We can use the same helper methods as before - weights = es.clean_weights() - print(weights) - es.portfolio_performance(verbose=True) - -The ``portfolio_performance`` method outputs the expected portfolio return, semivariance, -and the Sortino ratio (like the Sharpe ratio, but for downside deviation). - -Interested readers should refer to Estrada (2007) [2]_ for more details. I'd like to thank -`Philipp Schiele `_ for authoring the bulk -of the efficient semivariance functionality and documentation (all errors are my own). The -implementation is based on Markowitz et al (2019) [3]_. - -.. caution:: - - Finding portfolios on the mean-semivariance frontier is computationally harder - than standard mean-variance optimisation: our implementation uses ``2T + N`` optimisation variables, - meaning that for 50 assets and 3 years of data, there are about 1500 variables. - While :py:class:`EfficientSemivariance` allows for additional constraints/objectives in principle, - you are much more likely to run into solver errors. I suggest that you keep :py:class:`EfficientSemivariance` - problems small and minimally constrained. - -.. autoclass:: pypfopt.efficient_frontier.EfficientSemivariance - :members: - :exclude-members: max_sharpe, min_volatility - -Efficient CVaR -============== - -The **conditional value-at-risk** (a.k.a **expected shortfall**) is a popular measure of tail risk. The CVaR can be -thought of as the average of losses that occur on "very bad days", where "very bad" is quantified by the parameter -:math:`\beta`. - -For example, if we calculate the CVaR to be 10% for :math:`\beta = 0.95`, we can be 95% confident that the worst-case -average daily loss will be 3%. Put differently, the CVaR is the average of all losses so severe that they only occur -:math:`(1-\beta)\%` of the time. - -While CVaR is quite an intuitive concept, a lot of new notation is required to formulate it mathematically (see -the `wiki page `_ for more details). We will adopt the following -notation: - -- *w* for the vector of portfolio weights -- *r* for a vector of asset returns (daily), with probability distribution :math:`p(r)`. -- :math:`L(w, r) = - w^T r` for the loss of the portfolio -- :math:`\alpha` for the portfolio value-at-risk (VaR) with confidence :math:`\beta`. - -The CVaR can then be written as: - -.. math:: - CVaR(w, \beta) = \frac{1}{1-\beta} \int_{L(w, r) \geq \alpha (w)} L(w, r) p(r)dr. - -This is a nasty expression to optimise because we are essentially integrating over VaR values. The key insight -of Rockafellar and Uryasev (2001) [4]_ is that we can can equivalently optimise the following convex function: - -.. math:: - F_\beta (w, \alpha) = \alpha + \frac{1}{1-\beta} \int [-w^T r - \alpha]^+ p(r) dr, - -where :math:`[x]^+ = \max(x, 0)`. The authors prove that minimising :math:`F_\beta(w, \alpha)` over all -:math:`w, \alpha` minimises the CVaR. Suppose we have a sample of *T* daily returns (these -can either be historical or simulated). The integral in the expression becomes a sum, so the CVaR optimisation -problem reduces to a linear program: - -.. math:: - - \begin{equation*} - \begin{aligned} - & \underset{w, \alpha}{\text{minimise}} & & \alpha + \frac{1}{1-\beta} \frac 1 T \sum_{i=1}^T u_i \\ - & \text{subject to} & & u_i \geq 0 \\ - &&& u_i \geq -w^T r_i - \alpha. \\ - \end{aligned} - \end{equation*} - -This formulation introduces a new variable for each datapoint (similar to Efficient Semivariance), so -you may run into performance issues for long returns dataframes. At the same time, you should aim to -provide a sample of data that is large enough to include tail events. - -.. autoclass:: pypfopt.efficient_frontier.EfficientCVaR - :members: - :exclude-members: max_sharpe, min_volatility, max_quadratic_utility - - -.. _custom-optimisation: - -Custom optimisation problems -============================ - -Previously we described an API for adding constraints and objectives to one of the core -optimisation problems in the :py:class:`EfficientFrontier` class. However, what if you aren't interested -in anything related to ``max_sharpe()``, ``min_volatility()``, ``efficient_risk()`` etc and want to -set up a completely new problem to optimise for some custom objective? - -The :py:class:`EfficientFrontier` class inherits from the ``BaseConvexOptimizer``, which allows you to -define your own optimisation problem. You can either optimise some generic ``convex_objective`` -(which *must* be built using ``cvxpy`` atomic functions -- see `here `_) -or a ``nonconvex_objective``, which uses ``scipy.optimize`` as the backend and thus has a completely -different API. For more examples, check out this `cookbook recipe -`_. - - .. class:: pypfopt.base_optimizer.BaseConvexOptimizer - - .. automethod:: convex_objective - - .. automethod:: nonconvex_objective - - -References -========== - -.. [1] Boyd, S.; Vandenberghe, L. (2004). `Convex Optimization `_. -.. [2] Estrada, J (2007). `Mean-Semivariance Optimization: A Heuristic Approach `_. -.. [3] Markowitz, H.; Starer, D.; Fram, H.; Gerber, S. (2019). `Avoiding the Downside `_. -.. [4] Rockafellar, R.; Uryasev, D. (2001). `Optimization of conditional value-at-risk `_ diff --git a/docs/ExpectedReturns.rst b/docs/ExpectedReturns.rst index ac9395c8..a1c7247c 100755 --- a/docs/ExpectedReturns.rst +++ b/docs/ExpectedReturns.rst @@ -4,22 +4,22 @@ Expected Returns ################ -Mean-variance optimisation requires knowledge of the expected returns. In practice, +Mean-variance optimization requires knowledge of the expected returns. In practice, these are rather difficult to know with any certainty. Thus the best we can do is to come up with estimates, for example by extrapolating historical data, This is the -main flaw in mean-variance optimisation – the optimisation procedure is sound, and provides +main flaw in mean-variance optimization – the optimization procedure is sound, and provides strong mathematical guarantees, *given the correct inputs*. This is one of the reasons why I have emphasised modularity: users should be able to come up with their own -superior models and feed them into the optimiser. +superior models and feed them into the optimizer. .. caution:: - In my experience, supplying expected returns often does more harm than good. If + Supplying expected returns can do more harm than good. If predicting stock returns were as easy as calculating the mean historical return, we'd all be rich! For most use-cases, I would suggest that you focus your efforts on choosing an appropriate risk model (see :ref:`risk-models`). - As of v0.5.0, you can use :ref:`black-litterman` to greatly improve the quality of + As of v0.5.0, you can use :ref:`black-litterman` to significantly improve the quality of your estimate of the expected returns. .. automodule:: pypfopt.expected_returns @@ -33,7 +33,7 @@ superior models and feed them into the optimiser. This is probably the default textbook approach. It is intuitive and easily interpretable, however the estimates are subject to large uncertainty. This is a problem especially in the - context of a quadratic optimiser, which will maximise the erroneous inputs. + context of a mean-variance optimizer, which will maximise the erroneous inputs. .. autofunction:: ema_historical_return @@ -50,8 +50,6 @@ superior models and feed them into the optimiser. .. autofunction:: returns_from_prices - .. autofunction:: log_returns_from_prices - .. autofunction:: prices_from_returns diff --git a/docs/FAQ.rst b/docs/FAQ.rst new file mode 100644 index 00000000..0df6614d --- /dev/null +++ b/docs/FAQ.rst @@ -0,0 +1,40 @@ +.. _faq: + +#### +FAQs +#### + +Constraining the number of assets +--------------------------------- + +Unfortunately, cardinality constraints are not convex, making them difficult to implement. + +However, we can treat it as a mixed-integer program and solve (provided you have access to a solver). +or small problems with less than 1000 variables and constraints, you can use the community version of CPLEX: +``pip install cplex``. In the below example, we limit the portfolio to at most 10 assets:: + + ef = EfficientFrontier(mu, S, solver=cp.CPLEX) + booleans = cp.Variable(len(ef.tickers), boolean=True) + ef.add_constraint(lambda x: x <= booleans) + ef.add_constraint(lambda x: cp.sum(booleans) <= 10) + ef.min_volatility() + +This does not play well with ``max_sharpe``, and needs to be modified for different bounds. +See `this issue `_ for further discussion. + +Tracking error +-------------- + +Tracking error can either be used as an objective (as described in :ref:`efficient-frontier`) or +as a constraint. This is an example of adding a tracking error constraint:: + + from objective functions import ex_ante_tracking_error + + benchmark_weights = ... # benchmark + + ef = EfficientFrontier(mu, S) + ef.add_constraint(ex_ante_tracking_error, cov_matrix=ef.cov_matrix, + benchmark_weights=benchmark_weights) + ef.min_volatility() + + diff --git a/docs/GeneralEfficientFrontier.rst b/docs/GeneralEfficientFrontier.rst new file mode 100644 index 00000000..64f16983 --- /dev/null +++ b/docs/GeneralEfficientFrontier.rst @@ -0,0 +1,212 @@ +.. _efficient-frontier: + +########################## +General Efficient Frontier +########################## + +The mean-variance optimization methods described previously can be used whenever you have a vector +of expected returns and a covariance matrix. The objective and constraints will be some combination +of the portfolio return and portfolio volatility. + +However, you may want to construct the efficient frontier for an entirely different type of risk model +(one that doesn't depend on covariance matrices), or optimize an objective unrelated to portfolio +return (e.g tracking error). PyPortfolioOpt comes with several popular alternatives and provides support +for custom optimization problems. + +Efficient Semivariance +====================== + +Instead of penalising volatility, mean-semivariance optimization seeks to only penalise +downside volatility, since upside volatility may be desirable. + +There are two approaches to the mean-semivariance optimization problem. The first is to use a +heuristic (i.e "quick and dirty") solution: pretending that the semicovariance matrix +(implemented in :py:mod:`risk_models`) is a typical covariance matrix and doing standard +mean-variance optimization. It can be shown that this *does not* yield a portfolio that +is efficient in mean-semivariance space (though it might be a good-enough approximation). + +Fortunately, it is possible to write mean-semivariance optimization as a convex problem +(albeit one with many variables), that can be solved to give an "exact" solution. +For example, to maximise return for a target semivariance +:math:`s^*` (long-only), we would solve the following problem: + +.. math:: + + \begin{equation*} + \begin{aligned} + & \underset{w}{\text{maximise}} & & w^T \mu \\ + & \text{subject to} & & n^T n \leq s^* \\ + &&& B w - p + n = 0 \\ + &&& w^T \mathbf{1} = 1 \\ + &&& n \geq 0 \\ + &&& p \geq 0. \\ + \end{aligned} + \end{equation*} + +Here, **B** is the :math:`T \times N` (scaled) matrix of excess returns: +``B = (returns - benchmark) / sqrt(T)``. Additional linear equality constraints and +convex inequality constraints can be added. + +PyPortfolioOpt allows users to optimize along the efficient semivariance frontier +via the :py:class:`EfficientSemivariance` class. :py:class:`EfficientSemivariance` inherits from +:py:class:`EfficientFrontier`, so it has the same utility methods +(e.g :py:func:`add_constraint`, :py:func:`portfolio_performance`), but finds portfolios on the mean-semivariance +frontier. Note that some of the parent methods, like :py:func:`max_sharpe` and :py:func:`min_volatility` +are not applicable to mean-semivariance portfolios, so calling them returns ``NotImplementedError``. + +:py:class:`EfficientSemivariance` has a slightly different API to :py:class:`EfficientFrontier`. Instead of passing +in a covariance matrix, you should past in a dataframe of historical/simulated returns (this can be constructed +from your price dataframe using the helper method :py:func:`expected_returns.returns_from_prices`). Here +is a full example, in which we seek the portfolio that minimises the semivariance for a target +annual return of 20%:: + + from pypfopt import expected_returns, EfficientSemivariance + + df = ... # your dataframe of prices + mu = expected_returns.mean_historical_returns(df) + historical_returns = expected_returns.returns_from_prices(df) + + es = EfficientSemivariance(mu, historical_returns) + es.efficient_return(0.20) + + # We can use the same helper methods as before + weights = es.clean_weights() + print(weights) + es.portfolio_performance(verbose=True) + +The ``portfolio_performance`` method outputs the expected portfolio return, semivariance, +and the Sortino ratio (like the Sharpe ratio, but for downside deviation). + +Interested readers should refer to Estrada (2007) [1]_ for more details. I'd like to thank +`Philipp Schiele `_ for authoring the bulk +of the efficient semivariance functionality and documentation (all errors are my own). The +implementation is based on Markowitz et al (2019) [2]_. + +.. caution:: + + Finding portfolios on the mean-semivariance frontier is computationally harder + than standard mean-variance optimization: our implementation uses ``2T + N`` optimization variables, + meaning that for 50 assets and 3 years of data, there are about 1500 variables. + While :py:class:`EfficientSemivariance` allows for additional constraints/objectives in principle, + you are much more likely to run into solver errors. I suggest that you keep :py:class:`EfficientSemivariance` + problems small and minimally constrained. + +.. autoclass:: pypfopt.efficient_frontier.EfficientSemivariance + :members: + :exclude-members: max_sharpe, min_volatility + +Efficient CVaR +============== + +The **conditional value-at-risk** (a.k.a **expected shortfall**) is a popular measure of tail risk. The CVaR can be +thought of as the average of losses that occur on "very bad days", where "very bad" is quantified by the parameter +:math:`\beta`. + +For example, if we calculate the CVaR to be 10% for :math:`\beta = 0.95`, we can be 95% confident that the worst-case +average daily loss will be 3%. Put differently, the CVaR is the average of all losses so severe that they only occur +:math:`(1-\beta)\%` of the time. + +While CVaR is quite an intuitive concept, a lot of new notation is required to formulate it mathematically (see +the `wiki page `_ for more details). We will adopt the following +notation: + +- *w* for the vector of portfolio weights +- *r* for a vector of asset returns (daily), with probability distribution :math:`p(r)`. +- :math:`L(w, r) = - w^T r` for the loss of the portfolio +- :math:`\alpha` for the portfolio value-at-risk (VaR) with confidence :math:`\beta`. + +The CVaR can then be written as: + +.. math:: + CVaR(w, \beta) = \frac{1}{1-\beta} \int_{L(w, r) \geq \alpha (w)} L(w, r) p(r)dr. + +This is a nasty expression to optimize because we are essentially integrating over VaR values. The key insight +of Rockafellar and Uryasev (2001) [3]_ is that we can can equivalently optimize the following convex function: + +.. math:: + F_\beta (w, \alpha) = \alpha + \frac{1}{1-\beta} \int [-w^T r - \alpha]^+ p(r) dr, + +where :math:`[x]^+ = \max(x, 0)`. The authors prove that minimising :math:`F_\beta(w, \alpha)` over all +:math:`w, \alpha` minimises the CVaR. Suppose we have a sample of *T* daily returns (these +can either be historical or simulated). The integral in the expression becomes a sum, so the CVaR optimization +problem reduces to a linear program: + +.. math:: + + \begin{equation*} + \begin{aligned} + & \underset{w, \alpha}{\text{minimise}} & & \alpha + \frac{1}{1-\beta} \frac 1 T \sum_{i=1}^T u_i \\ + & \text{subject to} & & u_i \geq 0 \\ + &&& u_i \geq -w^T r_i - \alpha. \\ + \end{aligned} + \end{equation*} + +This formulation introduces a new variable for each datapoint (similar to Efficient Semivariance), so +you may run into performance issues for long returns dataframes. At the same time, you should aim to +provide a sample of data that is large enough to include tail events. + +I am grateful to `Nicolas Knudde `_ for the initial draft (all errors are my own). +The implementation is based on Rockafellar and Uryasev (2001) [3]_. + + +.. autoclass:: pypfopt.efficient_frontier.EfficientCVaR + :members: + :exclude-members: max_sharpe, min_volatility, max_quadratic_utility + + +.. _custom-optimization: + +Custom optimization problems +============================ + +We have seen previously that it is easy to add constraints to ``EfficientFrontier`` objects (and +by extension, other general efficient frontier objects like ``EfficientSemivariance``). However, what if you aren't interested +in anything related to ``max_sharpe()``, ``min_volatility()``, ``efficient_risk()`` etc and want to +set up a completely new problem to optimize for some custom objective? + +For example, perhaps our objective is to construct a basket of assets that best replicates a +particular index, in otherwords, to minimise the **tracking error**. This does not fit within +a mean-variance optimization paradigm, but we can still implement it in PyPortfolioOpt:: + + from pypfopt.base_optimizer import BaseConvexOptimizer + from pypfopt.objective_functions import ex_post_tracking error + + historic_returns = ... # dataframe of historic asset returns + S = risk_models.sample_cov(historic_returns, returns_data=True) + + opt = BaseConvexOptimizer( + n_assets=len(historic_returns.columns), + tickers=historic_returns.index, + weight_bounds=(0, 1) + ) + opt.convex_objective( + objective_functions.ex_post_tracking_error, + historic_returns=historical_rets, + benchmark_returns=benchmark_rets, + ) + weights = opt.clean_weights() + +The ``EfficientFrontier`` class inherits from ``BaseConvexOptimizer``. It may be more convenient +to call ``convex_objective`` from an ``EfficientFrontier`` instance than from ``BaseConvexOptimizer``, +particularly if your objective depends on the mean returns or covariance matrix. + +You can either optimize some generic ``convex_objective`` +(which *must* be built using ``cvxpy`` atomic functions -- see `here `_) +or a ``nonconvex_objective``, which uses ``scipy.optimize`` as the backend and thus has a completely +different API. For more examples, check out this `cookbook recipe +`_. + + .. class:: pypfopt.base_optimizer.BaseConvexOptimizer + + .. automethod:: convex_objective + + .. automethod:: nonconvex_objective + + + +References +========== + +.. [1] Estrada, J (2007). `Mean-Semivariance Optimization: A Heuristic Approach `_. +.. [2] Markowitz, H.; Starer, D.; Fram, H.; Gerber, S. (2019). `Avoiding the Downside `_. +.. [3] Rockafellar, R.; Uryasev, D. (2001). `Optimization of conditional value-at-risk `_ diff --git a/docs/MeanVariance.rst b/docs/MeanVariance.rst new file mode 100644 index 00000000..a87ee447 --- /dev/null +++ b/docs/MeanVariance.rst @@ -0,0 +1,197 @@ +.. _mean-variance: + +########################## +Mean-Variance Optimization +########################## + +Mathematical optimization is a very difficult problem in general, particularly when we are dealing +with complex objectives and constraints. However, **convex optimization** problems are a well-understood +class of problems, which happen to be incredibly useful for finance. A convex problem has the following form: + +.. math:: + + \begin{equation*} + \begin{aligned} + & \underset{\mathbf{x}}{\text{minimise}} & & f(\mathbf{x}) \\ + & \text{subject to} & & g_i(\mathbf{x}) \leq 0, i = 1, \ldots, m\\ + &&& A\mathbf{x} = b,\\ + \end{aligned} + \end{equation*} + +where :math:`\mathbf{x} \in \mathbb{R}^n`, and :math:`f(\mathbf{x}), g_i(\mathbf{x})` are convex functions. [1]_ + +Fortunately, portfolio optimization problems (with standard objectives and constraints) are convex. This +allows us to immediately apply the vast body of theory as well as the refined solving routines -- accordingly, +the main difficulty is inputting our specific problem into a solver. + +PyPortfolioOpt aims to do the hard work for you, allowing for one-liners like ``ef.min_volatility()`` +to generate a portfolio that minimises the volatility, while at the same time allowing for more +complex problems to be built up from modular units. This is all possible thanks to +`cvxpy `_, the *fantastic* python-embedded modelling +language for convex optimization upon which PyPortfolioOpt's efficient frontier functionality lies. + +.. tip:: + + You can find complete examples in the relevant cookbook `recipe `_. + + +Structure +========= + +As shown in the definition of a convex problem, there are essentially two things we need to specify: +the optimization objective, and the optimization constraints. For example, the classic portfolio +optimization problem is to **minimise risk** subject to a **return constraint** (i.e the portfolio +must return more than a certain amount). From an implementation perspective, however, there is +not much difference between an objective and a constraint. Consider a similar problem, which is to +**maximize return** subject to a **risk constraint** -- now, the role of risk and return have swapped. + +To that end, PyPortfolioOpt defines an :py:mod:`objective_functions` module that contains objective functions +(which can also act as constraints, as we have just seen). The actual optimization occurs in the :py:class:`efficient_frontier.EfficientFrontier` class. +This class provides straightforward methods for optimising different objectives (all documented below). + +However, PyPortfolioOpt was designed so that you can easily add new constraints or objective terms to an existing problem. +For example, adding a regularisation objective (explained below) to a minimum volatility objective is as simple as:: + + ef = EfficientFrontier(expected_returns, cov_matrix) # setup + ef.add_objective(objective_functions.L2_reg) # add a secondary objective + ef.min_volatility() # find the portfolio that minimises volatility and L2_reg + +.. tip:: + + If you would like to plot the efficient frontier, take a look at the :ref:`plotting` module. + +Basic Usage +=========== + +.. automodule:: pypfopt.efficient_frontier + + .. autoclass:: EfficientFrontier + + .. automethod:: __init__ + + .. note:: + + As of v0.5.0, you can pass a collection (list or tuple) of (min, max) pairs + representing different bounds for different assets. + + .. tip:: + + If you want to generate short-only portfolios, there is a quick hack. Multiply + your expected returns by -1, then optimize a long-only portfolio. + + .. automethod:: min_volatility + + .. automethod:: max_sharpe + + .. caution:: + + Because ``max_sharpe()`` makes a variable substitution, additional objectives may + not work as intended. + + + .. automethod:: max_quadratic_utility + + .. note:: + + ``pypfopt.black_litterman`` provides a method for calculating the market-implied + risk-aversion parameter, which gives a useful estimate in the absence of other + information! + + .. automethod:: efficient_risk + + .. caution:: + + If you pass an unreasonable target into :py:meth:`efficient_risk` or + :py:meth:`efficient_return`, the optimizer will fail silently and return + weird weights. *Caveat emptor* applies! + + .. automethod:: efficient_return + + .. automethod:: portfolio_performance + + .. tip:: + + If you would like to use the ``portfolio_performance`` function independently of any + optimizer (e.g for debugging purposes), you can use:: + + from pypfopt import base_optimizer + + base_optimizer.portfolio_performance( + weights, expected_returns, cov_matrix, verbose=True, risk_free_rate=0.02 + ) + +.. note:: + + PyPortfolioOpt defers to cvxpy's default choice of solver. If you would like to explicitly + choose the solver, simply pass the optional ``solver = "ECOS"`` kwarg to the constructor. + You can choose from any of the `supported solvers `_, + and pass in solver params via ``solver_options`` (a ``dict``). + + +Adding objectives and constraints +================================= + +EfficientFrontier inherits from the BaseConvexOptimizer class. In particular, the functions to +add constraints and objectives are documented below: + + +.. class:: pypfopt.base_optimizer.BaseConvexOptimizer + :noindex: + + .. automethod:: add_constraint + + .. automethod:: add_sector_constraints + + .. automethod:: add_objective + + +Objective functions +=================== + +.. automodule:: pypfopt.objective_functions + :members: + + + +.. _L2-Regularisation: + +More on L2 Regularisation +========================= + +As has been discussed in the :ref:`user-guide`, mean-variance optimization often +results in many weights being negligible, i.e the efficient portfolio does not end up +including most of the assets. This is expected behaviour, but it may be undesirable +if you need a certain number of assets in your portfolio. + +In order to coerce the mean-variance optimizer to produce more non-negligible +weights, we add what can be thought of as a "small weights penalty" to all +of the objective functions, parameterised by :math:`\gamma` (``gamma``). Considering +the minimum variance objective for instance, we have: + +.. math:: + \underset{w}{\text{minimise}} ~ \left\{w^T \Sigma w \right\} ~~~ \longrightarrow ~~~ + \underset{w}{\text{minimise}} ~ \left\{w^T \Sigma w + \gamma w^T w \right\} + +Note that :math:`w^T w` is the same as the sum of squared weights (I didn't +write this explicitly to reduce confusion caused by :math:`\Sigma` denoting both the +covariance matrix and the summation operator). This term reduces the number of +negligible weights, because it has a minimum value when all weights are +equally distributed, and maximum value in the limiting case where the entire portfolio +is allocated to one asset. I refer to it as **L2 regularisation** because it has +exactly the same form as the L2 regularisation term in machine learning, though +a slightly different purpose (in ML it is used to keep weights small while here it is +used to make them larger). + +.. note:: + + In practice, :math:`\gamma` must be tuned to achieve the level + of regularisation that you want. However, if the universe of assets is small + (less than 20 assets), then ``gamma=1`` is a good starting point. For larger + universes, or if you want more non-negligible weights in the final portfolio, + increase ``gamma``. + + +References +========== + +.. [1] Boyd, S.; Vandenberghe, L. (2004). `Convex Optimization `_. diff --git a/docs/OtherOptimisers.rst b/docs/OtherOptimizers.rst similarity index 54% rename from docs/OtherOptimisers.rst rename to docs/OtherOptimizers.rst index 5a43c2db..6a70067c 100644 --- a/docs/OtherOptimisers.rst +++ b/docs/OtherOptimizers.rst @@ -1,16 +1,16 @@ -.. _other-optimisers: +.. _other-optimizers: ################ -Other Optimisers +Other Optimizers ################ -In addition to optimisers that rely on the covariance matrix in the style of -Markowitz, recent developments in portfolio optimisation have seen a number -of alternative optimisation schemes. PyPortfolioOpt implements some of these, -though please note that the implementations may be slightly unstable. +Efficient frontier methods involve the direct optimization of an objective subject to constraints. +However, there are some portfolio optimization schemes that are completely different in character. +PyPortfolioOpt provides support for these alternatives, while still giving you access to the same +pre and post-processing API. .. note:: - As of v0.4, these other optimisers now inherit from ``BaseOptimizer`` or + As of v0.4, these other optimizers now inherit from ``BaseOptimizer`` or ``BaseConvexOptimizer``, so you no longer have to implement pre-processing and post-processing methods on your own. You can thus easily swap out, say, ``EfficientFrontier`` for ``HRPOpt``. @@ -18,7 +18,7 @@ though please note that the implementations may be slightly unstable. Hierarchical Risk Parity (HRP) ============================== -Hierarchical Risk Parity is a novel portfolio optimisation method developed by +Hierarchical Risk Parity is a novel portfolio optimization method developed by Marcos Lopez de Prado [1]_. Though a detailed explanation can be found in the linked paper, here is a rough overview of how HRP works: @@ -33,7 +33,7 @@ linked paper, here is a rough overview of how HRP works: The advantages of this are that it does not require the inversion of the covariance -matrix as with traditional quadratic optimisers, and seems to produce diverse +matrix as with traditional mean-variance optimization, and seems to produce diverse portfolios that perform well out of sample. .. image:: ../media/dendrogram.png @@ -57,8 +57,8 @@ The Critical Line Algorithm =========================== This is a robust alternative to the quadratic solver used to find mean-variance optimal portfolios, -that is especially advantageous when we apply linear inequalities. Unlike generic quadratic optimisers, -the CLA is specially designed for portfolio optimisation. It is guaranteed to converge after a certain +that is especially advantageous when we apply linear inequalities. Unlike generic convex optimization routines, +the CLA is specially designed for portfolio optimization. It is guaranteed to converge after a certain number of iterations, and can efficiently derive the entire efficient frontier. .. image:: ../media/cla_plot.png @@ -69,7 +69,7 @@ number of iterations, and can efficiently derive the entire efficient frontier. .. tip:: In general, unless you have specific requirements e.g you would like to efficiently compute the entire - efficient frontier for plotting, I would go with the standard ``EfficientFrontier`` optimiser. + efficient frontier for plotting, I would go with the standard ``EfficientFrontier`` optimizer. I am most grateful to Marcos López de Prado and David Bailey for providing the implementation [2]_. Permission for its distribution has been received by email. It has been modified such that it has @@ -84,15 +84,16 @@ the same API, though as of v0.5.0 we only support ``max_sharpe()`` and ``min_vol .. automethod:: __init__ -Implementing your own optimiser + +Implementing your own optimizer =============================== -Please note that this is quite different to implementing :ref:`custom-optimisation`, because in -that case we are still using the same quadratic optimiser. However, HRP and CLA optimisation -have a fundamentally different optimisation method. In general, these are much more difficult +Please note that this is quite different to implementing :ref:`custom-optimization`, because in +that case we are still using the same convex optimization structure. However, HRP and CLA optimization +have a fundamentally different optimization method. In general, these are much more difficult to code up compared to custom objective functions. -To implement a custom optimiser that is compatible with the rest of PyPortfolioOpt, just +To implement a custom optimizer that is compatible with the rest of PyPortfolioOpt, just extend ``BaseOptimizer`` (or ``BaseConvexOptimizer`` if you want to use ``cvxpy``), both of which can be found in ``base_optimizer.py``. This gives you access to utility methods like ``clean_weights()``, as well as making sure that any output is compatible @@ -111,53 +112,6 @@ with ``portfolio_performance()`` and post-processing methods. .. automethod:: __init__ -.. Value-at-Risk -.. ============= - -.. .. warning:: -.. Caveat emptor: this functionality is still experimental. Although I have -.. used the CVaR optimisation, I've noticed that it is very inconsistent -.. (which to some extent is expected because of its stochastic nature). -.. However, the optimiser doesn't always find a minimum, and it fails -.. silently. Additionally, the weight bounds are not treated as hard bounds. - - -.. The value-at-risk is a measure of tail risk that estimates how much a portfolio -.. will lose in a day with a given probability. Alternatively, it is the maximum -.. loss with a confidence of beta. In fact, a more useful measure is the -.. **expected shortfall**, or **conditional value-at-risk** (CVaR), which is the -.. mean of all losses so severe that they only occur with a probability -.. :math:`1-\beta`. - -.. .. math:: -.. CVaR_\beta = \frac{1}{1-\beta} \int_0^{1-\beta} VaR_\gamma(X) d\gamma - -.. To approximate the CVaR for a portfolio, we will follow these steps: - -.. 1. Generate the portfolio returns, i.e the weighted sum of individual asset returns. -.. 2. Fit a Gaussian KDE to these returns, then resample. -.. 3. Compute the value-at-risk as the :math:`1-\beta` quantile of sampled returns. -.. 4. Calculate the mean of all the sample returns that are below the value-at-risk. - -.. Though CVaR optimisation can be transformed into a linear programming problem [3]_, I -.. have opted to keep things simple using the `NoisyOpt `_ -.. library, which is suited for optimising noisy functions. - - -.. .. automodule:: pypfopt.value_at_risk - -.. .. autoclass:: CVAROpt -.. :members: - -.. .. automethod:: __init__ - -.. .. caution:: -.. Currently, we have not implemented any performance function. If you -.. would like to calculate the actual CVaR of the resulting portfolio, -.. please import the function from `objective_functions`. - - - References ========== diff --git a/docs/Plotting.rst b/docs/Plotting.rst index acc5a080..8d123368 100644 --- a/docs/Plotting.rst +++ b/docs/Plotting.rst @@ -4,7 +4,7 @@ Plotting ######## -All of the optimisation functions in :py:class:`EfficientFrontier` produce a single optimal portfolio. +All of the optimization functions in :py:class:`EfficientFrontier` produce a single optimal portfolio. However, you may want to plot the entire efficient frontier. This efficient frontier can be thought of in several different ways: @@ -14,7 +14,7 @@ of in several different ways: The :py:mod:`plotting` module provides support for all three of these approaches. To produce a plot of the efficient frontier, you should instantiate your :py:class:`EfficientFrontier` object -and add constraints like you normally would, but *before* calling an optimisation function (e.g with +and add constraints like you normally would, but *before* calling an optimization function (e.g with :py:func:`ef.max_sharpe`), you should pass this the instantiated object into :py:func:`plot.plot_efficient_frontier`:: ef = EfficientFrontier(mu, S, weight_bounds=(None, None)) diff --git a/docs/Postprocessing.rst b/docs/Postprocessing.rst index 78b0a47b..7cff2d5a 100644 --- a/docs/Postprocessing.rst +++ b/docs/Postprocessing.rst @@ -6,12 +6,12 @@ Post-processing weights After optimal weights have been generated, it is often necessary to do some post-processing before they can be used practically. In particular, you are -likely using portfolio optimisation techniques to generate a +likely using portfolio optimization techniques to generate a **portfolio allocation** – a list of tickers and corresponding integer quantities that you could go and purchase at a broker. However, it is not trivial to convert the continuous weights (output by any of our -optimisation methods) into an actionable allocation. For example, let us say that we +optimization methods) into an actionable allocation. For example, let us say that we have $10,000 that we would like to allocate. If we multiply the weights by this total portfolio value, the result will be dollar amounts of each asset. So if the optimal weight for Apple is 0.15, we need $1500 worth of Apple stock. However, Apple shares come @@ -74,7 +74,7 @@ closest to our desired weights. We will use the following notation: - :math:`x \in \mathbb{Z}^n` is the integer allocation (i.e the result) - :math:`r \in \mathbb{R}` is the remaining unallocated value, i.e :math:`r = T - x \cdot p`. -The optimisation problem is then given by: +The optimization problem is then given by: .. math:: @@ -85,9 +85,7 @@ The optimisation problem is then given by: \end{aligned} \end{equation*} -This is straightforward to translate into ``cvxpy``. The initial implementation used -`PuLP `_, but this caused numerous packaging issues and -the code was a lot more verbose. +This is straightforward to translate into ``cvxpy``. .. caution:: @@ -107,8 +105,8 @@ Practically, this means that you would go long $10,000 of some stocks, short $30 stocks, then use the proceeds from the shorts to go long another $3000. Thus the total value of the resulting portfolio would be $13,000. -Usage -===== +Documentation reference +======================== .. automodule:: pypfopt.discrete_allocation diff --git a/docs/RiskModels.rst b/docs/RiskModels.rst index beb0c002..72d15724 100644 --- a/docs/RiskModels.rst +++ b/docs/RiskModels.rst @@ -4,12 +4,11 @@ Risk Models ########### -In addition to the expected returns, mean-variance optimisation requires a +In addition to the expected returns, mean-variance optimization requires a **risk model**, some way of quantifying asset risk. The most commonly-used risk model -is the covariance matrix, a statistical entity that describes the -volatility of asset returns and how they vary with one another. This is +is the covariance matrix, which describes asset volatilities and their co-dependence. This is important because one of the principles of diversification is that risk can be -reduced by making many uncorrelated bets (and correlation is just normalised +reduced by making many uncorrelated bets (correlation is just normalised covariance). .. image:: ../media/corrplot.png @@ -19,7 +18,7 @@ covariance). In many ways, the subject of risk models is far more important than that of -expected returns because historical variance is generally a much more predictive +expected returns because historical variance is generally a much more persistent statistic than mean historical returns. In fact, research by Kritzman et al. (2010) [1]_ suggests that minimum variance portfolios, formed by optimising without providing expected returns, actually perform much better out of sample. @@ -38,8 +37,8 @@ covariance. Estimation of the covariance matrix is a very deep and actively-researched topic that involves statistics, econometrics, and numerical/computational - approaches. I have made an effort to familiarise myself with the seminal papers in the field - and implement a few basic options, but there is a lot of room for more sophistication. + approaches. PyPortfolioOpt implements several options, but there is a lot of room + for more sophistication. .. automodule:: pypfopt.risk_models @@ -64,7 +63,7 @@ covariance. variances). Although the sample covariance matrix is an unbiased estimator of the covariance matrix, i.e :math:`E(S) = \Sigma`, in practice it suffers from misspecification error and a lack of robustness. This is particularly problematic - in mean-variance optimisation, because the optimiser may give extra credence to + in mean-variance optimization, because the optimizer may give extra credence to the erroneous values. .. note:: @@ -94,13 +93,6 @@ covariance. `blog post `_ on my academic website. - .. autofunction:: min_cov_determinant - - The minimum covariance determinant (MCD) estimator is designed to be robust to - outliers and 'contaminated' data [3]_. An efficient estimator is implemented in the - :py:mod:`sklearn.covariance` module, which is based on the algorithm presented in - Rousseeuw (1999) [4]_. - .. autofunction:: cov_to_corr .. autofunction:: corr_to_cov @@ -111,11 +103,11 @@ Shrinkage estimators ==================== A great starting point for those interested in understanding shrinkage estimators is -*Honey, I Shrunk the Sample Covariance Matrix* [5]_ by Ledoit and Wolf, which does a +*Honey, I Shrunk the Sample Covariance Matrix* [3]_ by Ledoit and Wolf, which does a good job at capturing the intuition behind them – we will adopt the notation used therein. I have written a summary of this article, which is available on my `website `_. -A more rigorous reference can be found in Ledoit and Wolf (2001) [6]_. +A more rigorous reference can be found in Ledoit and Wolf (2001) [4]_. The essential idea is that the unbiased but often poorly estimated sample covariance can be combined with a structured estimator :math:`F`, using the below formula (where @@ -137,11 +129,11 @@ the following shrinkage methods: asset variances on the diagonals and zeroes elsewhere. This is the shrinkage offered by ``sklearn.LedoitWolf``. - ``single_factor`` shrinkage. Based on Sharpe's single-index model which effectively uses - a stock's beta to the market as a risk model. See Ledoit and Wolf 2001 [6]_. + a stock's beta to the market as a risk model. See Ledoit and Wolf 2001 [4]_. - ``constant_correlation`` shrinkage, in which all pairwise correlations are set to - the average correlation (sample variances are unchanged). See Ledoit and Wolf 2003 [5]_ + the average correlation (sample variances are unchanged). See Ledoit and Wolf 2003 [3]_ -- Oracle approximating shrinkage (OAS), invented by Chen et al. (2010) [7]_, which +- Oracle approximating shrinkage (OAS), invented by Chen et al. (2010) [5]_, which has a lower mean-squared error than Ledoit-Wolf shrinkage when samples are Gaussian or near-Gaussian. @@ -168,8 +160,6 @@ References .. [1] Kritzman, Page & Turkington (2010) `In defense of optimization: The fallacy of 1/N `_. Financial Analysts Journal, 66(2), 31-39. .. [2] Estrada (2006), `Mean-Semivariance Optimization: A Heuristic Approach `_ -.. [3] Rousseeuw, P., J (1984). `Least median of squares regression `_. The Journal of the American Statistical Association, 79, 871-880. -.. [4] Rousseeuw, P., J (1999). `A Fast Algorithm for the Minimum Covariance Determinant Estimator `_. The Journal of the American Statistical Association, 41, 212-223. -.. [5] Ledoit, O., & Wolf, M. (2003). `Honey, I Shrunk the Sample Covariance Matrix `_ The Journal of Portfolio Management, 30(4), 110–119. https://doi.org/10.3905/jpm.2004.110 -.. [6] Ledoit, O., & Wolf, M. (2001). `Improved estimation of the covariance matrix of stock returns with an application to portfolio selection `_, 10, 603–621. -.. [7] Chen et al. (2010), `Shrinkage Algorithms for MMSE Covariance Estimation `_, IEEE Transactions on Signals Processing, 58(10), 5016-5029. +.. [3] Ledoit, O., & Wolf, M. (2003). `Honey, I Shrunk the Sample Covariance Matrix `_ The Journal of Portfolio Management, 30(4), 110–119. https://doi.org/10.3905/jpm.2004.110 +.. [4] Ledoit, O., & Wolf, M. (2001). `Improved estimation of the covariance matrix of stock returns with an application to portfolio selection `_, 10, 603–621. +.. [5] Chen et al. (2010), `Shrinkage Algorithms for MMSE Covariance Estimation `_, IEEE Transactions on Signals Processing, 58(10), 5016-5029. diff --git a/docs/Roadmap.rst b/docs/Roadmap.rst index 63763dd0..3faf854c 100644 --- a/docs/Roadmap.rst +++ b/docs/Roadmap.rst @@ -15,15 +15,16 @@ discuss. If you have any other feature requests, please raise them using GitHub - Open-source backtests using either `Backtrader `_ or `Zipline `_. +- Risk parity - Optimising for higher moments (i.e skew and kurtosis) - Factor modelling - this is conceptually doable, but a lot of thought needs to be put into the API. -- Monte Carlo optimisation with custom distributions +- Monte Carlo optimization with custom distributions - Further support for different risk/return models 1.4.0 ===== -- Finally implemented CVaR optimisation! This has been one of the most requested features. Many thanks +- Finally implemented CVaR optimization! This has been one of the most requested features. Many thanks to `Nicolas Knudde `_ for the initial draft. - Re-architected plotting so users can pass an ax, allowing for complex plots (see cookbook). - Helper method to compute the max-return portfolio (thanks to `Philipp Schiele `_) @@ -144,7 +145,7 @@ Matplotlib now required dependency; support for pandas 1.0. - Replaced ``BaseScipyOptimizer`` with ``BaseConvexOptimizer`` - ``hierarchical_risk_parity`` was replaced by ``hierarchical_portfolios`` to leave the door open for other hierarchical methods. - - Sadly, removed CVaR optimisation for the time being until I can properly fix it. + - Sadly, removed CVaR optimization for the time being until I can properly fix it. 1.0.1 ----- @@ -155,7 +156,7 @@ Fixed minor issues in CLA: weight bound bug, ``efficient_frontier`` needed weigh ----- Fixed small but important bug where passing ``expected_returns=None`` fails. According to the docs, users -should be able to only pass covariance if they want to only optimise min volatility. +should be able to only pass covariance if they want to only optimize min volatility. 0.5.0 @@ -165,7 +166,7 @@ should be able to only pass covariance if they want to only optimise min volatil - Custom bounds per asset - Improved ``BaseOptimizer``, adding a method that writes weights to text and fixing a bug in ``set_weights``. -- Unconstrained quadratic utility optimisation (analytic) +- Unconstrained quadratic utility optimization (analytic) - Revamped docs, with information on types of attributes and more examples. @@ -183,9 +184,9 @@ experience. 0.5.3 ----- -- Fixed an optimisation bug in ``EfficientFrontier.efficient_risk``. An error is now - thrown if optimisation fails. -- Added a hidden API to change the scipy optimiser method. +- Fixed an optimization bug in ``EfficientFrontier.efficient_risk``. An error is now + thrown if optimization fails. +- Added a hidden API to change the scipy optimizer method. 0.5.4 ----- @@ -208,7 +209,7 @@ Began migration to cvxpy by changing the discrete allocation backend from PuLP t modified the linear programming method suggested by `Dingyuan Wang `_; added postprocessing section to User Guide. - Further refactoring and docs for ``HRPOpt``. -- Major documentation update, e.g to support custom optimisers +- Major documentation update, e.g to support custom optimizers 0.4.1 ----- @@ -264,13 +265,13 @@ Refactored shrinkage models, including single factor and constant correlation. 0.2.0 ===== -- Hierarchical Risk Parity optimisation +- Hierarchical Risk Parity optimization - Semicovariance matrix - Exponential covariance matrix -- CVaR optimisation +- CVaR optimization - Better support for custom objective functions - Multiple bug fixes (including minimum volatility vs minimum variance) -- Refactored so all optimisers inherit from a ``BaseOptimizer``. +- Refactored so all optimizers inherit from a ``BaseOptimizer``. 0.2.1 ----- diff --git a/docs/UserGuide.rst b/docs/UserGuide.rst index ee712f8e..d174b803 100644 --- a/docs/UserGuide.rst +++ b/docs/UserGuide.rst @@ -7,15 +7,15 @@ User Guide This is designed to be a practical guide, mostly aimed at users who are interested in a quick way of optimally combining some assets (most likely stocks). However, when necessary I do introduce the required theory and also point out areas that may be -suitable springboards for more advanced optimisation techniques. Details about the +suitable springboards for more advanced optimization techniques. Details about the parameters can be found in the respective documentation pages (please see the sidebar). -For this guide, we will be focusing on mean-variance optimisation (MVO), which is what -most people think of when they hear "portfolio optimisation". MVO forms the core of +For this guide, we will be focusing on mean-variance optimization (MVO), which is what +most people think of when they hear "portfolio optimization". MVO forms the core of PyPortfolioOpt's offering, though it should be noted that MVO comes in many flavours, which can have very different performance characteristics. Please refer to the sidebar -to get a feeling for the possibilities, as well as the other optimisation methods -offered. But for now, we will continue with the Efficient Frontier. +to get a feeling for the possibilities, as well as the other optimization methods +offered. But for now, we will continue with the standard Efficient Frontier. PyPortfolioOpt is designed with modularity in mind; the below flowchart sums up the current functionality and overall layout of PyPortfolioOpt. @@ -26,7 +26,7 @@ current functionality and overall layout of PyPortfolioOpt. Processing historical prices ============================ -Efficient frontier optimisation requires two things: the expected returns of the assets, +Mean-variance optimization requires two things: the expected returns of the assets, and the covariance matrix (or more generally, a *risk model* quantifying asset risk). PyPortfolioOpt provides methods for estimating both (located in :py:mod:`expected_returns` and :py:mod:`risk_models` respectively), but also supports @@ -81,13 +81,13 @@ and ``S`` will be the estimated covariance matrix (part of it is shown below):: Now that we have expected returns and a risk model, we are ready to move on to the -actual portfolio optimisation. +actual portfolio optimization. -Efficient Frontier Optimisation +Mean-variance optimization =============================== -Efficient Frontier Optimisation is based on Harry Markowitz's 1952 classic paper [1]_, which +Mean-variance optimization is based on Harry Markowitz's 1952 classic paper [1]_, which spearheaded the transformation of portfolio management from an art into a science. The key insight is that by combining assets with different expected returns and volatilities, one can decide on a mathematically optimal allocation. @@ -95,10 +95,10 @@ mathematically optimal allocation. If :math:`w` is the weight vector of stocks with expected returns :math:`\mu`, then the portfolio return is equal to each stock's weight multiplied by its return, i.e :math:`w^T \mu`. The portfolio risk in terms of the covariance matrix :math:`\Sigma` -is given by :math:`w^T \Sigma w`. Portfolio optimisation can then be regarded as a -convex optimisation problem, and a solution can be found using quadratic programming. +is given by :math:`w^T \Sigma w`. Portfolio optimization can then be regarded as a +convex optimization problem, and a solution can be found using quadratic programming. If we denote the target return as :math:`\mu^*`, the precise statement of the long-only -portfolio optimisation problem is as follows: +portfolio optimization problem is as follows: .. math:: @@ -122,9 +122,9 @@ portfolio) – the set of all these optimal portfolios is referred to as the Each dot on this diagram represents a different possible portfolio, with darker blue corresponding to 'better' portfolios (in terms of the Sharpe Ratio). The dotted black line is the efficient frontier itself. The triangular markers represent the -best portfolios for different optimisation objectives. +best portfolios for different optimization objectives. -The Sharpe ratio is the portfolio's return less the risk-free rate, per unit risk +The Sharpe ratio is the portfolio's return in excess of the risk-free rate, per unit risk (volatility). .. math:: @@ -143,7 +143,7 @@ dataframe ``S`` from before:: weights = ef.max_sharpe() If you print these weights, you will get quite an ugly result, because they will -be the raw output from the optimiser. As such, it is recommended that you use +be the raw output from the optimizer. As such, it is recommended that you use the :py:meth:`clean_weights` method, which truncates tiny weights to zero and rounds the rest:: @@ -186,7 +186,7 @@ weights ``w``, we can use the :py:meth:`portfolio_performance` method:: Annual volatility: 21.7% Sharpe Ratio: 1.43 -A detailed discussion of optimisation parameters is presented in +A detailed discussion of optimization parameters is presented in :ref:`efficient-frontier`. However, there are two main variations which are discussed below. @@ -201,7 +201,7 @@ with bounds that allow negative weights, for example:: This can be extended to generate **market neutral portfolios** (with weights summing to zero), but these are only available for the :py:meth:`efficient_risk` -and :py:meth:`efficient_return` optimisation methods for mathematical reasons. +and :py:meth:`efficient_return` optimization methods for mathematical reasons. If you want a market neutral portfolio, pass ``market_neutral=True`` as shown below:: ef.efficient_return(target_return=0.2, market_neutral=True) @@ -209,13 +209,13 @@ If you want a market neutral portfolio, pass ``market_neutral=True`` as shown be Dealing with many negligible weights ------------------------------------ -From experience, I have found that efficient frontier optimisation often sets many +From experience, I have found that mean-variance optimization often sets many of the asset weights to be zero. This may not be ideal if you need to have a certain number of positions in your portfolio, for diversification purposes or otherwise. To combat this, I have introduced an objective function which borrows the idea of regularisation from machine learning. Essentially, by adding an additional cost -function to the objective, you can 'encourage' the optimiser to choose different +function to the objective, you can 'encourage' the optimizer to choose different weights (mathematical details are provided in the :ref:`L2-Regularisation` section). To use this feature, change the ``gamma`` parameter:: @@ -284,8 +284,8 @@ Improving performance Let's say you have conducted backtests and the results aren't spectacular. What should you try? -- Try the Hierarchical Risk Parity model (see :ref:`other-optimisers`) – which seems - to robustly outperform mean-variance optimisation out of sample. +- Try the Hierarchical Risk Parity model (see :ref:`other-optimizers`) – which seems + to robustly outperform mean-variance optimization out of sample. - Use the Black-Litterman model to construct a more stable model of expected returns. Alternatively, just drop the expected returns altogether! There is a large body of research that suggests that minimum variance portfolios (``ef.min_volatility()``) consistently outperform @@ -299,7 +299,7 @@ in the sidebar to learn more about the parameters and theoretical details of the different models offered by PyPortfolioOpt. If you have any questions, please raise an issue on GitHub and I will try to respond promptly. -If you'd like even more examples, check out the cookbook `recipe `_. +If you'd like even more examples, check out the cookbook `recipe `_. References diff --git a/docs/index.rst b/docs/index.rst index eaa2375e..5a0fbdc9 100755 --- a/docs/index.rst +++ b/docs/index.rst @@ -6,7 +6,7 @@ .. raw:: html - + @@ -27,7 +27,7 @@ -PyPortfolioOpt is a library that implements portfolio optimisation methods, including +PyPortfolioOpt is a library that implements portfolio optimization methods, including classical efficient frontier techniques and Black-Litterman allocation, as well as more recent developments in the field like shrinkage and Hierarchical Risk Parity, along with some novel experimental features like exponentially-weighted covariance matrices. @@ -133,7 +133,7 @@ that's fine too:: mu = expected_returns.mean_historical_return(df) S = risk_models.sample_cov(df) - # Optimise for maximal Sharpe ratio + # Optimize for maximal Sharpe ratio ef = EfficientFrontier(mu, S) weights = ef.max_sharpe() ef.portfolio_performance(verbose=True) @@ -156,9 +156,10 @@ Contents UserGuide ExpectedReturns RiskModels - EfficientFrontier + MeanVariance + GeneralEfficientFrontier BlackLitterman - OtherOptimisers + OtherOptimizers Postprocessing Plotting @@ -166,6 +167,7 @@ Contents :maxdepth: 1 :caption: Other information + FAQ Roadmap Contributing About @@ -173,10 +175,10 @@ Contents Project principles and design decisions ======================================= -- It should be easy to swap out individual components of the optimisation process +- It should be easy to swap out individual components of the optimization process with the user's proprietary improvements. - Usability is everything: it is better to be self-explanatory than consistent. -- There is no point in portfolio optimisation unless it can be practically +- There is no point in portfolio optimization unless it can be practically applied to real asset prices. - Everything that has been implemented should be tested. - Inline documentation is good: dedicated (separate) documentation is better. diff --git a/poetry.lock b/poetry.lock index 8278ac6f..7638f161 100644 --- a/poetry.lock +++ b/poetry.lock @@ -132,12 +132,23 @@ category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +[[package]] +name = "coverage" +version = "5.4" +description = "Code coverage measurement for Python" +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4" + +[package.extras] +toml = ["toml"] + [[package]] name = "cvxopt" version = "1.2.5" description = "Convex optimization package" category = "main" -optional = false +optional = true python-versions = "*" [[package]] @@ -849,6 +860,21 @@ wcwidth = "*" [package.extras] testing = ["argcomplete", "hypothesis (>=3.56)", "nose", "requests", "mock"] +[[package]] +name = "pytest-cov" +version = "2.11.1" +description = "Pytest plugin for measuring coverage." +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" + +[package.dependencies] +coverage = ">=5.2.1" +pytest = ">=4.6" + +[package.extras] +testing = ["fields", "hunter", "process-tests (==2.0.2)", "six", "pytest-xdist", "virtualenv"] + [[package]] name = "python-dateutil" version = "2.8.1" @@ -1125,12 +1151,12 @@ docs = ["sphinx", "jaraco.packaging (>=3.2)", "rst.linker (>=1.9)"] testing = ["pytest (>=3.5,!=3.7.3)", "pytest-checkdocs (>=1.2.3)", "pytest-flake8", "pytest-cov", "jaraco.test (>=3.2.0)", "jaraco.itertools", "func-timeout", "pytest-black (>=0.3.7)", "pytest-mypy"] [extras] -optionals = ["scikit-learn", "matplotlib"] +optionals = ["scikit-learn", "matplotlib", "cvxopt"] [metadata] lock-version = "1.1" python-versions = "^3.6.1" -content-hash = "b4c771e08142d2c77cca2e64bedf9615bf8c7897e8f168d27aeb2ef7a494ce93" +content-hash = "763ef45b15be851b4a994160d27675c8113c363f3f5fd9235a34c55610c46b33" [metadata.files] appdirs = [ @@ -1158,8 +1184,6 @@ argon2-cffi = [ {file = "argon2_cffi-20.1.0-cp37-cp37m-win_amd64.whl", hash = "sha256:6678bb047373f52bcff02db8afab0d2a77d83bde61cfecea7c5c62e2335cb203"}, {file = "argon2_cffi-20.1.0-cp38-cp38-win32.whl", hash = "sha256:77e909cc756ef81d6abb60524d259d959bab384832f0c651ed7dcb6e5ccdbb78"}, {file = "argon2_cffi-20.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:9dfd5197852530294ecb5795c97a823839258dfd5eb9420233c7cfedec2058f2"}, - {file = "argon2_cffi-20.1.0-cp39-cp39-win32.whl", hash = "sha256:e2db6e85c057c16d0bd3b4d2b04f270a7467c147381e8fd73cbbe5bc719832be"}, - {file = "argon2_cffi-20.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:8a84934bd818e14a17943de8099d41160da4a336bcc699bb4c394bbb9b94bd32"}, ] async-generator = [ {file = "async_generator-1.10-py3-none-any.whl", hash = "sha256:01c7bf666359b4967d2cda0000cc2e4af16a0ae098cbffcb8472fb9e8ad6585b"}, @@ -1231,6 +1255,57 @@ colorama = [ {file = "colorama-0.4.4-py2.py3-none-any.whl", hash = "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2"}, {file = "colorama-0.4.4.tar.gz", hash = "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b"}, ] +coverage = [ + {file = "coverage-5.4-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:6d9c88b787638a451f41f97446a1c9fd416e669b4d9717ae4615bd29de1ac135"}, + {file = "coverage-5.4-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:66a5aae8233d766a877c5ef293ec5ab9520929c2578fd2069308a98b7374ea8c"}, + {file = "coverage-5.4-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:9754a5c265f991317de2bac0c70a746efc2b695cf4d49f5d2cddeac36544fb44"}, + {file = "coverage-5.4-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:fbb17c0d0822684b7d6c09915677a32319f16ff1115df5ec05bdcaaee40b35f3"}, + {file = "coverage-5.4-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:b7f7421841f8db443855d2854e25914a79a1ff48ae92f70d0a5c2f8907ab98c9"}, + {file = "coverage-5.4-cp27-cp27m-win32.whl", hash = "sha256:4a780807e80479f281d47ee4af2eb2df3e4ccf4723484f77da0bb49d027e40a1"}, + {file = "coverage-5.4-cp27-cp27m-win_amd64.whl", hash = "sha256:87c4b38288f71acd2106f5d94f575bc2136ea2887fdb5dfe18003c881fa6b370"}, + {file = "coverage-5.4-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:c6809ebcbf6c1049002b9ac09c127ae43929042ec1f1dbd8bb1615f7cd9f70a0"}, + {file = "coverage-5.4-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:ba7ca81b6d60a9f7a0b4b4e175dcc38e8fef4992673d9d6e6879fd6de00dd9b8"}, + {file = "coverage-5.4-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:89fc12c6371bf963809abc46cced4a01ca4f99cba17be5e7d416ed7ef1245d19"}, + {file = "coverage-5.4-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:4a8eb7785bd23565b542b01fb39115a975fefb4a82f23d407503eee2c0106247"}, + {file = "coverage-5.4-cp35-cp35m-macosx_10_9_x86_64.whl", hash = "sha256:7e40d3f8eb472c1509b12ac2a7e24158ec352fc8567b77ab02c0db053927e339"}, + {file = "coverage-5.4-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:1ccae21a076d3d5f471700f6d30eb486da1626c380b23c70ae32ab823e453337"}, + {file = "coverage-5.4-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:755c56beeacac6a24c8e1074f89f34f4373abce8b662470d3aa719ae304931f3"}, + {file = "coverage-5.4-cp35-cp35m-manylinux2010_i686.whl", hash = "sha256:322549b880b2d746a7672bf6ff9ed3f895e9c9f108b714e7360292aa5c5d7cf4"}, + {file = "coverage-5.4-cp35-cp35m-manylinux2010_x86_64.whl", hash = "sha256:60a3307a84ec60578accd35d7f0c71a3a971430ed7eca6567399d2b50ef37b8c"}, + {file = "coverage-5.4-cp35-cp35m-win32.whl", hash = "sha256:1375bb8b88cb050a2d4e0da901001347a44302aeadb8ceb4b6e5aa373b8ea68f"}, + {file = "coverage-5.4-cp35-cp35m-win_amd64.whl", hash = "sha256:16baa799ec09cc0dcb43a10680573269d407c159325972dd7114ee7649e56c66"}, + {file = "coverage-5.4-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:2f2cf7a42d4b7654c9a67b9d091ec24374f7c58794858bff632a2039cb15984d"}, + {file = "coverage-5.4-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:b62046592b44263fa7570f1117d372ae3f310222af1fc1407416f037fb3af21b"}, + {file = "coverage-5.4-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:812eaf4939ef2284d29653bcfee9665f11f013724f07258928f849a2306ea9f9"}, + {file = "coverage-5.4-cp36-cp36m-manylinux2010_i686.whl", hash = "sha256:859f0add98707b182b4867359e12bde806b82483fb12a9ae868a77880fc3b7af"}, + {file = "coverage-5.4-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:04b14e45d6a8e159c9767ae57ecb34563ad93440fc1b26516a89ceb5b33c1ad5"}, + {file = "coverage-5.4-cp36-cp36m-win32.whl", hash = "sha256:ebfa374067af240d079ef97b8064478f3bf71038b78b017eb6ec93ede1b6bcec"}, + {file = "coverage-5.4-cp36-cp36m-win_amd64.whl", hash = "sha256:84df004223fd0550d0ea7a37882e5c889f3c6d45535c639ce9802293b39cd5c9"}, + {file = "coverage-5.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:1b811662ecf72eb2d08872731636aee6559cae21862c36f74703be727b45df90"}, + {file = "coverage-5.4-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:6b588b5cf51dc0fd1c9e19f622457cc74b7d26fe295432e434525f1c0fae02bc"}, + {file = "coverage-5.4-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:3fe50f1cac369b02d34ad904dfe0771acc483f82a1b54c5e93632916ba847b37"}, + {file = "coverage-5.4-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:32ab83016c24c5cf3db2943286b85b0a172dae08c58d0f53875235219b676409"}, + {file = "coverage-5.4-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:68fb816a5dd901c6aff352ce49e2a0ffadacdf9b6fae282a69e7a16a02dad5fb"}, + {file = "coverage-5.4-cp37-cp37m-win32.whl", hash = "sha256:a636160680c6e526b84f85d304e2f0bb4e94f8284dd765a1911de9a40450b10a"}, + {file = "coverage-5.4-cp37-cp37m-win_amd64.whl", hash = "sha256:bb32ca14b4d04e172c541c69eec5f385f9a075b38fb22d765d8b0ce3af3a0c22"}, + {file = "coverage-5.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4d7165a4e8f41eca6b990c12ee7f44fef3932fac48ca32cecb3a1b2223c21f"}, + {file = "coverage-5.4-cp38-cp38-manylinux1_i686.whl", hash = "sha256:a565f48c4aae72d1d3d3f8e8fb7218f5609c964e9c6f68604608e5958b9c60c3"}, + {file = "coverage-5.4-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:fff1f3a586246110f34dc762098b5afd2de88de507559e63553d7da643053786"}, + {file = "coverage-5.4-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:a839e25f07e428a87d17d857d9935dd743130e77ff46524abb992b962eb2076c"}, + {file = "coverage-5.4-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:6625e52b6f346a283c3d563d1fd8bae8956daafc64bb5bbd2b8f8a07608e3994"}, + {file = "coverage-5.4-cp38-cp38-win32.whl", hash = "sha256:5bee3970617b3d74759b2d2df2f6a327d372f9732f9ccbf03fa591b5f7581e39"}, + {file = "coverage-5.4-cp38-cp38-win_amd64.whl", hash = "sha256:03ed2a641e412e42cc35c244508cf186015c217f0e4d496bf6d7078ebe837ae7"}, + {file = "coverage-5.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:14a9f1887591684fb59fdba8feef7123a0da2424b0652e1b58dd5b9a7bb1188c"}, + {file = "coverage-5.4-cp39-cp39-manylinux1_i686.whl", hash = "sha256:9564ac7eb1652c3701ac691ca72934dd3009997c81266807aef924012df2f4b3"}, + {file = "coverage-5.4-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:0f48fc7dc82ee14aeaedb986e175a429d24129b7eada1b7e94a864e4f0644dde"}, + {file = "coverage-5.4-cp39-cp39-manylinux2010_i686.whl", hash = "sha256:107d327071061fd4f4a2587d14c389a27e4e5c93c7cba5f1f59987181903902f"}, + {file = "coverage-5.4-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:0cdde51bfcf6b6bd862ee9be324521ec619b20590787d1655d005c3fb175005f"}, + {file = "coverage-5.4-cp39-cp39-win32.whl", hash = "sha256:c67734cff78383a1f23ceba3b3239c7deefc62ac2b05fa6a47bcd565771e5880"}, + {file = "coverage-5.4-cp39-cp39-win_amd64.whl", hash = "sha256:c669b440ce46ae3abe9b2d44a913b5fd86bb19eb14a8701e88e3918902ecd345"}, + {file = "coverage-5.4-pp36-none-any.whl", hash = "sha256:c0ff1c1b4d13e2240821ef23c1efb1f009207cb3f56e16986f713c2b0e7cd37f"}, + {file = "coverage-5.4-pp37-none-any.whl", hash = "sha256:cd601187476c6bed26a0398353212684c427e10a903aeafa6da40c63309d438b"}, + {file = "coverage-5.4.tar.gz", hash = "sha256:6d2e262e5e8da6fa56e774fb8e2643417351427604c2b177f8e8c5f75fc928ca"}, +] cvxopt = [ {file = "cvxopt-1.2.5-cp27-cp27m-macosx_10_6_intel.whl", hash = "sha256:78d0ea93f71f4b5b882ecd741ddc1e0cc8b06f4a2881f9d3fa55631d633273b6"}, {file = "cvxopt-1.2.5-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:fcc447eb3a0465f9d6b78f3df9861cb6fdb18ff7ad008e17f614da580e1fdbee"}, @@ -1250,8 +1325,6 @@ cvxopt = [ {file = "cvxopt-1.2.5-cp37-cp37m-win_amd64.whl", hash = "sha256:dd2c97ace2739f31bceadd29e0e9535fe7457d756a6d9d207e1a654fd1ad1dc2"}, {file = "cvxopt-1.2.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:37768ccefd3aec0dd9428d5eaa1a075d2db1d485e9b93c4d5d02464297a6530a"}, {file = "cvxopt-1.2.5-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:3446b360c4123949511b4aef0b3d0bcdd3ce485b8e682c221a39c886945eacd3"}, - {file = "cvxopt-1.2.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:75e136dca30e8d69bda891ec59fa21877553ba54c095f7345f7030bef1cdbf93"}, - {file = "cvxopt-1.2.5-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:5a7b4ee00b1c3030a6e1a4707f636a44534def69286265f15214a3a3957fbc08"}, {file = "cvxopt-1.2.5.tar.gz", hash = "sha256:94ec8c36bd6628a11de9014346692daeeef99b3b7bae28cef30c7490bbcb2d72"}, ] cvxpy = [ @@ -1439,11 +1512,6 @@ markupsafe = [ {file = "MarkupSafe-1.1.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:ba59edeaa2fc6114428f1637ffff42da1e311e29382d81b339c1817d37ec93c6"}, {file = "MarkupSafe-1.1.1-cp37-cp37m-win32.whl", hash = "sha256:b00c1de48212e4cc9603895652c5c410df699856a2853135b3967591e4beebc2"}, {file = "MarkupSafe-1.1.1-cp37-cp37m-win_amd64.whl", hash = "sha256:9bf40443012702a1d2070043cb6291650a0841ece432556f784f004937f0f32c"}, - {file = "MarkupSafe-1.1.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6788b695d50a51edb699cb55e35487e430fa21f1ed838122d722e0ff0ac5ba15"}, - {file = "MarkupSafe-1.1.1-cp38-cp38-manylinux1_i686.whl", hash = "sha256:cdb132fc825c38e1aeec2c8aa9338310d29d337bebbd7baa06889d09a60a1fa2"}, - {file = "MarkupSafe-1.1.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:13d3144e1e340870b25e7b10b98d779608c02016d5184cfb9927a9f10c689f42"}, - {file = "MarkupSafe-1.1.1-cp38-cp38-win32.whl", hash = "sha256:596510de112c685489095da617b5bcbbac7dd6384aeebeda4df6025d0256a81b"}, - {file = "MarkupSafe-1.1.1-cp38-cp38-win_amd64.whl", hash = "sha256:e8313f01ba26fbbe36c7be1966a7b7424942f670f38e666995b88d012765b9be"}, {file = "MarkupSafe-1.1.1.tar.gz", hash = "sha256:29872e92839765e546828bb7754a68c418d927cd064fd4708fab9fe9c8bb116b"}, ] matplotlib = [ @@ -1698,6 +1766,10 @@ pytest = [ {file = "pytest-4.6.11-py2.py3-none-any.whl", hash = "sha256:a00a7d79cbbdfa9d21e7d0298392a8dd4123316bfac545075e6f8f24c94d8c97"}, {file = "pytest-4.6.11.tar.gz", hash = "sha256:50fa82392f2120cc3ec2ca0a75ee615be4c479e66669789771f1758332be4353"}, ] +pytest-cov = [ + {file = "pytest-cov-2.11.1.tar.gz", hash = "sha256:359952d9d39b9f822d9d29324483e7ba04a3a17dd7d05aa6beb7ea01e359e5f7"}, + {file = "pytest_cov-2.11.1-py2.py3-none-any.whl", hash = "sha256:bdb9fdb0b85a7cc825269a4c56b48ccaa5c7e365054b6038772c32ddcdc969da"}, +] python-dateutil = [ {file = "python-dateutil-2.8.1.tar.gz", hash = "sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c"}, {file = "python_dateutil-2.8.1-py2.py3-none-any.whl", hash = "sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a"}, diff --git a/pypfopt/base_optimizer.py b/pypfopt/base_optimizer.py index 1d57a154..fc479fc5 100644 --- a/pypfopt/base_optimizer.py +++ b/pypfopt/base_optimizer.py @@ -1,7 +1,7 @@ """ The ``base_optimizer`` module houses the parent classes ``BaseOptimizer`` from which all -optimisers will inherit. ``BaseConvexOptimizer`` is the base class for all ``cvxpy`` (and ``scipy``) -optimisation. +optimizers will inherit. ``BaseConvexOptimizer`` is the base class for all ``cvxpy`` (and ``scipy``) +optimization. Additionally, we define a general utility function ``portfolio_performance`` to evaluate return and risk for a given set of portfolio weights. @@ -119,7 +119,7 @@ class BaseConvexOptimizer(BaseOptimizer): """ The BaseConvexOptimizer contains many private variables for use by - ``cvxpy``. For example, the immutable optimisation variable for weights + ``cvxpy``. For example, the immutable optimization variable for weights is stored as self._w. Interacting directly with these variables directly is discouraged. @@ -134,8 +134,8 @@ class BaseConvexOptimizer(BaseOptimizer): Public methods: - - ``add_objective()`` adds a (convex) objective to the optimisation problem - - ``add_constraint()`` adds a (linear) constraint to the optimisation problem + - ``add_objective()`` adds a (convex) objective to the optimization problem + - ``add_constraint()`` adds a constraint to the optimization problem - ``convex_objective()`` solves for a generic convex objective with linear constraints - ``nonconvex_objective()`` solves for a generic nonconvex objective using the scipy backend. This is prone to getting stuck in local minima and is generally *not* recommended. @@ -167,7 +167,7 @@ def __init__( """ super().__init__(n_assets, tickers) - # Optimisation variables + # Optimization variables self._w = cp.Variable(n_assets) self._objective = None self._additional_objectives = [] @@ -266,8 +266,8 @@ def L1_norm(w, k=1): def add_constraint(self, new_constraint): """ - Add a new constraint to the optimisation problem. This constraint must be linear and - must be either an equality or simple inequality. + Add a new constraint to the optimization problem. This constraint must satisfy DCP rules, + i.e be either a linear equality constraint or convex inequality constraint. Examples:: @@ -323,12 +323,12 @@ def add_sector_constraints(self, sector_mapper, sector_lower, sector_upper): def convex_objective(self, custom_objective, weights_sum_to_one=True, **kwargs): """ - Optimise a custom convex objective function. Constraints should be added with - ``ef.add_constraint()``. Optimiser arguments must be passed as keyword-args. Example:: + Optimize a custom convex objective function. Constraints should be added with + ``ef.add_constraint()``. Optimizer arguments must be passed as keyword-args. Example:: # Could define as a lambda function instead def logarithmic_barrier(w, cov_matrix, k=0.1): - # 60 Years of Portfolio Optimisation, Kolm et al (2014) + # 60 Years of Portfolio Optimization, Kolm et al (2014) return cp.quad_form(w, cov_matrix) - k * cp.sum(cp.log(w)) w = ef.convex_objective(logarithmic_barrier, cov_matrix=ef.cov_matrix) @@ -363,7 +363,7 @@ def nonconvex_objective( initial_guess=None, ): """ - Optimise some objective function using the scipy backend. This can + Optimize some objective function using the scipy backend. This can support nonconvex objectives and nonlinear constraints, but may get stuck at local minima. Example:: @@ -392,11 +392,11 @@ def nonconvex_objective( :param constraints: list of constraints in the scipy format (i.e dicts) :type constraints: dict list :param solver: which SCIPY solver to use, e.g "SLSQP", "COBYLA", "BFGS". - User beware: different optimisers require different inputs. + User beware: different optimizers require different inputs. :type solver: string :param initial_guess: the initial guess for the weights, shape (n,) or (n, 1) :type initial_guess: np.ndarray - :return: asset weights that optimise the custom objective + :return: asset weights that optimize the custom objective :rtype: OrderedDict """ # Sanitise inputs diff --git a/pypfopt/black_litterman.py b/pypfopt/black_litterman.py index 3472da95..50ee3c85 100644 --- a/pypfopt/black_litterman.py +++ b/pypfopt/black_litterman.py @@ -168,7 +168,7 @@ def __init__( :param risk_free_rate: (kwarg) risk_free_rate is needed in some methods :type risk_free_rate: float, defaults to 0.02 """ - if sys.version_info[1] == 5: # if python 3.5 + if sys.version_info[1] == 5: # pragma: no cover warnings.warn( "When using python 3.5 you must explicitly construct the Black-Litterman inputs" ) @@ -176,7 +176,7 @@ def __init__( # Keep raw dataframes self._raw_cov_matrix = cov_matrix - #  Initialise base optimiser + #  Initialise base optimizer if isinstance(cov_matrix, np.ndarray): self.cov_matrix = cov_matrix super().__init__(len(cov_matrix), list(range(len(cov_matrix)))) @@ -430,7 +430,7 @@ def bl_weights(self, risk_aversion=None): Compute the weights implied by the posterior returns, given the market price of risk. Technically this can be applied to any estimate of the expected returns, and is in fact a special case - of efficient frontier optimisation. + of mean-variance optimization .. math:: diff --git a/pypfopt/cla.py b/pypfopt/cla.py index c2079047..9dd8374e 100644 --- a/pypfopt/cla.py +++ b/pypfopt/cla.py @@ -24,7 +24,7 @@ class CLA(base_optimizer.BaseOptimizer): - ``lb`` - np.ndarray - ``ub`` - np.ndarray - - Optimisation parameters: + - Optimization parameters: - ``w`` - np.ndarray list - ``ls`` - float list @@ -38,11 +38,11 @@ class CLA(base_optimizer.BaseOptimizer): Public methods: - - ``max_sharpe()`` optimises for maximal Sharpe ratio (a.k.a the tangency portfolio) - - ``min_volatility()`` optimises for minimum volatility + - ``max_sharpe()`` optimizes for maximal Sharpe ratio (a.k.a the tangency portfolio) + - ``min_volatility()`` optimizes for minimum volatility - ``efficient_frontier()`` computes the entire efficient frontier - ``portfolio_performance()`` calculates the expected return, volatility and Sharpe ratio for - the optimised portfolio. + the optimized portfolio. - ``clean_weights()`` rounds the weights and clips near-zeros. - ``save_weights_to_file()`` saves the weights to csv, json, or txt. """ @@ -56,14 +56,14 @@ def __init__(self, expected_returns, cov_matrix, weight_bounds=(0, 1)): :type cov_matrix: pd.DataFrame or np.array :param weight_bounds: minimum and maximum weight of an asset, defaults to (0, 1). Must be changed to (-1, 1) for portfolios with shorting. - :type weight_bounds: tuple (float, float) or (list/ndarray, list/ndarray) + :type weight_bounds: tuple (float, float) or (list/ndarray, list/ndarray) or list(tuple(float, float)) :raises TypeError: if ``expected_returns`` is not a series, list or array :raises TypeError: if ``cov_matrix`` is not a dataframe or array """ # Initialize the class self.mean = np.array(expected_returns).reshape((len(expected_returns), 1)) - if (self.mean == np.ones(self.mean.shape) * self.mean.mean()).all(): - self.mean[-1, 0] += 1e-5 + # if (self.mean == np.ones(self.mean.shape) * self.mean.mean()).all(): + # self.mean[-1, 0] += 1e-5 self.expected_returns = self.mean.reshape((len(self.mean),)) self.cov_matrix = np.asarray(cov_matrix) @@ -159,7 +159,7 @@ def _compute_lambda(self, covarF_inv, covarFB, meanF, wB, i, bi): c3 = np.dot(np.dot(onesF.T, covarF_inv), meanF) c4 = np.dot(covarF_inv, onesF) c = -c1 * c2[i] + c3 * c4[i] - if c == 0: + if c == 0: # pragma: no cover return None, None # 2) bi if type(bi) == list: @@ -221,7 +221,7 @@ def _purge_num_err(self, tol): if ( self.w[i][j] - self.lB[j] < -tol or self.w[i][j] - self.uB[j] > tol - ): + ): #  pragma: no cover flag = True break if flag is True: diff --git a/pypfopt/discrete_allocation.py b/pypfopt/discrete_allocation.py index ce1fbbba..20ff4c62 100644 --- a/pypfopt/discrete_allocation.py +++ b/pypfopt/discrete_allocation.py @@ -162,9 +162,7 @@ def greedy_portfolio(self, reinvest=False, verbose=False): if verbose: print("\nAllocating long sub-portfolio...") da1 = DiscreteAllocation( - longs, - self.latest_prices[longs.keys()], - total_portfolio_value=long_val, + longs, self.latest_prices[longs.keys()], total_portfolio_value=long_val ) long_alloc, long_leftover = da1.greedy_portfolio() @@ -193,14 +191,13 @@ def greedy_portfolio(self, reinvest=False, verbose=False): # First round for ticker, weight in self.weights: price = self.latest_prices[ticker] - # Attempt to buy the lower integer number of shares + # Attempt to buy the lower integer number of shares, which could be zero. n_shares = int(weight * self.total_portfolio_value / price) cost = n_shares * price - if cost > available_funds: - # Buy as many as possible - n_shares = available_funds // price - if n_shares == 0: - print("Insufficient funds") + # As weights are all > 0 (long only) we always round down n_shares + # so the cost is always <= simple weighted share of portfolio value, + # so we can not run out of funds just here. + assert cost <= available_funds, "Unexpectedly insufficient funds." available_funds -= cost shares_bought.append(n_shares) buy_prices.append(price) @@ -235,7 +232,7 @@ def greedy_portfolio(self, reinvest=False, verbose=False): price = self.latest_prices[ticker] counter += 1 - if deficit[idx] <= 0 or counter == 10: + if deficit[idx] <= 0 or counter == 10: # pragma: no cover # Dirty solution to break out of both loops break @@ -286,9 +283,7 @@ def lp_portfolio(self, reinvest=False, verbose=False, solver="GLPK_MI"): if verbose: print("\nAllocating long sub-portfolio:") da1 = DiscreteAllocation( - longs, - self.latest_prices[longs.keys()], - total_portfolio_value=long_val, + longs, self.latest_prices[longs.keys()], total_portfolio_value=long_val ) long_alloc, long_leftover = da1.lp_portfolio() @@ -325,11 +320,11 @@ def lp_portfolio(self, reinvest=False, verbose=False, solver="GLPK_MI"): opt = cp.Problem(cp.Minimize(objective), constraints) - if solver not in cp.installed_solvers(): + if solver is not None and solver not in cp.installed_solvers(): raise NameError("Solver {} is not installed. ".format(solver)) opt.solve(solver=solver) - if opt.status not in {"optimal", "optimal_inaccurate"}: + if opt.status not in {"optimal", "optimal_inaccurate"}: # pragma: no cover raise exceptions.OptimizationError("Please try greedy_portfolio") vals = np.rint(x.value).astype(int) diff --git a/pypfopt/efficient_frontier.py b/pypfopt/efficient_frontier.py deleted file mode 100644 index 8c498992..00000000 --- a/pypfopt/efficient_frontier.py +++ /dev/null @@ -1,876 +0,0 @@ -""" -The ``efficient_frontier`` module houses the EfficientFrontier class and its descendants, -which generate optimal portfolios for various possible objective functions and parameters. -""" - -import warnings -import numpy as np -import pandas as pd -import cvxpy as cp - -from . import objective_functions, base_optimizer - - -class EfficientFrontier(base_optimizer.BaseConvexOptimizer): - - """ - An EfficientFrontier object (inheriting from BaseConvexOptimizer) contains multiple - optimisation methods that can be called (corresponding to different objective - functions) with various parameters. Note: a new EfficientFrontier object should - be instantiated if you want to make any change to objectives/constraints/bounds/parameters. - - Instance variables: - - - Inputs: - - - ``n_assets`` - int - - ``tickers`` - str list - - ``bounds`` - float tuple OR (float tuple) list - - ``cov_matrix`` - np.ndarray - - ``expected_returns`` - np.ndarray - - ``solver`` - str - - ``solver_options`` - {str: str} dict - - - Output: ``weights`` - np.ndarray - - Public methods: - - - ``min_volatility()`` optimises for minimum volatility - - ``max_sharpe()`` optimises for maximal Sharpe ratio (a.k.a the tangency portfolio) - - ``max_quadratic_utility()`` maximises the quadratic utility, given some risk aversion. - - ``efficient_risk()`` maximises return for a given target risk - - ``efficient_return()`` minimises risk for a given target return - - - ``add_objective()`` adds a (convex) objective to the optimisation problem - - ``add_constraint()`` adds a (linear) constraint to the optimisation problem - - ``convex_objective()`` solves for a generic convex objective with linear constraints - - - ``portfolio_performance()`` calculates the expected return, volatility and Sharpe ratio for - the optimised portfolio. - - ``set_weights()`` creates self.weights (np.ndarray) from a weights dict - - ``clean_weights()`` rounds the weights and clips near-zeros. - - ``save_weights_to_file()`` saves the weights to csv, json, or txt. - """ - - def __init__( - self, - expected_returns, - cov_matrix, - weight_bounds=(0, 1), - solver=None, - verbose=False, - solver_options=None, - ): - """ - :param expected_returns: expected returns for each asset. Can be None if - optimising for volatility only (but not recommended). - :type expected_returns: pd.Series, list, np.ndarray - :param cov_matrix: covariance of returns for each asset. This **must** be - positive semidefinite, otherwise optimisation will fail. - :type cov_matrix: pd.DataFrame or np.array - :param weight_bounds: minimum and maximum weight of each asset OR single min/max pair - if all identical, defaults to (0, 1). Must be changed to (-1, 1) - for portfolios with shorting. - :type weight_bounds: tuple OR tuple list, optional - :param solver: name of solver. list available solvers with: `cvxpy.installed_solvers()` - :type solver: str - :param verbose: whether performance and debugging info should be printed, defaults to False - :type verbose: bool, optional - :param solver_options: parameters for the given solver - :type solver_options: dict, optional - :raises TypeError: if ``expected_returns`` is not a series, list or array - :raises TypeError: if ``cov_matrix`` is not a dataframe or array - """ - # Inputs - self.cov_matrix = EfficientFrontier._validate_cov_matrix(cov_matrix) - self.expected_returns = EfficientFrontier._validate_expected_returns( - expected_returns - ) - - # Labels - if isinstance(expected_returns, pd.Series): - tickers = list(expected_returns.index) - elif isinstance(cov_matrix, pd.DataFrame): - tickers = list(cov_matrix.columns) - else: # use integer labels - tickers = list(range(len(expected_returns))) - - if expected_returns is not None and cov_matrix is not None: - if cov_matrix.shape != (len(expected_returns), len(expected_returns)): - raise ValueError("Covariance matrix does not match expected returns") - - super().__init__( - len(tickers), - tickers, - weight_bounds, - solver=solver, - verbose=verbose, - solver_options=solver_options, - ) - - @staticmethod - def _validate_expected_returns(expected_returns): - if expected_returns is None: - return None - elif isinstance(expected_returns, pd.Series): - return expected_returns.values - elif isinstance(expected_returns, list): - return np.array(expected_returns) - elif isinstance(expected_returns, np.ndarray): - return expected_returns.ravel() - else: - raise TypeError("expected_returns is not a series, list or array") - - @staticmethod - def _validate_cov_matrix(cov_matrix): - if cov_matrix is None: - raise ValueError("cov_matrix must be provided") - elif isinstance(cov_matrix, pd.DataFrame): - return cov_matrix.values - elif isinstance(cov_matrix, np.ndarray): - return cov_matrix - else: - raise TypeError("cov_matrix is not a dataframe or array") - - def _validate_returns(self, returns): - """ - Helper method to validate daily returns (needed for some efficient frontiers) - """ - if not isinstance(returns, (pd.DataFrame, np.ndarray)): - raise TypeError("returns should be a pd.Dataframe or np.ndarray") - - returns_df = pd.DataFrame(returns) - if returns_df.isnull().values.any(): - warnings.warn( - "Removing NaNs from returns", - UserWarning, - ) - returns_df = returns_df.dropna(axis=0, how="any") - - if self.expected_returns is not None: - if returns_df.shape[1] != len(self.expected_returns): - raise ValueError( - "returns columns do not match expected_returns. Please check your tickers." - ) - - return returns_df - - def _make_weight_sum_constraint(self, is_market_neutral): - """ - Helper method to make the weight sum constraint. If market neutral, - validate the weights proided in the constructor. - """ - if is_market_neutral: - #  Check and fix bounds - portfolio_possible = np.any(self._lower_bounds < 0) - if not portfolio_possible: - warnings.warn( - "Market neutrality requires shorting - bounds have been amended", - RuntimeWarning, - ) - self._map_bounds_to_constraints((-1, 1)) - # Delete original constraints - del self._constraints[0] - del self._constraints[0] - - self._constraints.append(cp.sum(self._w) == 0) - else: - self._constraints.append(cp.sum(self._w) == 1) - - def min_volatility(self): - """ - Minimise volatility. - - :return: asset weights for the volatility-minimising portfolio - :rtype: OrderedDict - """ - self._objective = objective_functions.portfolio_variance( - self._w, self.cov_matrix - ) - for obj in self._additional_objectives: - self._objective += obj - - self._constraints.append(cp.sum(self._w) == 1) - return self._solve_cvxpy_opt_problem() - - def _max_return(self, return_value=True): - """ - Helper method to maximise return. This should not be used to optimise a portfolio. - - :return: asset weights for the return-minimising portfolio - :rtype: OrderedDict - """ - self._objective = objective_functions.portfolio_return( - self._w, self.expected_returns - ) - - self._constraints.append(cp.sum(self._w) == 1) - - res = self._solve_cvxpy_opt_problem() - - #  Cleanup constraints since this is a helper method - del self._constraints[-1] - - if return_value: - return -self._opt.value - else: - return res - - def max_sharpe(self, risk_free_rate=0.02): - """ - Maximise the Sharpe Ratio. The result is also referred to as the tangency portfolio, - as it is the portfolio for which the capital market line is tangent to the efficient frontier. - - This is a convex optimisation problem after making a certain variable substitution. See - `Cornuejols and Tutuncu (2006) `_ for more. - - :param risk_free_rate: risk-free rate of borrowing/lending, defaults to 0.02. - The period of the risk-free rate should correspond to the - frequency of expected returns. - :type risk_free_rate: float, optional - :raises ValueError: if ``risk_free_rate`` is non-numeric - :return: asset weights for the Sharpe-maximising portfolio - :rtype: OrderedDict - """ - if not isinstance(risk_free_rate, (int, float)): - raise ValueError("risk_free_rate should be numeric") - self._risk_free_rate = risk_free_rate - - # max_sharpe requires us to make a variable transformation. - # Here we treat w as the transformed variable. - self._objective = cp.quad_form(self._w, self.cov_matrix) - k = cp.Variable() - - # Note: objectives are not scaled by k. Hence there are subtle differences - # between how these objectives work for max_sharpe vs min_volatility - if len(self._additional_objectives) > 0: - warnings.warn( - "max_sharpe transforms the optimisation problem so additional objectives may not work as expected." - ) - for obj in self._additional_objectives: - self._objective += obj - - new_constraints = [] - # Must rebuild the constraints - for constr in self._constraints: - if isinstance(constr, cp.constraints.nonpos.Inequality): - # Either the first or second item is the expression - if isinstance( - constr.args[0], cp.expressions.constants.constant.Constant - ): - new_constraints.append(constr.args[1] >= constr.args[0] * k) - else: - new_constraints.append(constr.args[0] <= constr.args[1] * k) - elif isinstance(constr, cp.constraints.zero.Equality): - new_constraints.append(constr.args[0] == constr.args[1] * k) - else: - raise TypeError( - "Please check that your constraints are in a suitable format" - ) - - # Transformed max_sharpe convex problem: - self._constraints = [ - (self.expected_returns - risk_free_rate).T @ self._w == 1, - cp.sum(self._w) == k, - k >= 0, - ] + new_constraints - - self._solve_cvxpy_opt_problem() - # Inverse-transform - self.weights = (self._w.value / k.value).round(16) + 0.0 - return self._make_output_weights() - - def max_quadratic_utility(self, risk_aversion=1, market_neutral=False): - r""" - Maximise the given quadratic utility, i.e: - - .. math:: - - \max_w w^T \mu - \frac \delta 2 w^T \Sigma w - - :param risk_aversion: risk aversion parameter (must be greater than 0), - defaults to 1 - :type risk_aversion: positive float - :param market_neutral: whether the portfolio should be market neutral (weights sum to zero), - defaults to False. Requires negative lower weight bound. - :param market_neutral: bool, optional - :return: asset weights for the maximum-utility portfolio - :rtype: OrderedDict - """ - if risk_aversion <= 0: - raise ValueError("risk aversion coefficient must be greater than zero") - - self._objective = objective_functions.quadratic_utility( - self._w, self.expected_returns, self.cov_matrix, risk_aversion=risk_aversion - ) - for obj in self._additional_objectives: - self._objective += obj - - self._make_weight_sum_constraint(market_neutral) - return self._solve_cvxpy_opt_problem() - - def efficient_risk(self, target_volatility, market_neutral=False): - """ - Maximise return for a target risk. The resulting portfolio will have a volatility - less than the target (but not guaranteed to be equal). - - :param target_volatility: the desired maximum volatility of the resulting portfolio. - :type target_volatility: float - :param market_neutral: whether the portfolio should be market neutral (weights sum to zero), - defaults to False. Requires negative lower weight bound. - :param market_neutral: bool, optional - :raises ValueError: if ``target_volatility`` is not a positive float - :raises ValueError: if no portfolio can be found with volatility equal to ``target_volatility`` - :raises ValueError: if ``risk_free_rate`` is non-numeric - :return: asset weights for the efficient risk portfolio - :rtype: OrderedDict - """ - if not isinstance(target_volatility, (float, int)) or target_volatility < 0: - raise ValueError("target_volatility should be a positive float") - - global_min_volatility = np.sqrt(1 / np.sum(np.linalg.inv(self.cov_matrix))) - - if target_volatility < global_min_volatility: - raise ValueError( - "The minimum volatility is {:.3f}. Please use a higher target_volatility".format( - global_min_volatility - ) - ) - - self._objective = objective_functions.portfolio_return( - self._w, self.expected_returns - ) - variance = objective_functions.portfolio_variance(self._w, self.cov_matrix) - - for obj in self._additional_objectives: - self._objective += obj - - self._constraints.append(variance <= target_volatility ** 2) - self._make_weight_sum_constraint(market_neutral) - return self._solve_cvxpy_opt_problem() - - def efficient_return(self, target_return, market_neutral=False): - """ - Calculate the 'Markowitz portfolio', minimising volatility for a given target return. - - :param target_return: the desired return of the resulting portfolio. - :type target_return: float - :param market_neutral: whether the portfolio should be market neutral (weights sum to zero), - defaults to False. Requires negative lower weight bound. - :type market_neutral: bool, optional - :raises ValueError: if ``target_return`` is not a positive float - :raises ValueError: if no portfolio can be found with return equal to ``target_return`` - :return: asset weights for the Markowitz portfolio - :rtype: OrderedDict - """ - if not isinstance(target_return, float) or target_return < 0: - raise ValueError("target_return should be a positive float") - if target_return > self._max_return(): - raise ValueError( - "target_return must be lower than the maximum possible return" - ) - - self._objective = objective_functions.portfolio_variance( - self._w, self.cov_matrix - ) - ret = objective_functions.portfolio_return( - self._w, self.expected_returns, negative=False - ) - - for obj in self._additional_objectives: - self._objective += obj - - self._constraints.append(ret >= target_return) - self._make_weight_sum_constraint(market_neutral) - return self._solve_cvxpy_opt_problem() - - def portfolio_performance(self, verbose=False, risk_free_rate=0.02): - """ - After optimising, calculate (and optionally print) the performance of the optimal - portfolio. Currently calculates expected return, volatility, and the Sharpe ratio. - - :param verbose: whether performance should be printed, defaults to False - :type verbose: bool, optional - :param risk_free_rate: risk-free rate of borrowing/lending, defaults to 0.02. - The period of the risk-free rate should correspond to the - frequency of expected returns. - :type risk_free_rate: float, optional - :raises ValueError: if weights have not been calcualted yet - :return: expected return, volatility, Sharpe ratio. - :rtype: (float, float, float) - """ - if self._risk_free_rate is not None: - if risk_free_rate != self._risk_free_rate: - warnings.warn( - "The risk_free_rate provided to portfolio_performance is different" - " to the one used by max_sharpe. Using the previous value.", - UserWarning, - ) - risk_free_rate = self._risk_free_rate - - return base_optimizer.portfolio_performance( - self.weights, - self.expected_returns, - self.cov_matrix, - verbose, - risk_free_rate, - ) - - -class EfficientSemivariance(EfficientFrontier): - """ - EfficientSemivariance objects allow for optimisation along the mean-semivariance frontier. - This may be relevant for users who are more concerned about downside deviation. - - Instance variables: - - - Inputs: - - - ``n_assets`` - int - - ``tickers`` - str list - - ``bounds`` - float tuple OR (float tuple) list - - ``returns`` - pd.DataFrame - - ``expected_returns`` - np.ndarray - - ``solver`` - str - - ``solver_options`` - {str: str} dict - - - - Output: ``weights`` - np.ndarray - - Public methods: - - - ``min_semivariance()`` minimises the portfolio semivariance (downside deviation) - - ``max_quadratic_utility()`` maximises the "downside quadratic utility", given some risk aversion. - - ``efficient_risk()`` maximises return for a given target semideviation - - ``efficient_return()`` minimises semideviation for a given target return - - ``add_objective()`` adds a (convex) objective to the optimisation problem - - ``add_constraint()`` adds a (linear) constraint to the optimisation problem - - ``convex_objective()`` solves for a generic convex objective with linear constraints - - - ``portfolio_performance()`` calculates the expected return, semideviation and Sortino ratio for - the optimised portfolio. - - ``set_weights()`` creates self.weights (np.ndarray) from a weights dict - - ``clean_weights()`` rounds the weights and clips near-zeros. - - ``save_weights_to_file()`` saves the weights to csv, json, or txt. - """ - - def __init__( - self, - expected_returns, - returns, - frequency=252, - benchmark=0, - weight_bounds=(0, 1), - solver=None, - verbose=False, - solver_options=None, - ): - """ - :param expected_returns: expected returns for each asset. Can be None if - optimising for semideviation only. - :type expected_returns: pd.Series, list, np.ndarray - :param returns: (historic) returns for all your assets (no NaNs). - See ``expected_returns.returns_from_prices``. - :type returns: pd.DataFrame or np.array - :param frequency: number of time periods in a year, defaults to 252 (the number - of trading days in a year). This must agree with the frequency - parameter used in your ``expected_returns``. - :type frequency: int, optional - :param benchmark: the return threshold to distinguish "downside" and "upside". - This should match the frequency of your ``returns``, - i.e this should be a benchmark daily returns if your - ``returns`` are also daily. - :param weight_bounds: minimum and maximum weight of each asset OR single min/max pair - if all identical, defaults to (0, 1). Must be changed to (-1, 1) - for portfolios with shorting. - :type weight_bounds: tuple OR tuple list, optional - :param solver: name of solver. list available solvers with: `cvxpy.installed_solvers()` - :type solver: str - :param verbose: whether performance and debugging info should be printed, defaults to False - :type verbose: bool, optional - :param solver_options: parameters for the given solver - :type solver_options: dict, optional - :raises TypeError: if ``expected_returns`` is not a series, list or array - """ - # Instantiate parent - super().__init__( - expected_returns=expected_returns, - cov_matrix=np.zeros((len(expected_returns),) * 2), # dummy - weight_bounds=weight_bounds, - solver=solver, - verbose=verbose, - solver_options=solver_options, - ) - - self.returns = self._validate_returns(returns) - self.benchmark = benchmark - self.frequency = frequency - self._T = self.returns.shape[0] - - def min_volatility(self): - raise NotImplementedError("Please use min_semivariance instead.") - - def max_sharpe(self, risk_free_rate=0.02): - raise NotImplementedError("Method not available in EfficientSemivariance") - - def min_semivariance(self, market_neutral=False): - """ - Minimise portfolio semivariance (see docs for further explanation). - - :param market_neutral: whether the portfolio should be market neutral (weights sum to zero), - defaults to False. Requires negative lower weight bound. - :param market_neutral: bool, optional - :return: asset weights for the volatility-minimising portfolio - :rtype: OrderedDict - """ - p = cp.Variable(self._T, nonneg=True) - n = cp.Variable(self._T, nonneg=True) - self._objective = cp.sum(cp.square(n)) - - for obj in self._additional_objectives: - self._objective += obj - - B = (self.returns.values - self.benchmark) / np.sqrt(self._T) - self._constraints.append(B @ self._w - p + n == 0) - self._make_weight_sum_constraint(market_neutral) - return self._solve_cvxpy_opt_problem() - - def max_quadratic_utility(self, risk_aversion=1, market_neutral=False): - """ - Maximise the given quadratic utility, using portfolio semivariance instead - of variance. - - :param risk_aversion: risk aversion parameter (must be greater than 0), - defaults to 1 - :type risk_aversion: positive float - :param market_neutral: whether the portfolio should be market neutral (weights sum to zero), - defaults to False. Requires negative lower weight bound. - :param market_neutral: bool, optional - :return: asset weights for the maximum-utility portfolio - :rtype: OrderedDict - """ - if risk_aversion <= 0: - raise ValueError("risk aversion coefficient must be greater than zero") - - p = cp.Variable(self._T, nonneg=True) - n = cp.Variable(self._T, nonneg=True) - mu = objective_functions.portfolio_return(self._w, self.expected_returns) - mu /= self.frequency - self._objective = mu + 0.5 * risk_aversion * cp.sum(cp.square(n)) - for obj in self._additional_objectives: - self._objective += obj - - B = (self.returns.values - self.benchmark) / np.sqrt(self._T) - self._constraints.append(B @ self._w - p + n == 0) - self._make_weight_sum_constraint(market_neutral) - return self._solve_cvxpy_opt_problem() - - def efficient_risk(self, target_semideviation, market_neutral=False): - """ - Maximise return for a target semideviation (downside standard deviation). - The resulting portfolio will have a semideviation less than the target - (but not guaranteed to be equal). - - :param target_semideviation: the desired maximum semideviation of the resulting portfolio. - :type target_semideviation: float - :param market_neutral: whether the portfolio should be market neutral (weights sum to zero), - defaults to False. Requires negative lower weight bound. - :param market_neutral: bool, optional - :return: asset weights for the efficient risk portfolio - :rtype: OrderedDict - """ - self._objective = objective_functions.portfolio_return( - self._w, self.expected_returns - ) - for obj in self._additional_objectives: - self._objective += obj - - p = cp.Variable(self._T, nonneg=True) - n = cp.Variable(self._T, nonneg=True) - - self._constraints.append( - self.frequency * cp.sum(cp.square(n)) <= (target_semideviation ** 2) - ) - B = (self.returns.values - self.benchmark) / np.sqrt(self._T) - self._constraints.append(B @ self._w - p + n == 0) - self._make_weight_sum_constraint(market_neutral) - return self._solve_cvxpy_opt_problem() - - def efficient_return(self, target_return, market_neutral=False): - """ - Minimise semideviation for a given target return. - - :param target_return: the desired return of the resulting portfolio. - :type target_return: float - :param market_neutral: whether the portfolio should be market neutral (weights sum to zero), - defaults to False. Requires negative lower weight bound. - :type market_neutral: bool, optional - :raises ValueError: if ``target_return`` is not a positive float - :raises ValueError: if no portfolio can be found with return equal to ``target_return`` - :return: asset weights for the optimal portfolio - :rtype: OrderedDict - """ - if not isinstance(target_return, float) or target_return < 0: - raise ValueError("target_return should be a positive float") - if target_return > np.abs(self.expected_returns).max(): - raise ValueError( - "target_return must be lower than the largest expected return" - ) - p = cp.Variable(self._T, nonneg=True) - n = cp.Variable(self._T, nonneg=True) - self._objective = cp.sum(cp.square(n)) - for obj in self._additional_objectives: - self._objective += obj - - self._constraints.append( - cp.sum(self._w @ self.expected_returns) >= target_return - ) - B = (self.returns.values - self.benchmark) / np.sqrt(self._T) - self._constraints.append(B @ self._w - p + n == 0) - self._make_weight_sum_constraint(market_neutral) - return self._solve_cvxpy_opt_problem() - - def portfolio_performance(self, verbose=False, risk_free_rate=0.02): - """ - After optimising, calculate (and optionally print) the performance of the optimal - portfolio, specifically: expected return, semideviation, Sortino ratio. - - :param verbose: whether performance should be printed, defaults to False - :type verbose: bool, optional - :param risk_free_rate: risk-free rate of borrowing/lending, defaults to 0.02. - The period of the risk-free rate should correspond to the - frequency of expected returns. - :type risk_free_rate: float, optional - :raises ValueError: if weights have not been calcualted yet - :return: expected return, semideviation, Sortino ratio. - :rtype: (float, float, float) - """ - mu = objective_functions.portfolio_return( - self.weights, self.expected_returns, negative=False - ) - - portfolio_returns = self.returns @ self.weights - drops = np.fmin(portfolio_returns - self.benchmark, 0) - semivariance = np.sum(np.square(drops)) / self._T * self.frequency - semi_deviation = np.sqrt(semivariance) - sortino_ratio = (mu - risk_free_rate) / semi_deviation - - if verbose: - print("Expected annual return: {:.1f}%".format(100 * mu)) - print("Annual semi-deviation: {:.1f}%".format(100 * semi_deviation)) - print("Sortino Ratio: {:.2f}".format(sortino_ratio)) - - return mu, semi_deviation, sortino_ratio - - -class EfficientCVaR(EfficientFrontier): - """ - The EfficientCVaR class allows for optimisation along the mean-CVaR frontier, using the - formulation of Rockafellar and Ursayev (2001). - - Instance variables: - - - Inputs: - - - ``n_assets`` - int - - ``tickers`` - str list - - ``bounds`` - float tuple OR (float tuple) list - - ``returns`` - pd.DataFrame - - ``expected_returns`` - np.ndarray - - ``solver`` - str - - ``solver_options`` - {str: str} dict - - - - Output: ``weights`` - np.ndarray - - Public methods: - - - ``min_cvar()`` minimises the CVaR - - ``efficient_risk()`` maximises return for a given CVaR - - ``efficient_return()`` minimises CVaR for a given target return - - ``add_objective()`` adds a (convex) objective to the optimisation problem - - ``add_constraint()`` adds a (linear) constraint to the optimisation problem - - - ``portfolio_performance()`` calculates the expected return and CVaR of the portfolio - - ``set_weights()`` creates self.weights (np.ndarray) from a weights dict - - ``clean_weights()`` rounds the weights and clips near-zeros. - - ``save_weights_to_file()`` saves the weights to csv, json, or txt. - """ - - def __init__( - self, - expected_returns, - returns, - beta=0.95, - weight_bounds=(0, 1), - solver=None, - verbose=False, - solver_options=None, - ): - """ - :param expected_returns: expected returns for each asset. Can be None if - optimising for semideviation only. - :type expected_returns: pd.Series, list, np.ndarray - :param returns: (historic) returns for all your assets (no NaNs). - See ``expected_returns.returns_from_prices``. - :type returns: pd.DataFrame or np.array - :param beta: confidence level, defauls to 0.95 (i.e expected loss on the worst (1-beta) days). - :param weight_bounds: minimum and maximum weight of each asset OR single min/max pair - if all identical, defaults to (0, 1). Must be changed to (-1, 1) - for portfolios with shorting. - :type weight_bounds: tuple OR tuple list, optional - :param solver: name of solver. list available solvers with: `cvxpy.installed_solvers()` - :type solver: str - :param verbose: whether performance and debugging info should be printed, defaults to False - :type verbose: bool, optional - :param solver_options: parameters for the given solver - :type solver_options: dict, optional - :raises TypeError: if ``expected_returns`` is not a series, list or array - """ - super().__init__( - expected_returns=expected_returns, - cov_matrix=np.zeros((len(expected_returns),) * 2), # dummy - weight_bounds=weight_bounds, - solver=solver, - verbose=verbose, - solver_options=solver_options, - ) - - self.returns = self._validate_returns(returns) - self._beta = self._validate_beta(beta) - self._alpha = cp.Variable() - self._u = cp.Variable(len(self.returns)) - - def _validate_beta(self, beta): - if not (0 <= beta < 1): - raise ValueError("beta must be between 0 and 1") - if beta <= 0.2: - warnings.warn( - "Warning: beta is the confidence-level, not the quantile. Typical values are 80%, 90%, 95%.", - UserWarning, - ) - return beta - - def min_volatility(self): - raise NotImplementedError("Please use min_cvar instead.") - - def max_sharpe(self, risk_free_rate=0.02): - raise NotImplementedError("Method not available in EfficientCVaR.") - - def max_quadratic_utility(self, risk_aversion=1, market_neutral=False): - raise NotImplementedError("Method not available in EfficientCVaR.") - - def min_cvar(self, market_neutral=False): - """ - Minimise portfolio CVaR (see docs for further explanation). - - :param market_neutral: whether the portfolio should be market neutral (weights sum to zero), - defaults to False. Requires negative lower weight bound. - :param market_neutral: bool, optional - :return: asset weights for the volatility-minimising portfolio - :rtype: OrderedDict - """ - self._objective = self._alpha + 1.0 / ( - len(self.returns) * (1 - self._beta) - ) * cp.sum(self._u) - - for obj in self._additional_objectives: - self._objective += obj - - self._constraints += [ - self._u >= 0.0, - self.returns.values @ self._w + self._alpha + self._u >= 0.0, - ] - - self._make_weight_sum_constraint(market_neutral) - return self._solve_cvxpy_opt_problem() - - def efficient_return(self, target_return, market_neutral=False): - """ - Minimise CVaR for a given target return. - - :param target_return: the desired return of the resulting portfolio. - :type target_return: float - :param market_neutral: whether the portfolio should be market neutral (weights sum to zero), - defaults to False. Requires negative lower weight bound. - :type market_neutral: bool, optional - :raises ValueError: if ``target_return`` is not a positive float - :raises ValueError: if no portfolio can be found with return equal to ``target_return`` - :return: asset weights for the optimal portfolio - :rtype: OrderedDict - """ - self._objective = self._alpha + 1.0 / ( - len(self.returns) * (1 - self._beta) - ) * cp.sum(self._u) - - for obj in self._additional_objectives: - self._objective += obj - - self._constraints += [ - self._u >= 0.0, - self.returns.values @ self._w + self._alpha + self._u >= 0.0, - ] - - ret = self.expected_returns.T @ self._w - self._constraints.append(ret >= target_return) - - self._make_weight_sum_constraint(market_neutral) - return self._solve_cvxpy_opt_problem() - - def efficient_risk(self, target_cvar, market_neutral=False): - """ - Maximise return for a target CVaR. - The resulting portfolio will have a CVaR less than the target - (but not guaranteed to be equal). - - :param target_cvar: the desired maximum semideviation of the resulting portfolio. - :type target_cvar: float - :param market_neutral: whether the portfolio should be market neutral (weights sum to zero), - defaults to False. Requires negative lower weight bound. - :param market_neutral: bool, optional - :return: asset weights for the efficient risk portfolio - :rtype: OrderedDict - """ - self._objective = objective_functions.portfolio_return( - self._w, self.expected_returns - ) - for obj in self._additional_objectives: - self._objective += obj - - cvar = self._alpha + 1.0 / (len(self.returns) * (1 - self._beta)) * cp.sum( - self._u - ) - self._constraints += [ - cvar <= target_cvar, - self._u >= 0.0, - self.returns.values @ self._w + self._alpha + self._u >= 0.0, - ] - - self._make_weight_sum_constraint(market_neutral) - return self._solve_cvxpy_opt_problem() - - def portfolio_performance(self, verbose=False): - """ - After optimising, calculate (and optionally print) the performance of the optimal - portfolio, specifically: expected return, CVaR - - :param verbose: whether performance should be printed, defaults to False - :type verbose: bool, optional - :raises ValueError: if weights have not been calcualted yet - :return: expected return, CVaR. - :rtype: (float, float) - """ - mu = objective_functions.portfolio_return( - self.weights, self.expected_returns, negative=False - ) - - cvar = self._alpha + 1.0 / (len(self.returns) * (1 - self._beta)) * cp.sum( - self._u - ) - cvar_val = cvar.value - - if verbose: - print("Expected annual return: {:.1f}%".format(100 * mu)) - print("Conditional Value at Risk: {:.2f}%".format(100 * cvar_val)) - - return mu, cvar_val diff --git a/pypfopt/efficient_frontier/__init__.py b/pypfopt/efficient_frontier/__init__.py new file mode 100644 index 00000000..44c6402d --- /dev/null +++ b/pypfopt/efficient_frontier/__init__.py @@ -0,0 +1,11 @@ +""" +The ``efficient_frontier`` module houses the EfficientFrontier class and its descendants, +which generate optimal portfolios for various possible objective functions and parameters. +""" + +from .efficient_frontier import EfficientFrontier +from .efficient_cvar import EfficientCVaR +from .efficient_semivariance import EfficientSemivariance + + +__all__ = ["EfficientFrontier", "EfficientCVaR", "EfficientSemivariance"] diff --git a/pypfopt/efficient_frontier/efficient_cvar.py b/pypfopt/efficient_frontier/efficient_cvar.py new file mode 100644 index 00000000..f070a86c --- /dev/null +++ b/pypfopt/efficient_frontier/efficient_cvar.py @@ -0,0 +1,225 @@ +""" +The ``efficient_cvar`` submodule houses the EfficientCVaR class, which +generates portfolios along the mean-CVaR frontier. +""" + +import warnings +import numpy as np +import cvxpy as cp + +from .. import objective_functions +from .efficient_frontier import EfficientFrontier + + +class EfficientCVaR(EfficientFrontier): + """ + The EfficientCVaR class allows for optimization along the mean-CVaR frontier, using the + formulation of Rockafellar and Ursayev (2001). + + Instance variables: + + - Inputs: + + - ``n_assets`` - int + - ``tickers`` - str list + - ``bounds`` - float tuple OR (float tuple) list + - ``returns`` - pd.DataFrame + - ``expected_returns`` - np.ndarray + - ``solver`` - str + - ``solver_options`` - {str: str} dict + + + - Output: ``weights`` - np.ndarray + + Public methods: + + - ``min_cvar()`` minimises the CVaR + - ``efficient_risk()`` maximises return for a given CVaR + - ``efficient_return()`` minimises CVaR for a given target return + - ``add_objective()`` adds a (convex) objective to the optimization problem + - ``add_constraint()`` adds a constraint to the optimization problem + + - ``portfolio_performance()`` calculates the expected return and CVaR of the portfolio + - ``set_weights()`` creates self.weights (np.ndarray) from a weights dict + - ``clean_weights()`` rounds the weights and clips near-zeros. + - ``save_weights_to_file()`` saves the weights to csv, json, or txt. + """ + + def __init__( + self, + expected_returns, + returns, + beta=0.95, + weight_bounds=(0, 1), + solver=None, + verbose=False, + solver_options=None, + ): + """ + :param expected_returns: expected returns for each asset. Can be None if + optimising for semideviation only. + :type expected_returns: pd.Series, list, np.ndarray + :param returns: (historic) returns for all your assets (no NaNs). + See ``expected_returns.returns_from_prices``. + :type returns: pd.DataFrame or np.array + :param beta: confidence level, defauls to 0.95 (i.e expected loss on the worst (1-beta) days). + :param weight_bounds: minimum and maximum weight of each asset OR single min/max pair + if all identical, defaults to (0, 1). Must be changed to (-1, 1) + for portfolios with shorting. + :type weight_bounds: tuple OR tuple list, optional + :param solver: name of solver. list available solvers with: `cvxpy.installed_solvers()` + :type solver: str + :param verbose: whether performance and debugging info should be printed, defaults to False + :type verbose: bool, optional + :param solver_options: parameters for the given solver + :type solver_options: dict, optional + :raises TypeError: if ``expected_returns`` is not a series, list or array + """ + super().__init__( + expected_returns=expected_returns, + cov_matrix=np.zeros((len(expected_returns),) * 2), # dummy + weight_bounds=weight_bounds, + solver=solver, + verbose=verbose, + solver_options=solver_options, + ) + + self.returns = self._validate_returns(returns) + self._beta = self._validate_beta(beta) + self._alpha = cp.Variable() + self._u = cp.Variable(len(self.returns)) + + @staticmethod + def _validate_beta(beta): + if not (0 <= beta < 1): + raise ValueError("beta must be between 0 and 1") + if beta <= 0.2: + warnings.warn( + "Warning: beta is the confidence-level, not the quantile. Typical values are 80%, 90%, 95%.", + UserWarning, + ) + return beta + + def min_volatility(self): + raise NotImplementedError("Please use min_cvar instead.") + + def max_sharpe(self, risk_free_rate=0.02): + raise NotImplementedError("Method not available in EfficientCVaR.") + + def max_quadratic_utility(self, risk_aversion=1, market_neutral=False): + raise NotImplementedError("Method not available in EfficientCVaR.") + + def min_cvar(self, market_neutral=False): + """ + Minimise portfolio CVaR (see docs for further explanation). + + :param market_neutral: whether the portfolio should be market neutral (weights sum to zero), + defaults to False. Requires negative lower weight bound. + :param market_neutral: bool, optional + :return: asset weights for the volatility-minimising portfolio + :rtype: OrderedDict + """ + self._objective = self._alpha + 1.0 / ( + len(self.returns) * (1 - self._beta) + ) * cp.sum(self._u) + + for obj in self._additional_objectives: + self._objective += obj + + self._constraints += [ + self._u >= 0.0, + self.returns.values @ self._w + self._alpha + self._u >= 0.0, + ] + + self._make_weight_sum_constraint(market_neutral) + return self._solve_cvxpy_opt_problem() + + def efficient_return(self, target_return, market_neutral=False): + """ + Minimise CVaR for a given target return. + + :param target_return: the desired return of the resulting portfolio. + :type target_return: float + :param market_neutral: whether the portfolio should be market neutral (weights sum to zero), + defaults to False. Requires negative lower weight bound. + :type market_neutral: bool, optional + :raises ValueError: if ``target_return`` is not a positive float + :raises ValueError: if no portfolio can be found with return equal to ``target_return`` + :return: asset weights for the optimal portfolio + :rtype: OrderedDict + """ + self._objective = self._alpha + 1.0 / ( + len(self.returns) * (1 - self._beta) + ) * cp.sum(self._u) + + for obj in self._additional_objectives: + self._objective += obj + + self._constraints += [ + self._u >= 0.0, + self.returns.values @ self._w + self._alpha + self._u >= 0.0, + ] + + ret = self.expected_returns.T @ self._w + self._constraints.append(ret >= target_return) + + self._make_weight_sum_constraint(market_neutral) + return self._solve_cvxpy_opt_problem() + + def efficient_risk(self, target_cvar, market_neutral=False): + """ + Maximise return for a target CVaR. + The resulting portfolio will have a CVaR less than the target + (but not guaranteed to be equal). + + :param target_cvar: the desired maximum semideviation of the resulting portfolio. + :type target_cvar: float + :param market_neutral: whether the portfolio should be market neutral (weights sum to zero), + defaults to False. Requires negative lower weight bound. + :param market_neutral: bool, optional + :return: asset weights for the efficient risk portfolio + :rtype: OrderedDict + """ + self._objective = objective_functions.portfolio_return( + self._w, self.expected_returns + ) + for obj in self._additional_objectives: + self._objective += obj + + cvar = self._alpha + 1.0 / (len(self.returns) * (1 - self._beta)) * cp.sum( + self._u + ) + self._constraints += [ + cvar <= target_cvar, + self._u >= 0.0, + self.returns.values @ self._w + self._alpha + self._u >= 0.0, + ] + + self._make_weight_sum_constraint(market_neutral) + return self._solve_cvxpy_opt_problem() + + def portfolio_performance(self, verbose=False): + """ + After optimising, calculate (and optionally print) the performance of the optimal + portfolio, specifically: expected return, CVaR + + :param verbose: whether performance should be printed, defaults to False + :type verbose: bool, optional + :raises ValueError: if weights have not been calcualted yet + :return: expected return, CVaR. + :rtype: (float, float) + """ + mu = objective_functions.portfolio_return( + self.weights, self.expected_returns, negative=False + ) + + cvar = self._alpha + 1.0 / (len(self.returns) * (1 - self._beta)) * cp.sum( + self._u + ) + cvar_val = cvar.value + + if verbose: + print("Expected annual return: {:.1f}%".format(100 * mu)) + print("Conditional Value at Risk: {:.2f}%".format(100 * cvar_val)) + + return mu, cvar_val diff --git a/pypfopt/efficient_frontier/efficient_frontier.py b/pypfopt/efficient_frontier/efficient_frontier.py new file mode 100644 index 00000000..a7b94e85 --- /dev/null +++ b/pypfopt/efficient_frontier/efficient_frontier.py @@ -0,0 +1,417 @@ +""" +The ``efficient_frontier`` submodule houses the EfficientFrontier class, which generates +classical mean-variance optimal portfolios for a variety of objectives and constraints +""" + +import warnings +import numpy as np +import pandas as pd +import cvxpy as cp + +from .. import objective_functions, base_optimizer + + +class EfficientFrontier(base_optimizer.BaseConvexOptimizer): + + """ + An EfficientFrontier object (inheriting from BaseConvexOptimizer) contains multiple + optimization methods that can be called (corresponding to different objective + functions) with various parameters. Note: a new EfficientFrontier object should + be instantiated if you want to make any change to objectives/constraints/bounds/parameters. + + Instance variables: + + - Inputs: + + - ``n_assets`` - int + - ``tickers`` - str list + - ``bounds`` - float tuple OR (float tuple) list + - ``cov_matrix`` - np.ndarray + - ``expected_returns`` - np.ndarray + - ``solver`` - str + - ``solver_options`` - {str: str} dict + + - Output: ``weights`` - np.ndarray + + Public methods: + + - ``min_volatility()`` optimizes for minimum volatility + - ``max_sharpe()`` optimizes for maximal Sharpe ratio (a.k.a the tangency portfolio) + - ``max_quadratic_utility()`` maximises the quadratic utility, given some risk aversion. + - ``efficient_risk()`` maximises return for a given target risk + - ``efficient_return()`` minimises risk for a given target return + + - ``add_objective()`` adds a (convex) objective to the optimization problem + - ``add_constraint()`` adds a constraint to the optimization problem + - ``convex_objective()`` solves for a generic convex objective with linear constraints + + - ``portfolio_performance()`` calculates the expected return, volatility and Sharpe ratio for + the optimized portfolio. + - ``set_weights()`` creates self.weights (np.ndarray) from a weights dict + - ``clean_weights()`` rounds the weights and clips near-zeros. + - ``save_weights_to_file()`` saves the weights to csv, json, or txt. + """ + + def __init__( + self, + expected_returns, + cov_matrix, + weight_bounds=(0, 1), + solver=None, + verbose=False, + solver_options=None, + ): + """ + :param expected_returns: expected returns for each asset. Can be None if + optimising for volatility only (but not recommended). + :type expected_returns: pd.Series, list, np.ndarray + :param cov_matrix: covariance of returns for each asset. This **must** be + positive semidefinite, otherwise optimization will fail. + :type cov_matrix: pd.DataFrame or np.array + :param weight_bounds: minimum and maximum weight of each asset OR single min/max pair + if all identical, defaults to (0, 1). Must be changed to (-1, 1) + for portfolios with shorting. + :type weight_bounds: tuple OR tuple list, optional + :param solver: name of solver. list available solvers with: `cvxpy.installed_solvers()` + :type solver: str + :param verbose: whether performance and debugging info should be printed, defaults to False + :type verbose: bool, optional + :param solver_options: parameters for the given solver + :type solver_options: dict, optional + :raises TypeError: if ``expected_returns`` is not a series, list or array + :raises TypeError: if ``cov_matrix`` is not a dataframe or array + """ + # Inputs + self.cov_matrix = EfficientFrontier._validate_cov_matrix(cov_matrix) + self.expected_returns = EfficientFrontier._validate_expected_returns( + expected_returns + ) + + # Labels + if isinstance(expected_returns, pd.Series): + tickers = list(expected_returns.index) + elif isinstance(cov_matrix, pd.DataFrame): + tickers = list(cov_matrix.columns) + else: # use integer labels + tickers = list(range(len(expected_returns))) + + if expected_returns is not None and cov_matrix is not None: + if cov_matrix.shape != (len(expected_returns), len(expected_returns)): + raise ValueError("Covariance matrix does not match expected returns") + + super().__init__( + len(tickers), + tickers, + weight_bounds, + solver=solver, + verbose=verbose, + solver_options=solver_options, + ) + + @staticmethod + def _validate_expected_returns(expected_returns): + if expected_returns is None: + return None + elif isinstance(expected_returns, pd.Series): + return expected_returns.values + elif isinstance(expected_returns, list): + return np.array(expected_returns) + elif isinstance(expected_returns, np.ndarray): + return expected_returns.ravel() + else: + raise TypeError("expected_returns is not a series, list or array") + + @staticmethod + def _validate_cov_matrix(cov_matrix): + if cov_matrix is None: + raise ValueError("cov_matrix must be provided") + elif isinstance(cov_matrix, pd.DataFrame): + return cov_matrix.values + elif isinstance(cov_matrix, np.ndarray): + return cov_matrix + else: + raise TypeError("cov_matrix is not a dataframe or array") + + def _validate_returns(self, returns): + """ + Helper method to validate daily returns (needed for some efficient frontiers) + """ + if not isinstance(returns, (pd.DataFrame, np.ndarray)): + raise TypeError("returns should be a pd.Dataframe or np.ndarray") + + returns_df = pd.DataFrame(returns) + if returns_df.isnull().values.any(): + warnings.warn( + "Removing NaNs from returns", + UserWarning, + ) + returns_df = returns_df.dropna(axis=0, how="any") + + if self.expected_returns is not None: + if returns_df.shape[1] != len(self.expected_returns): + raise ValueError( + "returns columns do not match expected_returns. Please check your tickers." + ) + + return returns_df + + def _make_weight_sum_constraint(self, is_market_neutral): + """ + Helper method to make the weight sum constraint. If market neutral, + validate the weights proided in the constructor. + """ + if is_market_neutral: + #  Check and fix bounds + portfolio_possible = np.any(self._lower_bounds < 0) + if not portfolio_possible: + warnings.warn( + "Market neutrality requires shorting - bounds have been amended", + RuntimeWarning, + ) + self._map_bounds_to_constraints((-1, 1)) + # Delete original constraints + del self._constraints[0] + del self._constraints[0] + + self._constraints.append(cp.sum(self._w) == 0) + else: + self._constraints.append(cp.sum(self._w) == 1) + + def min_volatility(self): + """ + Minimise volatility. + + :return: asset weights for the volatility-minimising portfolio + :rtype: OrderedDict + """ + self._objective = objective_functions.portfolio_variance( + self._w, self.cov_matrix + ) + for obj in self._additional_objectives: + self._objective += obj + + self._constraints.append(cp.sum(self._w) == 1) + return self._solve_cvxpy_opt_problem() + + def _max_return(self, return_value=True): + """ + Helper method to maximise return. This should not be used to optimize a portfolio. + + :return: asset weights for the return-minimising portfolio + :rtype: OrderedDict + """ + self._objective = objective_functions.portfolio_return( + self._w, self.expected_returns + ) + + self._constraints.append(cp.sum(self._w) == 1) + + res = self._solve_cvxpy_opt_problem() + + #  Cleanup constraints since this is a helper method + del self._constraints[-1] + + if return_value: + return -self._opt.value + else: + return res + + def max_sharpe(self, risk_free_rate=0.02): + """ + Maximise the Sharpe Ratio. The result is also referred to as the tangency portfolio, + as it is the portfolio for which the capital market line is tangent to the efficient frontier. + + This is a convex optimization problem after making a certain variable substitution. See + `Cornuejols and Tutuncu (2006) `_ for more. + + :param risk_free_rate: risk-free rate of borrowing/lending, defaults to 0.02. + The period of the risk-free rate should correspond to the + frequency of expected returns. + :type risk_free_rate: float, optional + :raises ValueError: if ``risk_free_rate`` is non-numeric + :return: asset weights for the Sharpe-maximising portfolio + :rtype: OrderedDict + """ + if not isinstance(risk_free_rate, (int, float)): + raise ValueError("risk_free_rate should be numeric") + self._risk_free_rate = risk_free_rate + + # max_sharpe requires us to make a variable transformation. + # Here we treat w as the transformed variable. + self._objective = cp.quad_form(self._w, self.cov_matrix) + k = cp.Variable() + + # Note: objectives are not scaled by k. Hence there are subtle differences + # between how these objectives work for max_sharpe vs min_volatility + if len(self._additional_objectives) > 0: + warnings.warn( + "max_sharpe transforms the optimization problem so additional objectives may not work as expected." + ) + for obj in self._additional_objectives: + self._objective += obj + + new_constraints = [] + # Must rebuild the constraints + for constr in self._constraints: + if isinstance(constr, cp.constraints.nonpos.Inequality): + # Either the first or second item is the expression + if isinstance( + constr.args[0], cp.expressions.constants.constant.Constant + ): + new_constraints.append(constr.args[1] >= constr.args[0] * k) + else: + new_constraints.append(constr.args[0] <= constr.args[1] * k) + elif isinstance(constr, cp.constraints.zero.Equality): + new_constraints.append(constr.args[0] == constr.args[1] * k) + else: + raise TypeError( + "Please check that your constraints are in a suitable format" + ) + + # Transformed max_sharpe convex problem: + self._constraints = [ + (self.expected_returns - risk_free_rate).T @ self._w == 1, + cp.sum(self._w) == k, + k >= 0, + ] + new_constraints + + self._solve_cvxpy_opt_problem() + # Inverse-transform + self.weights = (self._w.value / k.value).round(16) + 0.0 + return self._make_output_weights() + + def max_quadratic_utility(self, risk_aversion=1, market_neutral=False): + r""" + Maximise the given quadratic utility, i.e: + + .. math:: + + \max_w w^T \mu - \frac \delta 2 w^T \Sigma w + + :param risk_aversion: risk aversion parameter (must be greater than 0), + defaults to 1 + :type risk_aversion: positive float + :param market_neutral: whether the portfolio should be market neutral (weights sum to zero), + defaults to False. Requires negative lower weight bound. + :param market_neutral: bool, optional + :return: asset weights for the maximum-utility portfolio + :rtype: OrderedDict + """ + if risk_aversion <= 0: + raise ValueError("risk aversion coefficient must be greater than zero") + + self._objective = objective_functions.quadratic_utility( + self._w, self.expected_returns, self.cov_matrix, risk_aversion=risk_aversion + ) + for obj in self._additional_objectives: + self._objective += obj + + self._make_weight_sum_constraint(market_neutral) + return self._solve_cvxpy_opt_problem() + + def efficient_risk(self, target_volatility, market_neutral=False): + """ + Maximise return for a target risk. The resulting portfolio will have a volatility + less than the target (but not guaranteed to be equal). + + :param target_volatility: the desired maximum volatility of the resulting portfolio. + :type target_volatility: float + :param market_neutral: whether the portfolio should be market neutral (weights sum to zero), + defaults to False. Requires negative lower weight bound. + :param market_neutral: bool, optional + :raises ValueError: if ``target_volatility`` is not a positive float + :raises ValueError: if no portfolio can be found with volatility equal to ``target_volatility`` + :raises ValueError: if ``risk_free_rate`` is non-numeric + :return: asset weights for the efficient risk portfolio + :rtype: OrderedDict + """ + if not isinstance(target_volatility, (float, int)) or target_volatility < 0: + raise ValueError("target_volatility should be a positive float") + + global_min_volatility = np.sqrt(1 / np.sum(np.linalg.inv(self.cov_matrix))) + + if target_volatility < global_min_volatility: + raise ValueError( + "The minimum volatility is {:.3f}. Please use a higher target_volatility".format( + global_min_volatility + ) + ) + + self._objective = objective_functions.portfolio_return( + self._w, self.expected_returns + ) + variance = objective_functions.portfolio_variance(self._w, self.cov_matrix) + + for obj in self._additional_objectives: + self._objective += obj + + self._constraints.append(variance <= target_volatility ** 2) + self._make_weight_sum_constraint(market_neutral) + return self._solve_cvxpy_opt_problem() + + def efficient_return(self, target_return, market_neutral=False): + """ + Calculate the 'Markowitz portfolio', minimising volatility for a given target return. + + :param target_return: the desired return of the resulting portfolio. + :type target_return: float + :param market_neutral: whether the portfolio should be market neutral (weights sum to zero), + defaults to False. Requires negative lower weight bound. + :type market_neutral: bool, optional + :raises ValueError: if ``target_return`` is not a positive float + :raises ValueError: if no portfolio can be found with return equal to ``target_return`` + :return: asset weights for the Markowitz portfolio + :rtype: OrderedDict + """ + if not isinstance(target_return, float) or target_return < 0: + raise ValueError("target_return should be a positive float") + if target_return > self._max_return(): + raise ValueError( + "target_return must be lower than the maximum possible return" + ) + + self._objective = objective_functions.portfolio_variance( + self._w, self.cov_matrix + ) + ret = objective_functions.portfolio_return( + self._w, self.expected_returns, negative=False + ) + + for obj in self._additional_objectives: + self._objective += obj + + self._constraints.append(ret >= target_return) + self._make_weight_sum_constraint(market_neutral) + return self._solve_cvxpy_opt_problem() + + def portfolio_performance(self, verbose=False, risk_free_rate=0.02): + """ + After optimising, calculate (and optionally print) the performance of the optimal + portfolio. Currently calculates expected return, volatility, and the Sharpe ratio. + + :param verbose: whether performance should be printed, defaults to False + :type verbose: bool, optional + :param risk_free_rate: risk-free rate of borrowing/lending, defaults to 0.02. + The period of the risk-free rate should correspond to the + frequency of expected returns. + :type risk_free_rate: float, optional + :raises ValueError: if weights have not been calcualted yet + :return: expected return, volatility, Sharpe ratio. + :rtype: (float, float, float) + """ + if self._risk_free_rate is not None: + if risk_free_rate != self._risk_free_rate: + warnings.warn( + "The risk_free_rate provided to portfolio_performance is different" + " to the one used by max_sharpe. Using the previous value.", + UserWarning, + ) + risk_free_rate = self._risk_free_rate + + return base_optimizer.portfolio_performance( + self.weights, + self.expected_returns, + self.cov_matrix, + verbose, + risk_free_rate, + ) diff --git a/pypfopt/efficient_frontier/efficient_semivariance.py b/pypfopt/efficient_frontier/efficient_semivariance.py new file mode 100644 index 00000000..114a5181 --- /dev/null +++ b/pypfopt/efficient_frontier/efficient_semivariance.py @@ -0,0 +1,257 @@ +""" +The ``efficient_semivariance`` submodule houses the EfficientSemivariance class, which +generates portfolios along the mean-semivariance frontier. +""" + + +import numpy as np +import cvxpy as cp + +from .. import objective_functions +from .efficient_frontier import EfficientFrontier + + +class EfficientSemivariance(EfficientFrontier): + """ + EfficientSemivariance objects allow for optimization along the mean-semivariance frontier. + This may be relevant for users who are more concerned about downside deviation. + + Instance variables: + + - Inputs: + + - ``n_assets`` - int + - ``tickers`` - str list + - ``bounds`` - float tuple OR (float tuple) list + - ``returns`` - pd.DataFrame + - ``expected_returns`` - np.ndarray + - ``solver`` - str + - ``solver_options`` - {str: str} dict + + + - Output: ``weights`` - np.ndarray + + Public methods: + + - ``min_semivariance()`` minimises the portfolio semivariance (downside deviation) + - ``max_quadratic_utility()`` maximises the "downside quadratic utility", given some risk aversion. + - ``efficient_risk()`` maximises return for a given target semideviation + - ``efficient_return()`` minimises semideviation for a given target return + - ``add_objective()`` adds a (convex) objective to the optimization problem + - ``add_constraint()`` adds a constraint to the optimization problem + - ``convex_objective()`` solves for a generic convex objective with linear constraints + + - ``portfolio_performance()`` calculates the expected return, semideviation and Sortino ratio for + the optimized portfolio. + - ``set_weights()`` creates self.weights (np.ndarray) from a weights dict + - ``clean_weights()`` rounds the weights and clips near-zeros. + - ``save_weights_to_file()`` saves the weights to csv, json, or txt. + """ + + def __init__( + self, + expected_returns, + returns, + frequency=252, + benchmark=0, + weight_bounds=(0, 1), + solver=None, + verbose=False, + solver_options=None, + ): + """ + :param expected_returns: expected returns for each asset. Can be None if + optimising for semideviation only. + :type expected_returns: pd.Series, list, np.ndarray + :param returns: (historic) returns for all your assets (no NaNs). + See ``expected_returns.returns_from_prices``. + :type returns: pd.DataFrame or np.array + :param frequency: number of time periods in a year, defaults to 252 (the number + of trading days in a year). This must agree with the frequency + parameter used in your ``expected_returns``. + :type frequency: int, optional + :param benchmark: the return threshold to distinguish "downside" and "upside". + This should match the frequency of your ``returns``, + i.e this should be a benchmark daily returns if your + ``returns`` are also daily. + :param weight_bounds: minimum and maximum weight of each asset OR single min/max pair + if all identical, defaults to (0, 1). Must be changed to (-1, 1) + for portfolios with shorting. + :type weight_bounds: tuple OR tuple list, optional + :param solver: name of solver. list available solvers with: `cvxpy.installed_solvers()` + :type solver: str + :param verbose: whether performance and debugging info should be printed, defaults to False + :type verbose: bool, optional + :param solver_options: parameters for the given solver + :type solver_options: dict, optional + :raises TypeError: if ``expected_returns`` is not a series, list or array + """ + # Instantiate parent + super().__init__( + expected_returns=expected_returns, + cov_matrix=np.zeros((len(expected_returns),) * 2), # dummy + weight_bounds=weight_bounds, + solver=solver, + verbose=verbose, + solver_options=solver_options, + ) + + self.returns = self._validate_returns(returns) + self.benchmark = benchmark + self.frequency = frequency + self._T = self.returns.shape[0] + + def min_volatility(self): + raise NotImplementedError("Please use min_semivariance instead.") + + def max_sharpe(self, risk_free_rate=0.02): + raise NotImplementedError("Method not available in EfficientSemivariance") + + def min_semivariance(self, market_neutral=False): + """ + Minimise portfolio semivariance (see docs for further explanation). + + :param market_neutral: whether the portfolio should be market neutral (weights sum to zero), + defaults to False. Requires negative lower weight bound. + :param market_neutral: bool, optional + :return: asset weights for the volatility-minimising portfolio + :rtype: OrderedDict + """ + p = cp.Variable(self._T, nonneg=True) + n = cp.Variable(self._T, nonneg=True) + self._objective = cp.sum(cp.square(n)) + + for obj in self._additional_objectives: + self._objective += obj + + B = (self.returns.values - self.benchmark) / np.sqrt(self._T) + self._constraints.append(B @ self._w - p + n == 0) + self._make_weight_sum_constraint(market_neutral) + return self._solve_cvxpy_opt_problem() + + def max_quadratic_utility(self, risk_aversion=1, market_neutral=False): + """ + Maximise the given quadratic utility, using portfolio semivariance instead + of variance. + + :param risk_aversion: risk aversion parameter (must be greater than 0), + defaults to 1 + :type risk_aversion: positive float + :param market_neutral: whether the portfolio should be market neutral (weights sum to zero), + defaults to False. Requires negative lower weight bound. + :param market_neutral: bool, optional + :return: asset weights for the maximum-utility portfolio + :rtype: OrderedDict + """ + if risk_aversion <= 0: + raise ValueError("risk aversion coefficient must be greater than zero") + + p = cp.Variable(self._T, nonneg=True) + n = cp.Variable(self._T, nonneg=True) + mu = objective_functions.portfolio_return(self._w, self.expected_returns) + mu /= self.frequency + self._objective = mu + 0.5 * risk_aversion * cp.sum(cp.square(n)) + for obj in self._additional_objectives: + self._objective += obj + + B = (self.returns.values - self.benchmark) / np.sqrt(self._T) + self._constraints.append(B @ self._w - p + n == 0) + self._make_weight_sum_constraint(market_neutral) + return self._solve_cvxpy_opt_problem() + + def efficient_risk(self, target_semideviation, market_neutral=False): + """ + Maximise return for a target semideviation (downside standard deviation). + The resulting portfolio will have a semideviation less than the target + (but not guaranteed to be equal). + + :param target_semideviation: the desired maximum semideviation of the resulting portfolio. + :type target_semideviation: float + :param market_neutral: whether the portfolio should be market neutral (weights sum to zero), + defaults to False. Requires negative lower weight bound. + :param market_neutral: bool, optional + :return: asset weights for the efficient risk portfolio + :rtype: OrderedDict + """ + self._objective = objective_functions.portfolio_return( + self._w, self.expected_returns + ) + for obj in self._additional_objectives: + self._objective += obj + + p = cp.Variable(self._T, nonneg=True) + n = cp.Variable(self._T, nonneg=True) + + self._constraints.append( + self.frequency * cp.sum(cp.square(n)) <= (target_semideviation ** 2) + ) + B = (self.returns.values - self.benchmark) / np.sqrt(self._T) + self._constraints.append(B @ self._w - p + n == 0) + self._make_weight_sum_constraint(market_neutral) + return self._solve_cvxpy_opt_problem() + + def efficient_return(self, target_return, market_neutral=False): + """ + Minimise semideviation for a given target return. + + :param target_return: the desired return of the resulting portfolio. + :type target_return: float + :param market_neutral: whether the portfolio should be market neutral (weights sum to zero), + defaults to False. Requires negative lower weight bound. + :type market_neutral: bool, optional + :raises ValueError: if ``target_return`` is not a positive float + :raises ValueError: if no portfolio can be found with return equal to ``target_return`` + :return: asset weights for the optimal portfolio + :rtype: OrderedDict + """ + if not isinstance(target_return, float) or target_return < 0: + raise ValueError("target_return should be a positive float") + if target_return > np.abs(self.expected_returns).max(): + raise ValueError( + "target_return must be lower than the largest expected return" + ) + p = cp.Variable(self._T, nonneg=True) + n = cp.Variable(self._T, nonneg=True) + self._objective = cp.sum(cp.square(n)) + for obj in self._additional_objectives: + self._objective += obj + + self._constraints.append( + cp.sum(self._w @ self.expected_returns) >= target_return + ) + B = (self.returns.values - self.benchmark) / np.sqrt(self._T) + self._constraints.append(B @ self._w - p + n == 0) + self._make_weight_sum_constraint(market_neutral) + return self._solve_cvxpy_opt_problem() + + def portfolio_performance(self, verbose=False, risk_free_rate=0.02): + """ + After optimising, calculate (and optionally print) the performance of the optimal + portfolio, specifically: expected return, semideviation, Sortino ratio. + + :param verbose: whether performance should be printed, defaults to False + :type verbose: bool, optional + :param risk_free_rate: risk-free rate of borrowing/lending, defaults to 0.02. + The period of the risk-free rate should correspond to the + frequency of expected returns. + :type risk_free_rate: float, optional + :raises ValueError: if weights have not been calcualted yet + :return: expected return, semideviation, Sortino ratio. + :rtype: (float, float, float) + """ + mu = objective_functions.portfolio_return( + self.weights, self.expected_returns, negative=False + ) + + portfolio_returns = self.returns @ self.weights + drops = np.fmin(portfolio_returns - self.benchmark, 0) + semivariance = np.sum(np.square(drops)) / self._T * self.frequency + semi_deviation = np.sqrt(semivariance) + sortino_ratio = (mu - risk_free_rate) / semi_deviation + + if verbose: + print("Expected annual return: {:.1f}%".format(100 * mu)) + print("Annual semi-deviation: {:.1f}%".format(100 * semi_deviation)) + print("Sortino Ratio: {:.2f}".format(sortino_ratio)) + + return mu, semi_deviation, sortino_ratio diff --git a/pypfopt/expected_returns.py b/pypfopt/expected_returns.py index 23326e0c..86154034 100644 --- a/pypfopt/expected_returns.py +++ b/pypfopt/expected_returns.py @@ -1,6 +1,6 @@ """ The ``expected_returns`` module provides functions for estimating the expected returns of -the assets, which is a required input in mean-variance optimisation. +the assets, which is a required input in mean-variance optimization. By convention, the output of these methods is expected *annual* returns. It is assumed that *daily* prices are provided, though in reality the functions are agnostic @@ -43,22 +43,6 @@ def returns_from_prices(prices, log_returns=False): return prices.pct_change().dropna(how="all") -def log_returns_from_prices(prices): - """ - Calculate the log returns given prices. - - :param prices: adjusted (daily) closing prices of the asset, each row is a - date and each column is a ticker/id. - :type prices: pd.DataFrame - :return: (daily) returns - :rtype: pd.DataFrame - """ - warnings.warn( - "log_returns_from_prices is deprecated. Please use returns_from_prices(prices, log_returns=True)" - ) - return np.log(1 + prices.pct_change()).dropna(how="all") - - def prices_from_returns(returns, log_returns=False): """ Calculate the pseudo-prices given returns. These are not true prices because diff --git a/pypfopt/hierarchical_portfolio.py b/pypfopt/hierarchical_portfolio.py index 35ae33cf..fe43687c 100644 --- a/pypfopt/hierarchical_portfolio.py +++ b/pypfopt/hierarchical_portfolio.py @@ -1,6 +1,6 @@ """ The ``hierarchical_portfolio`` module seeks to implement one of the recent advances in -portfolio optimisation – the application of hierarchical clustering models in allocation. +portfolio optimization – the application of hierarchical clustering models in allocation. All of the hierarchical classes have a similar API to ``EfficientFrontier``, though since many hierarchical models currently don't support different objectives, the actual allocation @@ -43,7 +43,7 @@ class HRPOpt(base_optimizer.BaseOptimizer): - ``optimize()`` calculates weights using HRP - ``portfolio_performance()`` calculates the expected return, volatility and Sharpe ratio for - the optimised portfolio. + the optimized portfolio. - ``set_weights()`` creates self.weights (np.ndarray) from a weights dict - ``clean_weights()`` rounds the weights and clips near-zeros. - ``save_weights_to_file()`` saves the weights to csv, json, or txt. @@ -126,7 +126,7 @@ def _raw_hrp_allocation(cov, ordered_tickers): for j, k in ((0, len(i) // 2), (len(i) // 2, len(i))) if len(i) > 1 ] # bi-section - # For each pair, optimise locally. + # For each pair, optimize locally. for i in range(0, len(cluster_items), 2): first_cluster = cluster_items[i] second_cluster = cluster_items[i + 1] diff --git a/pypfopt/objective_functions.py b/pypfopt/objective_functions.py index 07c33ca8..3f6e86df 100644 --- a/pypfopt/objective_functions.py +++ b/pypfopt/objective_functions.py @@ -1,14 +1,14 @@ """ -The ``objective_functions`` module provides optimisation objectives, including the actual -objective functions called by the ``EfficientFrontier`` object's optimisation methods. -These methods are primarily designed for internal use during optimisation and each requires +The ``objective_functions`` module provides optimization objectives, including the actual +objective functions called by the ``EfficientFrontier`` object's optimization methods. +These methods are primarily designed for internal use during optimization and each requires a different signature (which is why they have not been factored into a class). For obvious reasons, any objective function must accept ``weights`` as an argument, and must also have at least one of ``expected_returns`` or ``cov_matrix``. The objective functions either compute the objective given a numpy array of weights, or they return a cvxpy *expression* when weights are a ``cp.Variable``. In this way, the same objective -function can be used both internally for optimisation and externally for computing the objective +function can be used both internally for optimization and externally for computing the objective given weights. ``_objective_value()`` automatically chooses between the two behaviours. ``objective_functions`` defaults to objectives for minimisation. In the cases of objectives diff --git a/pypfopt/plotting.py b/pypfopt/plotting.py index a06d1682..8759cd34 100644 --- a/pypfopt/plotting.py +++ b/pypfopt/plotting.py @@ -13,12 +13,13 @@ from . import risk_models, exceptions from . import EfficientFrontier, CLA import scipy.cluster.hierarchy as sch +import warnings try: import matplotlib.pyplot as plt plt.style.use("seaborn-deep") -except (ModuleNotFoundError, ImportError): +except (ModuleNotFoundError, ImportError): # pragma: no cover raise ImportError("Please install matplotlib via pip or poetry") @@ -40,7 +41,7 @@ def _plot_io(**kwargs): plt.tight_layout() if filename: plt.savefig(fname=filename, dpi=dpi) - if showfig: + if showfig: # pragma: no cover plt.show() @@ -100,6 +101,10 @@ def plot_dendrogram(hrp, ax=None, show_tickers=True, **kwargs): ax = ax or plt.gca() if hrp.clusters is None: + warnings.warn( + "hrp param has not been optimized. Attempting optimization.", + RuntimeWarning, + ) hrp.optimize() if show_tickers: @@ -152,7 +157,7 @@ def _ef_default_returns_range(ef, points): ef_minvol.min_volatility() min_ret = ef_minvol.portfolio_performance()[0] max_ret = ef_maxret._max_return() - return np.linspace(min_ret, max_ret, points) + return np.linspace(min_ret, max_ret - 0.0001, points) def _plot_ef(ef, ef_param, ef_param_range, ax, show_assets): @@ -208,7 +213,7 @@ def plot_efficient_frontier( """ Plot the efficient frontier based on either a CLA or EfficientFrontier object. - :param opt: an instantiated optimiser object BEFORE optimising an objective + :param opt: an instantiated optimizer object BEFORE optimising an objective :type opt: EfficientFrontier or CLA :param ef_param: [EfficientFrontier] whether to use a range over utility, risk, or return. Defaults to "return". @@ -252,7 +257,7 @@ def plot_weights(weights, ax=None, **kwargs): """ Plot the portfolio weights as a horizontal bar chart - :param weights: the weights outputted by any PyPortfolioOpt optimiser + :param weights: the weights outputted by any PyPortfolioOpt optimizer :type weights: {ticker: weight} dict :param ax: ax to plot to, optional :type ax: matplotlib.axes diff --git a/pypfopt/risk_models.py b/pypfopt/risk_models.py index 443025d8..1279185d 100644 --- a/pypfopt/risk_models.py +++ b/pypfopt/risk_models.py @@ -83,8 +83,10 @@ def fix_nonpositive_semidefinite(matrix, fix_method="spectral"): else: raise NotImplementedError("Method {} not implemented".format(fix_method)) - if not _is_positive_semidefinite(fixed_matrix): - warnings.warn("Could not fix matrix. Please try a different risk model.") + if not _is_positive_semidefinite(fixed_matrix): # pragma: no cover + warnings.warn( + "Could not fix matrix. Please try a different risk model.", UserWarning + ) # Rebuild labels if provided if isinstance(matrix, pd.DataFrame): @@ -109,7 +111,6 @@ def risk_matrix(prices, method="sample_cov", **kwargs): - ``sample_cov`` - ``semicovariance`` - ``exp_cov`` - - ``min_cov_determinant`` - ``ledoit_wolf`` - ``ledoit_wolf_constant_variance`` - ``ledoit_wolf_single_factor`` @@ -127,8 +128,6 @@ def risk_matrix(prices, method="sample_cov", **kwargs): return semicovariance(prices, **kwargs) elif method == "exp_cov": return exp_cov(prices, **kwargs) - elif method == "min_cov_determinant": - return min_cov_determinant(prices, **kwargs) elif method == "ledoit_wolf" or method == "ledoit_wolf_constant_variance": return CovarianceShrinkage(prices, **kwargs).ledoit_wolf() elif method == "ledoit_wolf_single_factor": @@ -272,24 +271,9 @@ def exp_cov(prices, returns_data=False, span=180, frequency=252, **kwargs): def min_cov_determinant( prices, returns_data=False, frequency=252, random_state=None, **kwargs -): - """ - Calculate the minimum covariance determinant, an estimator of the covariance matrix - that is more robust to noise. +): # pragma: no cover + warnings.warn("min_cov_determinant is deprecated and will be removed in v1.5") - :param prices: adjusted closing prices of the asset, each row is a date - and each column is a ticker/id. - :type prices: pd.DataFrame - :param returns_data: if true, the first argument is returns instead of prices. - :type returns_data: bool, defaults to False. - :param frequency: number of time periods in a year, defaults to 252 (the number - of trading days in a year) - :type frequency: int, optional - :param random_state: random seed to make results reproducible, defaults to None - :type random_state: int, optional - :return: annualised estimate of covariance matrix - :rtype: pd.DataFrame - """ if not isinstance(prices, pd.DataFrame): warnings.warn("data is not in a dataframe", RuntimeWarning) prices = pd.DataFrame(prices) @@ -303,10 +287,11 @@ def min_cov_determinant( assets = prices.columns if returns_data: - X = prices.dropna(how="all") + X = prices else: - X = prices.pct_change().dropna(how="all") - X = np.nan_to_num(X.values) + X = returns_from_prices(prices) + # X = np.nan_to_num(X.values) + X = X.dropna().values raw_cov_array = sklearn.covariance.fast_mcd(X, random_state=random_state)[1] cov = pd.DataFrame(raw_cov_array, index=assets, columns=assets) * frequency return fix_nonpositive_semidefinite(cov, kwargs.get("fix_method", "spectral")) @@ -342,7 +327,7 @@ def corr_to_cov(corr_matrix, stdevs): :rtype: pd.DataFrame """ if not isinstance(corr_matrix, pd.DataFrame): - warnings.warn("cov_matrix is not a dataframe", RuntimeWarning) + warnings.warn("corr_matrix is not a dataframe", RuntimeWarning) corr_matrix = pd.DataFrame(corr_matrix) return corr_matrix * np.outer(stdevs, stdevs) @@ -377,7 +362,7 @@ def __init__(self, prices, returns_data=False, frequency=252): from sklearn import covariance self.covariance = covariance - except (ModuleNotFoundError, ImportError): + except (ModuleNotFoundError, ImportError): # pragma: no cover raise ImportError("Please install scikit-learn via pip or poetry") if not isinstance(prices, pd.DataFrame): diff --git a/pyproject.toml b/pyproject.toml index 0866a022..e2dd93c7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [tool.poetry] name = "PyPortfolioOpt" version = "1.4.0" -description = "Financial portfolio optimisation in python" +description = "Financial portfolio optimization in python" license = "MIT" authors = ["Robert Andrew Martin "] readme = "README.md" @@ -36,10 +36,10 @@ python = "^3.6.1" numpy = "^1.12" scipy = "^1.3" pandas = ">=0.19" -cvxopt = "^1.2, !=1.2.5.post1" cvxpy = "^1.1.10" -scikit-learn = {version="^0.24.1", optional= true } -matplotlib = { version = "^3.2.0", optional = true } +cvxopt = {version="^1.2, !=1.2.5.post1", optional=true } +scikit-learn = {version = "^0.24.1", optional=true } +matplotlib = { version = "^3.2.0", optional=true } [tool.poetry.dev-dependencies] pytest = "^4.6" @@ -48,9 +48,10 @@ jupyter = "^1.0.0" black = "^20.8b1" ipykernel = "^5.4.3" jedi = "0.17.2" +pytest-cov = "^2.11.1" [tool.poetry.extras] -optionals = ["scikit-learn", "matplotlib"] +optionals = ["scikit-learn", "matplotlib", "cvxopt"] [build-system] requires = ["poetry>=0.12"] diff --git a/setup.py b/setup.py index 2e669438..9a43cfd2 100755 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ setup( name="PyPortfolioOpt", version="1.4.0", - description="Financial portfolio optimisation in python", + description="Financial portfolio optimization in python", long_description=desc, long_description_content_type="text/markdown", url="https://github.com/robertmartin8/PyPortfolioOpt", diff --git a/tests/test_base_optimizer.py b/tests/test_base_optimizer.py index a1ec9ce0..f1d1e9c0 100644 --- a/tests/test_base_optimizer.py +++ b/tests/test_base_optimizer.py @@ -6,10 +6,20 @@ import cvxpy as cp from pypfopt import EfficientFrontier from pypfopt import exceptions -from pypfopt.base_optimizer import portfolio_performance +from pypfopt.base_optimizer import portfolio_performance, BaseOptimizer from tests.utilities_for_tests import get_data, setup_efficient_frontier +def test_base_optimizer(): + """ Cover code not covered elsewhere.""" + # Test tickers not provided + bo = BaseOptimizer(2) + assert bo.tickers == [0, 1] + w = {0: 0.4, 1: 0.6} + bo.set_weights(w) + assert dict(bo.clean_weights()) == w + + def test_custom_bounds(): ef = EfficientFrontier( *setup_efficient_frontier(data_only=True), weight_bounds=(0.02, 0.13) @@ -83,7 +93,7 @@ def test_bound_input_types(): def test_bound_failure(): - # Ensure optimisation fails when lower bound is too high or upper bound is too low + # Ensure optimization fails when lower bound is too high or upper bound is too low ef = EfficientFrontier( *setup_efficient_frontier(data_only=True), weight_bounds=(0.06, 0.13) ) @@ -233,7 +243,12 @@ def test_portfolio_performance(): portfolio_performance(ef.weights, ef.expected_returns, ef.cov_matrix, True) == expected ) - + # including when used without expected returns too. + assert portfolio_performance(ef.weights, None, ef.cov_matrix, True) == ( + None, + expected[1], + None, + ) # Internal ticker creations when weights param is a dict and ... w_dict = dict(zip(ef.tickers, ef.weights)) # ... expected_returns is a Series diff --git a/tests/test_black_litterman.py b/tests/test_black_litterman.py index 4001565e..b4026b34 100644 --- a/tests/test_black_litterman.py +++ b/tests/test_black_litterman.py @@ -29,6 +29,10 @@ def test_input_errors(): # Because default_omega uses matrix mult on P BlackLittermanModel(S, Q=views, P=P) + # P not an DataFrame or ndarray + with pytest.raises(TypeError): + BlackLittermanModel(S, Q=views[:-1], P=1.0) + with pytest.raises(AssertionError): BlackLittermanModel(S, Q=views, P=P, omega=np.eye(len(views))) @@ -36,6 +40,22 @@ def test_input_errors(): with pytest.raises(AssertionError): BlackLittermanModel(S, Q=views, pi=df.pct_change().mean()[:-1]) + # If pi=="market" then market_caps must be supplied + with pytest.raises(ValueError): + BlackLittermanModel(S, Q=views, pi="market") + + # pi's valid numerical types are Series, DataFrame and ndarray + with pytest.raises(TypeError): + BlackLittermanModel(S, Q=views, pi=[0.1] * len(S)) + + # risk_aversion cannot be negative + with pytest.raises(ValueError): + BlackLittermanModel(S, Q=views, risk_aversion=-0.01) + + # omega must be ndarray, DataFrame and string + with pytest.raises(TypeError): + BlackLittermanModel(S, Q=views, omega=1.0) + def test_parse_views(): df = get_data() @@ -73,7 +93,8 @@ def test_dataframe_input(): # views on the first 10 assets view_df = pd.DataFrame(pd.Series(0.1, index=S.columns)[:10]) - picking = np.eye(len(S))[:10, :] + # P's index and columns labels are ignored when a DataFrame is used: + picking = pd.DataFrame(np.eye(len(S))[:10, :]) assert BlackLittermanModel(S, Q=view_df, P=picking) prior_df = df.pct_change().mean() @@ -82,6 +103,19 @@ def test_dataframe_input(): assert BlackLittermanModel(S, pi=prior_df, Q=view_df, P=picking, omega=omega_df) +def test_cov_ndarray(): + df = get_data() + prior_df = df.pct_change().mean() + S = risk_models.sample_cov(df) + views = pd.Series(0.1, index=S.columns) + bl = BlackLittermanModel(S, pi=prior_df, Q=views) + bl_nd = BlackLittermanModel(S.to_numpy(), pi=prior_df.to_numpy(), Q=views) + # Compare without missing ticker index values. + np.testing.assert_equal(bl_nd.bl_returns().to_numpy(), bl.bl_returns().to_numpy()) + np.testing.assert_equal(bl_nd.bl_cov().to_numpy(), bl.bl_cov().to_numpy()) + assert list(bl_nd.bl_weights().values()) == list(bl.bl_weights().values()) + + def test_default_omega(): df = get_data() S = risk_models.sample_cov(df) @@ -215,6 +249,11 @@ def test_market_risk_aversion(): delta = black_litterman.market_implied_risk_aversion(prices) assert np.round(delta.iloc[0], 5) == 2.68549 + # Check it raises for other types. + list_invalid = [100.0, 110.0, 120.0, 130.0] + with pytest.raises(TypeError): + delta = black_litterman.market_implied_risk_aversion(list_invalid) + def test_bl_weights(): df = get_data() diff --git a/tests/test_cla.py b/tests/test_cla.py index e0335370..978f3628 100644 --- a/tests/test_cla.py +++ b/tests/test_cla.py @@ -62,6 +62,12 @@ def test_cla_custom_bounds(): np.testing.assert_almost_equal(cla.weights.sum(), 1) assert (0.01 <= cla.weights[::2]).all() and (cla.weights[::2] <= 0.13).all() assert (0.02 <= cla.weights[1::2]).all() and (cla.weights[1::2] <= 0.11).all() + # Test polymorphism of the weight_bounds param. + bounds2 = ([bounds[0][0], bounds[1][0]] * 10, [bounds[0][1], bounds[1][1]] * 10) + cla2 = CLA(*setup_cla(data_only=True), weight_bounds=bounds2) + cla2.cov_matrix = risk_models.exp_cov(df).values + w2 = cla2.min_volatility() + assert dict(w2) == dict(w) def test_cla_min_volatility(): diff --git a/tests/test_custom_objectives.py b/tests/test_custom_objectives.py index 45955635..97cbad18 100644 --- a/tests/test_custom_objectives.py +++ b/tests/test_custom_objectives.py @@ -1,10 +1,11 @@ import numpy as np import cvxpy as cp import pytest -from pypfopt import EfficientFrontier +from pypfopt import EfficientFrontier, expected_returns, risk_models +from pypfopt.base_optimizer import BaseConvexOptimizer from pypfopt import objective_functions from pypfopt import exceptions -from tests.utilities_for_tests import setup_efficient_frontier +from tests.utilities_for_tests import setup_efficient_frontier, get_data def test_custom_convex_equal_weights(): @@ -17,6 +18,22 @@ def new_objective(w): np.testing.assert_allclose(ef.weights, np.array([1 / 20] * 20)) +def test_custom_convex_additional(): + ef = setup_efficient_frontier() + ef.add_objective(objective_functions.L2_reg, gamma=1) + w_co = ef.convex_objective( + objective_functions.portfolio_variance, cov_matrix=ef.cov_matrix + ) + + # Same as test_min_volatility_L2_reg: + ef = setup_efficient_frontier() + ef.add_objective(objective_functions.L2_reg, gamma=1) + w_mv = ef.min_volatility() + # Weights from custom convex objective match those from min_volatility where + # an additional objective is applied to both. + assert dict(w_co) == dict(w_mv) + + def test_custom_convex_abs_exposure(): ef = EfficientFrontier( *setup_efficient_frontier(data_only=True), weight_bounds=(None, None) @@ -53,7 +70,7 @@ def test_custom_convex_objective_market_neutral_efficient_risk(): ef.efficient_risk(target_risk, market_neutral=True) built_in = ef.weights - # Recreate the market-neutral efficient_risk optimiser using this API + # Recreate the market-neutral efficient_risk optimizer using this API ef = EfficientFrontier( *setup_efficient_frontier(data_only=True), weight_bounds=(-1, 1) ) @@ -75,8 +92,39 @@ def test_convex_sharpe_raises_error(): ) +def test_custom_tracking_error(): + df = get_data() + historical_rets = expected_returns.returns_from_prices(df).dropna() + benchmark_rets = historical_rets["AAPL"] + historical_rets = historical_rets.drop("AAPL", axis=1) + S = risk_models.sample_cov(historical_rets, returns_data=True) + + opt = BaseConvexOptimizer( + n_assets=len(historical_rets.columns), + tickers=list(historical_rets.columns), + weight_bounds=(0, 1), + ) + + opt.convex_objective( + objective_functions.ex_post_tracking_error, + historic_returns=historical_rets, + benchmark_returns=benchmark_rets, + ) + w = opt.clean_weights() + + ef = EfficientFrontier(None, S) + ef.convex_objective( + objective_functions.ex_post_tracking_error, + historic_returns=historical_rets, + benchmark_returns=benchmark_rets, + ) + w2 = ef.clean_weights() + + assert w == w2 + + def test_custom_convex_logarithmic_barrier(): - # 60 Years of Portfolio Optimisation, Kolm et al (2014) + # 60 Years of Portfolio Optimization, Kolm et al (2014) ef = setup_efficient_frontier() def logarithmic_barrier(w, cov_matrix, k=0.1): @@ -96,7 +144,7 @@ def logarithmic_barrier(w, cov_matrix, k=0.1): def test_custom_convex_deviation_risk_parity_error(): - # 60 Years of Portfolio Optimisation, Kolm et al (2014) + # 60 Years of Portfolio Optimization, Kolm et al (2014) ef = setup_efficient_frontier() def deviation_risk_parity(w, cov_matrix): @@ -147,7 +195,7 @@ def test_custom_nonconvex_min_var(): def test_custom_nonconvex_logarithmic_barrier(): - # 60 Years of Portfolio Optimisation, Kolm et al (2014) + # 60 Years of Portfolio Optimization, Kolm et al (2014) ef = setup_efficient_frontier() def logarithmic_barrier(weights, cov_matrix, k=0.1): @@ -162,7 +210,7 @@ def logarithmic_barrier(weights, cov_matrix, k=0.1): def test_custom_nonconvex_deviation_risk_parity_1(): - # 60 Years of Portfolio Optimisation, Kolm et al (2014) - first definition + # 60 Years of Portfolio Optimization, Kolm et al (2014) - first definition ef = setup_efficient_frontier() def deviation_risk_parity(w, cov_matrix): @@ -176,7 +224,7 @@ def deviation_risk_parity(w, cov_matrix): def test_custom_nonconvex_deviation_risk_parity_2(): - # 60 Years of Portfolio Optimisation, Kolm et al (2014) - second definition + # 60 Years of Portfolio Optimization, Kolm et al (2014) - second definition ef = setup_efficient_frontier() def deviation_risk_parity(w, cov_matrix): @@ -266,7 +314,7 @@ def utility_obj(weights, mu, cov_matrix, k=1): def test_custom_nonconvex_objective_market_neutral_efficient_risk(): - # Recreate the market-neutral efficient_risk optimiser using this API + # Recreate the market-neutral efficient_risk optimizer using this API target_risk = 0.19 ef = EfficientFrontier( *setup_efficient_frontier(data_only=True), weight_bounds=(-1, 1) diff --git a/tests/test_discrete_allocation.py b/tests/test_discrete_allocation.py index dafe77b0..68896d7e 100644 --- a/tests/test_discrete_allocation.py +++ b/tests/test_discrete_allocation.py @@ -25,7 +25,7 @@ def test_get_latest_prices_error(): def test_remove_zero_positions(): raw = {"MA": 14, "FB": 12, "XOM": 0, "PFE": 51, "BABA": 5, "GOOG": 0} - da = DiscreteAllocation({}, pd.Series()) + da = DiscreteAllocation({}, pd.Series(dtype=float)) assert da._remove_zero_positions(raw) == {"MA": 14, "FB": 12, "PFE": 51, "BABA": 5} @@ -40,14 +40,14 @@ def test_greedy_portfolio_allocation(): da = DiscreteAllocation(w, latest_prices, short_ratio=0.3) allocation, leftover = da.greedy_portfolio() - assert { - "MA": 14, + assert allocation == { + "MA": 20, "FB": 12, - "PFE": 51, - "BABA": 5, - "AAPL": 5, - "BBY": 9, - "SBUX": 6, + "PFE": 54, + "BABA": 4, + "AAPL": 4, + "BBY": 2, + "SBUX": 1, "GOOG": 1, } @@ -56,6 +56,11 @@ def test_greedy_portfolio_allocation(): total += num * latest_prices[ticker] np.testing.assert_almost_equal(total + leftover, 10000, decimal=4) + # Cover the verbose parameter, + allocation_verbose, leftover_verbose = da.greedy_portfolio(verbose=True) + assert allocation_verbose == allocation + assert leftover_verbose == leftover + def test_greedy_allocation_rmse_error(): df = get_data() @@ -114,6 +119,11 @@ def test_greedy_portfolio_allocation_short(): long_total + short_total + leftover, 13000, decimal=4 ) + # Cover the verbose parameter, + allocation_verbose, leftover_verbose = da.greedy_portfolio(verbose=True) + assert allocation_verbose == allocation + assert leftover_verbose == leftover + def test_greedy_allocation_rmse_error_short(): df = get_data() @@ -257,6 +267,11 @@ def test_lp_portfolio_allocation(): total += num * latest_prices[ticker] np.testing.assert_almost_equal(total + leftover, 10000, decimal=4) + # Cover the verbose parameter, + allocation_verbose, leftover_verbose = da.lp_portfolio(verbose=True) + assert allocation_verbose == allocation + assert leftover_verbose == leftover + def test_lp_allocation_rmse_error(): df = get_data() @@ -315,6 +330,11 @@ def test_lp_portfolio_allocation_short(): long_total + short_total + leftover, 13000, decimal=5 ) + # Cover the verbose parameter, + allocation_verbose, leftover_verbose = da.lp_portfolio(verbose=True) + assert allocation_verbose == allocation + assert leftover_verbose == leftover + def test_lp_portfolio_allocation_short_reinvest(): df = get_data() diff --git a/tests/test_efficient_cvar.py b/tests/test_efficient_cvar.py index 7544996a..08a988f1 100644 --- a/tests/test_efficient_cvar.py +++ b/tests/test_efficient_cvar.py @@ -434,6 +434,9 @@ def test_cvar_errors(): # Beta must be between 0 and 1 cv = EfficientCVaR(mu, historical_rets, 1) + with pytest.warns(UserWarning): + cv = EfficientCVaR(mu, historical_rets, 0.1) + with pytest.raises(OptimizationError): # Must be <= max expected return cv = EfficientCVaR(mu, historical_rets) diff --git a/tests/test_efficient_frontier.py b/tests/test_efficient_frontier.py index 27803f66..132eb36a 100644 --- a/tests/test_efficient_frontier.py +++ b/tests/test_efficient_frontier.py @@ -582,9 +582,10 @@ def test_max_sharpe_risk_free_portfolio_performance(): # max_sharpe ef = setup_efficient_frontier() ef.max_sharpe(risk_free_rate=0.05) - res = ef.portfolio_performance() - res2 = ef.portfolio_performance(risk_free_rate=0.05) - np.testing.assert_allclose(res, res2) + with pytest.warns(UserWarning): + res = ef.portfolio_performance() + res2 = ef.portfolio_performance(risk_free_rate=0.05) + np.testing.assert_allclose(res, res2) def test_min_vol_pair_constraint(): diff --git a/tests/test_efficient_semivariance.py b/tests/test_efficient_semivariance.py index d57191b6..f0aa68fc 100644 --- a/tests/test_efficient_semivariance.py +++ b/tests/test_efficient_semivariance.py @@ -165,8 +165,8 @@ def test_min_semivariance(): np.testing.assert_allclose( es.portfolio_performance(), (0.1024524845740464, 0.08497381732237187, 0.970328121911246), - rtol=1e-4, - atol=1e-4, + rtol=1e-3, + atol=1e-3, ) diff --git a/tests/test_expected_returns.py b/tests/test_expected_returns.py index d398a5ec..b2fccc95 100644 --- a/tests/test_expected_returns.py +++ b/tests/test_expected_returns.py @@ -57,9 +57,6 @@ def test_log_returns_from_prices(): new_nan = log_rets.isnull().sum(axis=1).sum() assert new_nan == old_nan np.testing.assert_almost_equal(log_rets.iloc[-1, -1], 0.0001682740081102576) - # Test the deprecated function, until it is removed. - deprecated_log_rets = expected_returns.log_returns_from_prices(df) - np.testing.assert_allclose(deprecated_log_rets, log_rets) def test_mean_historical_returns_dummy(): @@ -74,11 +71,11 @@ def test_mean_historical_returns_dummy(): ) mean = expected_returns.mean_historical_return(data, frequency=1) test_answer = pd.Series([0.0061922, 0.0241137, 0.0122722, -0.0421775]) - pd.testing.assert_series_equal(mean, test_answer, check_less_precise=4) + pd.testing.assert_series_equal(mean, test_answer, rtol=1e-3) mean = expected_returns.mean_historical_return(data, compounding=False, frequency=1) test_answer = pd.Series([0.0086560, 0.0250000, 0.0128697, -0.03632333]) - pd.testing.assert_series_equal(mean, test_answer, check_less_precise=4) + pd.testing.assert_series_equal(mean, test_answer, rtol=1e-3) def test_mean_historical_returns(): @@ -142,10 +139,11 @@ def test_ema_historical_return(): assert mean.notnull().all() assert mean.dtype == "float64" # Test the (warning triggering) case that input is not a dataFrame - mean_np = expected_returns.ema_historical_return(df.to_numpy()) - mean_np.name = mean.name # These will differ. - reset_mean = mean.reset_index(drop=True) # Index labels would be tickers. - pd.testing.assert_series_equal(mean_np, reset_mean) + with pytest.warns(RuntimeWarning): + mean_np = expected_returns.ema_historical_return(df.to_numpy()) + mean_np.name = mean.name # These will differ. + reset_mean = mean.reset_index(drop=True) # Index labels would be tickers. + pd.testing.assert_series_equal(mean_np, reset_mean) def test_ema_historical_return_frequency(): @@ -195,10 +193,11 @@ def test_capm_no_benchmark(): ) np.testing.assert_array_almost_equal(mu.values, correct_mu) # Test the (warning triggering) case that input is not a dataFrame - mu_np = expected_returns.capm_return(df.to_numpy()) - mu_np.name = mu.name # These will differ. - mu_np.index = mu.index # Index labels would be tickers. - pd.testing.assert_series_equal(mu_np, mu) + with pytest.warns(RuntimeWarning): + mu_np = expected_returns.capm_return(df.to_numpy()) + mu_np.name = mu.name # These will differ. + mu_np.index = mu.index # Index labels would be tickers. + pd.testing.assert_series_equal(mu_np, mu) def test_capm_with_benchmark(): @@ -236,6 +235,9 @@ def test_capm_with_benchmark(): ) np.testing.assert_array_almost_equal(mu.values, correct_mu) + mu2 = expected_returns.capm_return(df, market_prices=mkt_df, compounding=False) + assert (mu2 >= mu).all() + def test_risk_matrix_and_returns_data(): # Test the switcher method for simple calls @@ -272,3 +274,9 @@ def test_return_model_not_implemented(): df = get_data() with pytest.raises(NotImplementedError): expected_returns.return_model(df, method="fancy_new!") + + +def test_james_stein_shrinkage(): + df = get_data() + with pytest.raises(NotImplementedError): + expected_returns.james_stein_shrinkage(df) diff --git a/tests/test_hrp.py b/tests/test_hrp.py index 96987a8e..8b1eb3c5 100644 --- a/tests/test_hrp.py +++ b/tests/test_hrp.py @@ -6,6 +6,21 @@ from tests.utilities_for_tests import get_data, resource +def test_hrp_errors(): + with pytest.raises(ValueError): + hrp = HRPOpt() + + df = get_data() + returns = df.pct_change().dropna(how="all") + returns_np = returns.to_numpy() + with pytest.raises(TypeError): + hrp = HRPOpt(returns_np) + + hrp = HRPOpt(returns) + with pytest.raises(ValueError): + hrp.optimize(linkage_method="blah") + + def test_hrp_portfolio(): df = get_data() returns = df.pct_change().dropna(how="all") @@ -16,9 +31,7 @@ def test_hrp_portfolio(): # pd.Series(w).to_csv(resource("weights_hrp.csv")) x = pd.read_csv(resource("weights_hrp.csv"), squeeze=True, index_col=0) - pd.testing.assert_series_equal( - x, pd.Series(w), check_names=False, check_less_precise=True - ) + pd.testing.assert_series_equal(x, pd.Series(w), check_names=False, rtol=1e-2) assert isinstance(w, dict) assert set(w.keys()) == set(df.columns) diff --git a/tests/test_imports.py b/tests/test_imports.py new file mode 100644 index 00000000..1edc8d16 --- /dev/null +++ b/tests/test_imports.py @@ -0,0 +1,46 @@ +def test_import_modules(): + from pypfopt import ( + base_optimizer, + black_litterman, + cla, + discrete_allocation, + exceptions, + expected_returns, + hierarchical_portfolio, + objective_functions, + plotting, + risk_models, + ) + + +def test_explicit_import(): + from pypfopt.black_litterman import ( + market_implied_prior_returns, + market_implied_risk_aversion, + BlackLittermanModel, + ) + from pypfopt.cla import CLA + from pypfopt.discrete_allocation import get_latest_prices, DiscreteAllocation + from pypfopt.efficient_frontier import ( + EfficientFrontier, + EfficientSemivariance, + EfficientCVaR, + ) + from pypfopt.hierarchical_portfolio import HRPOpt + from pypfopt.risk_models import CovarianceShrinkage + + +def test_import_toplevel(): + from pypfopt import ( + market_implied_prior_returns, + market_implied_risk_aversion, + BlackLittermanModel, + CLA, + get_latest_prices, + DiscreteAllocation, + EfficientFrontier, + EfficientSemivariance, + EfficientCVaR, + HRPOpt, + CovarianceShrinkage, + ) diff --git a/tests/test_plotting.py b/tests/test_plotting.py index 7e6dc450..40b8e26b 100644 --- a/tests/test_plotting.py +++ b/tests/test_plotting.py @@ -1,6 +1,8 @@ import numpy as np import matplotlib import matplotlib.pyplot as plt +import os +import pytest from tests.utilities_for_tests import get_data, setup_efficient_frontier from pypfopt import plotting, risk_models, expected_returns from pypfopt import HRPOpt, CLA, EfficientFrontier @@ -9,14 +11,29 @@ def test_correlation_plot(): plt.figure() df = get_data() - S = risk_models.CovarianceShrinkage(df).ledoit_wolf() ax = plotting.plot_covariance(S, showfig=False) assert len(ax.findobj()) == 256 plt.clf() + ax = plotting.plot_covariance(S, plot_correlation=True, showfig=False) + assert len(ax.findobj()) == 256 + plt.clf() ax = plotting.plot_covariance(S, show_tickers=False, showfig=False) assert len(ax.findobj()) == 136 plt.clf() + ax = plotting.plot_covariance( + S, plot_correlation=True, show_tickers=False, showfig=False + ) + assert len(ax.findobj()) == 136 + plt.clf() + + plot_filename = "tests/plot.png" + ax = plotting.plot_covariance(S, filename=plot_filename, showfig=False) + assert len(ax.findobj()) == 256 + assert os.path.exists(plot_filename) + assert os.path.getsize(plot_filename) > 0 + os.remove(plot_filename) + plt.clf() plt.close() @@ -38,6 +55,21 @@ def test_dendrogram_plot(): plt.clf() plt.close() + # Test that passing an unoptimized HRPOpt works, but issues a warning as + # this should already have been optimized according to the API. + hrp = HRPOpt(returns) + with pytest.warns(RuntimeWarning) as w: + ax = plotting.plot_dendrogram(hrp, show_tickers=False, showfig=False) + assert len(w) == 1 + assert ( + str(w[0].message) + == "hrp param has not been optimized. Attempting optimization." + ) + assert len(ax.findobj()) == 65 + assert type(ax.findobj()[0]) == matplotlib.collections.LineCollection + plt.clf() + plt.close() + def test_cla_plot(): plt.figure() @@ -90,7 +122,7 @@ def test_default_ef_plot(): def test_ef_plot_utility(): plt.figure() ef = setup_efficient_frontier() - delta_range = np.arange(0.001, 100, 1) + delta_range = np.arange(0.001, 50, 1) ax = plotting.plot_efficient_frontier( ef, ef_param="utility", ef_param_range=delta_range, showfig=False ) @@ -99,6 +131,24 @@ def test_ef_plot_utility(): plt.close() +def test_ef_plot_errors(): + plt.figure() + ef = setup_efficient_frontier() + delta_range = np.arange(0.001, 50, 1) + # Test invalid ef_param + with pytest.raises(NotImplementedError): + plotting.plot_efficient_frontier( + ef, ef_param="blah", ef_param_range=delta_range, showfig=False + ) + # Test invalid optimizer + with pytest.raises(NotImplementedError): + plotting.plot_efficient_frontier( + None, ef_param_range=delta_range, showfig=False + ) + plt.clf() + plt.close() + + def test_ef_plot_risk(): plt.figure() ef = setup_efficient_frontier() @@ -106,7 +156,7 @@ def test_ef_plot_risk(): min_risk = ef.portfolio_performance()[1] ef = setup_efficient_frontier() - risk_range = np.linspace(min_risk + 0.05, 0.5, 50) + risk_range = np.linspace(min_risk + 0.05, 0.5, 30) ax = plotting.plot_efficient_frontier( ef, ef_param="risk", ef_param_range=risk_range, showfig=False ) @@ -115,10 +165,12 @@ def test_ef_plot_risk(): plt.close() -def ef_plot_return(): +def test_ef_plot_return(): plt.figure() ef = setup_efficient_frontier() - return_range = np.linspace(0, ef.expected_returns.max(), 50) + # Internally _max_return() is used, so subtract epsilon + max_ret = ef.expected_returns.max() - 0.0001 + return_range = np.linspace(0, max_ret, 30) ax = plotting.plot_efficient_frontier( ef, ef_param="return", ef_param_range=return_range, showfig=False ) @@ -132,7 +184,7 @@ def test_ef_plot_utility_short(): ef = EfficientFrontier( *setup_efficient_frontier(data_only=True), weight_bounds=(None, None) ) - delta_range = np.linspace(0.001, 20, 100) + delta_range = np.linspace(0.001, 20, 50) ax = plotting.plot_efficient_frontier( ef, ef_param="utility", ef_param_range=delta_range, showfig=False ) @@ -148,7 +200,7 @@ def test_constrained_ef_plot_utility(): ef.add_constraint(lambda w: w[2] == 0.15) ef.add_constraint(lambda w: w[3] + w[4] <= 0.10) - delta_range = np.linspace(0.001, 20, 100) + delta_range = np.linspace(0.001, 20, 50) ax = plotting.plot_efficient_frontier( ef, ef_param="utility", ef_param_range=delta_range, showfig=False ) @@ -168,7 +220,7 @@ def test_constrained_ef_plot_risk(): ef.add_constraint(lambda w: w[3] + w[4] <= 0.10) # 100 portfolios with risks between 0.10 and 0.30 - risk_range = np.linspace(0.157, 0.40, 100) + risk_range = np.linspace(0.157, 0.40, 50) ax = plotting.plot_efficient_frontier( ef, ef_param="risk", ef_param_range=risk_range, show_assets=True, showfig=False ) diff --git a/tests/test_risk_models.py b/tests/test_risk_models.py index 157443d6..d35372b8 100644 --- a/tests/test_risk_models.py +++ b/tests/test_risk_models.py @@ -69,6 +69,17 @@ def test_sample_cov_npd(): str(w[0].message) == "The covariance matrix is non positive semidefinite. Amending eigenvalues." ) + # Test works on DataFrame too, same results, index and columns rebuilt. + tickers = ["A", "B"] + S_df = pd.DataFrame(data=S, index=tickers, columns=tickers) + S2_df = risk_models.fix_nonpositive_semidefinite(S_df, fix_method=method) + assert isinstance(S2_df, pd.DataFrame) + np.testing.assert_equal(S2_df.to_numpy(), S2) + assert S2_df.index.equals(S_df.index) + assert S2_df.columns.equals(S_df.columns) + with pytest.warns(UserWarning): + with pytest.raises(NotImplementedError): + risk_models.fix_nonpositive_semidefinite(S, fix_method="blah") def test_fix_npd_different_method(): @@ -96,6 +107,10 @@ def test_semicovariance(): assert risk_models._is_positive_semidefinite(S) S2 = risk_models.semicovariance(df, frequency=2) pd.testing.assert_frame_equal(S / 126, S2) + # Cover that it works on np.ndarray, with a warning + with pytest.warns(RuntimeWarning): + S2_np = risk_models.semicovariance(df.to_numpy(), frequency=2) + np.testing.assert_equal(S2_np, S2.to_numpy()) def test_semicovariance_benchmark(): @@ -120,6 +135,13 @@ def test_exp_cov_matrix(): assert risk_models._is_positive_semidefinite(S) S2 = risk_models.exp_cov(df, frequency=2) pd.testing.assert_frame_equal(S / 126, S2) + # Cover that it works on np.ndarray, with a warning + with pytest.warns(RuntimeWarning): + S2_np = risk_models.exp_cov(df.to_numpy(), frequency=2) + np.testing.assert_equal(S2_np, S2.to_numpy()) + # Too short a span causes a warning. + with pytest.warns(UserWarning): + risk_models.exp_cov(df, frequency=2, span=9) def test_exp_cov_limits(): @@ -133,14 +155,20 @@ def test_exp_cov_limits(): assert np.abs(S2 - sample_cov).max().max() < 1e-3 -def test_min_cov_det(): - df = get_data() - S = risk_models.min_cov_determinant(df, random_state=8) - assert S.shape == (20, 20) - assert S.index.equals(df.columns) - assert S.index.equals(S.columns) - assert S.notnull().all().all() - # assert risk_models._is_positive_semidefinite(S) +# def test_min_cov_det(): +# df = get_data() +# S = risk_models.CovarianceShrinkage(df).ledoit_wolf() +# S = risk_models.min_cov_determinant(df, random_state=8) +# assert S.shape == (20, 20) +# assert S.index.equals(df.columns) +# assert S.index.equals(S.columns) +# assert S.notnull().all().all() +# # assert risk_models._is_positive_semidefinite(S) +# # Cover that it works on np.ndarray, with a warning +# with pytest.warns(RuntimeWarning): +# S2 = risk_models.min_cov_determinant(df.to_numpy(), random_state=8) +# assert isinstance(S2, pd.DataFrame) +# np.testing.assert_equal(S.to_numpy(), S2.to_numpy()) def test_cov_to_corr(): @@ -153,6 +181,7 @@ def test_cov_to_corr(): test_corr_numpy = risk_models.cov_to_corr(rets.cov().values) assert len(w) == 1 assert str(w[0].message) == "cov_matrix is not a dataframe" + assert isinstance(test_corr_numpy, pd.DataFrame) np.testing.assert_array_almost_equal(test_corr_numpy, rets.corr().values) @@ -163,6 +192,13 @@ def test_corr_to_cov(): new_cov = risk_models.corr_to_cov(test_corr, rets.std()) pd.testing.assert_frame_equal(new_cov, rets.cov()) + with pytest.warns(RuntimeWarning) as w: + cov_numpy = risk_models.corr_to_cov(test_corr.to_numpy(), rets.std()) + assert len(w) == 1 + assert str(w[0].message) == "corr_matrix is not a dataframe" + assert isinstance(cov_numpy, pd.DataFrame) + np.testing.assert_equal(cov_numpy.to_numpy(), new_cov.to_numpy()) + def test_covariance_shrinkage_init(): df = get_data() @@ -181,6 +217,13 @@ def test_shrunk_covariance(): assert list(shrunk_cov.columns) == list(df.columns) assert not shrunk_cov.isnull().any().any() assert risk_models._is_positive_semidefinite(shrunk_cov) + with pytest.warns(RuntimeWarning) as w: + cs_numpy = risk_models.CovarianceShrinkage(df.to_numpy()) + assert len(w) == 1 + assert str(w[0].message) == "data is not in a dataframe" + shrunk_cov_numpy = cs_numpy.shrunk_covariance(0.2) + assert isinstance(shrunk_cov_numpy, pd.DataFrame) + np.testing.assert_equal(shrunk_cov_numpy.to_numpy(), shrunk_cov.to_numpy()) def test_shrunk_covariance_extreme_delta(): diff --git a/tests/utilities_for_tests.py b/tests/utilities_for_tests.py index 3b3ebfbf..221146f5 100644 --- a/tests/utilities_for_tests.py +++ b/tests/utilities_for_tests.py @@ -119,7 +119,7 @@ def simple_ef_weights(expected_returns, cov_matrix, target_return, weights_sum): :type cov_matrix: np.ndarray :param target_return: the target return for the portfolio to achieve. :type target_return: float - :param weights_sum: the sum of the returned weights, optimisation constraint. + :param weights_sum: the sum of the returned weights, optimization constraint. :type weights_sum: float :return: weight for each asset, which sum to 1.0 :rtype: np.ndarray