From 30640e2c98e4bea2d5f11462a501ac0acad1c06e Mon Sep 17 00:00:00 2001 From: BrianWeiHaoMa Date: Sun, 22 Dec 2024 17:39:32 -0500 Subject: [PATCH] Updated documentation. --- CONTRIBUTING.md | 2 +- DEVELOPMENT.md | 38 +++------ DOCUMENTATION.md | 32 +++---- MISOReports/MISOReports.py | 5 +- MISOReports/test_MISOReports.py | 5 ++ README.md | 143 +++++++++++++++++--------------- 6 files changed, 115 insertions(+), 110 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 71e6a72..aec68f5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -21,7 +21,7 @@ If you encounter any bugs, please let us know! To make your bug report as effect - Steps to reproduce the bug - The expected vs. actual behavior - Screenshots, error messages, or code snippets if applicable -- Environment details (e.g., operating system, browser, version) +- Environment details You can report bugs by [opening an issue](https://github.com/BrianWeiHaoMa/MISOReports/issues/new?template=bug_report.md). We’ll do our best to address them as soon as possible diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 616cd9f..d5bfbfd 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -1,24 +1,6 @@ # Development Here you can find the general guidelines for the development of MISOReports. -## Github Workflow -Before commiting or pushing to github, remember to run these in the terminal and make sure everything passes: - -For running all tests: -``` -pytest -``` - -If you want to skip the completion and long tests: -``` -pytest -m "not completion and not long" -``` - -For checking type annotations: -``` -mypy --strict .\MISOReports\MISOReports.py -``` - ## Coding Style * We are using the vscode extension, autoDocstring's, one-line-sphinx documentation template. * Try to keep the style the same as the code that was previously there in all respects (naming schemes, character length per line, etc.) @@ -26,7 +8,7 @@ mypy --strict .\MISOReports\MISOReports.py ## Reports to Pandas Dataframe Mapping Logic Remember to make a parsing function in parsers.py and make a new Report entry in MISOReports.report_mappings. -As well, make sure to add the report's test in single_df_test_list or multiple_dfs_test_list in test_MISOReports.py. +As well, make sure to add the report's get_df test in test_MISOReports.py. Continue to use the same naming scheme as the previous code. When in doubt, check the entries for previously completed reports. @@ -37,7 +19,7 @@ Map every single column type to one of the below data pandas types: * **Float64** ex. 34.13. * **Int64** ex. 34. -When looking at the report, use this checklist: +When typing Dataframe columns, use this checklist: * Ignore null/empty values when deciding with the below guidelines. * If there is any string (ex. names, codes, etc.) in the column, the column type should be **string**. * Otherwise if the column is clearly meant to portray datetime/date/time, the column type should be **datetime64[ns]**. @@ -49,7 +31,7 @@ When looking at the report, use this checklist: csv ```python def parse_rt_lmp_prelim( - res: requests.Response, + res: requests.Response, ) -> pd.DataFrame: text = res.text csv_data = "\n".join(text.splitlines()[4:]) @@ -67,7 +49,7 @@ def parse_rt_lmp_prelim( json ```python def parse_SolarForecast( - res: requests.Response, + res: requests.Response, ) -> pd.DataFrame: text = res.text dictionary = json.loads(text) @@ -86,19 +68,23 @@ def parse_SolarForecast( zip with a csv file in the extracted folder ```python def parse_DA_LMPs( - res: requests.Response, + res: requests.Response, ) -> pd.DataFrame: with zipfile.ZipFile(file=io.BytesIO(res.content)) as z: text = z.read(z.namelist()[0]).decode("utf-8") - csv_data = "\n".join(text.splitlines()[1:]) + csv_data = "\n".join(text.lstrip().splitlines()) df = pd.read_csv( filepath_or_buffer=io.StringIO(csv_data), ) df[["MARKET_DAY"]] = df[["MARKET_DAY"]].apply(pd.to_datetime, format="%m/%d/%Y") - df[["HE1", "HE2", "HE3", "HE4", "HE5", "HE6", "HE7", "HE8", "HE9", "HE10", "HE11", "HE12", "HE13", "HE14", "HE15", "HE16", "HE17", "HE18", "HE19", "HE20", "HE21", "HE22", "HE23", "HE24"]] = df[["HE1", "HE2", "HE3", "HE4", "HE5", "HE6", "HE7", "HE8", "HE9", "HE10", "HE11", "HE12", "HE13", "HE14", "HE15", "HE16", "HE17", "HE18", "HE19", "HE20", "HE21", "HE22", "HE23", "HE24"]].astype("Float64") + + float_columns = ["HE1", "HE2", "HE3", "HE4", "HE5", "HE6", "HE7", "HE8", "HE9", "HE10", "HE11", "HE12", "HE13", "HE14", "HE15", "HE16", "HE17", "HE18", "HE19", "HE20", "HE21", "HE22", "HE23", "HE24"] + df[float_columns] = df[float_columns].astype("string") + df[float_columns] = df[float_columns].apply(lambda x: x.str.replace(',', '')) + df[float_columns] = df[float_columns].astype("Float64") df[["NODE", "TYPE", "VALUE"]] = df[["NODE", "TYPE", "VALUE"]].astype("string") return df @@ -107,7 +93,7 @@ def parse_DA_LMPs( excel ```python def parse_5min_exante_lmp( - res: requests.Response, + res: requests.Response, ) -> pd.DataFrame: df = pd.read_excel( io=io.BytesIO(res.content), diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index cf1f7d1..691333c 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -19,7 +19,7 @@ This is the documentation for MISOReports. - [Real-Time](#real-time) - [Resource Adequacy](#resource-adequacy) - [Summary](#summary) -- [Useful Tricks](#useful-tricks) + - [Disclaimer](#disclaimer) ## Data Types All dataframe columns are categorized into one of the following data types: @@ -248,21 +248,18 @@ def parse_datetime_from_text(text: str): # Report link: https://api.misoenergy.org/MISORTWDDataBroker/DataBrokerServices.asmx?messageType=getNAI&returnType=csv. report_name = "NAI" -# Download the raw data to get the datetime. -res = MISOReports.get_response( +# Download the relevant data. +data = MISOReports.get_data( report_name=report_name, file_extension="csv", ) +text = data.response.text +df = data.df print("Raw data:") -print(res.text) +print(text) -# Download the dataframe. -df = MISOReports.get_df( - report_name=report_name, -) - -df["datetime"] = parse_datetime_from_text(res.text) +df["datetime"] = parse_datetime_from_text(text) print("Final dataframe:") print(df) @@ -271,12 +268,19 @@ print(df) Executing the above gives: ``` Raw data: -RefId,21-Nov-2024 - Interval 20:50 EST +RefId,22-Dec-2024 - Interval 17:30 EST Name,Value -MISO,-32.77 +MISO,2247.37 Final dataframe: - Name Value datetime -0 MISO -32.77 2024-11-21 20:50:00 + Name Value datetime +0 MISO 2247.37 2024-12-22 17:30:00 ``` + +## Disclaimer +MISO might make changes to the structure of their reports in the future and this could lead to the current parsers breaking. +It is reasonable to say that the majority of the reports supported by this library do not change very frequently +but if you want to make sure that the parsers are updated as quickly as possible, we suggest cloning the repository +and maintaining the parsers yourself. This way, you can leverage the rest of the library, while ensuring that the parsers +stay updated. \ No newline at end of file diff --git a/MISOReports/MISOReports.py b/MISOReports/MISOReports.py index c36e6f5..767c606 100644 --- a/MISOReports/MISOReports.py +++ b/MISOReports/MISOReports.py @@ -502,6 +502,7 @@ def get_data( url: str | None = None, ddatetime: datetime.datetime | None = None, timeout: int | None = None, + file_extension: str | None = None, ) -> Data: """Gets the relevant data for the report. @@ -510,6 +511,8 @@ def get_data( :param datetime.datetime | None ddatetime: The target datetime to download the report for, defaults to None :param int | None timeout: The timeout for the request, defaults to None + :param str | None file_extension: The file extension to download, defaults + to None in which case the default file extension is used. :return Data: An object containing the DataFrame and the response. """ report = MISOReports.report_mappings[report_name] @@ -522,7 +525,7 @@ def get_data( else: response = MISOReports.get_response( report_name=report_name, - file_extension=report.type_to_parse, + file_extension=file_extension, ddatetime=ddatetime, timeout=timeout, ) diff --git a/MISOReports/test_MISOReports.py b/MISOReports/test_MISOReports.py index 50e8b8e..63624ca 100644 --- a/MISOReports/test_MISOReports.py +++ b/MISOReports/test_MISOReports.py @@ -84,6 +84,11 @@ def try_to_get_dfs( direction=1, ) increment_cnt += 1 + except Exception as e: + raise Exception( + f"Unexpected exception for {report_name} at " + + f"{curr_target_datetime}: {e}" + ) if increment_cnt > datetime_increment_limit: if len(dfs) == 0: diff --git a/README.md b/README.md index c42e5ec..bc3fa43 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # MISOReports A comprehensive Python library for downloading Midcontinent Independent System Operator (MISO) public reports into pandas dataframes. -As of 2024-11-21, MISOReports supports reports from +As of 2024-12-22, MISOReports supports reports from [MISORTWDDataBroker](https://api.misoenergy.org/MISORTWDDataBroker/), [MISORTWDBIReporter](https://api.misoenergy.org/MISORTWDBIReporter/), and [MISO Market Reports](https://www.misoenergy.org/markets-and-operations/real-time--market-data/market-reports/), totalling to well over 120 different reports. @@ -12,34 +12,26 @@ For documentation and information on currently supported reports, please check o ## Features MISOReports supports these features and more: -- Downloading reports by date for reports that offer a date option +- Downloading reports by datetime for reports that offer a datetime option - Downloading live reports for reports without a date option - Downloading raw report content in any of their supported formats (csv, xml, json, xls, xlsx, etc.) - Generating target URLs for the report of your choice -- Thoroughly tested and well annotated functions and classes with >= 98% test coverage and a pass with mypy --strict - -## Installation -To install, use -``` -pip install MISOReports -``` ## Examples ### Example 1: -Download a single-table report from [MISORTWDDataBroker](https://api.misoenergy.org/MISORTWDDataBroker/). +Download a single-table report with datetime option from [MISO Market Reports](https://www.misoenergy.org/markets-and-operations/real-time--market-data/market-reports/). #### Input: ```python +import datetime from MISOReports.MISOReports import MISOReports -# Single table reports will be stored -# directly as the dataframe result. - # Downloads the data offered at -# https://api.misoenergy.org/MISORTWDDataBroker/DataBrokerServices.asmx?messageType=getlmpconsolidatedtable&returnType=csv. +# https://docs.misoenergy.org/marketreports/20241030_da_expost_ramp_mcp.xlsx. df = MISOReports.get_df( - report_name="lmpconsolidatedtable", + report_name="da_expost_ramp_mcp", + ddatetime=datetime.datetime(year=2024, month=10, day=30), ) print(df) @@ -47,20 +39,33 @@ print(df) #### Output: ``` - Name LMP MLC MCC REGMCP REGMILEAGEMCP SPINMCP SUPPMCP STRMCP RCUPMCP RCDOWNMCP LMP.1 MLC.1 MCC.1 LMP.2 MLC.2 MCC.2 LMP.3 MLC.3 MCC.3 -0 EES.AXIALL 20.58 -0.56 0.00 13.55 0.75 2.72 0.01 0.0 0.0 0.0 20.37 -0.74 0.00 20.65 -0.35 0.52 20.65 -0.35 0.52 -1 EES.CALCAS1_CT 20.72 -0.42 0.00 13.55 0.75 2.72 0.01 0.0 0.0 0.0 20.51 -0.60 0.00 20.75 -0.25 0.52 20.75 -0.25 0.52 -2 EES.CALCAS2_CT 20.72 -0.42 0.00 13.55 0.75 2.72 0.01 0.0 0.0 0.0 20.51 -0.60 0.00 20.75 -0.25 0.52 20.75 -0.25 0.52 -3 EES.CARV_A 21.00 -0.14 0.00 13.55 0.75 2.72 0.01 0.0 0.0 0.0 20.77 -0.34 0.00 21.13 0.16 0.49 21.13 0.16 0.49 -4 EES.CARV_BC 21.00 -0.14 0.00 13.55 0.75 2.72 0.01 0.0 0.0 0.0 20.77 -0.34 0.00 21.13 0.16 0.49 21.13 0.16 0.49 -.. ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... -322 EAI.WH_BLUFF2 20.64 -0.50 0.00 13.55 0.75 2.72 0.01 0.0 0.0 0.0 20.53 -0.58 0.00 19.90 -0.69 0.11 19.90 -0.69 0.11 -323 EDE 20.53 -0.59 -0.02 0.00 0.00 0.00 0.00 0.0 0.0 0.0 20.62 -0.44 -0.06 21.24 -0.93 1.69 21.24 -0.93 1.69 -324 EES.ACAD2_CT1 20.50 -0.64 0.00 13.55 0.75 2.72 0.01 0.0 0.0 0.0 20.26 -0.85 0.00 20.58 -0.41 0.51 20.58 -0.41 0.51 -325 EES.ACAD2_CT2 20.50 -0.64 0.00 13.55 0.75 2.72 0.01 0.0 0.0 0.0 20.26 -0.85 0.00 20.58 -0.41 0.51 20.58 -0.41 0.51 -326 EES.ACAD2_ST 20.50 -0.64 0.00 13.55 0.75 2.72 0.01 0.0 0.0 0.0 20.26 -0.85 0.00 20.58 -0.41 0.51 20.58 -0.41 0.51 - -[327 rows x 20 columns] + Hour Ending Reserve Zone 1 - DA MCP Ramp Up Ex-Post 1 Hour ... Reserve Zone 8 - DA MCP Ramp Up Ex-Post 1 Hour Reserve Zone 8 - DA MCP Ramp Down Ex-Post 1 Hour +0 1 0.00 ... 0.00 0.0 +1 2 0.00 ... 0.00 0.0 +2 3 0.00 ... 0.00 0.0 +3 4 0.00 ... 0.00 0.0 +4 5 0.00 ... 0.00 0.0 +5 6 0.17 ... 0.17 0.0 +6 7 1.48 ... 1.48 0.0 +7 8 0.00 ... 0.00 0.0 +8 9 0.00 ... 0.00 0.0 +9 10 0.00 ... 0.00 0.0 +10 11 0.00 ... 0.00 0.0 +11 12 1.08 ... 1.08 0.0 +12 13 1.81 ... 1.81 0.0 +13 14 2.56 ... 2.56 0.0 +14 15 3.13 ... 3.13 0.0 +15 16 5.00 ... 5.00 0.0 +16 17 5.00 ... 5.00 0.0 +17 18 12.85 ... 12.85 0.0 +18 19 5.17 ... 5.17 0.0 +19 20 0.00 ... 0.00 0.0 +20 21 0.00 ... 0.00 0.0 +21 22 0.00 ... 0.00 0.0 +22 23 0.00 ... 0.00 0.0 +23 24 0.00 ... 0.00 0.0 + +[24 rows x 17 columns] ``` ### Example 2: @@ -109,56 +114,47 @@ FiveMinTotalLoad ``` ### Example 3: -Download a single-table report with datetime option from [MISO Market Reports](https://www.misoenergy.org/markets-and-operations/real-time--market-data/market-reports/). +Download a multi-table report from [MISORTWDDataBroker](https://api.misoenergy.org/MISORTWDDataBroker/). #### Input: ```python -import datetime from MISOReports.MISOReports import MISOReports # Downloads the data offered at -# https://docs.misoenergy.org/marketreports/20241030_da_expost_ramp_mcp.xlsx. +# https://api.misoenergy.org/MISORTWDDataBroker/DataBrokerServices.asmx?messageType=getlmpconsolidatedtable&returnType=csv. df = MISOReports.get_df( - report_name="da_expost_ramp_mcp", - ddatetime=datetime.datetime(year=2024, month=10, day=30), + report_name="lmpconsolidatedtable", ) -print(df) +# For multi-table reports, use a for-loop +# to iterate across the tables. +for i, table_name in enumerate(df["table_names"]): + print(table_name) + print(df["dataframes"].iloc[i].head(3)) + print() + print() ``` #### Output: ``` - Hour Ending Reserve Zone 1 - DA MCP Ramp Up Ex-Post 1 Hour ... Reserve Zone 8 - DA MCP Ramp Up Ex-Post 1 Hour Reserve Zone 8 - DA MCP Ramp Down Ex-Post 1 Hour -0 1 0.00 ... 0.00 0.0 -1 2 0.00 ... 0.00 0.0 -2 3 0.00 ... 0.00 0.0 -3 4 0.00 ... 0.00 0.0 -4 5 0.00 ... 0.00 0.0 -5 6 0.17 ... 0.17 0.0 -6 7 1.48 ... 1.48 0.0 -7 8 0.00 ... 0.00 0.0 -8 9 0.00 ... 0.00 0.0 -9 10 0.00 ... 0.00 0.0 -10 11 0.00 ... 0.00 0.0 -11 12 1.08 ... 1.08 0.0 -12 13 1.81 ... 1.81 0.0 -13 14 2.56 ... 2.56 0.0 -14 15 3.13 ... 3.13 0.0 -15 16 5.00 ... 5.00 0.0 -16 17 5.00 ... 5.00 0.0 -17 18 12.85 ... 12.85 0.0 -18 19 5.17 ... 5.17 0.0 -19 20 0.00 ... 0.00 0.0 -20 21 0.00 ... 0.00 0.0 -21 22 0.00 ... 0.00 0.0 -22 23 0.00 ... 0.00 0.0 -23 24 0.00 ... 0.00 0.0 +Metadata + Type Timing +0 FiveMinLMP 1900-01-01 16:45:00 +1 HourlyIntegratedLmp 1900-01-01 16:00:00 +2 DayAheadExAnteLmp 1900-01-01 17:00:00 -[24 rows x 17 columns] + +Data + Name LMP - FiveMinLMP MLC - FiveMinLMP MCC - FiveMinLMP REGMCP - FiveMinLMP ... MLC - DayAheadExAnteLmp MCC - DayAheadExAnteLmp LMP - DayAheadExPostLmp MLC - DayAheadExPostLmp MCC - DayAheadExPostLmp +1 EES.PERVL2_CT 17.49 -1.69 -12.64 15.0 ... -1.15 -6.1 21.0 -1.15 -6.1 +2 EES.RICE1 17.91 -1.25 -12.66 15.0 ... -0.06 -6.21 21.98 -0.06 -6.21 +3 EES.RVRBEND1 18.42 -0.98 -12.42 15.0 ... -0.38 -5.83 22.04 -0.38 -5.83 + +[3 rows x 20 columns] ``` ### Example 4: -Download the raw, unparsed data of a report from [MISO Market Reports](https://www.misoenergy.org/markets-and-operations/real-time--market-data/market-reports/). +Download a single-table report along with its text content from [MISO Market Reports](https://www.misoenergy.org/markets-and-operations/real-time--market-data/market-reports/). #### Input: ```python @@ -166,20 +162,31 @@ from MISOReports.MISOReports import MISOReports # Downloads the data offered at # https://api.misoenergy.org/MISORTWDDataBroker/DataBrokerServices.asmx?messageType=getNAI&returnType=csv. -res = MISOReports.get_response( +data = MISOReports.get_data( report_name="NAI", file_extension="csv", ) -print(res.text) +print("Text Content:") +print(data.response.text) +print() + +print("Dataframe:") +print(data.df) ``` #### Output: ``` -RefId,17-Nov-2024 - Interval 15:50 EST +Text Content: +RefId,22-Dec-2024 - Interval 16:40 EST Name,Value -MISO,-2494.86 +MISO,2212.89 + + +Dataframe: + Name Value +0 MISO 2212.89 ``` ### Example 5: @@ -195,12 +202,12 @@ from MISOReports.MISOReports import MISOReports # Note: the above link's 304 represents # the number of days past the start of the year, # 2024, which aligns with the ddatetime given below. -df = MISOReports.get_df( +data = MISOReports.get_data( report_name="MISOdaily", ddatetime=datetime.datetime(year=2024, month=10, day=30), ) -print(df) +print(data.df) ``` #### Output: