diff --git a/dataflow_amp/system/Cx/Cx_dump_market_data.py b/dataflow_amp/system/Cx/Cx_dump_market_data.py index 4cd2f49de1..d60f894283 100755 --- a/dataflow_amp/system/Cx/Cx_dump_market_data.py +++ b/dataflow_amp/system/Cx/Cx_dump_market_data.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # TODO(Grisha): the script might become more general purpose, e.g. dump any data from db. """ -The script saves market data from the DB to a file. +The script saves market data from the DB to a file. ``` > dataflow_amp/system/Cx/Cx_dump_market_data.py \ --dst_dir '/shared_data/prod_reconciliation' \ @@ -17,10 +17,10 @@ import pandas as pd import dataflow_amp.system.Cx as dtfamsysc +import helpers.hdatetime as hdateti import helpers.hdbg as hdbg import helpers.hparser as hparser import im_v2.common.universe as ivcu -import oms _LOG = logging.getLogger(__name__) @@ -57,8 +57,9 @@ def dump_market_data_from_db( """ Save market data from the DB to a file. """ - start_timestamp = oms.timestamp_as_str_to_timestamp(start_timestamp_as_str) - end_timestamp = oms.timestamp_as_str_to_timestamp(end_timestamp_as_str) + tz = "America/New_York" + start_timestamp = hdateti.str_to_timestamp(start_timestamp_as_str, tz) + end_timestamp = hdateti.str_to_timestamp(end_timestamp_as_str, tz) # We need to use exactly the same data that the prod system ran against # in production. asset_ids = _get_universe() diff --git a/dev_scripts/cleanup_scripts/SorrTask584_Use_str_to_timestamp_instead_of_timestamp_as_str_to_timestamp.sh b/dev_scripts/cleanup_scripts/SorrTask584_Use_str_to_timestamp_instead_of_timestamp_as_str_to_timestamp.sh new file mode 100755 index 0000000000..111e97ab6c --- /dev/null +++ b/dev_scripts/cleanup_scripts/SorrTask584_Use_str_to_timestamp_instead_of_timestamp_as_str_to_timestamp.sh @@ -0,0 +1,21 @@ +#!/bin/bash -xe + +dir_names="amp/dev_scripts/cleanup_scripts dev_scripts/cleanup_scripts" + +replace_text.py \ + --old "(?" ] @@ -2003,7 +2004,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.10" + "version": "3.9.5" }, "toc": { "base_numbering": 1, diff --git a/oms/notebooks/Master_multiday_system_reconciliation.py b/oms/notebooks/Master_multiday_system_reconciliation.py index 7ec7aa0c5b..c69906e8f2 100644 --- a/oms/notebooks/Master_multiday_system_reconciliation.py +++ b/oms/notebooks/Master_multiday_system_reconciliation.py @@ -5,7 +5,7 @@ # extension: .py # format_name: percent # format_version: '1.3' -# jupytext_version: 1.14.1 +# jupytext_version: 1.15.0 # kernelspec: # display_name: Python 3 (ipykernel) # language: python @@ -35,6 +35,7 @@ import core.config as cconfig import core.plotting as coplotti import dataflow.model as dtfmod +import helpers.hdatetime as hdateti import helpers.hdbg as hdbg import helpers.henv as henv import helpers.hpandas as hpandas @@ -193,10 +194,9 @@ def compute_research_portfolio( ) # Compute research portfolio. dag_df_prod = get_prod_dag_output_for_last_node(system_log_path_dict) - start_timestamp = omreconc.timestamp_as_str_to_timestamp( - start_timestamp_as_str - ) - end_timestamp = omreconc.timestamp_as_str_to_timestamp(end_timestamp_as_str) + tz = "America/New_York" + start_timestamp = hdateti.str_to_timestamp(start_timestamp_as_str, tz) + end_timestamp = hdateti.str_to_timestamp(end_timestamp_as_str, tz) forecast_evaluator_from_prices_dict = reconciliation_config[ "research_forecast_evaluator_from_prices" ] diff --git a/oms/reconciliation.py b/oms/reconciliation.py index 788486080c..5e76af253a 100644 --- a/oms/reconciliation.py +++ b/oms/reconciliation.py @@ -234,8 +234,9 @@ def build_multiday_system_reconciliation_config( :param end_timestamp_as_str: string representation of timestamp at which to end reconcile run, e.g. "20221010_080000" """ - start_timestamp = timestamp_as_str_to_timestamp(start_timestamp_as_str) - end_timestamp = timestamp_as_str_to_timestamp(end_timestamp_as_str) + tz = "America/New_York" + start_timestamp = hdateti.str_to_timestamp(start_timestamp_as_str, tz) + end_timestamp = hdateti.str_to_timestamp(end_timestamp_as_str, tz) config = { "dst_root_dir": dst_root_dir, "dag_builder_name": dag_builder_name, @@ -570,16 +571,18 @@ def get_system_run_timestamps( system_run_timestamps = [tuple(ts.split(".")) for ts in timestamp_dirs] # Keep timestamps within the `[start_date, end_date]` range. tz = "UTC" + datetime_format = "%Y%m%d_%H%M%S" # Filter by start / end timestamps using system run start timestamp. if start_timestamp is not None: - # TODO(Grisha): use `str_to_timestamp()` from `helpers.hdatetime.py`. system_run_timestamps = [ (system_run_start_timestamp, system_run_end_timestamp) for ( system_run_start_timestamp, system_run_end_timestamp, ) in system_run_timestamps - if timestamp_as_str_to_timestamp(system_run_start_timestamp, tz=tz) + if hdateti.str_to_timestamp( + system_run_start_timestamp, tz, datetime_format=datetime_format + ) >= start_timestamp ] _LOG.info("Filtered by `start_timestamp`: %s.", system_run_timestamps) @@ -590,7 +593,9 @@ def get_system_run_timestamps( system_run_start_timestamp, system_run_end_timestamp, ) in system_run_timestamps - if timestamp_as_str_to_timestamp(system_run_start_timestamp, tz=tz) + if hdateti.str_to_timestamp( + system_run_start_timestamp, tz, datetime_format=datetime_format + ) <= end_timestamp ] _LOG.info("Filtered by `end_timestamp`: %s.", system_run_timestamps)