SorrTask584_Use_str_to_timestamp_instead_of_timestamp_as_str_to_times…

…tamp (#585)

* Replaced timestamp_as_str_to_timestamp() with str_to_timestamp()

* Update Resolve

* Function parameter update

* Update issues

* Changes

* Working script

* Fixed tests

* Ran linter

* Added datetime_format parameter

---------

Co-authored-by: Shabarish Nair <shabarishnair2020@gmail.com>
Co-authored-by: Nina Lee <68428519+rheenina@users.noreply.github.com>

dataflow_amp/system/Cx/Cx_dump_market_data.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # TODO(Grisha): the script might become more general purpose, e.g. dump any data from db.
 """
-The script saves market data from the DB to a file.
+The script saves market data from the DB to a file. 
 ```
 > dataflow_amp/system/Cx/Cx_dump_market_data.py \
     --dst_dir '/shared_data/prod_reconciliation' \
@@ -17,10 +17,10 @@
 import pandas as pd
 
 import dataflow_amp.system.Cx as dtfamsysc
+import helpers.hdatetime as hdateti
 import helpers.hdbg as hdbg
 import helpers.hparser as hparser
 import im_v2.common.universe as ivcu
-import oms
 
 _LOG = logging.getLogger(__name__)
 
@@ -57,8 +57,9 @@ def dump_market_data_from_db(
     """
     Save market data from the DB to a file.
     """
-    start_timestamp = oms.timestamp_as_str_to_timestamp(start_timestamp_as_str)
-    end_timestamp = oms.timestamp_as_str_to_timestamp(end_timestamp_as_str)
+    tz = "America/New_York"
+    start_timestamp = hdateti.str_to_timestamp(start_timestamp_as_str, tz)
+    end_timestamp = hdateti.str_to_timestamp(end_timestamp_as_str, tz)
     # We need to use exactly the same data that the prod system ran against
     # in production.
     asset_ids = _get_universe()

dev_scripts/cleanup_scripts/SorrTask584_Use_str_to_timestamp_instead_of_timestamp_as_str_to_timestamp.sh

@@ -0,0 +1,21 @@
+#!/bin/bash -xe
+
+dir_names="amp/dev_scripts/cleanup_scripts dev_scripts/cleanup_scripts"
+
+replace_text.py \
+  --old "(?<!def\s)(?<!\.)(\btimestamp_as_str_to_timestamp\b)" \
+  --new "hdateti.str_to_timestamp" \
+  --exclude_dirs "$dir_names" \
+  --ext "py,ipynb"
+
+replace_text.py \
+  --old "omreconc.timestamp_as_str_to_timestamp" \
+  --new "hdateti.str_to_timestamp" \
+  --exclude_dirs "$dir_names" \
+  --ext "py,ipynb"
+
+replace_text.py \
+  --old "oms.timestamp_as_str_to_timestamp" \
+  --new "hdateti.str_to_timestamp" \
+  --exclude_dirs "$dir_names" \
+  --ext "py,ipynb"

oms/notebooks/Master_multiday_system_reconciliation.py

@@ -5,7 +5,7 @@
 #       extension: .py
 #       format_name: percent
 #       format_version: '1.3'
-#       jupytext_version: 1.14.1
+#       jupytext_version: 1.15.0
 #   kernelspec:
 #     display_name: Python 3 (ipykernel)
 #     language: python
@@ -35,6 +35,7 @@
 import core.config as cconfig
 import core.plotting as coplotti
 import dataflow.model as dtfmod
+import helpers.hdatetime as hdateti
 import helpers.hdbg as hdbg
 import helpers.henv as henv
 import helpers.hpandas as hpandas
@@ -193,10 +194,9 @@ def compute_research_portfolio(
     )
     # Compute research portfolio.
     dag_df_prod = get_prod_dag_output_for_last_node(system_log_path_dict)
-    start_timestamp = omreconc.timestamp_as_str_to_timestamp(
-        start_timestamp_as_str
-    )
-    end_timestamp = omreconc.timestamp_as_str_to_timestamp(end_timestamp_as_str)
+    tz = "America/New_York"
+    start_timestamp = hdateti.str_to_timestamp(start_timestamp_as_str, tz)
+    end_timestamp = hdateti.str_to_timestamp(end_timestamp_as_str, tz)
     forecast_evaluator_from_prices_dict = reconciliation_config[
         "research_forecast_evaluator_from_prices"
     ]

oms/reconciliation.py

@@ -234,8 +234,9 @@ def build_multiday_system_reconciliation_config(
     :param end_timestamp_as_str: string representation of timestamp
         at which to end reconcile run, e.g. "20221010_080000"
     """
-    start_timestamp = timestamp_as_str_to_timestamp(start_timestamp_as_str)
-    end_timestamp = timestamp_as_str_to_timestamp(end_timestamp_as_str)
+    tz = "America/New_York"
+    start_timestamp = hdateti.str_to_timestamp(start_timestamp_as_str, tz)
+    end_timestamp = hdateti.str_to_timestamp(end_timestamp_as_str, tz)
     config = {
         "dst_root_dir": dst_root_dir,
         "dag_builder_name": dag_builder_name,
@@ -570,16 +571,18 @@ def get_system_run_timestamps(
     system_run_timestamps = [tuple(ts.split(".")) for ts in timestamp_dirs]
     # Keep timestamps within the `[start_date, end_date]` range.
     tz = "UTC"
+    datetime_format = "%Y%m%d_%H%M%S"
     # Filter by start / end timestamps using system run start timestamp.
     if start_timestamp is not None:
-        # TODO(Grisha): use `str_to_timestamp()` from `helpers.hdatetime.py`.
         system_run_timestamps = [
             (system_run_start_timestamp, system_run_end_timestamp)
             for (
                 system_run_start_timestamp,
                 system_run_end_timestamp,
             ) in system_run_timestamps
-            if timestamp_as_str_to_timestamp(system_run_start_timestamp, tz=tz)
+            if hdateti.str_to_timestamp(
+                system_run_start_timestamp, tz, datetime_format=datetime_format
+            )
             >= start_timestamp
         ]
         _LOG.info("Filtered by `start_timestamp`: %s.", system_run_timestamps)
@@ -590,7 +593,9 @@ def get_system_run_timestamps(
                 system_run_start_timestamp,
                 system_run_end_timestamp,
             ) in system_run_timestamps
-            if timestamp_as_str_to_timestamp(system_run_start_timestamp, tz=tz)
+            if hdateti.str_to_timestamp(
+                system_run_start_timestamp, tz, datetime_format=datetime_format
+            )
             <= end_timestamp
         ]
         _LOG.info("Filtered by `end_timestamp`: %s.", system_run_timestamps)