[COST-4155] - use datetimes for args instead of dates (#464)

MANIFEST.in

@@ -1,3 +1,2 @@
 include nise/aws-template-manifest.json
-include nise/ocp-template-manifest.json
 include nise/yaml_generators/static/*

nise/__init__.py

@@ -1,3 +1,3 @@
-__version__ = "4.3.3"
+__version__ = "4.4.0"
 
 VERSION = __version__.split(".")

nise/__main__.py

@@ -21,10 +21,12 @@
 import os
 import sys
 import time
+from datetime import timezone
 from pathlib import Path
 from pprint import pformat
 
 from dateutil import parser as date_parser
+from dateutil.parser import ParserError
 from dateutil.relativedelta import relativedelta
 from nise import __version__
 from nise.report import aws_create_marketplace_report
@@ -50,11 +52,13 @@ class NiseError(Exception):
 
 def valid_date(date_string):
     """Create date from date string."""
+    if "T" in date_string and not date_string.endswith("+0000"):
+        date_string += " +0000"
     try:
-        valid = datetime.datetime.strptime(date_string, "%Y-%m-%d")
-    except ValueError:
+        valid = date_parser.parse(date_string)
+    except ParserError as e:
         msg = f"{date_string} is an unsupported date format."
-        raise argparse.ArgumentTypeError(msg)
+        raise argparse.ArgumentTypeError(msg) from e
     return valid
 
 
@@ -85,7 +89,7 @@ def valid_currency(currency):
 
 def today():
     """Create the date of today."""
-    return datetime.datetime.now().replace(microsecond=0, second=0, minute=0)
+    return datetime.datetime.now(tz=timezone.utc).replace(microsecond=0, second=0, minute=0)
 
 
 def add_aws_parser_args(parser):
@@ -274,6 +278,13 @@ def add_ocp_parser_args(parser):
         action="store_true",
         help="Generate ROS for Openshift data",
     )
+    parser.add_argument(
+        "--daily-reports",
+        dest="daily_reports",
+        required=False,
+        action="store_true",
+        help="Flag used to add the `daily_reports` marker to manifests.",
+    )
 
 
 def add_oci_parser_args(parser):
@@ -324,21 +335,21 @@ def create_parser():
     parent_parser.add_argument(
         "-s",
         "--start-date",
-        metavar="YYYY-MM-DD",
+        metavar="YYYY-MM-DD[THH:MM:SS +0000]",
         dest="start_date",
         required=False,
         type=valid_date,
-        help="Date to start generating data (YYYY-MM-DD)",
+        help="Date to start generating data (YYYY-MM-DD[THH:MM:SS +0000])",
     )
     parent_parser.add_argument(
         "-e",
         "--end-date",
-        metavar="YYYY-MM-DD",
+        metavar="YYYY-MM-DD[THH:MM:SS +0000]",
         dest="end_date",
         required=False,
         type=valid_date,
         default=today(),
-        help="Date to end generating data (YYYY-MM-DD). Default is today.",
+        help="Date to end generating data (YYYY-MM-DD[THH:MM:SS +0000]). Default is today.",
     )
     parent_parser.add_argument(
         "--file-row-limit",
@@ -654,7 +665,6 @@ def _load_static_report_data(options):
             start_date = get_start_date(attributes, options)
             generated_start_date = calculate_start_date(start_date)
             start_dates.append(generated_start_date)
-
             if attributes.get("end_date"):
                 generated_end_date = calculate_end_date(generated_start_date, attributes.get("end_date"))
             elif options.get("end_date") and options.get("end_date").date() != today().date():
@@ -705,6 +715,8 @@ def calculate_start_date(start_date):
         generated_start_date = date_parser.parse(start_date)
     else:
         generated_start_date = today().replace(day=1, hour=0, minute=0, second=0)
+    if generated_start_date.tzinfo is None:
+        generated_start_date = generated_start_date.replace(tzinfo=timezone.utc)
     return generated_start_date
 
 
@@ -726,6 +738,8 @@ def calculate_end_date(start_date, end_date):
             generated_end_date = offset_date
         else:
             generated_end_date = min(start_date + relativedelta(days=offset), today())
+    if generated_end_date.tzinfo is None:
+        generated_end_date = generated_end_date.replace(tzinfo=timezone.utc)
     if generated_end_date < start_date:
         raise ValueError("Static yaml error: End date must be after start date.")
     return generated_end_date
@@ -734,9 +748,16 @@ def calculate_end_date(start_date, end_date):
 def fix_dates(options, provider_type):
     """Correct any unique dates."""
     # Azure end_date is always the following day
+    if options["start_date"].tzinfo is None:
+        options["start_date"] = options["start_date"].replace(tzinfo=timezone.utc)
+    if options["end_date"].tzinfo is None:
+        options["end_date"] = options["end_date"].replace(tzinfo=timezone.utc)
     if provider_type == "azure":
         options["end_date"] += relativedelta(days=1)
 
+    if options["end_date"] < options["start_date"]:
+        raise ValueError("End date must be after start date.")
+
 
 def run(provider_type, options):
     """Run nise."""
@@ -746,6 +767,8 @@ def run(provider_type, options):
     if not static_data_bool:
         fix_dates(options, provider_type)
 
+    LOG.debug("Options are: %s", pformat(options))
+
     LOG.info("Creating reports...")
     if provider_type == "aws":
         aws_create_report(options)

nise/generators/gcp/cloud_storage_generator.py

@@ -85,7 +85,7 @@ def _update_data(self, row):  # noqa: C901
         credit, credit_total = self._gen_credit(self.credit_total, self._credit_amount)
         self.credit_total = credit_total
         row["credits"] = credit
-        usage_date = datetime.strptime(row.get("usage_start_time"), "%Y-%m-%dT%H:%M:%S")
+        usage_date = datetime.strptime(row.get("usage_start_time")[:7], "%Y-%m")
         row["invoice.month"] = f"{usage_date.year}{usage_date.month:02d}"
 
         if self.attributes:
@@ -151,8 +151,8 @@ def _update_data(self, row):  # noqa: C901
         row["cost_type"] = "regular"
         row["currency_conversion_rate"] = 1
         invoice = {}
-        year = datetime.strptime(row.get("usage_start_time"), "%Y-%m-%dT%H:%M:%S").year
-        month = datetime.strptime(row.get("usage_start_time"), "%Y-%m-%dT%H:%M:%S").month
+        year = datetime.strptime(row.get("usage_start_time")[:7], "%Y-%m").year
+        month = datetime.strptime(row.get("usage_start_time")[:7], "%Y-%m").month
         invoice["month"] = f"{year}{month:02d}"
         row["invoice"] = invoice
         if self.resource_level:

nise/generators/gcp/compute_engine_generator.py

@@ -103,7 +103,7 @@ def _update_data(self, row):  # noqa: C901
         credit, credit_total = self._gen_credit(self.credit_total, self._credit_amount)
         self.credit_total = credit_total
         row["credits"] = credit
-        usage_date = datetime.strptime(row.get("usage_start_time"), "%Y-%m-%dT%H:%M:%S")
+        usage_date = datetime.strptime(row.get("usage_start_time")[:7], "%Y-%m")
         row["invoice.month"] = f"{usage_date.year}{usage_date.month:02d}"
 
         if self.attributes:
@@ -170,8 +170,8 @@ def _update_data(self, row):  # noqa: C901
         row["cost_type"] = "regular"
         row["currency_conversion_rate"] = 1
         invoice = {}
-        year = datetime.strptime(row.get("usage_start_time"), "%Y-%m-%dT%H:%M:%S").year
-        month = datetime.strptime(row.get("usage_start_time"), "%Y-%m-%dT%H:%M:%S").month
+        year = datetime.strptime(row.get("usage_start_time")[:7], "%Y-%m").year
+        month = datetime.strptime(row.get("usage_start_time")[:7], "%Y-%m").month
         invoice["month"] = f"{year}{month:02d}"
         row["invoice"] = invoice
         if self.resource_level:

nise/generators/gcp/gcp_database_generator.py

@@ -90,7 +90,7 @@ def _update_data(self, row):  # noqa: C901
         credit, credit_total = self._gen_credit(self.credit_total, self._credit_amount)
         self.credit_total = credit_total
         row["credits"] = credit
-        usage_date = datetime.strptime(row.get("usage_start_time"), "%Y-%m-%dT%H:%M:%S")
+        usage_date = datetime.strptime(row.get("usage_start_time")[:7], "%Y-%m")
         row["invoice.month"] = f"{usage_date.year}{usage_date.month:02d}"
 
         if self.attributes:
@@ -157,9 +157,9 @@ def _update_data(self, row):  # noqa: C901
         credit, credit_total = self._gen_credit(self.credit_total, self._credit_amount, True)
         self.credit_total = credit_total
         row["credits"] = credit
-        usage_date = datetime.strptime(row.get("usage_start_time"), "%Y-%m-%dT%H:%M:%S")
+        usage_date = datetime.strptime(row.get("usage_start_time")[:7], "%Y-%m")
         invoice = {}
-        usage_date = datetime.strptime(row.get("usage_start_time"), "%Y-%m-%dT%H:%M:%S")
+        usage_date = datetime.strptime(row.get("usage_start_time")[:7], "%Y-%m")
         invoice["month"] = f"{usage_date.year}{usage_date.month:02d}"
         row["invoice"] = invoice
         if self.resource_level:

nise/generators/gcp/gcp_network_generator.py

@@ -96,7 +96,7 @@ def _update_data(self, row):  # noqa: C901
         credit, credit_total = self._gen_credit(self.credit_total, self._credit_amount)
         self.credit_total = credit_total
         row["credits"] = credit
-        usage_date = datetime.strptime(row.get("usage_start_time"), "%Y-%m-%dT%H:%M:%S")
+        usage_date = datetime.strptime(row.get("usage_start_time")[:7], "%Y-%m")
         row["invoice.month"] = f"{usage_date.year}{usage_date.month:02d}"
 
         if self.attributes:
@@ -162,9 +162,9 @@ def _update_data(self, row):  # noqa: C901
         credit, credit_total = self._gen_credit(self.credit_total, self._credit_amount, True)
         self.credit_total = credit_total
         row["credits"] = credit
-        usage_date = datetime.strptime(row.get("usage_start_time"), "%Y-%m-%dT%H:%M:%S")
+        usage_date = datetime.strptime(row.get("usage_start_time")[:7], "%Y-%m")
         invoice = {}
-        usage_date = datetime.strptime(row.get("usage_start_time"), "%Y-%m-%dT%H:%M:%S")
+        usage_date = datetime.strptime(row.get("usage_start_time")[:7], "%Y-%m")
         invoice["month"] = f"{usage_date.year}{usage_date.month:02d}"
         row["invoice"] = invoice
         if self.resource_level:

nise/generators/generator.py

@@ -27,10 +27,11 @@
 class AbstractGenerator(ABC):
     """Defines a abstract class for generators."""
 
-    def __init__(self, start_date, end_date):
+    def __init__(self, start_date, end_date, hour_delta=datetime.timedelta(minutes=60)):
         """Initialize the generator."""
         self.start_date = start_date
         self.end_date = end_date
+        self.hour_delta = hour_delta
         self.hours = self._set_hours()
         self.quarter_hours = self._set_quarter_hours()
         self.days = self._set_days()
@@ -49,12 +50,11 @@ def _set_hours(self):
         if self.end_date < self.start_date:
             raise ValueError("start_date must be a date object less than end_date.")
 
-        one_hour = datetime.timedelta(minutes=60)
         cur_date = self.start_date
-        while (cur_date + one_hour) <= self.end_date:
-            cur_hours = {"start": cur_date, "end": cur_date + one_hour}
+        while (cur_date + self.hour_delta) <= self.end_date:
+            cur_hours = {"start": cur_date, "end": cur_date + self.hour_delta}
             hours.append(cur_hours)
-            cur_date = cur_date + one_hour
+            cur_date = cur_date + datetime.timedelta(minutes=60)
         return hours
 
     def _set_quarter_hours(self):

nise/generators/ocp/ocp_generator.py

@@ -195,7 +195,7 @@ def __init__(self, start_date, end_date, attributes, ros_ocp_info=False):
         if attributes:
             self._nodes = attributes.get("nodes")
 
-        super().__init__(start_date, end_date)
+        super().__init__(start_date, end_date, hour_delta=datetime.timedelta(minutes=59, seconds=59))
         self.apps = [
             self.fake.word(),
             self.fake.word(),

nise/manifest.py

@@ -24,7 +24,6 @@
 
 TEMPLATE_DIR = os.path.dirname(__file__)
 AWS_TEMPLATE_FILE = "aws-template-manifest.json"
-OCP_TEMPLATE_FILE = "ocp-template-manifest.json"
 
 
 def _manifest_datetime_str(date_time):
@@ -51,7 +50,7 @@ def _manifest_datetime_range(start, end):
     """
     start_str = start.strftime("%Y%m%d")
     end_str = end.strftime("%Y%m%d")
-    return start_str + "-" + end_str
+    return f"{start_str}-{end_str}"
 
 
 def aws_generate_manifest(fake, template_data):
@@ -111,8 +110,5 @@ def ocp_generate_manifest(template_data):
         (String): Rendered template data
 
     """
-    template_loader = jinja2.FileSystemLoader(searchpath=TEMPLATE_DIR)
-    template_env = jinja2.Environment(loader=template_loader)
-    template = template_env.get_template(OCP_TEMPLATE_FILE)
-    output = template.render(template_data)
-    return output
+    # template_data["cr_status"] = json.dumps(template_data["cr_status"], indent=2)
+    return json.dumps(template_data, indent=2)