-
Notifications
You must be signed in to change notification settings - Fork 3
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Initial refactoring of process_report
This refactor commit is the first of a few, to lay out intial structure A new submodule, `invoices`, is added, containing a base class `Invoice` which is inherited by all other invoices. Currently, only the lenovo and nonbillable invoice has classes which inherits from `Invoice` Also created and partially populated an `util.py` file, containing functions placed above `main()` in `process_report.py` After the refactoring process is fully complete, these utility functions will be completely removed from `process_report.py`
- Loading branch information
Showing
6 changed files
with
188 additions
and
56 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,78 @@ | ||
| from dataclasses import dataclass | ||
| import pandas | ||
|
|
||
| import process_report.util as util | ||
|
|
||
|
|
||
| ### Invoice field names | ||
| INVOICE_DATE_FIELD = "Invoice Month" | ||
| PROJECT_FIELD = "Project - Allocation" | ||
| PROJECT_ID_FIELD = "Project - Allocation ID" | ||
| PI_FIELD = "Manager (PI)" | ||
| INVOICE_EMAIL_FIELD = "Invoice Email" | ||
| INVOICE_ADDRESS_FIELD = "Invoice Address" | ||
| INSTITUTION_FIELD = "Institution" | ||
| INSTITUTION_ID_FIELD = "Institution - Specific Code" | ||
| SU_HOURS_FIELD = "SU Hours (GBhr or SUhr)" | ||
| SU_TYPE_FIELD = "SU Type" | ||
| COST_FIELD = "Cost" | ||
| CREDIT_FIELD = "Credit" | ||
| CREDIT_CODE_FIELD = "Credit Code" | ||
| SUBSIDY_FIELD = "Subsidy" | ||
| BALANCE_FIELD = "Balance" | ||
| ### | ||
|
|
||
|
|
||
| @dataclass | ||
| class Invoice: | ||
| name: str | ||
| invoice_month: str | ||
| data: pandas.DataFrame | ||
|
|
||
| def process(self): | ||
| self._prepare() | ||
| self._process() | ||
| self._prepare_export() | ||
|
|
||
| @property | ||
| def output_path(self) -> str: | ||
| return f"{self.name} {self.invoice_month}.csv" | ||
|
|
||
| @property | ||
| def output_s3_key(self) -> str: | ||
| return f"Invoices/{self.invoice_month}/{self.name} {self.invoice_month}.csv" | ||
|
|
||
| @property | ||
| def output_s3_archive_key(self): | ||
| return f"Invoices/{self.invoice_month}/Archive/{self.name} {self.invoice_month} {util.get_iso8601_time()}.csv" | ||
|
|
||
| def _prepare(self): | ||
| """Prepares the data for processing. | ||
| Implement in subclass if necessary. May add or remove columns | ||
| necessary for processing, add or remove rows, validate the data, or | ||
| perform simple substitutions. | ||
| """ | ||
| pass | ||
|
|
||
| def _process(self): | ||
| """Processes the data. | ||
| Implement in subclass if necessary. Performs necessary calculations | ||
| on the data, e.g. applying subsidies or credits. | ||
| """ | ||
| pass | ||
|
|
||
| def _prepare_export(self): | ||
| """Prepares the data for export. | ||
| Implement in subclass if necessary. May add or remove columns or rows | ||
| that should or should not be exported after processing.""" | ||
| pass | ||
|
|
||
| def export(self): | ||
| self.data.to_csv(self.output_path) | ||
|
|
||
| def export_s3(self, s3_bucket): | ||
| s3_bucket.upload_file(self.output_path, self.output_s3_key) | ||
| s3_bucket.upload_file(self.output_path, self.output_s3_archive_key) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,28 @@ | ||
| from dataclasses import dataclass | ||
|
|
||
| import process_report.invoices.invoice as invoice | ||
|
|
||
|
|
||
| @dataclass | ||
| class LenovoInvoice(invoice.Invoice): | ||
| LENOVO_SU_TYPES = ["OpenShift GPUA100SXM4", "OpenStack GPUA100SXM4"] | ||
| SU_CHARGE_MULTIPLIER = 1 | ||
|
|
||
| def _prepare(self): | ||
| self.data = self.data[ | ||
| self.data[invoice.SU_TYPE_FIELD].isin(self.LENOVO_SU_TYPES) | ||
| ][ | ||
| [ | ||
| invoice.INVOICE_DATE_FIELD, | ||
| invoice.PROJECT_FIELD, | ||
| invoice.INSTITUTION_FIELD, | ||
| invoice.SU_HOURS_FIELD, | ||
| invoice.SU_TYPE_FIELD, | ||
| ] | ||
| ].copy() | ||
|
|
||
| self.data.rename(columns={invoice.SU_HOURS_FIELD: "SU Hours"}, inplace=True) | ||
| self.data.insert(len(self.data.columns), "SU Charge", self.SU_CHARGE_MULTIPLIER) | ||
|
|
||
| def _process(self): | ||
| self.data["Charge"] = self.data["SU Hours"] * self.data["SU Charge"] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,15 @@ | ||
| from dataclasses import dataclass | ||
|
|
||
| import process_report.invoices.invoice as invoice | ||
|
|
||
|
|
||
| @dataclass | ||
| class NonbillableInvoice(invoice.Invoice): | ||
| nonbillable_pis: list[str] | ||
| nonbillable_projects: list[str] | ||
|
|
||
| def _prepare_export(self): | ||
| self.data = self.data[ | ||
| self.data[invoice.PI_FIELD].isin(self.nonbillable_pis) | ||
| | self.data[invoice.PROJECT_FIELD].isin(self.nonbillable_projects) | ||
| ] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,35 @@ | ||
| import datetime | ||
| import json | ||
| import logging | ||
|
|
||
|
|
||
| logger = logging.getLogger(__name__) | ||
| logging.basicConfig(level=logging.INFO) | ||
|
|
||
|
|
||
| def get_institution_from_pi(institute_map, pi_uname): | ||
| institution_key = pi_uname.split("@")[-1] | ||
| institution_name = institute_map.get(institution_key, "") | ||
|
|
||
| if institution_name == "": | ||
| logger.warn(f"PI name {pi_uname} does not match any institution!") | ||
|
|
||
| return institution_name | ||
|
|
||
|
|
||
| def load_institute_map() -> dict: | ||
| with open("process_report/institute_map.json", "r") as f: | ||
| institute_map = json.load(f) | ||
|
|
||
| return institute_map | ||
|
|
||
|
|
||
| def get_iso8601_time(): | ||
| return datetime.datetime.now().strftime("%Y%m%dT%H%M%SZ") | ||
|
|
||
|
|
||
| def compare_invoice_month(month_1, month_2): | ||
| """Returns True if 1st date is later than 2nd date""" | ||
| dt1 = datetime.datetime.strptime(month_1, "%Y-%m") | ||
| dt2 = datetime.datetime.strptime(month_2, "%Y-%m") | ||
| return dt1 > dt2 |