new JSON output format

Author: rpreen

acro/acro.py

@@ -271,11 +271,10 @@ def crosstab(  # pylint: disable=too-many-arguments,too-many-locals
             mask.replace({0: False, 1: True}, inplace=True)
             masks[name] = mask
 
-        # build the properties dictionary
-        properties: dict = {"method": "crosstab", "suppressed": self.suppress}
-        utils.update_table_properties(masks, properties)
+        # build the sdc dictionary
+        sdc: dict = utils.get_table_sdc(masks, self.suppress)
         # get the status and summary
-        status, summary = utils.get_summary(properties)
+        status, summary = utils.get_summary(sdc)
         # apply the suppression
         safe_table, outcome = utils.apply_suppression(table, masks)
         if self.suppress:
@@ -284,7 +283,8 @@ def crosstab(  # pylint: disable=too-many-arguments,too-many-locals
         self.results.add(
             status=status,
             output_type="table",
-            properties=properties,
+            properties={"method": "crosstab"},
+            sdc=sdc,
             command=command,
             summary=summary,
             outcome=outcome,
@@ -410,11 +410,10 @@ def pivot_table(  # pylint: disable=too-many-arguments,too-many-locals
                     data, values, index, columns, aggfunc=agg
                 )
 
-        # build the properties dictionary
-        properties: dict = {"method": "pivot_table", "suppressed": self.suppress}
-        utils.update_table_properties(masks, properties)
+        # build the sdc dictionary
+        sdc: dict = utils.get_table_sdc(masks, self.suppress)
         # get the status and summary
-        status, summary = utils.get_summary(properties)
+        status, summary = utils.get_summary(sdc)
         # apply the suppression
         safe_table, outcome = utils.apply_suppression(table, masks)
         if self.suppress:
@@ -423,7 +422,8 @@ def pivot_table(  # pylint: disable=too-many-arguments,too-many-locals
         self.results.add(
             status=status,
             output_type="table",
-            properties=properties,
+            properties={"method": "pivot_table"},
+            sdc=sdc,
             command=command,
             summary=summary,
             outcome=outcome,
@@ -503,6 +503,7 @@ def ols(  # pylint: disable=too-many-locals
             status=status,
             output_type="regression",
             properties={"method": "ols", "dof": dof},
+            sdc={},
             command=command,
             summary=summary,
             outcome=DataFrame(),
@@ -566,6 +567,7 @@ def olsr(  # pylint: disable=too-many-locals,keyword-arg-before-vararg
             status=status,
             output_type="regression",
             properties={"method": "olsr", "dof": dof},
+            sdc={},
             command=command,
             summary=summary,
             outcome=DataFrame(),
@@ -614,6 +616,7 @@ def logit(  # pylint: disable=too-many-arguments,too-many-locals
             status=status,
             output_type="regression",
             properties={"method": "logit", "dof": dof},
+            sdc={},
             command=command,
             summary=summary,
             outcome=DataFrame(),
@@ -677,6 +680,7 @@ def logitr(  # pylint: disable=too-many-locals,keyword-arg-before-vararg
             status=status,
             output_type="regression",
             properties={"method": "logitr", "dof": dof},
+            sdc={},
             command=command,
             summary=summary,
             outcome=DataFrame(),
@@ -725,6 +729,7 @@ def probit(  # pylint: disable=too-many-arguments,too-many-locals
             status=status,
             output_type="regression",
             properties={"method": "probit", "dof": dof},
+            sdc={},
             command=command,
             summary=summary,
             outcome=DataFrame(),
@@ -788,6 +793,7 @@ def probitr(  # pylint: disable=too-many-locals,keyword-arg-before-vararg
             status=status,
             output_type="regression",
             properties={"method": "probitr", "dof": dof},
+            sdc={},
             command=command,
             summary=summary,
             outcome=DataFrame(),

acro/record.py

@@ -69,6 +69,8 @@ class Record:  # pylint: disable=too-many-instance-attributes,too-few-public-met
         Type of output, e.g., "regression"
     properties : dict
         Dictionary containing structured output data.
+    sdc : dict
+        Dictionary containing SDC results.
     command : str
         String representation of the operation performed.
     summary : str
@@ -91,6 +93,7 @@ def __init__(  # pylint: disable=too-many-arguments
         status: str,
         output_type: str,
         properties: dict,
+        sdc: dict,
         command: str,
         summary: str,
         outcome: DataFrame,
@@ -109,6 +112,8 @@ def __init__(  # pylint: disable=too-many-arguments
             Type of output, e.g., "regression"
         properties : dict
             Dictionary containing structured output data.
+        sdc : dict
+            Dictionary containing SDC results.
         command : str
             String representation of the operation performed.
         summary : str
@@ -124,6 +129,7 @@ def __init__(  # pylint: disable=too-many-arguments
         self.status: str = status
         self.output_type: str = output_type
         self.properties: dict = properties
+        self.sdc: dict = sdc
         self.command: str = command
         self.summary: str = summary
         self.outcome: DataFrame = outcome
@@ -182,6 +188,7 @@ def __str__(self) -> str:
             f"status: {self.status}\n"
             f"type: {self.output_type}\n"
             f"properties: {self.properties}\n"
+            f"sdc: {self.sdc}\n"
             f"command: {self.command}\n"
             f"summary: {self.summary}\n"
             f"outcome: {self.outcome}\n"
@@ -205,6 +212,7 @@ def add(  # pylint: disable=too-many-arguments
         status: str,
         output_type: str,
         properties: dict,
+        sdc: dict,
         command: str,
         summary: str,
         outcome: DataFrame,
@@ -221,6 +229,8 @@ def add(  # pylint: disable=too-many-arguments
             Type of output, e.g., "regression"
         properties : dict
             Dictionary containing structured output data.
+        sdc : dict
+            Dictionary containing SDC results.
         command : str
             String representation of the operation performed.
         summary : str
@@ -237,6 +247,7 @@ def add(  # pylint: disable=too-many-arguments
             status=status,
             output_type=output_type,
             properties=properties,
+            sdc=sdc,
             command=command,
             summary=summary,
             outcome=outcome,
@@ -319,6 +330,7 @@ def add_custom(self, filename: str, comment: str | None = None) -> None:
             status="review",
             output_type="custom",
             properties={},
+            sdc={},
             command="custom",
             summary="review",
             outcome=DataFrame(),
@@ -439,18 +451,22 @@ def finalise_json(self, path: str) -> None:
                 "status": val.status,
                 "type": val.output_type,
                 "properties": val.properties,
+                "files": [],
+                "outcome": json.loads(val.outcome.to_json()),
                 "command": val.command,
                 "summary": val.summary,
-                "outcome": json.loads(val.outcome.to_json()),
-                "output": val.serialize_output(path),
                 "timestamp": val.timestamp,
                 "comments": val.comments,
                 "exception": val.exception,
             }
+            files: list[str] = val.serialize_output(path)
+            for file in files:
+                outputs[key]["files"].append({"name": file, "sdc": val.sdc})
+
         results: dict = {"version": __version__, "results": outputs}
         filename: str = os.path.normpath(f"{path}/results.json")
-        with open(filename, "w", newline="", encoding="utf-8") as file:
-            json.dump(results, file, indent=4, sort_keys=False)
+        with open(filename, "w", newline="", encoding="utf-8") as handle:
+            json.dump(results, handle, indent=4, sort_keys=False)
 
     def finalise_excel(self, path: str) -> None:
         """Writes outputs to an excel spreadsheet.
@@ -539,21 +555,29 @@ def load_records(path: str) -> Records:
     """
     records = Records()
     filename = os.path.normpath(f"{path}/results.json")
-    with open(filename, newline="", encoding="utf-8") as file:
-        data = json.load(file)
+    with open(filename, newline="", encoding="utf-8") as handle:
+        data = json.load(handle)
         if data["version"] != __version__:  # pragma: no cover
             raise ValueError("error loading output")
         for key, val in data["results"].items():
+            files: list[dict] = val["files"]
+            filenames: list = []
+            sdcs: list = []
+            for file in files:
+                filenames.append(file["name"])
+                sdcs.append(file["sdc"])
             records.results[key] = Record(
-                val["uid"],
-                val["status"],
-                val["type"],
-                val["properties"],
-                val["command"],
-                val["summary"],
-                load_outcome(val["outcome"]),
-                load_output(path, val["output"]),
-                val["comments"],
+                uid=val["uid"],
+                status=val["status"],
+                output_type=val["type"],
+                properties=val["properties"],
+                sdc=sdcs[0],
+                command=val["command"],
+                summary=val["summary"],
+                outcome=load_outcome(val["outcome"]),
+                output=load_output(path, filenames),
+                comments=val["comments"],
             )
+            records.results[key].exception = val["exception"]
             records.results[key].timestamp = val["timestamp"]
     return records

acro/utils.py

@@ -218,44 +218,45 @@ def apply_suppression(
     return safe_df, outcome_df
 
 
-def update_table_properties(masks: dict[str, DataFrame], properties: dict) -> None:
-    """Updates the properties dictionary using the suppression masks.
+def get_table_sdc(masks: dict[str, DataFrame], suppress: bool) -> dict:
+    """Returns the SDC dictionary using the suppression masks.
 
     Parameters
     ----------
     masks : dict[str, DataFrame]
         Dictionary of tables specifying suppression masks for application.
-    properties : dict
-        Properties of the SDC checks.
+    suppress : bool
+        Whether suppression has been applied.
     """
     # summary of cells to be suppressed
-    properties["negative"] = 0
-    properties["missing"] = 0
-    properties["threshold"] = 0
-    properties["p-ratio"] = 0
-    properties["nk-rule"] = 0
+    sdc: dict = {"summary": {"suppressed": suppress}, "cells": {}}
+    sdc["summary"]["negative"] = 0
+    sdc["summary"]["missing"] = 0
+    sdc["summary"]["threshold"] = 0
+    sdc["summary"]["p-ratio"] = 0
+    sdc["summary"]["nk-rule"] = 0
     for name, mask in masks.items():
-        properties[name] = int(mask.to_numpy().sum())
+        sdc["summary"][name] = int(mask.to_numpy().sum())
     # positions of cells to be suppressed
-    properties["sdc"] = {}
-    properties["sdc"]["negative"] = []
-    properties["sdc"]["missing"] = []
-    properties["sdc"]["threshold"] = []
-    properties["sdc"]["p-ratio"] = []
-    properties["sdc"]["nk-rule"] = []
+    sdc["cells"]["negative"] = []
+    sdc["cells"]["missing"] = []
+    sdc["cells"]["threshold"] = []
+    sdc["cells"]["p-ratio"] = []
+    sdc["cells"]["nk-rule"] = []
     for name, mask in masks.items():
         true_positions = np.column_stack(np.where(mask.values))
         for pos in true_positions:
             row_index, col_index = pos
-            properties["sdc"][name].append([int(row_index), int(col_index)])
+            sdc["cells"][name].append([int(row_index), int(col_index)])
+    return sdc
 
 
-def get_summary(properties: dict) -> tuple[str, str]:
+def get_summary(sdc: dict) -> tuple[str, str]:
     """Returns the status and summary of the suppression masks.
 
     Parameters
     ----------
-    properties : dict
+    sdc : dict
         Properties of the SDC checks.
 
     Returns
@@ -267,22 +268,23 @@ def get_summary(properties: dict) -> tuple[str, str]:
     """
     status: str = "pass"
     summary: str = ""
-    sup: str = "suppressed" if properties["suppressed"] else "may need suppressing"
-    if properties["negative"]:
+    sdc_summary = sdc["summary"]
+    sup: str = "suppressed" if sdc_summary["suppressed"] else "may need suppressing"
+    if sdc_summary["negative"] > 0:
         summary += "negative values found"
         status = "review"
-    elif properties["missing"]:
+    elif sdc_summary["missing"] > 0:
         summary += "missing values found"
         status = "review"
     else:
-        if properties["threshold"] > 0:
-            summary += f"threshold: {properties['threshold']} cells {sup}; "
+        if sdc_summary["threshold"] > 0:
+            summary += f"threshold: {sdc_summary['threshold']} cells {sup}; "
             status = "fail"
-        if properties["p-ratio"] > 0:
-            summary += f"p-ratio: {properties['p-ratio']} cells {sup}; "
+        if sdc_summary["p-ratio"] > 0:
+            summary += f"p-ratio: {sdc_summary['p-ratio']} cells {sup}; "
             status = "fail"
-        if properties["nk-rule"] > 0:
-            summary += f"nk-rule: {properties['nk-rule']} cells {sup}; "
+        if sdc_summary["nk-rule"] > 0:
+            summary += f"nk-rule: {sdc_summary['nk-rule']} cells {sup}; "
             status = "fail"
     if summary != "":
         summary = f"{status}; {summary}"

test/test_initial.py

@@ -67,7 +67,7 @@ def test_crosstab_threshold(data, acro):
     output = acro.results.get_index(0)
     total_nan: int = output.output[0]["R/G"].isnull().sum()
     assert total_nan == 6
-    positions = output.properties["sdc"]["threshold"]
+    positions = output.sdc["cells"]["threshold"]
     for pos in positions:
         row, col = pos
         assert np.isnan(output.output[0].iloc[row, col])
@@ -287,11 +287,17 @@ def test_finalise_json(data, acro):
     loaded: Records = load_records(PATH)
     orig = result.get_index(0)
     read = loaded.get_index(0)
+    print("*****************************")
+    print(orig)
+    print("*****************************")
+    print(read)
+    print("*****************************")
     # check equal
     assert orig.uid == read.uid
     assert orig.status == read.status
     assert orig.output_type == read.output_type
     assert orig.properties == read.properties
+    assert orig.sdc == read.sdc
     assert orig.command == read.command
     assert orig.summary == read.summary
     assert orig.comments == read.comments
@@ -301,16 +307,7 @@ def test_finalise_json(data, acro):
     with open(os.path.normpath(f"{PATH}/results.json"), encoding="utf-8") as file:
         json_data = json.load(file)
     results: dict = json_data["results"]
-    assert results[orig.uid]["output"][0] == f"{orig.uid}_0.csv"
-    # regression check: the outcome fields are dicts not strings
-    assert results[orig.uid]["outcome"]["R/G"] == {
-        "2010": "threshold; ",
-        "2011": "threshold; ",
-        "2012": "threshold; ",
-        "2013": "threshold; ",
-        "2014": "threshold; ",
-        "2015": "threshold; ",
-    }
+    assert results[orig.uid]["files"][0]["name"] == f"{orig.uid}_0.csv"
 
 
 def test_rename_output(data, acro):