Merge pull request #114 from AI-SDC/113_modify_json_schema

modify output JSON to be in a file-oriented schema

Author: rpreen

.pre-commit-config.yaml

@@ -80,6 +80,7 @@ repos:
           - --exclude=__init__.py
           - --ignore=E203,W503
         additional_dependencies: [flake8-bugbear, pep8-naming]
+        exclude: "docs"
 
   # Check types with mypy
   - repo: https://github.com/pre-commit/mirrors-mypy

CHANGELOG.md

@@ -7,6 +7,7 @@ Changes:
 *    Refactor load_json Records class function to static load_results ([#110](https://github.com/AI-SDC/ACRO/pull/110))
 *    Write SDC parameters to config.json upon finalise ([#111](https://github.com/AI-SDC/ACRO/pull/111))
 *    Add explicit exception handling and finalise prompt ([#112](https://github.com/AI-SDC/ACRO/pull/112))
+*    Add version number to JSON and use new schema ([#114](https://github.com/AI-SDC/ACRO/pull/114))
 
 ## Version 0.3.0 (Jul 04, 2023)
 

acro/acro.py

@@ -271,11 +271,10 @@ def crosstab(  # pylint: disable=too-many-arguments,too-many-locals
             mask.replace({0: False, 1: True}, inplace=True)
             masks[name] = mask
 
-        # build the properties dictionary
-        properties: dict = {"method": "crosstab", "suppressed": self.suppress}
-        utils.update_table_properties(masks, properties)
+        # build the sdc dictionary
+        sdc: dict = utils.get_table_sdc(masks, self.suppress)
         # get the status and summary
-        status, summary = utils.get_summary(properties)
+        status, summary = utils.get_summary(sdc)
         # apply the suppression
         safe_table, outcome = utils.apply_suppression(table, masks)
         if self.suppress:
@@ -284,7 +283,8 @@ def crosstab(  # pylint: disable=too-many-arguments,too-many-locals
         self.results.add(
             status=status,
             output_type="table",
-            properties=properties,
+            properties={"method": "crosstab"},
+            sdc=sdc,
             command=command,
             summary=summary,
             outcome=outcome,
@@ -410,11 +410,10 @@ def pivot_table(  # pylint: disable=too-many-arguments,too-many-locals
                     data, values, index, columns, aggfunc=agg
                 )
 
-        # build the properties dictionary
-        properties: dict = {"method": "pivot_table", "suppressed": self.suppress}
-        utils.update_table_properties(masks, properties)
+        # build the sdc dictionary
+        sdc: dict = utils.get_table_sdc(masks, self.suppress)
         # get the status and summary
-        status, summary = utils.get_summary(properties)
+        status, summary = utils.get_summary(sdc)
         # apply the suppression
         safe_table, outcome = utils.apply_suppression(table, masks)
         if self.suppress:
@@ -423,7 +422,8 @@ def pivot_table(  # pylint: disable=too-many-arguments,too-many-locals
         self.results.add(
             status=status,
             output_type="table",
-            properties=properties,
+            properties={"method": "pivot_table"},
+            sdc=sdc,
             command=command,
             summary=summary,
             outcome=outcome,
@@ -503,6 +503,7 @@ def ols(  # pylint: disable=too-many-locals
             status=status,
             output_type="regression",
             properties={"method": "ols", "dof": dof},
+            sdc={},
             command=command,
             summary=summary,
             outcome=DataFrame(),
@@ -566,6 +567,7 @@ def olsr(  # pylint: disable=too-many-locals,keyword-arg-before-vararg
             status=status,
             output_type="regression",
             properties={"method": "olsr", "dof": dof},
+            sdc={},
             command=command,
             summary=summary,
             outcome=DataFrame(),
@@ -614,6 +616,7 @@ def logit(  # pylint: disable=too-many-arguments,too-many-locals
             status=status,
             output_type="regression",
             properties={"method": "logit", "dof": dof},
+            sdc={},
             command=command,
             summary=summary,
             outcome=DataFrame(),
@@ -677,6 +680,7 @@ def logitr(  # pylint: disable=too-many-locals,keyword-arg-before-vararg
             status=status,
             output_type="regression",
             properties={"method": "logitr", "dof": dof},
+            sdc={},
             command=command,
             summary=summary,
             outcome=DataFrame(),
@@ -725,6 +729,7 @@ def probit(  # pylint: disable=too-many-arguments,too-many-locals
             status=status,
             output_type="regression",
             properties={"method": "probit", "dof": dof},
+            sdc={},
             command=command,
             summary=summary,
             outcome=DataFrame(),
@@ -788,6 +793,7 @@ def probitr(  # pylint: disable=too-many-locals,keyword-arg-before-vararg
             status=status,
             output_type="regression",
             properties={"method": "probitr", "dof": dof},
+            sdc={},
             command=command,
             summary=summary,
             outcome=DataFrame(),

acro/record.py

@@ -12,6 +12,8 @@
 import pandas as pd
 from pandas import DataFrame
 
+from .version import __version__
+
 logger = logging.getLogger("acro:records")
 
 
@@ -67,6 +69,8 @@ class Record:  # pylint: disable=too-many-instance-attributes,too-few-public-met
         Type of output, e.g., "regression"
     properties : dict
         Dictionary containing structured output data.
+    sdc : dict
+        Dictionary containing SDC results.
     command : str
         String representation of the operation performed.
     summary : str
@@ -89,6 +93,7 @@ def __init__(  # pylint: disable=too-many-arguments
         status: str,
         output_type: str,
         properties: dict,
+        sdc: dict,
         command: str,
         summary: str,
         outcome: DataFrame,
@@ -107,6 +112,8 @@ def __init__(  # pylint: disable=too-many-arguments
             Type of output, e.g., "regression"
         properties : dict
             Dictionary containing structured output data.
+        sdc : dict
+            Dictionary containing SDC results.
         command : str
             String representation of the operation performed.
         summary : str
@@ -122,6 +129,7 @@ def __init__(  # pylint: disable=too-many-arguments
         self.status: str = status
         self.output_type: str = output_type
         self.properties: dict = properties
+        self.sdc: dict = sdc
         self.command: str = command
         self.summary: str = summary
         self.outcome: DataFrame = outcome
@@ -180,6 +188,7 @@ def __str__(self) -> str:
             f"status: {self.status}\n"
             f"type: {self.output_type}\n"
             f"properties: {self.properties}\n"
+            f"sdc: {self.sdc}\n"
             f"command: {self.command}\n"
             f"summary: {self.summary}\n"
             f"outcome: {self.outcome}\n"
@@ -203,6 +212,7 @@ def add(  # pylint: disable=too-many-arguments
         status: str,
         output_type: str,
         properties: dict,
+        sdc: dict,
         command: str,
         summary: str,
         outcome: DataFrame,
@@ -219,6 +229,8 @@ def add(  # pylint: disable=too-many-arguments
             Type of output, e.g., "regression"
         properties : dict
             Dictionary containing structured output data.
+        sdc : dict
+            Dictionary containing SDC results.
         command : str
             String representation of the operation performed.
         summary : str
@@ -235,6 +247,7 @@ def add(  # pylint: disable=too-many-arguments
             status=status,
             output_type=output_type,
             properties=properties,
+            sdc=sdc,
             command=command,
             summary=summary,
             outcome=outcome,
@@ -317,6 +330,7 @@ def add_custom(self, filename: str, comment: str | None = None) -> None:
             status="review",
             output_type="custom",
             properties={},
+            sdc={},
             command="custom",
             summary="review",
             outcome=DataFrame(),
@@ -437,17 +451,22 @@ def finalise_json(self, path: str) -> None:
                 "status": val.status,
                 "type": val.output_type,
                 "properties": val.properties,
+                "files": [],
+                "outcome": json.loads(val.outcome.to_json()),
                 "command": val.command,
                 "summary": val.summary,
-                "outcome": json.loads(val.outcome.to_json()),
-                "output": val.serialize_output(path),
                 "timestamp": val.timestamp,
                 "comments": val.comments,
                 "exception": val.exception,
             }
+            files: list[str] = val.serialize_output(path)
+            for file in files:
+                outputs[key]["files"].append({"name": file, "sdc": val.sdc})
+
+        results: dict = {"version": __version__, "results": outputs}
         filename: str = os.path.normpath(f"{path}/results.json")
-        with open(filename, "w", newline="", encoding="utf-8") as file:
-            json.dump(outputs, file, indent=4, sort_keys=False)
+        with open(filename, "w", newline="", encoding="utf-8") as handle:
+            json.dump(results, handle, indent=4, sort_keys=False)
 
     def finalise_excel(self, path: str) -> None:
         """Writes outputs to an excel spreadsheet.
@@ -536,19 +555,29 @@ def load_records(path: str) -> Records:
     """
     records = Records()
     filename = os.path.normpath(f"{path}/results.json")
-    with open(filename, newline="", encoding="utf-8") as file:
-        data = json.load(file)
-        for key, val in data.items():
+    with open(filename, newline="", encoding="utf-8") as handle:
+        data = json.load(handle)
+        if data["version"] != __version__:  # pragma: no cover
+            raise ValueError("error loading output")
+        for key, val in data["results"].items():
+            files: list[dict] = val["files"]
+            filenames: list = []
+            sdcs: list = []
+            for file in files:
+                filenames.append(file["name"])
+                sdcs.append(file["sdc"])
             records.results[key] = Record(
-                val["uid"],
-                val["status"],
-                val["type"],
-                val["properties"],
-                val["command"],
-                val["summary"],
-                load_outcome(val["outcome"]),
-                load_output(path, val["output"]),
-                val["comments"],
+                uid=val["uid"],
+                status=val["status"],
+                output_type=val["type"],
+                properties=val["properties"],
+                sdc=sdcs[0],
+                command=val["command"],
+                summary=val["summary"],
+                outcome=load_outcome(val["outcome"]),
+                output=load_output(path, filenames),
+                comments=val["comments"],
             )
+            records.results[key].exception = val["exception"]
             records.results[key].timestamp = val["timestamp"]
     return records

acro/utils.py

@@ -218,44 +218,45 @@ def apply_suppression(
     return safe_df, outcome_df
 
 
-def update_table_properties(masks: dict[str, DataFrame], properties: dict) -> None:
-    """Updates the properties dictionary using the suppression masks.
+def get_table_sdc(masks: dict[str, DataFrame], suppress: bool) -> dict:
+    """Returns the SDC dictionary using the suppression masks.
 
     Parameters
     ----------
     masks : dict[str, DataFrame]
         Dictionary of tables specifying suppression masks for application.
-    properties : dict
-        Properties of the SDC checks.
+    suppress : bool
+        Whether suppression has been applied.
     """
     # summary of cells to be suppressed
-    properties["negative"] = 0
-    properties["missing"] = 0
-    properties["threshold"] = 0
-    properties["p-ratio"] = 0
-    properties["nk-rule"] = 0
+    sdc: dict = {"summary": {"suppressed": suppress}, "cells": {}}
+    sdc["summary"]["negative"] = 0
+    sdc["summary"]["missing"] = 0
+    sdc["summary"]["threshold"] = 0
+    sdc["summary"]["p-ratio"] = 0
+    sdc["summary"]["nk-rule"] = 0
     for name, mask in masks.items():
-        properties[name] = int(mask.to_numpy().sum())
+        sdc["summary"][name] = int(mask.to_numpy().sum())
     # positions of cells to be suppressed
-    properties["sdc"] = {}
-    properties["sdc"]["negative"] = []
-    properties["sdc"]["missing"] = []
-    properties["sdc"]["threshold"] = []
-    properties["sdc"]["p-ratio"] = []
-    properties["sdc"]["nk-rule"] = []
+    sdc["cells"]["negative"] = []
+    sdc["cells"]["missing"] = []
+    sdc["cells"]["threshold"] = []
+    sdc["cells"]["p-ratio"] = []
+    sdc["cells"]["nk-rule"] = []
     for name, mask in masks.items():
         true_positions = np.column_stack(np.where(mask.values))
         for pos in true_positions:
             row_index, col_index = pos
-            properties["sdc"][name].append([int(row_index), int(col_index)])
+            sdc["cells"][name].append([int(row_index), int(col_index)])
+    return sdc
 
 
-def get_summary(properties: dict) -> tuple[str, str]:
+def get_summary(sdc: dict) -> tuple[str, str]:
     """Returns the status and summary of the suppression masks.
 
     Parameters
     ----------
-    properties : dict
+    sdc : dict
         Properties of the SDC checks.
 
     Returns
@@ -267,22 +268,23 @@ def get_summary(properties: dict) -> tuple[str, str]:
     """
     status: str = "pass"
     summary: str = ""
-    sup: str = "suppressed" if properties["suppressed"] else "may need suppressing"
-    if properties["negative"]:
+    sdc_summary = sdc["summary"]
+    sup: str = "suppressed" if sdc_summary["suppressed"] else "may need suppressing"
+    if sdc_summary["negative"] > 0:
         summary += "negative values found"
         status = "review"
-    elif properties["missing"]:
+    elif sdc_summary["missing"] > 0:
         summary += "missing values found"
         status = "review"
     else:
-        if properties["threshold"] > 0:
-            summary += f"threshold: {properties['threshold']} cells {sup}; "
+        if sdc_summary["threshold"] > 0:
+            summary += f"threshold: {sdc_summary['threshold']} cells {sup}; "
             status = "fail"
-        if properties["p-ratio"] > 0:
-            summary += f"p-ratio: {properties['p-ratio']} cells {sup}; "
+        if sdc_summary["p-ratio"] > 0:
+            summary += f"p-ratio: {sdc_summary['p-ratio']} cells {sup}; "
             status = "fail"
-        if properties["nk-rule"] > 0:
-            summary += f"nk-rule: {properties['nk-rule']} cells {sup}; "
+        if sdc_summary["nk-rule"] > 0:
+            summary += f"nk-rule: {sdc_summary['nk-rule']} cells {sup}; "
             status = "fail"
     if summary != "":
         summary = f"{status}; {summary}"

acro/version.py

@@ -0,0 +1,2 @@
+"""ACRO version number."""
+__version__ = "0.4.0"

docs/source/conf.py

@@ -7,12 +7,14 @@
 
 sys.path.insert(0, os.path.abspath("../../"))
 
+from acro.version import __version__
+
 # -- Project information -----------------------------------------------------
 
 project = "ACRO"
 copyright = "2023, ACRO Project Team"
 author = "ACRO Project Team"
-release = "0.3.0"
+release = __version__
 
 # -- General configuration ---------------------------------------------------