Merge pull request #33 from OpenEnergyPlatform/release/v0.3.2

Release/v0.3.2

Author: jh-RLI

CHANGELOG.md

@@ -26,20 +26,34 @@ Template:
 ### Removed
 - basic description [#PR/#Issue/#Commit]
 ```
-## Current
+## [Unreleased] - 20XX-MM-DD
 
 ### Added
 
 ### Changed
----
-## [0.3.1] - 2022-10-24
+
+### Removed
+
+______________________________________________________________________
+## [0.3.2] - 2022-11-29
 
 ### Added
+- token can be passed as parameter (to support usage in APIs)
+
+### Changed
+- Error is raised if schema does not exist
+- Metadata compilance checks now inculue optinal jsonschema validation for metadata (PR#32)
+- MetadataError is thrown if uploading metadata to OEP fails
+
+______________________________________________________________________
+## [0.3.1] - 2022-10-24
 
 ### Changed
 - fix module not installed error after pip install and import of oem2orm PR(#26)
+
 ______________________________________________________________________
 ## [0.3.0] - 2022-10-24
+
 ### Added
 - Option to create tables from OEM JSON instead of file
 - New module to check if metadata is oep compliant. Can check (omi's 1 parse 2 compile) oemetadata v1.5 and v1.4 (PR#23)
@@ -55,10 +69,12 @@ ______________________________________________________________________
 - enable console usage [PR#8]
 - new package requirement "omi" 
 - support for sqlachemy "numeric" data type 
+
 ### Changed
 - fix missing dependency that made pip install fail [ISSUE#1;PR#8]
+
 ### Removed
-- concolse script, remove table delete 
+- console script, remove table delete 
 
 ______________________________________________________________________
 ## [0.2.6] - 2020-09-23
@@ -67,13 +83,13 @@ ______________________________________________________________________
 - support for datatypes "hstore" and "decimal"
 - provide new example files that work with oem2orm
 
-### Changed
-
+______________________________________________________________________
 ## [0.2.5] - 2020-08-06
 
 ### Changed
 - fix installation error caused by jmespath package dependency
 
+______________________________________________________________________
 ## [0.2.4] - 2020-07-20
 
 ### Added
@@ -85,20 +101,20 @@ ______________________________________________________________________
 ### Changed
 - change functions names
 
-
+______________________________________________________________________
 ## [0.2.3] - 2020-06-02
 
 ### Added
 - provide a minimal working example as jupyter notebook tutorial
 - New OEP-API related functions: Prepare the oemetadata string to send to api 
-    Simple User Input function to set the OEP-API-Token
+- Simple User Input function to set the OEP-API-Token
 
 ### Changed
 - Update README
 - include OEP public schema (whitelist) check
 - Spatial types from Geoalchemy2 do not set a spatial_index anymore
 
-
+______________________________________________________________________
 ## [0.2.2] - 2020-06-02
 
 ### Added
@@ -109,7 +125,7 @@ ______________________________________________________________________
 - extended description in changelog
 - Fix logging 
 
-
+______________________________________________________________________
 ## [0.2.0] - 2020-05-27
 
 ### Added
@@ -118,7 +134,7 @@ ______________________________________________________________________
 - new function: tables are collected and ordered by fk (increase usability)
 
 ### Changed
-- added docstrings 
+- added docstrings
 
 ### Removed
 - the user is no longer required to use a for loop in the main function to collect tables

oem2orm/oep_compliance.py

@@ -6,21 +6,23 @@
 
 
 """
-
-from distutils.errors import CompileError
+import logging
 import pathlib
+import json
+
 from omi.dialects.oep.parser import ParserException
 from omi.structure import Compilable
 
 from omi.dialects.oep import OEP_V_1_4_Dialect, OEP_V_1_5_Dialect
 from omi.dialects.oep.compiler import JSONCompiler
 
-import json
+from omi.dialects.oep.parser import JSONParser
 
 # instances of metadata parsers / compilers, order of priority
 METADATA_PARSERS = [OEP_V_1_5_Dialect(), OEP_V_1_4_Dialect()]
 METADATA_COMPILERS = [OEP_V_1_5_Dialect(), OEP_V_1_4_Dialect(), JSONCompiler()]
 
+logger = logging.getLogger()
 
 def read_input_json(file_path: pathlib.Path = "tests/data/metadata_v15.json"):
     with open(file_path, "r", encoding="utf-8") as f:
@@ -38,11 +40,6 @@ def try_parse_metadata(inp):
         The first component is the result of the parsing procedure or `None` if
         the parsing failed. The second component is None, if the parsing failed,
         otherwise an error message.
-    Examples:
-        >>> from api.actions import try_parse_metadata
-        >>> result, error = try_parse_metadata('{"id":"id"}')
-        >>> error is None
-        True
     """
 
     if isinstance(inp, Compilable):
@@ -126,12 +123,12 @@ def check_oemetadata_is_oep_compatible(metadata):
 # -------------------------------------------
 
 
-def run_metadata_checks(oemetadata: dict = None, oemetadata_path: str = None):
+def run_metadata_checks(oemetadata: dict = None, oemetadata_path: str = None,  check_jsonschema: bool = False):
     """
     Runs metadata checks includes:
         - basic oep compliant check - tested by using omi's parsing and compiling
 
-        Not included:
+        Optional - included:
         - jsonschema valdiation
 
     Args:
@@ -159,8 +156,19 @@ def run_metadata_checks(oemetadata: dict = None, oemetadata_path: str = None):
     if oemetadata is not None and oemetadata_path is None:
         metadata = oemetadata
 
+    logger.info("Check if metadata is parsable and compileable by omi.")
     check_oemetadata_is_oep_compatible(metadata=metadata)
 
+    if check_jsonschema:
+        logger.info("Check if metadata is valid against the jsonschema.")
+        parser_validation = JSONParser()
+        schema = parser_validation.get_schema_by_metadata_version(metadata=metadata)
+        result = parser_validation.is_valid(inp=metadata, schema=schema)
+        if result is False:
+            result = result, parser_validation.validate(metadata=metadata, save_report=True)
+        
+        return result
+            
 
 if __name__ == "__main__":
     correct_v15_test_data = "tests/data/metadata_v15.json"
@@ -169,7 +177,7 @@ def run_metadata_checks(oemetadata: dict = None, oemetadata_path: str = None):
 
     meta = read_input_json(file_path=correct_v15_test_data)
     print("Check v15 metadata from file!")
-    run_metadata_checks(oemetadata_path=correct_v15_test_data)
+    result = run_metadata_checks(oemetadata_path=correct_v15_test_data, check_jsonschema=True)
     print("Check v15 metadata from object!")
     run_metadata_checks(oemetadata=meta)
 

oem2orm/oep_oedialect_oem2orm.py

@@ -79,15 +79,15 @@ def setup_db_connection(engine="postgresql+oedialect", host="openenergy-platform
     return DB(engine, metadata)
 
 
-def setupApiAction(schema, table):
+def setupApiAction(schema, table, token=None):
     API_ACTION = namedtuple("API_Action", ["dest_url", "headers"])
     OEP_URL = "https://openenergy-platform.org"
 
     url = OEP_URL + "/api/v0/schema/{schema}/tables/{table}/meta/".format(
         schema=schema, table=table
     )
 
-    token = setUserToken()
+    token = token if token else setUserToken()
     headers = {
         "Authorization": "Token %s" % token,
         "Accept": "application/json",
@@ -108,20 +108,22 @@ def create_tables(db: DB, tables: List[sa.Table]):
     """
     for table in tables:
         if not db.engine.dialect.has_schema(db.engine, table.schema):
-            logging.info(
-                f'The provided database schema: "{table.schema}" does not exist. Please use an existing schema'
-            )
+            error_msg = f'The provided database schema: "{table.schema}" does not exist. Please use an existing schema'
+            logging.info(error_msg)
+            raise DatabaseError(error_msg)
         else:
             if not db.engine.dialect.has_table(db.engine, table.name, table.schema):
                 try:
                     table.create(checkfirst=True)
                     logging.info(f"Created table {table.name}")
                 except oedialect.engine.ConnectionException as ce:
-                    logging.error(f'Error when uploading table "{table.name}".')
-                    raise ce
-                except sa.exc.ProgrammingError:
-                    logging.error(f'Table "{table.name}" already exists')
-                    raise
+                    error_msg = f'Error when uploading table "{table.name}".'
+                    logging.error(error_msg)
+                    raise DatabaseError(error_msg) from ce
+                except sa.exc.ProgrammingError as pe:
+                    error_msg = f'Table "{table.name}" already exists.'
+                    logging.error(error_msg)
+                    raise DatabaseError(error_msg) from pe
 
 
 def delete_tables(db: DB, tables: List[sa.Table]):
@@ -366,27 +368,29 @@ def parseDatapackageToString(oem_folder_path, datapackage_name=None, table_name=
     raise NotImplemented
 
 
-def api_updateMdOnTable(metadata):
+def api_updateMdOnTable(metadata, token=None):
     """ """
     schema = getTableSchemaNameFromOEM(metadata)[0]
     table = getTableSchemaNameFromOEM(metadata)[1]
 
     logging.info("UPDATE METADATA")
-    api_action = setupApiAction(schema, table)
+    api_action = setupApiAction(schema, table, token)
     resp = requests.post(api_action.dest_url, json=metadata, headers=api_action.headers)
-    if resp.status_code == "200":
+    if resp.status_code == 200:
         logging.info("   ok.")
         logging.info(api_action.dest_url)
     else:
-        logging.info(resp.json())
+        error_msg = resp.json()
+        logging.info(error_msg)
         logging.info("HTTP status code: ")
         logging.info(resp.status_code)
+        raise MetadataError(f"Uploading of metadata failed. Response from OEP: {error_msg}")
 
 
-def api_downloadMd(schema, table):
+def api_downloadMd(schema, table, token=None):
     """ """
     logging.info("DOWNLOAD_METADATA")
-    api_action = setupApiAction(schema, table)
+    api_action = setupApiAction(schema, table, token)
     res = requests.get(api_action.dest_url)
     res = res.json()
     logging.info("   ok.")
@@ -404,8 +408,8 @@ def moveTableToSchema(engine, destination_schema):
 
 
 # TODO: rename or remove this function - functionality moved to oep_complicance module, keep to avoide 3. party implementation errors
-def omi_validateMd(data: dict):
-    run_metadata_checks(oemetadata=data)
+def omi_validateMd(data: dict, jsonschema_validation=False):
+    run_metadata_checks(oemetadata=data, check_jsonschema=jsonschema_validation)
 
 
 def getTableSchemaNameFromOEM(metadata):

setup.py

@@ -8,7 +8,7 @@
 
 setuptools.setup(
     name="oem2orm",
-    version="0.3.1",
+    version="0.3.2",
     author="henhuy, jh-RLI",
     author_email="Hendrik.Huyskens@rl-institut.de",
     description="SQLAlchemy module to generate ORM, read from data model (oedatamodel) in open-energy-metadata JSON format",