[MDS-5697] Orgbook Publisher, data typing issues and clean up (#3296)

* data typing issues and clean up

* better logging

* better logging of errors

services/core-api/.env-example

@@ -121,6 +121,7 @@ UNTP_DIGITAL_CONFORMITY_CREDENTIAL_CONTEXT=https://test.uncefact.org/vocabulary/
 UNTP_DIGITAL_CONFORMITY_CREDENTIAL_SCHEMA=https://test.uncefact.org/vocabulary/untp/dcc/untp-dcc-schema-0.5.0.json
 UNTP_BC_MINES_ACT_PERMIT_CONTEXT=https://bcgov.github.io/digital-trust-toolkit/contexts/BCMinesActPermit/v1.jsonld
 ORGBOOK_CREDENTIAL_BASE_URL=https://dev.orgbook.traceability.site/credentials
+ORGBOOK_PUBLISHER_API_KEY=ORGBOOK_PUBLISHER_API_KEY
 # Permit Search Service
 PERMITS_ENDPOINT=http://haystack
 PERMITS_CLIENT_ID=mds-core-api-internal-5194

services/core-api/app/api/services/traction_service.py

@@ -204,7 +204,11 @@ def sign_add_data_integrity_proof(self, verificationMethod: str, credential: Bas
 
         post_resp = requests.post(
             traction_vc_di_add_proof, json=payload, headers=self.get_headers())
-        return post_resp.json()
+        if post_resp.ok:
+            return post_resp.json()
+        else:
+            raise VerificableCredentialWorkflowError(
+                f"Error while adding data integrity signature, {post_resp.text}")
 
     def fetch_a_random_did_key(self):
         get_resp = requests.get(

services/core-api/app/api/verifiable_credentials/manager.py

@@ -1,15 +1,15 @@
 # for midware/business level actions between requests and data access
 import json
 import requests
-from datetime import date, datetime, timedelta
 
+from datetime import date, datetime
+from dateutil.relativedelta import relativedelta
 from uuid import uuid4, UUID
 from sqlalchemy.exc import IntegrityError
 from typing import List, Union, Tuple, Optional
 from pydantic import BaseModel, Field, ConfigDict
 from openlocationcode.openlocationcode import encode as plus_code_encode
 from hashlib import md5
-from datetime import datetime
 from zoneinfo import ZoneInfo
 from time import sleep
 from typing import List
@@ -81,6 +81,10 @@ class W3CCred(BaseModel):
     credentialSchema: List[dict]
 
 
+def convert_date_to_iso_datetime(date: date) -> str:
+    return datetime(date.year, date.month, date.day, 0, 0, 0, tzinfo=ZoneInfo("UTC")).isoformat()
+
+
 @celery.task()
 def revoke_all_credentials_for_permit(permit_guid: str, mine_guid: str, reason: str):
     cred_exch = PartyVerifiableCredentialMinesActPermit.find_by_permit_guid_and_mine_guid(
@@ -228,10 +232,10 @@ def forward_all_pending_untp_vc_to_orgbook():
     records_to_forward = PermitAmendmentOrgBookPublish.find_all_unpublished(unsafe=True)
     ORGBOOK_W3C_CRED_FORWARD = f"{Config.ORGBOOK_CREDENTIAL_BASE_URL}/forward"
 
-    current_app.logger.warning(f"going to publish {len(records_to_forward)} records to orgbook")
+    task_logger.warning(f"going to publish {len(records_to_forward)} records to orgbook")
 
     for record in records_to_forward:
-        current_app.logger.warning(f"publishing record={json.loads(record.signed_credential)}")
+        task_logger.warning(f"publishing record={json.loads(record.signed_credential)}")
         payload = {
             "verifiableCredential": json.loads(record.signed_credential),
             "options": {
@@ -254,11 +258,15 @@ def push_untp_map_data_to_publisher():
     ## This is a different process that passes the data to the publisher.
     ## the publisher structures the data and sends it to the orgbook.
     ## the publisher also manages the BitStringStatusLists.
-    ORGBOOK_W3C_CRED_PUBLISH = f"{Config.ORGBOOK_CREDENTIAL_BASE_URL}/publish"
+    ORGBOOK_W3C_CRED_PUBLISH = f"{Config.ORGBOOK_PUBLISHER_BASE_URL}/credentials/publish"
 
     records_to_publish = PermitAmendmentOrgBookPublish.find_all_unpublished(unsafe=True)
     permit_amendment_query_results = db.session.execute(
         permit_amendments_for_orgbook_query).fetchall()
+
+    failed_credentials: List[Tuple[str, bool]] = []
+    success_count = 0
+
     for row in permit_amendment_query_results:
         pa = PermitAmendment.find_by_permit_amendment_guid(row[0], unsafe=True)
         pa_cred, new_id = VerifiableCredentialManager.produce_untp_cc_map_payload(
@@ -270,34 +278,52 @@ def push_untp_map_data_to_publisher():
             "coreData": {
                 "entityId": pa_cred.credentialSubject.issuedToParty.registeredId,
                 "resourceId": pa_cred.credentialSubject.permitNumber,
-                "validFrom": pa_cred.validFrom,
-                "validUntil": date.fromisoformat(pa_cred.validFrom) + timedelta(years=5)
+                "validFrom": convert_date_to_iso_datetime(pa.issue_date),
+                "validUntil": convert_date_to_iso_datetime(pa.issue_date + relativedelta(years=5)),
             },
             "subjectData": {
                 "permitNumber": pa_cred.credentialSubject.permitNumber
             },
             "untpData": {
-                "assessedFacility": pa_cred.credentialSubject.assessment[0].model_dump(),
-                "assessedProduct": pa_cred.credentialSubject.assessment[0].model_dump(),
+                "assessedFacility": [
+                    f.model_dump(exclude_none=True)
+                    for f in pa_cred.credentialSubject.assessment[0].assessedFacility
+                ],
+                "assessedProduct": [
+                    p.model_dump(exclude_none=True)
+                    for p in pa_cred.credentialSubject.assessment[0].assessedProduct
+                ],
             }
         }
+        current_app.logger.warning(f"publishing record={publish_payload}")
         payload_hash = md5(json.dumps(publish_payload).encode('utf-8')).hexdigest()
 
-        current_app.logger.warning(f"publishing record={publish_payload}")
-        post_resp = requests.post(Config.ORGBOOK_W3C_CRED_PUBLISH, json=publish_payload)
+        post_resp = requests.post(
+            ORGBOOK_W3C_CRED_PUBLISH,
+            json=publish_payload,
+            headers={"X-API-KEY": Config.ORGBOOK_PUBLISHER_API_KEY})
 
         publish_record = PermitAmendmentOrgBookPublish(
-            payload_hash=payload_hash,
+            unsigned_payload_hash=payload_hash,
             permit_amendment_guid=row[0],
             party_guid=row[1],
             signed_credential="Produced by publisher",
             publish_state=post_resp.ok,
-            permitNumber=pa_cred.credentialSubject.permitNumber,
+            permit_number=pa_cred.credentialSubject.permitNumber,
             orgbook_entity_id=pa_cred.credentialSubject.issuedToParty.registeredId,
             orgbook_credential_id=new_id,
             error_msg=post_resp.text if not post_resp.ok else None)
 
         publish_record.save()
+        if publish_record.error_msg:
+            task_logger.warning(
+                f"failed to publish unsigned_payload_id={publish_record.unsigned_payload_hash} error={publish_record.error_msg}"
+            )
+        else:
+            success_count += success_count + 1
+        failed_credentials.append(publish_record.unsigned_payload_hash, publish_record.error_msg)
+
+    return f"num published={success_count}, num failed = {len(failed_credentials)}"
 
 
 class VerifiableCredentialManager():
@@ -427,7 +453,7 @@ def produce_untp_cc_map_payload(cls, did: str,
         curr_appt = permit_amendment.permittee_appointments[0]
         for pmt_appt in permit_amendment.permittee_appointments:
             #find the last permittee appointment relevant to the amendment issue date.
-            if pmt_appt.start_date <= permit_amendment.issue_date:
+            if (pmt_appt.start_date or date(year=1900)) <= permit_amendment.issue_date:
                 curr_appt = pmt_appt
             else:
                 break
@@ -468,9 +494,6 @@ def produce_untp_cc_map_payload(cls, did: str,
         ]
 
         issue_date = permit_amendment.issue_date
-        issuance_date_str = datetime(
-            issue_date.year, issue_date.month, issue_date.day, 0, 0, 0,
-            tzinfo=ZoneInfo("UTC")).isoformat()
 
         untp_assessment = cc.ConformityAssessment(
             id=None,
@@ -490,10 +513,6 @@ def produce_untp_cc_map_payload(cls, did: str,
             assessedFacility=[facility],
             assessedProduct=products)
 
-        issuance_date_str = datetime(
-            issue_date.year, issue_date.month, issue_date.day, 0, 0, 0,
-            tzinfo=ZoneInfo("UTC")).isoformat()
-
         cred = UNTPCCMinesActPermit(
             id=None,
             name="Credential for permitNumber=" + permit_amendment.permit_no,
@@ -514,12 +533,13 @@ def produce_untp_cc_map_payload(cls, did: str,
 
         new_credential_id = uuid4()
         w3c_cred = W3CCred(
-            id=f"{Config.ORGBOOK_CREDENTIAL_BASE_URL}/{new_credential_id}",
+            id=f"{Config.ORGBOOK_PUBLISHER_BASE_URL}/credentials/{new_credential_id}",
             type=[
                 "VerifiableCredential", "DigitalConformityCredential", "BCMinesActPermitCredential"
             ],
             issuer={"id": did},
-            validFrom=issuance_date_str,                                                            #vcdm1.1, will change to 'validFrom' in vcdm2.0
+            validFrom=convert_date_to_iso_datetime(
+                permit_amendment.issue_date),                                                       #vcdm1.1, will change to 'validFrom' in vcdm2.0
             credentialSubject=cred,
             credentialSchema=[{
                 "id": Config.UNTP_DIGITAL_CONFORMITY_CREDENTIAL_CONTEXT,

services/core-api/app/commands.py

@@ -161,7 +161,7 @@ def process_all_untp_map_for_orgbook():
             process_all_untp_map_for_orgbook, )
         auth.apply_security = False
         with current_app.app_context() as app:
-            result = process_all_untp_map_for_orgbook()
+            result = process_all_untp_map_for_orgbook().apply_async()
 
     @app.cli.command('forward_all_pending_untp_vc_to_orgbook')
     def forward_all_pending_untp_vc_to_orgbook():
@@ -170,7 +170,7 @@ def forward_all_pending_untp_vc_to_orgbook():
             forward_all_pending_untp_vc_to_orgbook, )
         auth.apply_security = False
         with current_app.app_context():
-            result = forward_all_pending_untp_vc_to_orgbook()
+            result = forward_all_pending_untp_vc_to_orgbook().apply_async()
 
     @app.cli.command('push_untp_map_data_to_publisher')
     def push_untp_map_data_to_publisher():
@@ -179,7 +179,7 @@ def push_untp_map_data_to_publisher():
             push_untp_map_data_to_publisher, )
         auth.apply_security = False
         with current_app.app_context():
-            result = push_untp_map_data_to_publisher()
+            result = push_untp_map_data_to_publisher().apply_async()
 
     @app.cli.command('generate_history_table_migration')
     @click.argument('table')

services/core-api/app/config.py

@@ -305,8 +305,10 @@ def JWT_ROLE_CALLBACK(jwt_dict):
     UNTP_BC_MINES_ACT_PERMIT_CONTEXT = os.environ.get("UNTP_BC_MINES_ACT_PERMIT_CONTEXT",
                                                       "UNTP_BC_MINES_ACT_PERMIT_CONTEXT")
 
-    ORGBOOK_CREDENTIAL_BASE_URL = os.environ.get(
-        "ORGBOOK_CREDENTIAL_BASE_URL", "https://dev.orgbook.traceability.site/credentials")
+    ORGBOOK_PUBLISHER_BASE_URL = os.environ.get("ORGBOOK_PUBLISHER_BASE_URL",
+                                                "https://dev.orgbook.traceability.site")
+    ORGBOOK_PUBLISHER_API_KEY = os.environ.get("ORGBOOK_PUBLISHER_API_KEY",
+                                               "ORGBOOK_PUBLISHER_API_KEY")
 
 
 class TestConfig(Config):