[MDS-5697] Add method to forward UNTP DCC's through Orgbook Publisher (…

…#3289)

* attestation type and use published vocab

* need tupe

* update dcc construction and library to dcc 0.3.10

* update test

* test

* remove/update logging

* ensure query doesn't include regional mines (yet

* make context files configurable.

based on UNTPDCC and BCMinesActPermitCredential versions

* real context extension

* remove type not defined in context files.

update context reference to 0.5.0

* add schema file and config, updated to match 0.1.0 of models package

* nullable but not defaulted.

* same with product

* same for ca

* link to credential, not orgbook identifer.

* real tdw registry, and schema extension

* Define verification method, strip did

stop using sovrin did, just use did:web from tdw server.

* add permit_number, remove name.

* extended class needs extended type

* name is required in model (But not in spec)

remove hardcode from id path

* bc prefix is removed

* vcdm 2.0 updates.

name is included, issuanceDate is now validFrom.

* update to use new DataIntegrity Proof

DI proofs are going to be required by orgbook, but maybe not others. removed other WIP code with different endpoints

* remove dup line from merge

* remove dup lines from merge.

* key 1 is different, signing key is now key-02-multikey

* use mine_no ad factility identifier

* remove unused urls

* new env var for target url and generate id.

* remove slash from env var

* optional id's are allowed in package

* return tuple in error case

* add columns and publish all pending records

* keep error response if success isn't there.

* better names, going to implement both processes

* new command to push to publisher for construction and publishing

* trying to sync

* prettier instead of eslint in vscode

migrations/sql/V2024.11.29.13.31__add_columns_to_vc_orgbook_table.sql

@@ -0,0 +1,4 @@
+ALTER TABLE permit_amendment_orgbook_publish_status 
+    ADD COLUMN IF NOT EXISTS permit_number VARCHAR(50),
+    ADD COLUMN IF NOT EXISTS orgbook_entity_id VARCHAR(50),
+    ADD COLUMN IF NOT EXISTS error_msg VARCHAR;

services/core-api/app/api/verifiable_credentials/manager.py

@@ -1,5 +1,7 @@
 # for midware/business level actions between requests and data access
 import json
+import requests
+from datetime import date, datetime, timedelta
 
 from uuid import uuid4, UUID
 from sqlalchemy.exc import IntegrityError
@@ -37,6 +39,26 @@ class UNTPCCMinesActPermit(cc.ConformityAttestation):
     permitNumber: str
 
 
+permit_amendments_for_orgbook_query = """
+    select pa.permit_amendment_guid, poe.party_guid
+
+    from party_orgbook_entity poe
+    inner join party p on poe.party_guid = p.party_guid
+    inner join mine_party_appt mpa on p.party_guid = mpa.party_guid
+    inner join permit pmt on pmt.permit_id = mpa.permit_id
+    inner join permit_amendment pa on pa.permit_id = pmt.permit_id
+    inner join mine m on pa.mine_guid = m.mine_guid
+    
+    where mpa.permit_id is not null
+    and mpa.mine_party_appt_type_code = 'PMT'
+    and mpa.deleted_ind = false
+    and m.major_mine_ind = true
+    
+    group by pa.permit_amendment_guid, pa.description, pa.issue_date, pa.permit_amendment_status_code, mpa.deleted_ind, pmt.permit_no, mpa.permit_id, poe.party_guid, p.party_name, poe.name_text, poe.registration_id
+    order by pmt.permit_no, pa.issue_date;
+"""
+
+
 #this should probably be imported from somewhere.
 class W3CCred(BaseModel):
     #based on VCDM 2.0. https://www.w3.org/TR/vc-data-model-2.0/
@@ -129,25 +151,8 @@ def process_all_untp_map_for_orgbook():
 
     # https://metabase-4c2ba9-prod.apps.silver.devops.gov.bc.ca/question/2937-permit-amendments-for-each-party-orgbook-entity
 
-    permit_amendment_query_results = db.session.execute("""
-                        select pa.permit_amendment_guid, poe.party_guid
-
-                        from party_orgbook_entity poe
-                        inner join party p on poe.party_guid = p.party_guid
-                        inner join mine_party_appt mpa on p.party_guid = mpa.party_guid
-                        inner join permit pmt on pmt.permit_id = mpa.permit_id
-                        inner join permit_amendment pa on pa.permit_id = pmt.permit_id
-                        inner join mine m on pa.mine_guid = m.mine_guid
-                        
-                        where mpa.permit_id is not null
-                        and mpa.mine_party_appt_type_code = 'PMT'
-                        and mpa.deleted_ind = false
-                        and m.major_mine_ind = true
-                        
-                        group by pa.permit_amendment_guid, pa.description, pa.issue_date, pa.permit_amendment_status_code, mpa.deleted_ind, pmt.permit_no, mpa.permit_id, poe.party_guid, p.party_name, poe.name_text, poe.registration_id
-                        order by pmt.permit_no, pa.issue_date;
-
-                       """).fetchall()
+    permit_amendment_query_results = db.session.execute(
+        permit_amendments_for_orgbook_query).fetchall()
 
     task_logger.info("Num of results from query to process:" +
                      str(len(permit_amendment_query_results)))
@@ -189,6 +194,8 @@ def process_all_untp_map_for_orgbook():
             permit_amendment_guid=row[0],
             party_guid=row[1],
             unsigned_payload_hash=payload_hash,
+            permit_number=pa_cred.credentialSubject.permitNumber,
+            orgbook_entity_id=pa_cred.credentialSubject.issuedToParty.registeredId,
             orgbook_credential_id=new_id,
         )
         records.append((pa_cred, paob))
@@ -215,16 +222,82 @@ def process_all_untp_map_for_orgbook():
     return [record for payload, record in records]
 
 
-def publish_all_pending_vc_to_orgbook():
+def forward_all_pending_untp_vc_to_orgbook():
     """STUB for celery job to publis all pending vc to orgbook."""
-    ## Orgbook doesn't have this functionality yet.
+    ## CORE signs and structures the credential, the publisher just validates and forwards it.
+    records_to_forward = PermitAmendmentOrgBookPublish.find_all_unpublished(unsafe=True)
+    ORGBOOK_W3C_CRED_FORWARD = f"{Config.ORGBOOK_CREDENTIAL_BASE_URL}/forward"
+
+    current_app.logger.warning(f"going to publish {len(records_to_forward)} records to orgbook")
+
+    for record in records_to_forward:
+        current_app.logger.warning(f"publishing record={json.loads(record.signed_credential)}")
+        payload = {
+            "verifiableCredential": json.loads(record.signed_credential),
+            "options": {
+                "entityId": record.orgbook_entity_id,
+                "resourceId": record.permit_number,
+                "credentialId": record.orgbook_credential_id,
+                "credentialType": "BCMinesActPermitCredential"
+            }
+        }
+        resp = requests.post(ORGBOOK_W3C_CRED_FORWARD, json=payload)
+        if resp.status_code == 201:
+            record.publish_state = True
+        else:
+            record.error_msg = resp.text
+        record.save()
+
+
+@celery.task()
+def push_untp_map_data_to_publisher():
+    ## This is a different process that passes the data to the publisher.
+    ## the publisher structures the data and sends it to the orgbook.
+    ## the publisher also manages the BitStringStatusLists.
+    ORGBOOK_W3C_CRED_PUBLISH = f"{Config.ORGBOOK_CREDENTIAL_BASE_URL}/publish"
+
     records_to_publish = PermitAmendmentOrgBookPublish.find_all_unpublished(unsafe=True)
+    permit_amendment_query_results = db.session.execute(
+        permit_amendments_for_orgbook_query).fetchall()
+    for row in permit_amendment_query_results:
+        pa = PermitAmendment.find_by_permit_amendment_guid(row[0], unsafe=True)
+        pa_cred, new_id = VerifiableCredentialManager.produce_untp_cc_map_payload(
+            Config.CHIEF_PERMITTING_OFFICER_DID_WEB, pa)
+
+        #only one assessment per credential
+        publish_payload = {
+            "type": "BCMinesActPermitCredential",
+            "coreData": {
+                "entityId": pa_cred.credentialSubject.issuedToParty.registeredId,
+                "resourceId": pa_cred.credentialSubject.permitNumber,
+                "validFrom": pa_cred.validFrom,
+                "validUntil": date.fromisoformat(pa_cred.validFrom) + timedelta(years=5)
+            },
+            "subjectData": {
+                "permitNumber": pa_cred.credentialSubject.permitNumber
+            },
+            "untpData": {
+                "assessedFacility": pa_cred.credentialSubject.assessment[0].model_dump(),
+                "assessedProduct": pa_cred.credentialSubject.assessment[0].model_dump(),
+            }
+        }
+        payload_hash = md5(json.dumps(publish_payload).encode('utf-8')).hexdigest()
+
+        current_app.logger.warning(f"publishing record={publish_payload}")
+        post_resp = requests.post(Config.ORGBOOK_W3C_CRED_PUBLISH, json=publish_payload)
+
+        publish_record = PermitAmendmentOrgBookPublish(
+            payload_hash=payload_hash,
+            permit_amendment_guid=row[0],
+            party_guid=row[1],
+            signed_credential="Produced by publisher",
+            publish_state=post_resp.ok,
+            permitNumber=pa_cred.credentialSubject.permitNumber,
+            orgbook_entity_id=pa_cred.credentialSubject.issuedToParty.registeredId,
+            orgbook_credential_id=new_id,
+            error_msg=post_resp.text if not post_resp.ok else None)
 
-    for record in records_to_publish:
-        current_app.logger.warning("NOT sending cred to orgbook")
-        current_app.logger.warning(record.signed_credential)
-        # resp = requests.post(ORGBOOK_W3C_CRED_POST, record.signed_credential)
-        # assert resp.status_code == 200, f"resp={resp.json()}"
+        publish_record.save()
 
 
 class VerifiableCredentialManager():

services/core-api/app/api/verifiable_credentials/models/orgbook_publish_status.py

@@ -1,5 +1,6 @@
 from sqlalchemy.dialects.postgresql import UUID
 from sqlalchemy.schema import FetchedValue
+from sqlalchemy.sql import true
 from app.extensions import db
 from typing import List
 from app.api.utils.models_mixins import AuditMixin, Base
@@ -18,8 +19,11 @@ class PermitAmendmentOrgBookPublish(AuditMixin, Base):
     signed_credential = db.Column(db.String, nullable=True)
     publish_state = db.Column(
         db.Boolean, nullable=True)                                 # null = not published, true = published, false = failed
+    permit_number = db.Column(db.String, nullable=False)
+    orgbook_entity_id = db.Column(db.String, nullable=False)
     orgbook_credential_id = db.Column(
         db.String, nullable=False)                                 # not sure this will be able to be populated
+    error_msg = db.Column(db.String, nullable=True)
 
     def __repr__(self):
         return f'<PermitAmendmentOrgBookPublishStatus unsigned_payload_hash={self.unsigned_payload_hash}, permit_amendment_guid={self.permit_amendment_guid}, sign_date={self.sign_date}, publish_state={self.publish_state}>'
@@ -35,4 +39,5 @@ def find_by_unsigned_payload_hash(cls,
     @classmethod
     def find_all_unpublished(cls, *, unsafe: bool = False) -> List["PermitAmendmentOrgBookPublish"]:
         query = cls.query.unbound_unsafe() if unsafe else cls.query
-        return query.filter(PermitAmendmentOrgBookPublish.publish_state != True).all()
+        results: List["PermitAmendmentOrgBookPublish"] = query.filter().all()
+        return [r for r in results if r.publish_state is not True]

services/core-api/app/commands.py

@@ -134,8 +134,7 @@ def notify_expiring_party_appointments():
     def notify_and_update_expired_party_appointments():
         from app import auth
         from app.api.parties.party_appt import (
-            notify_and_update_expired_party_appointments,
-        )
+            notify_and_update_expired_party_appointments, )
         auth.apply_security = False
 
         with current_app.app_context():
@@ -147,8 +146,7 @@ def notify_and_update_expired_party_appointments():
     def revoke_mines_act_permits_for_permit(credential_exchange_id, permit_guid):
         from app import auth
         from app.api.verifiable_credentials.manager import (
-            revoke_all_credentials_for_permit,
-        )
+            revoke_all_credentials_for_permit, )
         auth.apply_security = False
         with current_app.app_context():
             permit = Permit.query.unbound_unsafe().filter_by(permit_guid=permit_guid).first()
@@ -160,21 +158,28 @@ def revoke_mines_act_permits_for_permit(credential_exchange_id, permit_guid):
     def process_all_untp_map_for_orgbook():
         from app import auth
         from app.api.verifiable_credentials.manager import (
-            process_all_untp_map_for_orgbook,
-        )
+            process_all_untp_map_for_orgbook, )
         auth.apply_security = False
         with current_app.app_context() as app:
-            result = process_all_untp_map_for_orgbook.apply_async()
+            result = process_all_untp_map_for_orgbook()
 
-    @app.cli.command('publish_all_pending_vc_to_orgbook')
-    def publish_all_pending_vc_to_orgbook():
+    @app.cli.command('forward_all_pending_untp_vc_to_orgbook')
+    def forward_all_pending_untp_vc_to_orgbook():
         from app import auth
         from app.api.verifiable_credentials.manager import (
-            publish_all_pending_vc_to_orgbook,
-        )
+            forward_all_pending_untp_vc_to_orgbook, )
         auth.apply_security = False
         with current_app.app_context():
-            result = publish_all_pending_vc_to_orgbook()
+            result = forward_all_pending_untp_vc_to_orgbook()
+
+    @app.cli.command('push_untp_map_data_to_publisher')
+    def push_untp_map_data_to_publisher():
+        from app import auth
+        from app.api.verifiable_credentials.manager import (
+            push_untp_map_data_to_publisher, )
+        auth.apply_security = False
+        with current_app.app_context():
+            result = push_untp_map_data_to_publisher()
 
     @app.cli.command('generate_history_table_migration')
     @click.argument('table')

services/core-api/app/config.py

@@ -306,7 +306,7 @@ def JWT_ROLE_CALLBACK(jwt_dict):
                                                       "UNTP_BC_MINES_ACT_PERMIT_CONTEXT")
 
     ORGBOOK_CREDENTIAL_BASE_URL = os.environ.get(
-        "ORGBOOK_CREDENTIAL_BASE_URL", "https://dev.orgbook.traceability.site/credentials/")
+        "ORGBOOK_CREDENTIAL_BASE_URL", "https://dev.orgbook.traceability.site/credentials")
 
 
 class TestConfig(Config):

services/core-web/common/utils/helpers.ts

@@ -10,7 +10,6 @@ import moment from "moment-timezone";
 import { reset } from "redux-form";
 import { ItemMap } from "@mds/common";
 
-
 /**
  * Helper function to clear redux form after submissions
  *
@@ -107,8 +106,6 @@ export const timeAgo = (dateTime, unit = "day") => {
   }
 };
 
-
-
 export const formatPostalCode = (code) => code && code.replace(/.{3}$/, " $&");
 
 export const formatTitleString = (input) =>
@@ -593,7 +590,7 @@ export const getHighestConsequence = (tsf) => {
 
   const highestRankedDam = tsf.dams.reduce((prev, current) =>
     CONSEQUENCE_CLASSIFICATION_RANK_HASH[prev.consequence_classification] >
-      CONSEQUENCE_CLASSIFICATION_RANK_HASH[current.consequence_classification]
+    CONSEQUENCE_CLASSIFICATION_RANK_HASH[current.consequence_classification]
       ? prev
       : current
   );

services/minespace-web/common/utils/helpers.ts

@@ -10,7 +10,6 @@ import moment from "moment-timezone";
 import { reset } from "redux-form";
 import { ItemMap } from "@mds/common";
 
-
 /**
  * Helper function to clear redux form after submissions
  *
@@ -107,8 +106,6 @@ export const timeAgo = (dateTime, unit = "day") => {
   }
 };
 
-
-
 export const formatPostalCode = (code) => code && code.replace(/.{3}$/, " $&");
 
 export const formatTitleString = (input) =>
@@ -593,7 +590,7 @@ export const getHighestConsequence = (tsf) => {
 
   const highestRankedDam = tsf.dams.reduce((prev, current) =>
     CONSEQUENCE_CLASSIFICATION_RANK_HASH[prev.consequence_classification] >
-      CONSEQUENCE_CLASSIFICATION_RANK_HASH[current.consequence_classification]
+    CONSEQUENCE_CLASSIFICATION_RANK_HASH[current.consequence_classification]
       ? prev
       : current
   );