PPR API verify new CI script updates. (#2082)

* PPR API verify new CI script updates. Signed-off-by: Doug Lovett <doug@diamante.ca> * PPR API verify new CI script updates. Signed-off-by: Doug Lovett <doug@diamante.ca> * PPR API verify new CI script updates. Signed-off-by: Doug Lovett <doug@diamante.ca> * PPR API verify new CI script updates. Signed-off-by: Doug Lovett <doug@diamante.ca> * PPR API verify new CI script updates. Signed-off-by: Doug Lovett <doug@diamante.ca> * PPR API verify new CI script updates. Signed-off-by: Doug Lovett <doug@diamante.ca> * PPR API verify new CI script updates. Signed-off-by: Doug Lovett <doug@diamante.ca> * PPR API verify new CI script updates. Signed-off-by: Doug Lovett <doug@diamante.ca> --------- Signed-off-by: Doug Lovett <doug@diamante.ca>
bcgov · Dec 16, 2024 · ca17800 · ca17800
1 parent 80697fe
commit ca17800
Show file tree

Hide file tree

Showing 27 changed files with 196 additions and 934 deletions.
diff --git a/ppr-api/gunicorn_config.py b/ppr-api/gunicorn_config.py
@@ -17,8 +17,8 @@
 
 import os
 
-workers = int(os.environ.get('GUNICORN_PROCESSES', '1'))  # pylint: disable=invalid-name
-threads = int(os.environ.get('GUNICORN_THREADS', '1'))  # pylint: disable=invalid-name
+workers = int(os.environ.get("GUNICORN_PROCESSES", "1"))  # pylint: disable=invalid-name
+threads = int(os.environ.get("GUNICORN_THREADS", "1"))  # pylint: disable=invalid-name
 
-forwarded_allow_ips = '*'  # pylint: disable=invalid-name
-secure_scheme_headers = {'X-Forwarded-Proto': 'https'}  # pylint: disable=invalid-name
+forwarded_allow_ips = "*"  # pylint: disable=invalid-name
+secure_scheme_headers = {"X-Forwarded-Proto": "https"}  # pylint: disable=invalid-name
diff --git a/ppr-api/manage.py b/ppr-api/manage.py
@@ -0,0 +1,77 @@
+# Copyright © 2019 Province of British Columbia
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Manage the database and some other items required to run the API
+"""
+import logging
+import os
+
+from flask.cli import FlaskGroup  # replaces flask_script Manager
+from sqlalchemy.sql import text
+
+# models included so that migrate can build the database migrations
+from ppr_api import models  # pylint: disable=unused-import
+from ppr_api import create_app
+from ppr_api.models import db
+
+APP = create_app()
+CLI = FlaskGroup(APP)  # replaces MANAGER
+
+
+def execute_script(session, file_name):
+    """Execute a SQL script as one or more SQL statements in a single file."""
+    print("Executing SQL statements in file " + file_name)
+    with open(file_name, "r") as sql_file:
+        sql_command = ""
+        # Iterate over all lines in the sql file
+        for line in sql_file:
+            # Ignore commented lines
+            if not line.startswith("--") and line.strip("\n"):
+                # Append line to the command string
+                sql_command += line.strip("\n")
+
+                # If the command string ends with ';', it is a full statement
+                if sql_command.endswith(";"):
+                    sql_command = sql_command.replace(";", "")
+                    # print(sql_command)
+                    # Try to execute statement and commit it
+                    try:
+                        session.execute(text(sql_command))
+
+                    # Assert in case of error
+                    except Exception as ex:
+                        print(repr(ex))
+
+                    # Finally, clear command string
+                    finally:
+                        sql_command = ""
+
+        session.commit()
+        sql_file.close()
+
+
+@CLI.command("create_test_data")
+def create_test_data():
+    """Load unit test data in the dev/local environment. Delete all existing test data as a first step."""
+    execute_script(db.session, "test_data/postgres_test_reset.sql")
+    execute_script(db.session, "test_data/postgres_create_first.sql")
+    filenames = os.listdir(os.path.join(os.getcwd(), "test_data/postgres_data_files"))
+    sorted_names = sorted(filenames)
+    for filename in sorted_names:
+        execute_script(db.session, os.path.join(os.getcwd(), ("test_data/postgres_data_files/" + filename)))
+
+
+if __name__ == "__main__":
+    logging.log(logging.INFO, "Running the Flask CLI")
+    CLI()
diff --git a/ppr-api/migrations/versions/0001_rebase_2024-10.py b/ppr-api/migrations/versions/0001_rebase_2024-10.py
@@ -23,26 +23,9 @@
 
 def upgrade():
     # ### Manually install extensions. ###
-    # op.execute("CREATE EXTENSION IF NOT EXISTS fuzzystrmatch;")
-    # op.execute("CREATE EXTENSION IF NOT EXISTS pg_trgm;")
-    # op.execute("CREATE EXTENSION IF NOT EXISTS btree_gist;")
-    public_fuzzystrmatch = PGExtension(
-        schema="public",
-        signature="fuzzystrmatch"
-    )
-    op.create_entity(public_fuzzystrmatch)
-
-    public_btree_gist = PGExtension(
-        schema="public",
-        signature="btree_gist"
-    )
-    op.create_entity(public_btree_gist)
-
-    public_pg_trgm = PGExtension(
-        schema="public",
-        signature="pg_trgm"
-    )
-    op.create_entity(public_pg_trgm)
+    op.execute("CREATE EXTENSION IF NOT EXISTS fuzzystrmatch;")
+    op.execute("CREATE EXTENSION IF NOT EXISTS pg_trgm;")
+    op.execute("CREATE EXTENSION IF NOT EXISTS btree_gist;")
 
     # ### Manually create sequences and add them to pk columns. ###
     # PPR sequences
@@ -3626,25 +3609,6 @@ def downgrade():
     op.execute(DropSequence(Sequence('word_id_seq')))
     op.execute(DropSequence(Sequence('name_id_seq')))
 
-    # Manually added drop extensions
-    public_pg_trgm = PGExtension(
-        schema="public",
-        signature="pg_trgm"
-    )
-    op.drop_entity(public_pg_trgm)
-
-    public_btree_gist = PGExtension(
-        schema="public",
-        signature="btree_gist"
-    )
-    op.drop_entity(public_btree_gist)
-
-    public_fuzzystrmatch = PGExtension(
-        schema="public",
-        signature="fuzzystrmatch"
-    )
-    op.drop_entity(public_fuzzystrmatch)
-
     public_mhr_account_reg_vw = PGView(
         schema="public",
         signature="mhr_account_reg_vw",

diff --git a/ppr-api/src/ppr_api/__init__.py b/ppr-api/src/ppr_api/__init__.py
@@ -35,9 +35,7 @@
 from ppr_api.utils.auth import jwt
 from ppr_api.utils.logging import logger, setup_logging
 
-setup_logging(
-    os.path.join(os.path.abspath(os.path.dirname(__file__)), "logging.yaml")
-)  # important to do this first
+setup_logging(os.path.join(os.path.abspath(os.path.dirname(__file__)), "logging.yaml"))  # important to do this first
 
 
 def create_app(service_environment=APP_RUNNING_ENVIRONMENT, **kwargs):
@@ -50,16 +48,12 @@ def create_app(service_environment=APP_RUNNING_ENVIRONMENT, **kwargs):
 
     db.init_app(app)
     Migrate(app, db)
-    if (
-        app.config.get("DEPLOYMENT_ENV", "") == "unitTesting"
-    ):  # CI only run upgrade for unit testing.
+    if app.config.get("DEPLOYMENT_ENV", "") == "testing":  # CI only run upgrade for unit testing.
         logger.info("Running migration upgrade.")
         with app.app_context():
             upgrade(directory="migrations", revision="head", sql=False, tag=None)
         # Alembic has it's own logging config, we'll need to restore our logging here.
-        setup_logging(
-            os.path.join(os.path.abspath(os.path.dirname(__file__)), "logging.yaml")
-        )
+        setup_logging(os.path.join(os.path.abspath(os.path.dirname(__file__)), "logging.yaml"))
         logger.info("Finished migration upgrade.")
     else:
         logger.info("Logging, migrate set up.")
@@ -114,9 +108,7 @@ def setup_test_data():
     """Load unit test data in the dev/local environment. Delete all existing test data as a first step."""
     try:
         test_path = os.getcwd()
-        logger.info(
-            f"Executing DB scripts to create test data from test data dir {test_path}..."
-        )
+        logger.info(f"Executing DB scripts to create test data from test data dir {test_path}...")
         # execute_script(db.session, os.path.join(test_path, "test_data/postgres_test_reset.sql"))
         execute_script(db.session, "test_data/postgres_create_first.sql")
         filenames = os.listdir(os.path.join(test_path, "test_data/postgres_data_files"))

diff --git a/ppr-api/src/ppr_api/config.py b/ppr-api/src/ppr_api/config.py
@@ -82,7 +82,7 @@ class Config:  # pylint: disable=too-few-public-methods
     AUTH_API_VERSION = os.getenv("AUTH_API_VERSION", "")
     PAY_API_URL = os.getenv("PAY_API_URL", "")
     PAY_API_VERSION = os.getenv("PAY_API_VERSION", "")
-    REPORT_API_URL = os.getenv("REPORT_API_URL", "")
+    REPORT_API_URL = os.getenv("REPORT_API_URL", "https://gotenberg-p56lvhvsqa-nn.a.run.app")
 
     AUTH_SVC_URL = f"{AUTH_API_URL + AUTH_API_VERSION}"
     PAYMENT_SVC_URL = f"{PAY_API_URL + PAY_API_VERSION}"
@@ -101,9 +101,7 @@ class Config:  # pylint: disable=too-few-public-methods
     if DB_UNIX_SOCKET := os.getenv("DATABASE_UNIX_SOCKET", None):
         SQLALCHEMY_DATABASE_URI = f"postgresql+psycopg2://{DB_USER}:{DB_PASSWORD}@/{DB_NAME}?host={DB_UNIX_SOCKET}"
     else:
-        SQLALCHEMY_DATABASE_URI = (
-            f"postgresql://{DB_USER}:{DB_PASSWORD}@{DB_HOST}:{DB_PORT}/{DB_NAME}"
-        )
+        SQLALCHEMY_DATABASE_URI = f"postgresql://{DB_USER}:{DB_PASSWORD}@{DB_HOST}:{DB_PORT}/{DB_NAME}"
 
     # Connection pool settings
     DB_MIN_POOL_SIZE = os.getenv("DATABASE_MIN_POOL_SIZE", "2")
@@ -143,25 +141,15 @@ class Config:  # pylint: disable=too-few-public-methods
     # ACCOUNT_SVC_TIMEOUT = os.g
 
     # DB Query limits on result set sizes
-    ACCOUNT_REGISTRATIONS_MAX_RESULTS = os.getenv(
-        "ACCOUNT_REGISTRATIONS_MAX_RESULTS", "100"
-    )
+    ACCOUNT_REGISTRATIONS_MAX_RESULTS = os.getenv("ACCOUNT_REGISTRATIONS_MAX_RESULTS", "100")
     ACCOUNT_DRAFTS_MAX_RESULTS = os.getenv("ACCOUNT_DRAFTS_MAX_RESULTS", "1000")
     ACCOUNT_SEARCH_MAX_RESULTS = os.getenv("ACCOUNT_SEARCH_MAX_RESULTS", "1000")
 
     # DEBTOR search trgram similarity quotients
-    SIMILARITY_QUOTIENT_BUSINESS_NAME: float = float(
-        os.getenv("SIMILARITY_QUOTIENT_BUSINESS_NAME", "0.6")
-    )
-    SIMILARITY_QUOTIENT_FIRST_NAME: float = float(
-        os.getenv("SIMILARITY_QUOTIENT_FIRST_NAME", "0.4")
-    )
-    SIMILARITY_QUOTIENT_LAST_NAME: float = float(
-        os.getenv("SIMILARITY_QUOTIENT_LAST_NAME", "0.29")
-    )
-    SIMILARITY_QUOTIENT_DEFAULT: float = float(
-        os.getenv("SIMILARITY_QUOTIENT_DEFAULT", "0.5")
-    )
+    SIMILARITY_QUOTIENT_BUSINESS_NAME: float = float(os.getenv("SIMILARITY_QUOTIENT_BUSINESS_NAME", "0.6"))
+    SIMILARITY_QUOTIENT_FIRST_NAME: float = float(os.getenv("SIMILARITY_QUOTIENT_FIRST_NAME", "0.4"))
+    SIMILARITY_QUOTIENT_LAST_NAME: float = float(os.getenv("SIMILARITY_QUOTIENT_LAST_NAME", "0.29"))
+    SIMILARITY_QUOTIENT_DEFAULT: float = float(os.getenv("SIMILARITY_QUOTIENT_DEFAULT", "0.5"))
 
     # Search results report number of financing statements threshold for async requests.
     SEARCH_PDF_ASYNC_THRESHOLD: int = int(os.getenv("SEARCH_PDF_ASYNC_THRESHOLD", "75"))
@@ -174,36 +162,22 @@ class Config:  # pylint: disable=too-few-public-methods
 
     # Google APIs and cloud storage
     GOOGLE_DEFAULT_SA = os.getenv("GOOGLE_DEFAULT_SA")
-    GCP_CS_SA_SCOPES = os.getenv(
-        "GCP_CS_SA_SCOPES", "https://www.googleapis.com/auth/cloud-platform"
-    )
+    GCP_CS_SA_SCOPES = os.getenv("GCP_CS_SA_SCOPES", "https://www.googleapis.com/auth/cloud-platform")
     # Storage of search reports
     GCP_CS_BUCKET_ID = os.getenv("GCP_CS_BUCKET_ID", "ppr_search_results_dev")
     # Storage of verification mail reports
-    GCP_CS_BUCKET_ID_VERIFICATION = os.getenv(
-        "GCP_CS_BUCKET_ID_VERIFICATION", "ppr_verification_report_dev"
-    )
+    GCP_CS_BUCKET_ID_VERIFICATION = os.getenv("GCP_CS_BUCKET_ID_VERIFICATION", "ppr_verification_report_dev")
     # Storage of registration verification reports
-    GCP_CS_BUCKET_ID_REGISTRATION = os.getenv(
-        "GCP_CS_BUCKET_ID_REGISTRATION", "ppr_registration_report_dev"
-    )
+    GCP_CS_BUCKET_ID_REGISTRATION = os.getenv("GCP_CS_BUCKET_ID_REGISTRATION", "ppr_registration_report_dev")
     # Storage of mail verification reports
     GCP_CS_BUCKET_ID_MAIL = os.getenv("GCP_CS_BUCKET_ID_MAIL", "ppr_mail_report_dev")
 
     # Pub/Sub
     GCP_PS_PROJECT_ID = os.getenv("DEPLOYMENT_PROJECT", "eogruh-dev")
-    GCP_PS_SEARCH_REPORT_TOPIC = os.getenv(
-        "GCP_PS_SEARCH_REPORT_TOPIC", "ppr-search-report"
-    )
-    GCP_PS_NOTIFICATION_TOPIC = os.getenv(
-        "GCP_PS_NOTIFICATION_TOPIC", "ppr-api-notification"
-    )
-    GCP_PS_VERIFICATION_REPORT_TOPIC = os.getenv(
-        "GCP_PS_VERIFICATION_REPORT_TOPIC", "ppr-mail-report"
-    )
-    GCP_PS_REGISTRATION_REPORT_TOPIC = os.getenv(
-        "GCP_PS_REGISTRATION_REPORT_TOPIC", "ppr-registration-report"
-    )
+    GCP_PS_SEARCH_REPORT_TOPIC = os.getenv("GCP_PS_SEARCH_REPORT_TOPIC", "ppr-search-report")
+    GCP_PS_NOTIFICATION_TOPIC = os.getenv("GCP_PS_NOTIFICATION_TOPIC", "ppr-api-notification")
+    GCP_PS_VERIFICATION_REPORT_TOPIC = os.getenv("GCP_PS_VERIFICATION_REPORT_TOPIC", "ppr-mail-report")
+    GCP_PS_REGISTRATION_REPORT_TOPIC = os.getenv("GCP_PS_REGISTRATION_REPORT_TOPIC", "ppr-registration-report")
 
     GATEWAY_URL = os.getenv("GATEWAY_URL", "https://bcregistry-dev.apigee.net")
     SUBSCRIPTION_API_KEY = os.getenv("SUBSCRIPTION_API_KEY")
@@ -215,9 +189,7 @@ class Config:  # pylint: disable=too-few-public-methods
     MAX_SIZE_SEARCH_RT: int = int(os.getenv("MAX_SIZE_SEARCH_RT", "225000"))
     # Default 2, set to 1 to revert to original report api client
     REPORT_VERSION = os.getenv("REPORT_VERSION", "2")
-    REPORT_API_AUDIENCE = os.getenv(
-        "REPORT_API_AUDIENCE", "https://gotenberg-p56lvhvsqa-nn.a.run.app"
-    )
+    REPORT_API_AUDIENCE = os.getenv("REPORT_API_AUDIENCE", "https://gotenberg-p56lvhvsqa-nn.a.run.app")
     # Number of registrations threshold for search report light format.
     REPORT_SEARCH_LIGHT: int = int(os.getenv("REPORT_SEARCH_LIGHT", "700"))
 
@@ -257,9 +229,7 @@ class UnitTestingConfig(Config):  # pylint: disable=too-few-public-methods
     DB_HOST = os.getenv("DATABASE_TEST_HOST", "")
     DB_PORT = os.getenv("DATABASE_TEST_PORT", "5432")
     # SQLALCHEMY_DATABASE_URI = f"postgresql+pg8000://{DB_USER}:{DB_PASSWORD}@{DB_HOST}:{DB_PORT}/{DB_NAME}"
-    SQLALCHEMY_DATABASE_URI = (
-        f"postgresql://{DB_USER}:{DB_PASSWORD}@{DB_HOST}:{DB_PORT}/{DB_NAME}"
-    )
+    SQLALCHEMY_DATABASE_URI = f"postgresql://{DB_USER}:{DB_PASSWORD}@{DB_HOST}:{DB_PORT}/{DB_NAME}"
 
     # JWT OIDC settings
     # JWT_OIDC_TEST_MODE will set jwt_manager to use

diff --git a/ppr-api/src/ppr_api/resources/financing_utils.py b/ppr-api/src/ppr_api/resources/financing_utils.py
@@ -63,7 +63,7 @@
 }
 
 
-def pay_and_save(  # pylint: disable=too-many-arguments,too-many-locals,too-many-branches
+def pay_and_save(  # pylint: disable=too-many-positional-arguments,too-many-locals,too-many-branches,too-many-arguments
     req: request,
     request_json,
     registration_class,

diff --git a/ppr-api/src/ppr_api/services/payment/client/__init__.py b/ppr-api/src/ppr_api/services/payment/client/__init__.py
@@ -170,7 +170,7 @@ def __init__(self, jwt=None, account_id=None, api_key=None, details=None):
             self.detail_label = None
             self.detail_value = None
 
-    def call_api(  # pylint: disable=too-many-arguments
+    def call_api(  # pylint: disable=too-many-positional-arguments,too-many-arguments
         self,
         method,
         relative_path,
@@ -320,7 +320,7 @@ def create_payment_staff_search_data(transaction_info, client_reference_id=None)
 
         return data
 
-    def create_payment(  # pylint: disable=too-many-arguments
+    def create_payment(  # pylint: disable=too-many-positional-arguments,too-many-arguments
         self, transaction_type, quantity=1, ppr_id=None, client_reference_id=None, processing_fee=None
     ):
         """Submit a payment request for the PPR API transaction."""

diff --git a/ppr-api/src/ppr_api/services/payment/payment.py b/ppr-api/src/ppr_api/services/payment/payment.py
@@ -30,7 +30,7 @@ def __init__(self, jwt=None, account_id=None, api_key=None, details=None):
         self.api_key = api_key
         self.details = details
 
-    def create_payment(  # pylint: disable=too-many-arguments
+    def create_payment(  # pylint: disable=too-many-positional-arguments,too-many-arguments
         self, transaction_type, quantity, transaction_id=None, client_reference_id=None, processing_fee=None
     ):
         """Submit a payment request for the account_id-jwt pair.

diff --git a/ppr-api/src/ppr_api/utils/run_version.py b/ppr-api/src/ppr_api/utils/run_version.py