diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 5277de8f..6dc0091e 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -2,6 +2,7 @@ "dockerComposeFile": [ "../docker-compose.yml" ], + "name": "assesplication", "service": "assesplication-store", "workspaceFolder": "/assesplication-store", "shutdownAction": "none", diff --git a/.vscode/launch.json b/.vscode/launch.json index 5fb6c93c..07ad07e4 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -4,6 +4,22 @@ // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 "version": "0.2.0", "configurations": [ + { + // launch command is python -m debugpy --listen 0.0.0.0:5678 -m wsgi + "name": "Python Debugger: Remote Attach", + "type": "debugpy", + "request": "attach", + "connect": { + "host": "localhost", + "port": 5678 + }, + "pathMappings": [ + { + "localRoot": "${workspaceFolder}", + "remoteRoot": "." + } + ] + }, { "name": "Docker-Runner assessment-store", "type": "python", @@ -25,7 +41,10 @@ "type": "python", "request": "launch", "module": "flask", - "args": ["run", "--no-reload"], + "args": [ + "run", + "--no-reload" + ], "jinja": true, "justMyCode": false, "envFile": "${workspaceFolder}/.env.development", @@ -39,45 +58,47 @@ "host": "localhost", "port": 9091, "cwd": "${workspaceFolder}", - "env": { "PYTHONPATH":"${workspaceFolder}"}, + "env": { + "PYTHONPATH": "${workspaceFolder}" + }, "envFile": "${workspaceFolder}/.env.development", "justMyCode": false, - }, - { - "name": "Upgrade DB", - "type": "python", - "request": "launch", - "module": "flask", - "envFile": "${workspaceFolder}/.env.development", - "args": [ - "db", - "upgrade" - ] - }, - { - "name": "Downgrade DB", - "type": "python", - "request": "launch", - "module": "flask", - "envFile": "${workspaceFolder}/.env.development", - "args": [ - "db", - "downgrade", - "6c8205510de6" // modify the downgrade revision accordingly - ] - }, - { - "name": "Prepare DB Migration", - "type": "python", - "request": "launch", - "module": "flask", - "envFile": "${workspaceFolder}/.env.development", - "args": [ - "db", - "migrate" - ] - }, - { + }, + { + "name": "Upgrade DB", + "type": "python", + "request": "launch", + "module": "flask", + "envFile": "${workspaceFolder}/.env.development", + "args": [ + "db", + "upgrade" + ] + }, + { + "name": "Downgrade DB", + "type": "python", + "request": "launch", + "module": "flask", + "envFile": "${workspaceFolder}/.env.development", + "args": [ + "db", + "downgrade", + "6c8205510de6" // modify the downgrade revision accordingly + ] + }, + { + "name": "Prepare DB Migration", + "type": "python", + "request": "launch", + "module": "flask", + "envFile": "${workspaceFolder}/.env.development", + "args": [ + "db", + "migrate" + ] + }, + { "name": "Import Applications to Assessment", "type": "python", "request": "launch", @@ -88,14 +109,16 @@ "cwd": "${workspaceFolder}", "envFile": "${workspaceFolder}/.env.development", "env": { - "PYTHONPATH":"${workspaceFolder}", + "PYTHONPATH": "${workspaceFolder}", }, "justMyCode": false, // modify the args accordingly "args": [ - "--fundround", "NSTFR2"] - }, - { + "--fundround", + "NSTFR2" + ] + }, + { "name": "Seed Applications in assessment-store", "type": "python", "request": "launch", @@ -106,38 +129,43 @@ "cwd": "${workspaceFolder}", "envFile": "${workspaceFolder}/.env.development", "justMyCode": false, - "args": ["seed_dev_db"] - }, - { - "name": "Feed location in assessment-store", - "type": "python", - "request": "launch", - "program": "${workspaceFolder}/scripts/populate_location_data.py", - "console": "integratedTerminal", - "host": "localhost", - "port": 9091, - "cwd": "${workspaceFolder}", - "envFile": "${workspaceFolder}/.env.development", - "env": { - "PYTHONPATH":"${workspaceFolder}", + "args": [ + "seed_dev_db" + ] + }, + { + "name": "Feed location in assessment-store", + "type": "python", + "request": "launch", + "program": "${workspaceFolder}/scripts/populate_location_data.py", + "console": "integratedTerminal", + "host": "localhost", + "port": 9091, + "cwd": "${workspaceFolder}", + "envFile": "${workspaceFolder}/.env.development", + "env": { + "PYTHONPATH": "${workspaceFolder}", + }, + "justMyCode": false, + // modify the args accordingly + "args": [ + "--fundround", + "NSTFR2", + "--update_db", + "True", + "--write_csv", + "False" + ] }, - "justMyCode": false, - // modify the args accordingly - "args": [ - "--fundround", "NSTFR2", - "--update_db", "True", - "--write_csv", "False" - ] - }, - { + { "name": "Run Tests: All", "type": "python", "request": "launch", "module": "pytest", "console": "integratedTerminal", "envFile": "${workspaceFolder}/.env.development", - }, - { + }, + { "name": "Run Tests: Current File (debug)", "type": "python", "request": "launch", @@ -146,13 +174,13 @@ "cwd": "${workspaceFolder}", "envFile": "${workspaceFolder}/.env.development", "args": [ - "-c", - "pytest.ini", - "${file}" + "-c", + "pytest.ini", + "${file}" ], "justMyCode": false - }, - { + }, + { "name": "Run Tests: Current Function (debug)", "type": "python", "request": "launch", @@ -161,12 +189,12 @@ "cwd": "${workspaceFolder}", "envFile": "${workspaceFolder}/.env.development", "args": [ - "-c", - "pytest.ini", - "-k", - "test_get_application_fields_export" // modify this accordingly + "-c", + "pytest.ini", + "-k", + "test_get_application_fields_export" // modify this accordingly ], "justMyCode": false - }, + }, ] -} +} diff --git a/.vscode/settings.json b/.vscode/settings.json index b87a16f2..4df75ca2 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -10,4 +10,9 @@ "python.testing.unittestEnabled": false, "python.testing.pytestEnabled": true, "cSpell.language": "en-GB", + "editor.quickSuggestions": { + "other": true, + "comments": false, + "strings": false + }, } diff --git a/_helpers/__init__.py b/_helpers/__init__.py index 4ede8e6d..5f56b093 100644 --- a/_helpers/__init__.py +++ b/_helpers/__init__.py @@ -1 +1,3 @@ # noqa +from .application import order_applications # noqa +from .form import get_blank_forms # noqa diff --git a/_helpers/application.py b/_helpers/application.py new file mode 100644 index 00000000..4da40b99 --- /dev/null +++ b/_helpers/application.py @@ -0,0 +1,19 @@ +from operator import itemgetter + + +def order_applications(applications, order_by, order_rev): + """Returns a list of ordered applications.""" + if order_by and order_by in [ + "id", + "status", + "account_id", + "assessment_deadline", + "started_at", + "last_edited", + ]: + applications = sorted( + applications, + key=itemgetter(order_by), + reverse=int(order_rev), + ) + return applications diff --git a/_helpers/form.py b/_helpers/form.py new file mode 100644 index 00000000..d719c0d4 --- /dev/null +++ b/_helpers/form.py @@ -0,0 +1,39 @@ +from services.apply import get_application_sections + + +def get_form_name(section): + forms = set() + if section["children"]: + for child in section["children"]: + forms.update(get_form_name(child)) + if section["form_name"]: + forms.add(section["form_name"]) + return forms + + +def get_forms_from_sections(sections): + mint_form_list = set() + for section in sections: + mint_form_list.update(get_form_name(section)) + return mint_form_list + + +def get_blank_forms(fund_id: str, round_id: str, language: str): + """Get the list of forms required to populate a blank application for a fund + round. + + Args: + fund_id: (str) The id of the fund + round_id: (str) The id of the fund round + + Returns: + A list of json forms to populate the form + + """ + application_sections = get_application_sections(fund_id, round_id, language) + if application_sections: + forms = get_forms_from_sections(application_sections) + if not forms: + raise Exception(f"Could not find forms for {fund_id} - {round_id}") + return forms + raise Exception(f"Could not find fund round for {fund_id} - {round_id} in fund store.") diff --git a/app.py b/app.py index dec908c5..143b3b78 100644 --- a/app.py +++ b/app.py @@ -47,6 +47,8 @@ def create_app() -> FlaskApp: health.add_check(FlaskRunningChecker()) health.add_check(DbChecker(db)) + # TODO work out sqs stuff where same app is writing to and reading from the queue + # Initialize sqs extended client # create_sqs_extended_client(flask_app) diff --git a/apply/api/__init__.py b/apply/api/__init__.py new file mode 100644 index 00000000..b592f3a2 --- /dev/null +++ b/apply/api/__init__.py @@ -0,0 +1,2 @@ +from apply.api.routes.application.routes import ApplicationsView # noqa +from apply.api.routes.queues.routes import QueueView # noqa diff --git a/apply/api/routes/__init__.py b/apply/api/routes/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/apply/api/routes/application/__init__.py b/apply/api/routes/application/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/apply/api/routes/application/routes.py b/apply/api/routes/application/routes.py new file mode 100644 index 00000000..d259f706 --- /dev/null +++ b/apply/api/routes/application/routes.py @@ -0,0 +1,432 @@ +import json +import time +from typing import Optional +from uuid import uuid4 + +from _helpers import get_blank_forms +from _helpers import order_applications +from config import Config +from config.key_report_mappings.mappings import ROUND_ID_TO_KEY_REPORT_MAPPING +from db.models.application.enums import Status +from db.queries.apply import add_new_forms +from db.queries.apply import create_application +from db.queries.apply import export_json_to_csv +from db.queries.apply import export_json_to_excel +from db.queries.apply import get_application +from db.queries.apply import get_feedback +from db.queries.apply import get_fund_id +from db.queries.apply import get_general_status_applications_report +from db.queries.apply import get_key_report_field_headers +from db.queries.apply import get_report_for_applications +from db.queries.apply import search_applications +from db.queries.apply import submit_application +from db.queries.apply import update_form +from db.queries.apply import upsert_feedback +from db.queries.apply.application import create_qa_base64file +from db.queries.apply.feedback import retrieve_all_feedbacks_and_surveys +from db.queries.apply.feedback import retrieve_end_of_application_survey_data +from db.queries.apply.feedback import upsert_end_of_application_survey_data +from db.queries.apply.reporting.queries import export_application_statuses_to_csv +from db.queries.apply.reporting.queries import map_application_key_fields +from db.queries.apply.research import retrieve_research_survey_data +from db.queries.apply.research import upsert_research_survey_data +from db.queries.apply.statuses import check_is_fund_round_open +from db.queries.apply.statuses import update_statuses +from flask import current_app +from flask import jsonify +from flask import request +from flask import send_file +from flask.views import MethodView +from fsd_utils import Decision +from fsd_utils import evaluate_response +from fsd_utils.config.notify_constants import NotifyConstants +from services.apply import get_account +from services.apply import get_fund +from services.apply import get_round +from services.apply import get_round_eoi_schema +from services.apply.exceptions import NotificationError +from services.apply.exceptions import SubmitError +from services.apply.models.notification import Notification +from sqlalchemy.orm.exc import NoResultFound + + +class ApplicationsView(MethodView): + def get(self, **kwargs): + response_headers = { + "Access-Control-Allow-Origin": "*", + "Access-Control-Allow-Credentials": True, + } + matching_applications = search_applications(**kwargs) + order_by = kwargs.get("order_by", None) + order_rev = kwargs.get("order_rev", None) + sorted_applications = order_applications(matching_applications, order_by, order_rev) + return sorted_applications, 200, response_headers + + def post(self): + args = request.get_json() + account_id = args["account_id"] + round_id = args["round_id"] + fund_id = args["fund_id"] + language = args["language"] + fund = get_fund(fund_id=fund_id) + if language == "cy" and not fund.welsh_available: + language = "en" + empty_forms = get_blank_forms(fund_id=fund_id, round_id=round_id, language=language) + application = create_application( + account_id=account_id, + fund_id=fund_id, + round_id=round_id, + language=language, + ) + add_new_forms(forms=empty_forms, application_id=application.id) + return application.as_dict(), 201 + + def get_by_id(self, application_id, with_questions_file=False): + try: + return_dict = get_application(application_id, as_json=True, include_forms=True) + return_dict = create_qa_base64file(return_dict, with_questions_file) + return return_dict, 200 + except ValueError as e: + current_app.logger.error("Value error getting application ID: {application_id}") + raise e + except NoResultFound as e: + return {"code": 404, "message": str(e)}, 404 + + def get_key_application_data_report(self, application_id): + try: + return send_file( + export_json_to_csv(get_report_for_applications(application_ids=[application_id])), + "text/csv", + as_attachment=True, + download_name="required_data.csv", + ) + except NoResultFound as e: + return {"code": 404, "message": str(e)}, 404 + + def get_applications_statuses_report( + self, + round_id: Optional[list] = [], + fund_id: Optional[list] = [], + format: Optional[str] = "csv", + ): + print("hello") + current_app.logger.warning("hello") + return jsonify( + { + "metrics": [ + {"fund_id": "asdf", "rounds": [{"round_id": "123", "application_statuses": {"NOT_STARTED": 12}}]} + ] + } + ) + print("hello") + current_app.logger.warning("hello") + if not round_id and not fund_id: + print("No params") + return jsonify({"code": 404, "message": "No funds or rounds specified"}) + try: + report_data = get_general_status_applications_report( + round_id or None, + fund_id or None, + ) + except NoResultFound as e: + print("No result") + return {"code": 404, "message": str(e)}, 404 + + if format.lower() == "json": + response = jsonify({"metrics": report_data}) + response.headers["Content-Type"] = "application/json" + return response + else: + return send_file( + export_application_statuses_to_csv(report_data), + "text/csv", + as_attachment=True, + download_name="required_data.csv", + ) + + def get_key_applications_data_report( + self, + status=Status.SUBMITTED.name, + round_id: Optional[str] = None, + fund_id: Optional[str] = None, + ): + try: + return send_file( + export_json_to_csv( + get_report_for_applications(status=status, round_id=round_id, fund_id=fund_id), + get_key_report_field_headers(round_id), + ), + "text/csv", + as_attachment=True, + download_name="required_data.csv", + ) + except NoResultFound as e: + return {"code": 404, "message": str(e)}, 404 + + def put(self): + request_json = request.get_json(force=True) + form_dict = { + "application_id": request_json["metadata"]["application_id"], + "form_name": request_json["metadata"].get("form_name"), + "question_json": request_json["questions"], + "is_summary_page_submit": request_json["metadata"].get("isSummaryPageSubmit", False), + } + try: + updated_form = update_form(**form_dict) + is_round_open = check_is_fund_round_open(form_dict["application_id"]) + if not is_round_open: + current_app.logger.info("Round is closed so user will be redirected") + return {}, 301 + return updated_form, 201 + except NoResultFound as e: + return {"code": 404, "message": str(e)}, 404 + + def submit(self, application_id): + should_send_email = True + if request.args.get("dont_send_email") == "true": + should_send_email = False + + try: + fund_id = get_fund_id(application_id) + fund_data = get_fund(fund_id) + application = submit_application(application_id) + account = get_account(account_id=application.account_id) + round_data = get_round(fund_id, application.round_id) + application_with_form_json = get_application(application_id, as_json=True, include_forms=True) + language = application_with_form_json["language"] + fund_name = fund_data.name_json[language] + round_name = round_data.title_json[language] + application_with_form_json_and_fund_name = { + **application_with_form_json, + "fund_name": fund_name, + "round_name": round_name, + } + + self._send_submit_queue(application_id, application_with_form_json) + + if round_data.is_expression_of_interest: + full_name = ( + account.full_name + if account.full_name + else map_application_key_fields( + application_with_form_json, + ROUND_ID_TO_KEY_REPORT_MAPPING[application.round_id], + application.round_id, + ).get("lead_contact_name", "") + ) + eoi_results = self.get_application_eoi_response(application_with_form_json) + eoi_decision = eoi_results["decision"] + contents = { + NotifyConstants.APPLICATION_FIELD: application_with_form_json_and_fund_name, + NotifyConstants.MAGIC_LINK_CONTACT_HELP_EMAIL_FIELD: round_data.contact_email, + NotifyConstants.APPLICATION_CAVEATS: eoi_results["caveats"], + } + if Decision(eoi_decision) == Decision.PASS: # EOI Full pass + notify_template = Config.NOTIFY_TEMPLATE_EOI_PASS + + elif Decision(eoi_decision) == Decision.PASS_WITH_CAVEATS: # EOI Pass with caveats + notify_template = Config.NOTIFY_TEMPLATE_EOI_PASS_W_CAVEATS + else: + notify_template = None + should_send_email = False + else: + notify_template = Config.NOTIFY_TEMPLATE_SUBMIT_APPLICATION + eoi_decision = None + full_name = account.full_name + contents = { + NotifyConstants.APPLICATION_FIELD: application_with_form_json_and_fund_name, + NotifyConstants.MAGIC_LINK_CONTACT_HELP_EMAIL_FIELD: round_data.contact_email, + } + + if should_send_email: + contents["application"] = create_qa_base64file(contents.get("application"), True) + del contents["application"]["forms"] + message_id = Notification.send( + notify_template, + account.email, + full_name.title() if full_name else None, + contents, + ) + current_app.logger.info(f"Message added to the queue msg_id: [{message_id}]") + return { + "id": application_id, + "reference": application_with_form_json["reference"], + "email": account.email, + "eoi_decision": eoi_decision, + }, 201 + except KeyError as e: + current_app.logger.exception( + f"Key error on processing application submissionfor application: '{application_id}'" + ) + return str(e), 500, {"x-error": "key error"} + except NotificationError as e: + current_app.logger.exception( + f"Notification error on sending SUBMIT notification for application {application_id}" + ) + return str(e), 500, {"x-error": "notification error"} + except SubmitError as e: + current_app.logger.exception(f"Submit error on sending SUBMIT application {application_id}") + return str(e), 500, {"x-error": "Submit error"} + except Exception as e: + current_app.logger.exception(f"Error on sending SUBMIT notification for application {application_id}") + return str(e), 500, {"x-error": "Error"} + + def _send_submit_queue(self, application_id, application_with_form_json): + """Send message to sqs queue once application is submitted.""" + application_attributes = { + "application_id": {"StringValue": application_id, "DataType": "String"}, + "S3Key": { + "StringValue": "submit", + "DataType": "String", + }, + } + try: + sqs_extended_client = self._get_sqs_client() + message_id = sqs_extended_client.submit_single_message( + queue_url=Config.AWS_SQS_IMPORT_APP_PRIMARY_QUEUE_URL, + message=json.dumps(application_with_form_json), + message_group_id="import_applications_group", + message_deduplication_id=str(uuid4()), # ensures message uniqueness + extra_attributes=application_attributes, + ) + current_app.logger.info(f"Message sent to SQS queue and message id is [{message_id}]") + except Exception as e: + current_app.logger.error("An error occurred while sending message") + current_app.logger.error(e) + raise SubmitError(message="Sorry, cannot submit the message") + + def post_feedback(self): + args = request.get_json() + application_id = args["application_id"] + fund_id = args["fund_id"] + round_id = args["round_id"] + section_id = args["section_id"] + feedback_json = args["feedback_json"] + status = args["status"] + + feedback = upsert_feedback( + application_id=application_id, + fund_id=fund_id, + round_id=round_id, + section_id=section_id, + feedback_json=feedback_json, + status=status, + ) + + update_statuses(application_id, form_name=None) + + return feedback.as_dict(), 201 + + def get_feedback_for_section(self, application_id, section_id): + feedback = get_feedback(application_id, section_id) + if feedback: + return feedback.as_dict(), 200 + + return { + "code": 404, + "message": f"Feedback not fund for {application_id}, {section_id}", + }, 404 + + def post_end_of_application_survey_data(self): + args = request.get_json() + application_id = args["application_id"] + fund_id = args["fund_id"] + round_id = args["round_id"] + page_number = args["page_number"] + data = args["data"] + + survey_data = upsert_end_of_application_survey_data( + application_id=application_id, + fund_id=fund_id, + round_id=round_id, + page_number=page_number, + data=data, + ) + + update_statuses(application_id, form_name=None) + + return survey_data.as_dict(), 201 + + def get_end_of_application_survey_data(self, application_id, page_number): + survey_data = retrieve_end_of_application_survey_data(application_id, int(page_number)) + if survey_data: + return survey_data.as_dict(), 200 + + return { + "code": 404, + "message": f"End of application feedback survey data for {application_id}, {page_number} not found", + }, 404 + + def get_all_feedbacks_and_survey_report(self, **params): + fund_id = params.get("fund_id") + round_id = params.get("round_id") + status = params.get("status_only") + + try: + return send_file( + path_or_file=export_json_to_excel(retrieve_all_feedbacks_and_surveys(fund_id, round_id, status)), + mimetype="application/vnd.ms-excel", + as_attachment=True, + download_name=f"fsd_feedback_{str(int(time.time()))}.xlsx", + ) + except NoResultFound as e: + return {"code": 404, "message": str(e)}, 404 + + def get_application_eoi_response(self, application): + eoi_schema = get_round_eoi_schema(application["fund_id"], application["round_id"], application["language"]) + result = evaluate_response(eoi_schema, application["forms"]) + return result + + def _get_sqs_client(self): + sqs_extended_client = current_app.extensions["sqs_extended_client"] + if sqs_extended_client is not None: + return sqs_extended_client + current_app.logger.error("An error occurred while sending message since client is not available") + + def post_research_survey_data(self): + """Endpoint to post research survey data. + + This method retrieves application_id, fund_id, round_id, and (form) data and will either + create or update the research survey associated with that application. Finally the + application status is checked. + + Returns: + Research survey data in dict form and HTTP status code 201 (Created). + + """ + args = request.get_json() + application_id = args["application_id"] + fund_id = args["fund_id"] + round_id = args["round_id"] + data = args["data"] + + survey_data = upsert_research_survey_data( + application_id=application_id, + fund_id=fund_id, + round_id=round_id, + data=data, + ) + + update_statuses(application_id, form_name=None) + + return survey_data.as_dict(), 201 + + def get_research_survey_data(self, application_id): + """Endpoint to retrieve research survey data for a given application_id. + + Args: + application_id (str): The ID of the application for which survey data is requested. + + Returns: + If found, survey data in dict form is returned with 200 HTTP code + Else an error message with HTTP status code 404. + + """ + survey_data = retrieve_research_survey_data(application_id) + if survey_data: + return survey_data.as_dict(), 200 + + return { + "code": 404, + "message": f"Research survey data for {application_id} not found", + }, 404 diff --git a/apply/api/routes/queues/routes.py b/apply/api/routes/queues/routes.py new file mode 100644 index 00000000..89d85c51 --- /dev/null +++ b/apply/api/routes/queues/routes.py @@ -0,0 +1,54 @@ +# from config import Config +import json +from uuid import uuid4 + +from config import Config +from db.queries.apply import get_application +from flask import current_app +from flask.views import MethodView + + +class QueueView(MethodView): + def post_submitted_application_to_assessment(self, application_id=None): + application_with_form_json = get_application(application_id, as_json=True, include_forms=True) + # check to see if application has status submitted + if application_with_form_json["status"] == "SUBMITTED": + application_attributes = { + "application_id": {"StringValue": application_id, "DataType": "String"}, + "S3Key": { + "StringValue": "assessment", + "DataType": "String", + }, + } + """Submit message to queue, in a future state this can trigger the + assessment service to import the application (currently assessment is + using a CRON timer to pick up messages, not a webhook for triggers)""" + try: + sqs_extended_client = self._get_sqs_client() + message_id = sqs_extended_client.submit_single_message( + queue_url=Config.AWS_SQS_IMPORT_APP_PRIMARY_QUEUE_URL, + message=json.dumps(application_with_form_json), + message_group_id="import_applications_group", + message_deduplication_id=str(uuid4()), # ensures message uniqueness + extra_attributes=application_attributes, + ) + current_app.logger.info(f"Message sent to SQS queue and message id is [{message_id}]") + return f"Message queued, message_id is: {message_id}.", 201 + except Exception as e: + current_app.logger.error("An error occurred while sending message") + current_app.logger.error(e) + return { + "code": 500, + "message": "Message failed", + }, 500 + else: + return { + "code": 400, + "message": "Application must be submitted before it can be assessed", + }, 400 + + def _get_sqs_client(self): + sqs_extended_client = current_app.extensions["sqs_extended_client"] + if sqs_extended_client is not None: + return sqs_extended_client + current_app.logger.error("An error occurred while sending message since client is not available") diff --git a/config/envs/default.py b/config/envs/default.py index a49348ba..f0d13294 100644 --- a/config/envs/default.py +++ b/config/envs/default.py @@ -5,6 +5,7 @@ from config.mappings.assessment_mapping_fund_round import ( fund_round_to_assessment_mapping, ) +from distutils.util import strtobool from fsd_utils import CommonConfig from fsd_utils import configclass @@ -92,3 +93,87 @@ class DefaultConfig: # --------------- AWS_SQS_NOTIF_APP_PRIMARY_QUEUE_URL = environ.get("AWS_SQS_NOTIF_APP_PRIMARY_QUEUE_URL") AWS_SQS_NOTIF_APP_SECONDARY_QUEUE_URL = environ.get("AWS_SQS_NOTIF_APP_SECONDARY_QUEUE_URL") + + # ------------------------------ + # ------------------------------ + # From application-store + # ------------------------------ + # ------------------------------ + + # Application Config + # FLASK_ENV = CommonConfig.FLASK_ENV + # SECRET_KEY = CommonConfig.SECRET_KEY + # SESSION_COOKIE_NAME = environ.get("SESSION_COOKIE_NAME", "session_cookie") + # FLASK_ROOT = str(Path(__file__).parent.parent.parent) + + # FSD_LOGGING_LEVEL = logging.WARN + + # APIs + # TEST_FUND_STORE_API_HOST = "fund_store" + # TEST_ACCOUNT_STORE_API_HOST = "account_store" + USE_LOCAL_DATA = strtobool(environ.get("USE_LOCAL_DATA", "False")) + + # FUND_STORE_API_HOST = environ.get("FUND_STORE_API_HOST", TEST_FUND_STORE_API_HOST) + # ACCOUNT_STORE_API_HOST = environ.get("ACCOUNT_STORE_API_HOST", TEST_ACCOUNT_STORE_API_HOST) + + # Notification Service + NOTIFY_TEMPLATE_SUBMIT_APPLICATION = "APPLICATION_RECORD_OF_SUBMISSION" + NOTIFY_TEMPLATE_INCOMPLETE_APPLICATION = "INCOMPLETE_APPLICATION_RECORDS" + NOTIFY_TEMPLATE_APPLICATION_DEADLINE_REMINDER = "APPLICATION_DEADLINE_REMINDER" + NOTIFY_TEMPLATE_APPLICATION_DEADLINE_REMINDER = environ.get( + "NOTIFY_TEMPLATE_APPLICATION_DEADLINE_REMINDER", + NOTIFY_TEMPLATE_APPLICATION_DEADLINE_REMINDER, + ) + NOTIFY_TEMPLATE_EOI_PASS = "Full pass" + NOTIFY_TEMPLATE_EOI_PASS_W_CAVEATS = "Pass with caveats" + + # if "PRIMARY_QUEUE_URL" in os.environ: + # AWS_REGION = AWS_SQS_REGION = os.environ.get("AWS_REGION") + # AWS_BUCKET_NAME = os.environ.get("AWS_BUCKET_NAME") + # AWS_SQS_IMPORT_APP_PRIMARY_QUEUE_URL = os.environ.get("PRIMARY_QUEUE_URL") + # AWS_SQS_IMPORT_APP_SECONDARY_QUEUE_URL = os.environ.get("DEAD_LETTER_QUEUE_URL") + # else: + # AWS_ACCESS_KEY_ID = AWS_SQS_ACCESS_KEY_ID = os.environ.get("AWS_ACCESS_KEY_ID") + # AWS_SECRET_ACCESS_KEY = AWS_SQS_SECRET_ACCESS_KEY = os.environ.get("AWS_SECRET_ACCESS_KEY") + # AWS_BUCKET_NAME = os.environ.get("AWS_BUCKET_NAME") + # AWS_REGION = AWS_SQS_REGION = os.environ.get("AWS_REGION") + # AWS_SQS_IMPORT_APP_PRIMARY_QUEUE_URL = os.environ.get("AWS_SQS_IMPORT_APP_PRIMARY_QUEUE_URL") + # AWS_SQS_IMPORT_APP_SECONDARY_QUEUE_URL = os.environ.get("AWS_SQS_IMPORT_APP_SECONDARY_QUEUE_URL") + + # Account Store Endpoints + # ACCOUNTS_ENDPOINT = "/accounts" + + # Fund Store Endpoints + # FUNDS_ENDPOINT = CommonConfig.FUNDS_ENDPOINT + # FUND_ENDPOINT = CommonConfig.FUND_ENDPOINT + FUND_ROUNDS_ENDPOINT = CommonConfig.ROUNDS_ENDPOINT + FUND_ROUND_ENDPOINT = CommonConfig.ROUND_ENDPOINT + FUND_ROUND_APPLICATION_SECTIONS_ENDPOINT = ( + "/funds/{fund_id}/rounds/{round_id}/sections/application?language={language}" + ) + FUND_ROUND_APPLICATION_REMINDER_STATUS = "/funds/{round_id}/application_reminder_status?status=true" + FUND_ROUND_EOI_SCHEMA_ENDPOINT = FUND_STORE_API_HOST + "/funds/{fund_id}/rounds/{round_id}/eoi_decision_schema" + + # SQLALCHEMY_DATABASE_URI = environ.get("DATABASE_URL") + # SQLALCHEMY_TRACK_MODIFICATIONS = False + # SQLALCHEMY_ENGINE_OPTIONS = {"future": True} + DOCUMENT_UPLOAD_SIZE_LIMIT = 2 * 1024 * 1024 + + # --------------- + # AWS Overall Config + # --------------- + # AWS_ACCESS_KEY_ID = AWS_SQS_ACCESS_KEY_ID = environ.get("AWS_ACCESS_KEY_ID") + # AWS_SECRET_ACCESS_KEY = AWS_SQS_SECRET_ACCESS_KEY = environ.get("AWS_SECRET_ACCESS_KEY") + # AWS_REGION = AWS_SQS_REGION = environ.get("AWS_REGION") + # AWS_ENDPOINT_OVERRIDE = environ.get("AWS_ENDPOINT_OVERRIDE") + + # --------------- + # S3 Config + # --------------- + # AWS_MSG_BUCKET_NAME = environ.get("AWS_MSG_BUCKET_NAME") + + # --------------- + # SQS Config + # --------------- + # AWS_SQS_NOTIF_APP_PRIMARY_QUEUE_URL = environ.get("AWS_SQS_NOTIF_APP_PRIMARY_QUEUE_URL") + # AWS_SQS_NOTIF_APP_SECONDARY_QUEUE_URL = environ.get("AWS_SQS_NOTIF_APP_SECONDARY_QUEUE_URL") diff --git a/config/key_report_mappings/cof25_eoi_key_report_mapping.py b/config/key_report_mappings/cof25_eoi_key_report_mapping.py new file mode 100644 index 00000000..65b45579 --- /dev/null +++ b/config/key_report_mappings/cof25_eoi_key_report_mapping.py @@ -0,0 +1,46 @@ +from config.key_report_mappings.model import extract_postcode +from config.key_report_mappings.model import FormMappingItem +from config.key_report_mappings.model import KeyReportMapping + +COF25_EOI_KEY_REPORT_MAPPING = KeyReportMapping( + round_id="9104d809-0fb0-4144-b514-55e81cc2b6fa", + mapping=[ + FormMappingItem( + form_name="organisation-details-25", + form_name_cy="manylion-y-sefydliad-25", + key="SMRWjl", + return_field="organisation_name", + ), + FormMappingItem( + form_name="development-support-provider-25", + form_name_cy="darparwr-cymorth-datblygu-25", + key="xWnVof", + return_field="lead_contact_name", + ), + FormMappingItem( + form_name="about-your-asset-25", + form_name_cy="ynglyn-ach-ased-25", + key="Ihjjyi", + return_field="asset_type", + ), + FormMappingItem( + form_name="about-your-asset-25", + form_name_cy="ynglyn-ach-ased-25", + key="dnqIdW", + return_field="geography", + formatter=extract_postcode, + ), + FormMappingItem( + form_name="your-funding-request-25", + form_name_cy="eich-cais-am-gyllid-25", + key="fZAMFv", + return_field="capital", + ), + FormMappingItem( + form_name="development-support-provider-25", + form_name_cy="darparwr-cymorth-datblygu-25", + key="NQoGIm", + return_field="applicant_email", + ), + ], +) diff --git a/config/key_report_mappings/cof_eoi_key_report_mapping.py b/config/key_report_mappings/cof_eoi_key_report_mapping.py new file mode 100644 index 00000000..5f36ba64 --- /dev/null +++ b/config/key_report_mappings/cof_eoi_key_report_mapping.py @@ -0,0 +1,46 @@ +from config.key_report_mappings.model import extract_postcode +from config.key_report_mappings.model import FormMappingItem +from config.key_report_mappings.model import KeyReportMapping + +COF_EOI_KEY_REPORT_MAPPING = KeyReportMapping( + round_id="6a47c649-7bac-4583-baed-9c4e7a35c8b3", + mapping=[ + FormMappingItem( + form_name="organisation-details", + form_name_cy="manylion-y-sefydliad", + key="SMRWjl", + return_field="organisation_name", + ), + FormMappingItem( + form_name="development-support-provider", + form_name_cy="darparwr-cymorth-datblygu", + key="xWnVof", + return_field="lead_contact_name", + ), + FormMappingItem( + form_name="about-your-asset", + form_name_cy="ynglyn-ach-ased", + key="Ihjjyi", + return_field="asset_type", + ), + FormMappingItem( + form_name="about-your-asset", + form_name_cy="ynglyn-ach-ased", + key="dnqIdW", + return_field="geography", + formatter=extract_postcode, + ), + FormMappingItem( + form_name="your-funding-request", + form_name_cy="eich-cais-am-gyllid", + key="fZAMFv", + return_field="capital", + ), + FormMappingItem( + form_name="development-support-provider", + form_name_cy="darparwr-cymorth-datblygu", + key="NQoGIm", + return_field="applicant_email", + ), + ], +) diff --git a/config/key_report_mappings/cof_key_report_mapping.py b/config/key_report_mappings/cof_key_report_mapping.py new file mode 100644 index 00000000..d3b99125 --- /dev/null +++ b/config/key_report_mappings/cof_key_report_mapping.py @@ -0,0 +1,80 @@ +from config.key_report_mappings.model import ApplicationColumnMappingItem +from config.key_report_mappings.model import extract_postcode +from config.key_report_mappings.model import FormMappingItem +from config.key_report_mappings.model import KeyReportMapping + +COF_KEY_REPORT_MAPPING = KeyReportMapping( + round_id=["4efc3263-aefe-4071-b5f4-0910abec12d2", "33726b63-efce-4749-b149-20351346c76e"], + mapping=[ + FormMappingItem( + form_name="applicant-information-cof", + form_name_cy="gwybodaeth-am-yr-ymgeisydd-cof", + key="NlHSBg", + return_field="applicant_email", + ), + FormMappingItem( + form_name="organisation-information-cof", + form_name_cy="gwybodaeth-am-y-sefydliad-cof", + key="WWWWxy", + return_field="eoi_reference", + ), + FormMappingItem( + form_name="organisation-information-cof", + form_name_cy="gwybodaeth-am-y-sefydliad-cof", + key="YdtlQZ", + return_field="organisation_name", + ), + FormMappingItem( + form_name="organisation-information-cof", + form_name_cy="gwybodaeth-am-y-sefydliad-cof", + key="lajFtB", + return_field="organisation_type", + ), + FormMappingItem( + form_name="asset-information-cof", + form_name_cy="gwybodaeth-am-yr-ased-cof", + key="oXGwlA", + return_field="asset_type", + ), + FormMappingItem( + form_name="asset-information-cof", + form_name_cy="gwybodaeth-am-yr-ased-cof", + key="aJGyCR", + return_field="asset_type_other", + ), + FormMappingItem( + form_name="project-information-cof", + form_name_cy="gwybodaeth-am-y-prosiect-cof", + key="EfdliG", + return_field="geography", + formatter=extract_postcode, + ), + FormMappingItem( + form_name="funding-required-cof", + form_name_cy="cyllid-sydd-ei-angen-cof", + key="ABROnB", + return_field="capital", + ), + FormMappingItem( + form_name="funding-required-cof", + form_name_cy="cyllid-sydd-ei-angen-cof", + key="tSKhQQ", + return_field="revenue", + formatter=lambda answer: sum([x["UyaAHw"] for x in answer or []]), + ), + ApplicationColumnMappingItem( + column_name="reference", + return_field="ref", + ), + ApplicationColumnMappingItem( + column_name="id", + return_field="link", + ), # noqa + FormMappingItem( + form_name="project-information-cof", + form_name_cy="gwybodaeth-am-y-prosiect-cof", + key="apGjFS", + return_field="project_name", + ), + ], +) diff --git a/config/key_report_mappings/cof_r2_key_report_mapping.py b/config/key_report_mappings/cof_r2_key_report_mapping.py new file mode 100644 index 00000000..668c60e6 --- /dev/null +++ b/config/key_report_mappings/cof_r2_key_report_mapping.py @@ -0,0 +1,57 @@ +from config.key_report_mappings.model import extract_postcode +from config.key_report_mappings.model import FormMappingItem +from config.key_report_mappings.model import KeyReportMapping + +COF_R2_KEY_REPORT_MAPPING = KeyReportMapping( + round_id="c603d114-5364-4474-a0c4-c41cbf4d3bbd", + mapping=[ + FormMappingItem( + form_name="organisation-information", + form_name_cy="gwybodaeth-am-y-sefydliad", + key="WWWWxy", + return_field="eoi_reference", + ), + FormMappingItem( + form_name="organisation-information", + form_name_cy="gwybodaeth-am-y-sefydliad", + key="YdtlQZ", + return_field="organisation_name", + ), + FormMappingItem( + form_name="organisation-information", + form_name_cy="gwybodaeth-am-y-sefydliad", + key="lajFtB", + return_field="organisation_type", + ), + FormMappingItem( + form_name="asset-information", + form_name_cy="gwybodaeth-am-yr-ased", + key="yaQoxU", + return_field="asset_type", + ), + FormMappingItem( + form_name="project-information", + form_name_cy="gwybodaeth-am-y-prosiect", + key="yEmHpp", + return_field="geography", + formatter=extract_postcode, + ), + FormMappingItem( + form_name="funding-required", + form_name_cy="cyllid-sydd-ei-angen", + key="JzWvhj", + return_field="capital", + ), + FormMappingItem( + form_name="funding-required", + form_name_cy="cyllid-sydd-ei-angen", + key="jLIgoi", + return_field="revenue", + ), + FormMappingItem( + form_name="organisation-information-ns", + key="opFJRm", + return_field="organisation_name_nstf", + ), + ], +) diff --git a/config/key_report_mappings/cof_r3w2_key_report_mapping.py b/config/key_report_mappings/cof_r3w2_key_report_mapping.py new file mode 100644 index 00000000..130d3c8a --- /dev/null +++ b/config/key_report_mappings/cof_r3w2_key_report_mapping.py @@ -0,0 +1,84 @@ +from config.key_report_mappings.model import ApplicationColumnMappingItem +from config.key_report_mappings.model import extract_postcode +from config.key_report_mappings.model import FormMappingItem +from config.key_report_mappings.model import KeyReportMapping + +COF_R3W2_KEY_REPORT_MAPPING = KeyReportMapping( + round_id="6af19a5e-9cae-4f00-9194-cf10d2d7c8a7", + mapping=[ + FormMappingItem( + form_name="applicant-information-cof-r3-w2", + form_name_cy="gwybodaeth-am-yr-ymgeisydd-cof-r3-w2", + key="NlHSBg", + return_field="applicant_email", + ), + FormMappingItem( + form_name="organisation-information-cof-r3-w2", + form_name_cy="gwybodaeth-am-y-sefydliad-cof-r3-w2", + key="WWWWxy", + return_field="eoi_reference", + ), + FormMappingItem( + form_name="organisation-information-cof-r3-w2", + form_name_cy="gwybodaeth-am-y-sefydliad-cof-r3-w2", + key="YdtlQZ", + return_field="organisation_name", + ), + FormMappingItem( + form_name="organisation-information-cof-r3-w2", + form_name_cy="gwybodaeth-am-y-sefydliad-cof-r3-w2", + key="lajFtB", + return_field="organisation_type", + ), + FormMappingItem( + form_name="asset-information-cof-r3-w2", + form_name_cy="gwybodaeth-am-yr-ased-cof-r3-w2", + key="oXGwlA", + return_field="asset_type", + ), + FormMappingItem( + form_name="asset-information-cof-r3-w2", + form_name_cy="gwybodaeth-am-yr-ased-cof-r3-w2", + key="aJGyCR", + return_field="asset_type_other", + ), + FormMappingItem( + form_name="project-information-cof-r3-w2", + form_name_cy="gwybodaeth-am-y-prosiect-cof-r3-w2", + key="EfdliG", + return_field="geography", + formatter=extract_postcode, + ), + FormMappingItem( + form_name="funding-required-cof-r3-w2", + form_name_cy="cyllid-sydd-ei-angen-cof-r3-w2", + key="ABROnB", + return_field="capital", + ), + FormMappingItem( + form_name="funding-required-cof-r3-w2", + form_name_cy="cyllid-sydd-ei-angen-cof-r3-w2", + key="tSKhQQ", + return_field="revenue", + formatter=lambda answer: sum([x["UyaAHw"] for x in answer or []]), + ), + ApplicationColumnMappingItem( + column_name="reference", + return_field="ref", + ), + ApplicationColumnMappingItem( + column_name="id", + return_field="link", + ), + # ApplicationColumnMappingItem( # think we'd need to add a concept for grabbing email by account id # noqa + # column_name="email", # however that data belongs in the account-store # noqa + # return_field="account_id" # noqa + # ), # noqa + FormMappingItem( + form_name="project-information-cof-r3-w2", + form_name_cy="gwybodaeth-am-y-prosiect-cof-r3-w2", + key="apGjFS", + return_field="project_name", + ), + ], +) diff --git a/config/key_report_mappings/cyp_r1_key_report_mapping.py b/config/key_report_mappings/cyp_r1_key_report_mapping.py new file mode 100644 index 00000000..31f48ac9 --- /dev/null +++ b/config/key_report_mappings/cyp_r1_key_report_mapping.py @@ -0,0 +1,18 @@ +from config.key_report_mappings.model import FormMappingItem +from config.key_report_mappings.model import KeyReportMapping + +CYP_R1_KEY_REPORT_MAPPING = KeyReportMapping( + round_id="888aae3d-7e2c-4523-b9c1-95952b3d1644", + mapping=[ + FormMappingItem( + form_name="applicant-information-cyp", + key="BKOHaM", + return_field="applicant_email", + ), + FormMappingItem( + form_name="about-your-organisation-cyp", + key="JbmcJE", + return_field="organisation_name", + ), + ], +) diff --git a/config/key_report_mappings/dpif_r2_key_report_mapping.py b/config/key_report_mappings/dpif_r2_key_report_mapping.py new file mode 100644 index 00000000..e70b2ce1 --- /dev/null +++ b/config/key_report_mappings/dpif_r2_key_report_mapping.py @@ -0,0 +1,18 @@ +from config.key_report_mappings.model import FormMappingItem +from config.key_report_mappings.model import KeyReportMapping + +DPIF_R2_KEY_REPORT_MAPPING = KeyReportMapping( + round_id="0059aad4-5eb5-11ee-8c99-0242ac120002", + mapping=[ + FormMappingItem( + form_name="organisation-information-dpi", + key="IRugBv", + return_field="applicant_email", + ), + FormMappingItem( + form_name="organisation-information-dpi", + key="nYJiWy", + return_field="organisation_name", + ), + ], +) diff --git a/config/key_report_mappings/mappings.py b/config/key_report_mappings/mappings.py new file mode 100644 index 00000000..12fb60b1 --- /dev/null +++ b/config/key_report_mappings/mappings.py @@ -0,0 +1,41 @@ +from collections import defaultdict + +from config.key_report_mappings.cof25_eoi_key_report_mapping import ( + COF25_EOI_KEY_REPORT_MAPPING, +) +from config.key_report_mappings.cof_eoi_key_report_mapping import ( + COF_EOI_KEY_REPORT_MAPPING, +) +from config.key_report_mappings.cof_key_report_mapping import ( + COF_KEY_REPORT_MAPPING, +) +from config.key_report_mappings.cof_r2_key_report_mapping import ( + COF_R2_KEY_REPORT_MAPPING, +) +from config.key_report_mappings.cof_r3w2_key_report_mapping import ( + COF_R3W2_KEY_REPORT_MAPPING, +) +from config.key_report_mappings.cyp_r1_key_report_mapping import ( + CYP_R1_KEY_REPORT_MAPPING, +) +from config.key_report_mappings.dpif_r2_key_report_mapping import ( + DPIF_R2_KEY_REPORT_MAPPING, +) + + +ROUND_ID_TO_KEY_REPORT_MAPPING = defaultdict( + lambda: COF_R2_KEY_REPORT_MAPPING.mapping, + { + CYP_R1_KEY_REPORT_MAPPING.round_id: CYP_R1_KEY_REPORT_MAPPING.mapping, + DPIF_R2_KEY_REPORT_MAPPING.round_id: DPIF_R2_KEY_REPORT_MAPPING.mapping, + COF_EOI_KEY_REPORT_MAPPING.round_id: COF_EOI_KEY_REPORT_MAPPING.mapping, + COF25_EOI_KEY_REPORT_MAPPING.round_id: COF25_EOI_KEY_REPORT_MAPPING.mapping, + COF_R2_KEY_REPORT_MAPPING.round_id: COF_R2_KEY_REPORT_MAPPING.mapping, + COF_R3W2_KEY_REPORT_MAPPING.round_id: COF_R3W2_KEY_REPORT_MAPPING.mapping, + **({key: COF_KEY_REPORT_MAPPING.mapping for key in COF_KEY_REPORT_MAPPING.round_id}), + }, +) + + +def get_report_mapping_for_round(round_id): + return ROUND_ID_TO_KEY_REPORT_MAPPING[round_id] diff --git a/config/key_report_mappings/model.py b/config/key_report_mappings/model.py new file mode 100644 index 00000000..153d4626 --- /dev/null +++ b/config/key_report_mappings/model.py @@ -0,0 +1,53 @@ +import re +from dataclasses import dataclass +from typing import Any +from typing import Callable + + +@dataclass +class MappingItem: + return_field: str + formatter: Callable[[Any], Any] = None + + +@dataclass +class FormMappingItem(MappingItem): + key: str | None = None + form_name: str | None = None + form_name_cy: str | None = None + + def get_form_name(self, language: str = "en"): + if language == "cy": + return self.form_name_cy + return self.form_name + + def format_answer(self, field: dict) -> str: + if (answer := field.get("answer")) and self.formatter: + return self.formatter(answer) + return answer # no formatting required by default + + +@dataclass +class ApplicationColumnMappingItem(MappingItem): + column_name: str | None = None + + def format_answer(self, data: Any) -> str: + if data and self.formatter: + return self.formatter(data) + return data + + +@dataclass +class KeyReportMapping: + round_id: str + mapping: list[MappingItem] + + +# this was extracted from existing functionality, not a fan of regex for this +def extract_postcode(postcode: str) -> str | None: + postcode = re.search( + "([A-Za-z][A-Ha-hJ-Yj-y]?[0-9][A-Za-z0-9]? ?[0-9][A-Za-z]{2}|[Gg][Ii][Rr] ?0[Aa]{2})", # noqa + postcode, + ) + if postcode: + return postcode.group() diff --git a/db/exceptions/__init__.py b/db/exceptions/__init__.py new file mode 100644 index 00000000..b4b66213 --- /dev/null +++ b/db/exceptions/__init__.py @@ -0,0 +1,3 @@ +from .application import ApplicationError + +__all__ = ["ApplicationError"] diff --git a/db/exceptions/application.py b/db/exceptions/application.py new file mode 100644 index 00000000..bbadfcaf --- /dev/null +++ b/db/exceptions/application.py @@ -0,0 +1,11 @@ +class ApplicationError(Exception): + """Exception raised for errors in Application management. + + Attributes: + message -- explanation of the error + + """ + + def __init__(self, message="Sorry, there was a problem, please try later"): + self.message = message + super().__init__(self.message) diff --git a/db/queries/apply/__init__.py b/db/queries/apply/__init__.py new file mode 100644 index 00000000..1edcb66c --- /dev/null +++ b/db/queries/apply/__init__.py @@ -0,0 +1,41 @@ +from .application import create_application +from .application import get_application +from .application import get_applications +from .application import get_count_by_status +from .application import get_fund_id +from .application import search_applications +from .application import submit_application +from .feedback import get_feedback +from .feedback import upsert_feedback +from .form import add_new_forms +from .form import get_form +from .form import get_forms_by_app_id +from .reporting import export_json_to_csv +from .reporting import export_json_to_excel +from .reporting import get_general_status_applications_report +from .reporting import get_key_report_field_headers +from .reporting import get_report_for_applications +from .updating import update_application_and_related_form +from .updating import update_form + +__all__ = [ + create_application, + get_application, + get_applications, + get_count_by_status, + search_applications, + submit_application, + add_new_forms, + get_form, + get_forms_by_app_id, + export_json_to_csv, + export_json_to_excel, + get_general_status_applications_report, + get_key_report_field_headers, + get_report_for_applications, + update_application_and_related_form, + update_form, + get_fund_id, + upsert_feedback, + get_feedback, +] diff --git a/db/queries/apply/application/__init__.py b/db/queries/apply/application/__init__.py new file mode 100644 index 00000000..af302725 --- /dev/null +++ b/db/queries/apply/application/__init__.py @@ -0,0 +1 @@ +from .queries import * # noqa diff --git a/db/queries/apply/application/queries.py b/db/queries/apply/application/queries.py new file mode 100644 index 00000000..eda4f762 --- /dev/null +++ b/db/queries/apply/application/queries.py @@ -0,0 +1,301 @@ +import base64 +import random +import string +from datetime import datetime +from datetime import timezone +from io import BytesIO +from itertools import groupby +from typing import Optional + +from config import Config +from db import db +from db.exceptions import ApplicationError +from db.models import Applications +from db.models.application.enums import Status as ApplicationStatus +from db.schemas import ApplicationSchema +from flask import current_app +from fsd_utils import extract_questions_and_answers +from fsd_utils import generate_text_of_application +from services.apply import get_fund +from services.apply import get_round +from services.apply.aws import FileData +from services.apply.aws import list_files_by_prefix +from sqlalchemy import func +from sqlalchemy import select +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import joinedload +from sqlalchemy.orm import noload +from sqlalchemy.sql.expression import Select + + +def get_application(app_id, include_forms=False, as_json=False) -> dict | Applications: + stmt: Select = select(Applications).filter(Applications.id == app_id) + + if include_forms: + stmt.options(joinedload(Applications.forms)) + serialiser = ApplicationSchema() + else: + stmt.options(noload(Applications.forms)) + serialiser = ApplicationSchema(exclude=["forms"]) + + row: Applications = db.session.scalars(stmt).unique().one() + + if as_json: + json_row = serialiser.dump(row) + return json_row + else: + return row + + +def get_applications(filters=None, include_forms=False, as_json=False) -> list[dict] | list[Applications]: + if filters is None: + filters = [] + stmt: Select = select(Applications) + + if len(filters) > 0: + stmt = stmt.where(*filters) + + if include_forms: + stmt = stmt.options(joinedload(Applications.forms)) + serialiser = ApplicationSchema() + else: + stmt = stmt.options(noload(Applications.forms)) + serialiser = ApplicationSchema(exclude=["forms"]) + + rows: Applications = db.session.scalars(stmt).unique().all() + + if as_json: + return [serialiser.dump(row) for row in rows] + else: + return rows + + +def get_application_status(app_id): + application = get_application(app_id) + return application.status + + +def random_key_generator(length: int = 6): + key = "".join(random.choices(string.ascii_uppercase, k=length)) + while True: + yield key + + +def _create_application_try(account_id, fund_id, round_id, key, language, reference, attempt) -> Applications: + try: + new_application_row = Applications( + account_id=account_id, + fund_id=fund_id, + round_id=round_id, + key=key, + language=language, + reference=reference, + ) + db.session.add(new_application_row) + db.session.commit() + return new_application_row + except IntegrityError: + db.session.remove() + current_app.logger.error( + f"Failed {attempt} attempt(s) to create application with" + f" application reference {reference}, for fund_id" + f" {fund_id} and round_id {round_id}" + ) + + +def create_application(account_id, fund_id, round_id, language) -> Applications: + fund = get_fund(fund_id) + fund_round = get_round(fund_id, round_id) + application_start_language = language + if language == "cy" and not fund.welsh_available: + application_start_language = "en" + if fund and fund_round and fund.short_name and fund_round.short_name: + new_application = None + max_tries = 10 + attempt = 0 + key = None + app_key_gen = random_key_generator() + while attempt < max_tries and new_application is None: + key = next(app_key_gen) + new_application = _create_application_try( + account_id=account_id, + fund_id=fund_id, + round_id=round_id, + key=key, + language=application_start_language, + reference="-".join([fund.short_name, fund_round.short_name, key]), + attempt=attempt, + ) + attempt += 1 + + if not new_application: + raise ApplicationError( + f"Max ({max_tries}) tries exceeded for create application" + f" with application key {key}, for fund.short_name" + f" {fund.short_name} and round.short_name" + f" {fund_round.short_name}" + ) + return new_application + else: + raise ApplicationError(f"Failed to create application. Fund round {round_id} for fund {fund_id} not found") + + +def get_all_applications() -> list: + application_list = db.session.query(Applications).all() + return application_list + + +def get_count_by_status(round_ids: Optional[list] = [], fund_ids: Optional[list] = []) -> dict[str, int]: + query = db.session.query( + Applications.fund_id, + Applications.round_id, + Applications.status, + func.count(Applications.status), + ) + + if round_ids: + query = query.filter(Applications.round_id.in_(round_ids)) + if fund_ids: + query = query.filter(Applications.fund_id.in_(fund_ids)) + + grouped_by_fund_round_result = ( + query.group_by(Applications.fund_id).group_by(Applications.round_id).group_by(Applications.status).all() + ) + results = [] + unique_funds = {f[0] for f in grouped_by_fund_round_result}.union(fund_ids or []) + for fund_id in unique_funds: + unique_rounds = {row[1] for row in grouped_by_fund_round_result if row[0] == fund_id}.union(round_ids or []) + rounds = [] + for round_id in unique_rounds: + this_round_statuses = { + s[2].name: s[3] for s in grouped_by_fund_round_result if s[0] == fund_id and s[1] == round_id + } + rounds.append( + { + "round_id": round_id, + "application_statuses": { + **{s.name: 0 for s in ApplicationStatus}, + **this_round_statuses, + }, + } + ) + results.append({"fund_id": fund_id, "rounds": rounds}) + return results + + +def create_qa_base64file(application_data: dict, with_questions_file: bool): + """If the query param with_questions_file is True then it will get the + application questions ans answers, and then it will generate a formatted text + document for an application with questions and answers. + + and this file will be base64 encoded. + + """ + if with_questions_file: + fund_details = get_fund(application_data["fund_id"]) + q_and_a = extract_questions_and_answers(application_data["forms"], application_data["language"]) + contents = BytesIO( + bytes(generate_text_of_application(q_and_a, fund_details.name, application_data["language"]), "utf-8") + ).read() + if len(contents) > Config.DOCUMENT_UPLOAD_SIZE_LIMIT: + raise ValueError("File is larger than 2MB") + application_data = { + **application_data, + "questions_file": base64.b64encode(contents).decode("ascii"), + } + current_app.logger.info("Sending the Q and A base64 encoded file with the response") + return application_data + + +def search_applications(**params): + """Returns a list of applications matching required params.""" + # datetime_start = params.get("datetime_start") + # datetime_end = params.get("datetime_end") + fund_id = params.get("fund_id") + round_id = params.get("round_id") + account_id = params.get("account_id") + status_only = params.get("status_only") + application_id = params.get("application_id") + forms = params.get("forms") + + filters = [] + if fund_id: + filters.append(Applications.fund_id == fund_id) + if round_id: + filters.append(Applications.round_id == round_id) + if account_id: + filters.append(Applications.account_id == account_id) + if status_only: + if " " in status_only: + status_only = status_only.replace(" ", "_") + if isinstance(status_only, list): + filters.append(Applications.status.in_(status_only)) + else: + filters.append(Applications.status == status_only) + if application_id: + filters.append(Applications.id == application_id) + found_apps = get_applications(filters, include_forms=forms, as_json=True) + return found_apps + + +def submit_application(application_id) -> Applications: + current_app.logger.info(f"Processing database submission for application_id: '{application_id}.") + application = get_application(application_id) + application.date_submitted = datetime.now(timezone.utc).isoformat() + + all_application_files = list_files_by_prefix(application_id) + application = process_files(application, all_application_files) + + application.status = "SUBMITTED" + db.session.commit() + return application + + +def process_files(application: Applications, all_files: list[FileData]) -> Applications: + comp_id_to_files = {comp_id: list(files) for comp_id, files in groupby(all_files, key=lambda x: x.component_id)} + for form in application.forms: + for component in form.json: + for field in component["fields"]: + comp_id = field["key"] + if files := comp_id_to_files.get(comp_id): + field["answer"] = ", ".join(state.filename for state in files) + return application + + +def update_project_name(form_name, question_json, application) -> None: + if form_name.startswith("project-information") or form_name.startswith("gwybodaeth-am-y-prosiect"): + for question in question_json: + for field in question["fields"]: + # field id for project name in json + if field["title"] == "Project name": + try: + application.project_name = field["answer"] + except KeyError: + current_app.logger.info("Project name was not edited") + continue + + +def get_fund_id(application_id): + """Function takes an application_id and returns the fund_id of that + application.""" + try: + application = db.session.query(Applications).filter_by(id=application_id).first() + if application: + return application.fund_id + else: + return None + except Exception: + current_app.logger.error(f"Incorrect application id: {application_id}") + return None + + +def attempt_to_find_and_update_project_name(question_json, application) -> None: + """Updates the applications project name if the updated question_json contains + a field_id match on the pre-configured project_name field_id.""" + round = get_round(application.fund_id, application.round_id) + project_name_field_id = round.project_name_field_id + + for question in question_json: + for field in question["fields"]: + if field["key"] == project_name_field_id and "answer" in field.keys(): + return field["answer"] diff --git a/db/queries/apply/feedback/__init__.py b/db/queries/apply/feedback/__init__.py new file mode 100644 index 00000000..af302725 --- /dev/null +++ b/db/queries/apply/feedback/__init__.py @@ -0,0 +1 @@ +from .queries import * # noqa diff --git a/db/queries/apply/feedback/queries.py b/db/queries/apply/feedback/queries.py new file mode 100644 index 00000000..a82f05f3 --- /dev/null +++ b/db/queries/apply/feedback/queries.py @@ -0,0 +1,162 @@ +from datetime import datetime + +from config.key_report_mappings.mappings import get_report_mapping_for_round +from db import db +from db.models import Applications +from db.models import Feedback +from db.models.feedback import EndOfApplicationSurveyFeedback +from db.queries.apply.application.queries import get_applications +from db.queries.apply.reporting.queries import map_application_key_fields +from db.schemas.application import ApplicationSchema +from db.schemas.end_of_application_survey import EndOfApplicationSurveyFeedbackSchema +from flask import current_app +from services.apply.data import get_application_sections + + +def upsert_feedback(application_id, fund_id, round_id, section_id, feedback_json, status): + existing_feedback = Feedback.query.filter_by( + application_id=application_id, + fund_id=fund_id, + round_id=round_id, + section_id=section_id, + ).first() + + if existing_feedback: + existing_feedback.feedback_json = feedback_json + existing_feedback.status = status + existing_feedback.date_submitted = datetime.now() + db.session.commit() + return existing_feedback + else: + new_feedback_row = Feedback( + application_id=application_id, + fund_id=fund_id, + round_id=round_id, + section_id=section_id, + feedback_json=feedback_json, + status=status, + date_submitted=datetime.now(), + ) + db.session.add(new_feedback_row) + db.session.commit() + return new_feedback_row + + +def get_feedback(application_id, section_id): + return ( + db.session.query(Feedback) + .filter(Feedback.application_id == application_id, Feedback.section_id == section_id) + .one_or_none() + ) + + +def upsert_end_of_application_survey_data(application_id, fund_id, round_id, page_number, data): + existing_survey_data = EndOfApplicationSurveyFeedback.query.filter_by( + application_id=application_id, + fund_id=fund_id, + round_id=round_id, + page_number=page_number, + ).first() + + if existing_survey_data: + existing_survey_data.data = data + existing_survey_data.date_submitted = datetime.now() + db.session.commit() + return existing_survey_data + else: + new_survey_data = EndOfApplicationSurveyFeedback( + application_id=application_id, + fund_id=fund_id, + round_id=round_id, + page_number=page_number, + data=data, + date_submitted=datetime.now(), + ) + db.session.add(new_survey_data) + db.session.commit() + return new_survey_data + + +def retrieve_end_of_application_survey_data(application_id, page_number): + return ( + db.session.query(EndOfApplicationSurveyFeedback) + .filter( + EndOfApplicationSurveyFeedback.application_id == application_id, + EndOfApplicationSurveyFeedback.page_number == page_number, + ) + .one_or_none() + ) + + +def retrieve_all_feedbacks_and_surveys(fund_id, round_id, status): + filters = [] + section_names = {} + sections_feedback = [] + end_of_application_survey_data = [] + + # get applications + if fund_id: + filters.append(Applications.fund_id == fund_id) + if round_id: + filters.append(Applications.round_id == round_id) + if status: + filters.append(Applications.status == status) + applications = get_applications(filters=filters, include_forms=True) + + mapping_report = get_report_mapping_for_round(round_id) + + # get section id & names map + application_sections = get_application_sections(fund_id, round_id, language="en") + for section in application_sections: + section_names[str(section["id"])] = section["title"] + + # extract section feedbacks & end of survey feedbacks for all applications + eoas_serialiser = EndOfApplicationSurveyFeedbackSchema() + applicant_serialiser = ApplicationSchema() + + for application in applications: + # extract applicant email & organisation + try: + result = map_application_key_fields( + applicant_serialiser.dump(application), + mapping_report, + round_id, + ) + applicant_email = result["applicant_email"] + applicant_organisation = result["organisation_name"] + except Exception as e: + current_app.logger.error(f"Coudn't extract applicant email & organisation. Exception :{e}") + applicant_email = "" + applicant_organisation = "" + + # extract sections feedback + for feedback in application.feedbacks: + sections_feedback.append( + { + "application_id": str(application.id), + "applicant_email": applicant_email, + "applicant_organisation": applicant_organisation, + "section": section_names[feedback.section_id], + "comment": feedback.feedback_json["comment"], + "rating": feedback.feedback_json["rating"], + } + ) + + # extract end of survey feedback + eoas_list = [eoas_serialiser.dump(row) for row in application.end_of_application_survey] + total_feedback = { + "application_id": str(application.id), + "applicant_email": applicant_email, + "applicant_organisation": applicant_organisation, + } + for eoas in eoas_list: + for key, value in eoas["data"].items(): + if key != "csrf_token": + total_feedback[key] = value + + end_of_application_survey_data.append(total_feedback) + + return { + "sections_feedback": sections_feedback, + "end_of_application_survey_data": end_of_application_survey_data, + } diff --git a/db/queries/apply/form/__init__.py b/db/queries/apply/form/__init__.py new file mode 100644 index 00000000..af302725 --- /dev/null +++ b/db/queries/apply/form/__init__.py @@ -0,0 +1 @@ +from .queries import * # noqa diff --git a/db/queries/apply/form/queries.py b/db/queries/apply/form/queries.py new file mode 100644 index 00000000..7284bad9 --- /dev/null +++ b/db/queries/apply/form/queries.py @@ -0,0 +1,27 @@ +from db import db +from db.models import Forms + + +def add_new_forms(forms, application_id): + for form in forms: + new_form_row = Forms( + application_id=application_id, + json=[], + name=form, + status="NOT_STARTED", + ) + db.session.add(new_form_row) + db.session.commit() + return {"forms": forms} + + +def get_forms_by_app_id(application_id, as_json=True): + forms = db.session.query(Forms).filter(Forms.application_id == application_id).all() + if as_json: + return [form.as_json() for form in forms] + else: + return forms + + +def get_form(application_id, form_name) -> Forms: + return db.session.query(Forms).filter(Forms.application_id == application_id, Forms.name == form_name).one() diff --git a/db/queries/apply/reporting/__init__.py b/db/queries/apply/reporting/__init__.py new file mode 100644 index 00000000..af302725 --- /dev/null +++ b/db/queries/apply/reporting/__init__.py @@ -0,0 +1 @@ +from .queries import * # noqa diff --git a/db/queries/apply/reporting/queries.py b/db/queries/apply/reporting/queries.py new file mode 100644 index 00000000..01970833 --- /dev/null +++ b/db/queries/apply/reporting/queries.py @@ -0,0 +1,151 @@ +import csv +import io +from typing import Any +from typing import Optional + +import pandas as pd +from config.key_report_mappings.mappings import ROUND_ID_TO_KEY_REPORT_MAPPING +from config.key_report_mappings.model import ApplicationColumnMappingItem +from config.key_report_mappings.model import FormMappingItem +from config.key_report_mappings.model import MappingItem +from db.models import Applications +from db.queries.apply import get_applications +from db.queries.apply.application import get_count_by_status + +APPLICATION_STATUS_HEADERS = [ + "fund_id", + "round_id", + "NOT_STARTED", + "IN_PROGRESS", + "COMPLETED", + "SUBMITTED", +] + + +def export_application_statuses_to_csv(return_data): + output = io.StringIO() + + w = csv.DictWriter(output, APPLICATION_STATUS_HEADERS) + w.writeheader() + for fund in return_data: + fund_id = fund["fund_id"] + for round in fund["rounds"]: + round_id = round["round_id"] + w.writerow( + { + "fund_id": fund_id, + "round_id": round_id, + **round["application_statuses"], + } + ) + + bytes_object = bytes(output.getvalue(), encoding="utf-8") + bytes_output = io.BytesIO(bytes_object) + return bytes_output + + +def export_json_to_csv(return_data, headers=None): + output = io.StringIO() + if isinstance(return_data, list): + if not headers: + headers = return_data[0].keys() + w = csv.DictWriter(output, headers) + w.writeheader() + w.writerows(return_data) + else: + w = csv.DictWriter(output, return_data.keys()) + w.writeheader() + w.writerow(return_data) + bytes_object = bytes(output.getvalue(), encoding="utf-8") + bytes_output = io.BytesIO(bytes_object) + return bytes_output + + +def export_json_to_excel(return_data: dict): + output = io.BytesIO() + + if not return_data: + return output + + with pd.ExcelWriter(output, engine="openpyxl") as writer: + for key in return_data.keys(): + df = pd.DataFrame(return_data[key]) + df.to_excel(writer, sheet_name=key) + + # seeking is necessary + output.seek(0) + return output + + +def get_general_status_applications_report(round_id: Optional[str] = None, fund_id: Optional[str] = None): + return get_count_by_status(round_id, fund_id) + + +def get_key_report_field_headers(round_id: str) -> list[str]: + mapping: list[MappingItem] = ROUND_ID_TO_KEY_REPORT_MAPPING[round_id] + return [field.return_field for field in mapping] + + +def get_report_for_applications( + *, # kwargs only + status: Optional[str] = None, + application_ids: Optional[list[str]] = None, + round_id: Optional[str] = None, + fund_id: Optional[str] = None, +): + filters = [] + if status: + filters.append(Applications.status == status) + if application_ids: + filters.append(Applications.id.in_(application_ids)) + if fund_id: + filters.append(Applications.fund_id == fund_id) + if round_id: + filters.append(Applications.round_id == round_id) + applications = get_applications( + filters=filters, + include_forms=True, + as_json=True, + ) + + return_json_list: list[dict[str, Any]] = [] + mapping: list[MappingItem] = ROUND_ID_TO_KEY_REPORT_MAPPING[round_id] + for application in applications: + return_json = map_application_key_fields(application, mapping, round_id) + return_json_list.append(return_json) + return return_json_list + + +def map_application_key_fields( + application: dict[str, Any], mapping: list[MappingItem], round_id: str +) -> dict[str, Any]: + return_json: dict[str, Any] = {field: None for field in get_key_report_field_headers(round_id)} + language: str = application["language"] + + form_mapping_items = [item for item in mapping if isinstance(item, FormMappingItem)] + report_config_forms: list[str] = [report_config.get_form_name(language) for report_config in form_mapping_items] + report_config_keys: list[str] = [report_config.key for report_config in form_mapping_items] + + form_mapping_items = [item for item in mapping if isinstance(item, FormMappingItem)] + for application_form in application["forms"]: + # skip any forms that are not in the report config + if application_form.get("name") not in report_config_forms: + continue + + for field in (f for q in application_form["questions"] for f in q["fields"]): + # skip any fields that are not in the report config + if field.get("key") not in report_config_keys: + continue + + for mapping_item in form_mapping_items: + if mapping_item.key == field.get("key"): + return_json[mapping_item.return_field] = mapping_item.format_answer(field) + + application_column_mapping_items = [item for item in mapping if isinstance(item, ApplicationColumnMappingItem)] + for mapping_item in application_column_mapping_items: + return_json[mapping_item.return_field] = mapping_item.format_answer(application.get(mapping_item.column_name)) + + return_fields_ordered = [item.return_field for item in mapping] + sorted_result_json = {k: return_json.get(k) for k in return_fields_ordered} + + return sorted_result_json diff --git a/db/queries/apply/research/__init__.py b/db/queries/apply/research/__init__.py new file mode 100644 index 00000000..af302725 --- /dev/null +++ b/db/queries/apply/research/__init__.py @@ -0,0 +1 @@ +from .queries import * # noqa diff --git a/db/queries/apply/research/queries.py b/db/queries/apply/research/queries.py new file mode 100644 index 00000000..609518c6 --- /dev/null +++ b/db/queries/apply/research/queries.py @@ -0,0 +1,42 @@ +from copy import deepcopy +from datetime import datetime + +from db import db +from db.models.research import ResearchSurvey + + +def upsert_research_survey_data(application_id, fund_id, round_id, data) -> ResearchSurvey: + existing_survey_data = ResearchSurvey.query.filter_by( + application_id=application_id, + fund_id=fund_id, + round_id=round_id, + ).first() + + if existing_survey_data: + existing_form_data = deepcopy(existing_survey_data.data) + existing_form_data.update(data) + existing_survey_data.data = existing_form_data + existing_survey_data.date_submitted = datetime.now() + db.session.commit() + return existing_survey_data + + new_survey_data = ResearchSurvey( + application_id=application_id, + fund_id=fund_id, + round_id=round_id, + data=data, + date_submitted=datetime.now(), + ) + db.session.add(new_survey_data) + db.session.commit() + return new_survey_data + + +def retrieve_research_survey_data(application_id) -> ResearchSurvey: + return ( + db.session.query(ResearchSurvey) + .filter( + ResearchSurvey.application_id == application_id, + ) + .one_or_none() + ) diff --git a/db/queries/apply/statuses/__init__.py b/db/queries/apply/statuses/__init__.py new file mode 100644 index 00000000..af302725 --- /dev/null +++ b/db/queries/apply/statuses/__init__.py @@ -0,0 +1 @@ +from .queries import * # noqa diff --git a/db/queries/apply/statuses/queries.py b/db/queries/apply/statuses/queries.py new file mode 100644 index 00000000..859d0cf7 --- /dev/null +++ b/db/queries/apply/statuses/queries.py @@ -0,0 +1,264 @@ +from datetime import datetime + +from db import db +from db.models import Applications +from db.models.forms.forms import Forms +from db.queries.apply import get_feedback +from db.queries.apply.application import get_application +from db.queries.apply.feedback import retrieve_end_of_application_survey_data +from db.queries.apply.form.queries import get_form +from db.queries.apply.research import retrieve_research_survey_data +from services.apply import get_round +from services.apply.data import get_application_sections +from services.apply.models.round import FeedbackSurveyConfig + + +def _is_all_sections_feedback_complete(application_id, fund_id, round_id, language: str): + sections = get_application_sections(fund_id, round_id, language) + all_feedback_completed = all( + get_feedback(application_id, str(s["id"])) for s in sections if s.get("requires_feedback") + ) + return all_feedback_completed + + +def _is_feedback_survey_complete(application_id): + is_survey_completed = all(retrieve_end_of_application_survey_data(application_id, pn) for pn in "1234") + return is_survey_completed + + +def _is_research_survey_complete(application_id): + research_survey = retrieve_research_survey_data(application_id) + is_survey_completed = False + if opt_in := research_survey.data.get("research_opt_in"): + if opt_in == "disagree" or ( + research_survey.data.get("contact_name") and research_survey.data.get("contact_email") + ): + is_survey_completed = True + return is_survey_completed + + +def update_application_status( + application_with_forms: Applications, feedback_survey_config: FeedbackSurveyConfig +) -> str: + """Updates the status of the supplied application based on the status of all + forms with that application, and the status of all sections feedback and + survey status (if the round requires feedback) + + Parameters: + application_with_forms (`Applications`): Application record to update, with the form jsons populated + This object should be within a db context as the function updates it. + feedback_survey_config (`FeedbackSurveyConfig`): feedback_survey_config of the round + + """ + + all_feedback_and_survey_completed = True + if feedback_survey_config.has_section_feedback: + all_feedback_and_survey_completed = ( + feedback_survey_config.is_section_feedback_optional + or _is_all_sections_feedback_complete( + application_with_forms.id, + application_with_forms.fund_id, + application_with_forms.round_id, + application_with_forms.language.name, + ) + ) + + if feedback_survey_config.has_feedback_survey: + all_feedback_and_survey_completed = all_feedback_and_survey_completed and ( + feedback_survey_config.is_feedback_survey_optional + or _is_feedback_survey_complete(application_with_forms.id) + ) + + if feedback_survey_config.has_research_survey: + all_feedback_and_survey_completed = all_feedback_and_survey_completed and ( + feedback_survey_config.is_research_survey_optional + or _is_research_survey_complete(application_with_forms.id) + ) + + form_statuses = [form.status.name for form in application_with_forms.forms] + if "IN_PROGRESS" in form_statuses: + status = "IN_PROGRESS" + elif "COMPLETED" in form_statuses and ("NOT_STARTED" in form_statuses or not all_feedback_and_survey_completed): + status = "IN_PROGRESS" + elif "COMPLETED" in form_statuses and all_feedback_and_survey_completed: + status = "COMPLETED" + elif "SUBMITTED" in form_statuses: + status = "SUBMITTED" + else: + status = "NOT_STARTED" + application_with_forms.status = status + + +def update_form_status( + form_to_update: Forms, + round_mark_as_complete_enabled: bool, + is_summary_page_submitted: bool = False, +): + """Updates the status of a whole form based on the statuses of all questions + within the form. + + Parameters: + form_to_update (`Forms'): Forms object that we want to update. + This object is updated by this function so needs to be within the DB context + is_summary_page_submit (`bool`): Whether or not this is an update from submitting the summary page + + """ + + if round_mark_as_complete_enabled: + mark_as_complete_question = next( + (question_page for question_page in form_to_update.json if question_page["question"] == "MarkAsComplete"), + None, + ) + is_marked_as_complete = mark_as_complete_question["fields"][0]["answer"] if mark_as_complete_question else False + else: + is_marked_as_complete = False + + status_list = [ + question_page["status"] + for question_page in form_to_update.json + if question_page["question"] != "MarkAsComplete" + ] + if "COMPLETED" not in status_list: + # If no single question page is complete + form_to_update.status = "NOT_STARTED" + elif "NOT_STARTED" not in status_list and is_summary_page_submitted: + # If every question page has answers and this is submit on summary page + if round_mark_as_complete_enabled: + if is_marked_as_complete: + form_to_update.status = "COMPLETED" + form_to_update.has_completed = True + else: + form_to_update.status = "IN_PROGRESS" + form_to_update.has_completed = False + else: + form_to_update.status = "COMPLETED" + form_to_update.has_completed = True + elif "NOT_STARTED" not in status_list and form_to_update.has_completed: + # All question pages have answers and form has previously completed + form_to_update.status = "COMPLETED" + else: + form_to_update.status = "IN_PROGRESS" + + +def _is_field_answered(field: dict) -> bool: + """Determines whether or not an answer has been provided for the supplied + field. + + Parameters: + field (`dict`): The field we want to find and answer for + + Returns: + bool: Whether or not the field has been answered + + """ + answer_or_not_specified = field.get("answer") + match answer_or_not_specified: # noqa + case "": + return False + case []: # noqa (E211) + return False + # optional questions return None (not string) + # when submitted with no answer + case None: + return True + # default case when there is an answer + case _: + return True + + +def _determine_answer_status_for_fields( + fields_in_question_page: list[dict], +) -> list[bool]: + """Builds a list of bools representing which fields have been answered in the + supplied question page. + + Parameters: + fields_in_question_page (`list[dict]`): The fields element from a question in the form + + Returns: + list: True or False for each field in the question, eg. `[True, True, False]` + + """ + answer_found_list = [_is_field_answered(field) for field in fields_in_question_page] + return answer_found_list + + +def _determine_question_page_status_from_answers(answer_found_list: list[bool]) -> str: + """Determines the status of this question page (could contain multiple + questions/responses), based on the supplied list of whether each field is + answered. + + Parameters: + answer_found_list (`list[bool]`): Whether or not each field in the question has an answer, + eg. `[True, True, False]` + + Returns: + str: The status of this question page + + """ + + # If we found no answers + if not answer_found_list: + return "NOT_STARTED" + # If all answers are given + if all(answer_found_list): + return "COMPLETED" + # If some answers are given + elif any(answer_found_list): + return "IN_PROGRESS" + # If no answers are given + else: + return "NOT_STARTED" + + +def update_question_page_statuses(stored_form_json: dict): + """Updates `status` field in every question page in this form object, based on + whether or not answers are supplied. + + Parameters: + stored_form_json: The json object, within the db context, representing the form to update. + This same object will be updated by this function. + + """ + for question_page in stored_form_json: + question_page["status"] = question_page.get("status", "NOT_STARTED") + + answer_found_list = _determine_answer_status_for_fields(question_page["fields"]) + question_page["status"] = _determine_question_page_status_from_answers(answer_found_list) + + +def update_statuses(application_id: str, form_name: str, is_summary_page_submitted: bool = False): + """Updates the status of questions, forms, and the application, based on the + state of the supplied form. If no form supplied, just updates the status of + the application (based on feedback and form status) + + Parameters: + application_id (`str`): ID of the application to update the status + form_name (`str`): Name of the form that has been updated, uses this form as the basis for the update + is_summary_page_submitted (`bool`): If this is as a result of submitting from the summary page of a form. + + """ + application = get_application(application_id, include_forms=True) + round = get_round(application.fund_id, application.round_id) + if form_name: + form_to_update = get_form(application_id=application_id, form_name=form_name) + update_question_page_statuses(stored_form_json=form_to_update.json) + update_form_status(form_to_update, round.mark_as_complete_enabled, is_summary_page_submitted) + db.session.commit() + + update_application_status(application, round.feedback_survey_config) + db.session.commit() + + +def check_is_fund_round_open(application_id: str): + """ + Check is given fund round is closed for the following application id + Parameters: + application_id (`str`): ID of the application to update the status + """ + application = get_application(application_id, include_forms=True) + round_obj = get_round(application.fund_id, application.round_id) + deadline = datetime.strptime(round_obj.deadline, "%Y-%m-%dT%H:%M:%S") + if datetime.now() > deadline: + return False + return True diff --git a/db/queries/apply/updating/__init__.py b/db/queries/apply/updating/__init__.py new file mode 100644 index 00000000..af302725 --- /dev/null +++ b/db/queries/apply/updating/__init__.py @@ -0,0 +1 @@ +from .queries import * # noqa diff --git a/db/queries/apply/updating/queries.py b/db/queries/apply/updating/queries.py new file mode 100644 index 00000000..d576a2a4 --- /dev/null +++ b/db/queries/apply/updating/queries.py @@ -0,0 +1,59 @@ +import sqlalchemy +from db import db +from db.models.application.enums import Status as ApplicationStatus +from db.queries.apply.application import attempt_to_find_and_update_project_name +from db.queries.apply.application import get_application +from db.queries.apply.form import get_form +from db.queries.apply.statuses import update_statuses +from flask import abort +from flask import current_app +from sqlalchemy import func + + +def update_application_and_related_form(application_id, question_json, form_name, is_summary_page_submit): + application = get_application(application_id) + if application.status == ApplicationStatus.SUBMITTED: + current_app.logger.error( + f"Not allowed. Attempted to PUT data into a SUBMITTED application with an application_id: {application_id}." + ) + abort(400, "Not allowed to edit a submitted application.") + + application.last_edited = func.now() + form_sql_row = get_form(application_id, form_name) + if (project_name := attempt_to_find_and_update_project_name(question_json, application)) is not None: + application.project_name = project_name + form_sql_row.json = question_json + update_statuses(application_id, form_name, is_summary_page_submit) + db.session.commit() + current_app.logger.info(f"Application updated for application_id: '{application_id}.") + + +def update_form(application_id, form_name, question_json, is_summary_page_submit): + try: + form_sql_row = get_form(application_id, form_name) + # Running update form for the first time + if question_json and not form_sql_row.json: + update_application_and_related_form( + application_id, + question_json, + form_name, + is_summary_page_submit, + ) + # Removing all data in the form (should not be allowed) + elif form_sql_row.json and not question_json: + current_app.logger.error( + f"Application update aborted for application_id: '{application_id}. Invalid data supplied" + ) + raise Exception("ABORTING UPDATE, INVALID DATA GIVEN") + # Updating form subsequent times + elif form_sql_row.json and form_sql_row.json != question_json: + update_application_and_related_form( + application_id, + question_json, + form_name, + is_summary_page_submit, + ) + except sqlalchemy.orm.exc.NoResultFound as e: + raise e + db.session.commit() + return form_sql_row.as_json() diff --git a/db/schemas/__init__.py b/db/schemas/__init__.py index 905322bd..8a4dd244 100644 --- a/db/schemas/__init__.py +++ b/db/schemas/__init__.py @@ -1,3 +1,6 @@ +from .application import ApplicationSchema +from .eligibility import EligibilitySchema +from .form import FormsRunnerSchema from .schemas import AssessmentRecordMetadata from .schemas import AssessmentRoundMetadata from .schemas import AssessmentSubCriteriaMetadata @@ -14,4 +17,7 @@ "CommentMetadata", "AssessmentRoundMetadata", "ScoringSystemMetadata", + "ApplicationSchema", + "FormsRunnerSchema", + "EligibilitySchema", ] diff --git a/db/schemas/application.py b/db/schemas/application.py new file mode 100644 index 00000000..074c5df2 --- /dev/null +++ b/db/schemas/application.py @@ -0,0 +1,45 @@ +from db.models import Applications +from db.models.application.enums import Language +from db.models.application.enums import Status +from marshmallow import post_dump +from marshmallow.fields import DateTime +from marshmallow.fields import Enum +from marshmallow.fields import Method +from marshmallow_sqlalchemy import auto_field +from marshmallow_sqlalchemy import SQLAlchemyAutoSchema +from marshmallow_sqlalchemy.fields import Nested +from services.apply import get_round_name + +from .form import FormsRunnerSchema + + +class ApplicationSchema(SQLAlchemyAutoSchema): + class Meta: + model = Applications + exclude = ["key"] + + @post_dump + def handle_nones(self, data, **kwargs): + if data["last_edited"] is None: + data["last_edited"] = data["started_at"] + if data["date_submitted"] is None: + data["date_submitted"] = "null" + if data["language"] is None: + data["language"] = "en" + return data + + def get_round_name(self, obj): + # TODO: is this actually being used at all? + # If we're not using this, it would be good to remove it or exclude it. + # i.e exclude = ["round_name"], as we don't want it to run during each + # serialization - it makes a GET request per application. The request + # is LRU cached for now, incase this is actually used. + return get_round_name(obj.fund_id, obj.round_id) + + language = Enum(Language, default=Language.en) + project_name = auto_field() + started_at = DateTime(format="iso") + status = Enum(Status) + last_edited = DateTime(format="iso") + round_name = Method("get_round_name") + forms = Nested(FormsRunnerSchema, many=True, allow_none=True) diff --git a/db/schemas/eligibility.py b/db/schemas/eligibility.py new file mode 100644 index 00000000..33a00e15 --- /dev/null +++ b/db/schemas/eligibility.py @@ -0,0 +1,20 @@ +from db.models import Eligibility +from db.models.eligibility.eligibility_trail import EligibilityUpdate +from marshmallow import post_dump +from marshmallow_sqlalchemy import SQLAlchemyAutoSchema + + +class EligibilitySchema(SQLAlchemyAutoSchema): + class Meta: + model = Eligibility + exclude = ["key"] + + @post_dump + def handle_nones(self, data, **kwargs): + if data["date_submitted"] is None: + data["date_submitted"] = "null" + + +class EligibilityUpdateSchema(SQLAlchemyAutoSchema): + class Meta: + model = EligibilityUpdate diff --git a/db/schemas/end_of_application_survey.py b/db/schemas/end_of_application_survey.py new file mode 100644 index 00000000..3f87c04b --- /dev/null +++ b/db/schemas/end_of_application_survey.py @@ -0,0 +1,14 @@ +from db.models import EndOfApplicationSurveyFeedback +from marshmallow_sqlalchemy import auto_field +from marshmallow_sqlalchemy import SQLAlchemySchema + + +class EndOfApplicationSurveyFeedbackSchema(SQLAlchemySchema): + class Meta: + model = EndOfApplicationSurveyFeedback + + application_id = auto_field() + fund_id = auto_field() + round_id = auto_field() + data = auto_field() + page_number = auto_field() diff --git a/db/schemas/feedback.py b/db/schemas/feedback.py new file mode 100644 index 00000000..d81faefb --- /dev/null +++ b/db/schemas/feedback.py @@ -0,0 +1,19 @@ +from db.models import Feedback +from db.models.feedback.enums import Status +from marshmallow.fields import DateTime +from marshmallow.fields import Enum +from marshmallow_sqlalchemy import auto_field +from marshmallow_sqlalchemy import SQLAlchemySchema + + +class FeedbackSchema(SQLAlchemySchema): + class Meta: + model = Feedback + + application_id = auto_field() + fund_id = auto_field() + round_id = auto_field() + status = Enum(Status) + section_id = auto_field() + feedback_json = auto_field() + date_submitted = DateTime(format="iso") diff --git a/db/schemas/form.py b/db/schemas/form.py new file mode 100644 index 00000000..41c60b08 --- /dev/null +++ b/db/schemas/form.py @@ -0,0 +1,14 @@ +from db.models import Forms +from db.models.forms.enums import Status +from marshmallow.fields import Enum +from marshmallow_sqlalchemy import auto_field +from marshmallow_sqlalchemy import SQLAlchemySchema + + +class FormsRunnerSchema(SQLAlchemySchema): + class Meta: + model = Forms + + status = Enum(Status) + name = auto_field() + questions = auto_field("json") diff --git a/openapi/api.yml b/openapi/api.yml index 05895121..60a172c0 100644 --- a/openapi/api.yml +++ b/openapi/api.yml @@ -1,8 +1,8 @@ openapi: "3.0.0" info: - description: Assessment API for DLUHC Funding Service Design - version: "1.0.0" - title: Funding Service Design - Assessment Store + description: Assesplication poc API + version: "0.0.1" + title: Funding Service - Assesplication Store tags: - name: assessments description: Assessment operations @@ -10,6 +10,8 @@ tags: description: Score operations - name: flags description: Flag operations + - name: application-store + description: Application store operations paths: '/application_overviews/{fund_id}/{round_id}': @@ -1520,3 +1522,607 @@ paths: type: array items: $ref: 'components.yml#/components/schemas/UserAssociation' + +# ---------------------------------------------------- +# Imported from application-store +# ---------------------------------------------------- + + /applications/reporting/applications_statuses_data: + get: + tags: + - reporting + summary: Get report on started and submitted applications + description: Get report on started and submitted applications + operationId: apply.api.ApplicationsView.get_applications_statuses_report + parameters: + - name: round_id + in: query + description: Optional round ID to filter by + schema: + type: array + items: + type: string + - name: fund_id + in: query + description: Optional fund ID to filter by + schema: + type: array + items: + type: string + - name: format + in: query + description: Optional format specifier, csv or json + schema: + type: string + enum: [csv,json] + responses: + 200: + description: SUCCESS - Here is the status report on applications + content: + application/json: + schema: + $ref: 'apply_components.yml#/components/schemas/StatusReport' + 404: + description: ERROR - Could not get report + content: + application/json: + schema: + $ref: 'apply_components.yml#/components/schemas/Error' + + /applications/reporting/key_application_metrics/{application_id}: + get: + tags: + - reporting + summary: Get the key data report on an application + description: Get the key data report on an application + operationId: apply.api.ApplicationsView.get_key_application_data_report + responses: + 200: + description: SUCCESS - Here is the report on requested application + content: + text/csv: {} + 404: + description: ERROR - Could not get report + content: + application/json: + schema: + $ref: 'apply_components.yml#/components/schemas/Error' + parameters: + - name: application_id + in: path + required: true + schema: + type: string + format: path + + /applications/reporting/key_application_metrics: + get: + tags: + - reporting + summary: Get the key data report on applications + description: Get the key data report on applications + operationId: apply.api.ApplicationsView.get_key_applications_data_report + parameters: + - name: status + in: query + required: false + schema: + type: string + - name: round_id + in: query + description: Optional round ID to filter by + schema: + type: string + - name: fund_id + in: query + description: Optional fund ID to filter by + schema: + type: string + responses: + 200: + description: SUCCESS - Here is the report on applications + content: + text/csv: {} + 404: + description: ERROR - Could not get report + content: + application/json: + schema: + $ref: 'apply_components.yml#/components/schemas/Error' + + /applications/forms: + put: + requestBody: + description: Update application with new forms state + required: true + content: + application/json: + schema: + $ref: 'apply_components.yml#/components/schemas/PutForms' + tags: + - applications + summary: Update an application with new forms state + description: Updates the form state of an application + operationId: apply.api.ApplicationsView.put + responses: + 201: + description: SUCCESS - Application updated + content: + application/json: + schema: + $ref: 'apply_components.yml#/components/schemas/UpdatedForms' + 404: + description: ERROR - Form cannot be updated + content: + application/json: + schema: + $ref: 'apply_components.yml#/components/schemas/Error' + example: + code: 404 + status: 'error' + message: 'Form is not updated' + + /applications/get_all_feedbacks_and_survey_report: + get: + tags: + - feedback + - survey + summary: Retrive all section feedbacks & survey + description: Retrive all section feedbacks & survey + operationId: apply.api.ApplicationsView.get_all_feedbacks_and_survey_report + responses: + 200: + description: SUCCESS - Here is the report on requested applicants feedback & survey + content: + application/vnd.ms-excel: {} + 404: + description: ERROR - Could not get report + content: + application/json: + schema: + $ref: 'apply_components.yml#/components/schemas/Error' + parameters: + - in: query + name: fund_id + style: form + schema: + type: string + required: true + explode: false + - in: query + name: round_id + style: form + schema: + type: string + required: true + explode: false + - in: query + name: status_only + style: form + schema: + type: string + required: false + explode: false + + /applications: + get: + tags: + - applications + summary: Search applications + description: List all applications + operationId: apply.api.ApplicationsView.get + responses: + 200: + description: SUCCESS - A list of applications + content: + application/json: + schema: + type: array + items: + $ref: 'apply_components.yml#/components/schemas/Application' + parameters: + - in: query + name: application_id + style: form + schema: + type: string + required: false + explode: false + - in: query + name: account_id + style: form + schema: + type: string + required: false + explode: false + - in: query + name: fund_id + style: form + schema: + type: string + required: false + explode: false + - in: query + name: round_id + style: form + schema: + type: string + required: false + explode: false + - in: query + name: status_only + style: form + schema: + type: array + items: + type: string + required: false + explode: true + # - filtering applications + - in: query + name: order_by + style: form + schema: + type: string + required: false + explode: false + - in: query + name: order_rev + style: form + schema: + type: string + required: false + explode: false + - in: query + name: forms + style: form + schema: + type: boolean + required: false + explode: false + post: + tags: + - applications + summary: Post a new application + description: Registers a new application for a user + operationId: apply.api.ApplicationsView.post + requestBody: + description: Application creation parameters + required: true + content: + application/json: + schema: + $ref: 'apply_components.yml#/components/schemas/PostApplication' + example: + account_id: 'usera' + fund_id: '47aef2f5-3fcb-4d45-acb5-f0152b5f03c4' + round_id: 'c603d114-5364-4474-a0c4-c41cbf4d3bbd' + language: 'en' + responses: + 201: + description: SUCCESS - Application created + content: + application/json: + schema: + $ref: 'apply_components.yml#/components/schemas/CreatedApplication' + 401: + description: ERROR - Could not create application + content: + application/json: + schema: + $ref: 'apply_components.yml#/components/schemas/Error' + example: + code: 401 + status: 'error' + message: 'An assessment for this application already exists' + + /applications/{application_id}: + get: + tags: + - applications + summary: Get a specific application + description: Get a specific application by application id + operationId: apply.api.ApplicationsView.get_by_id + responses: + 200: + description: SUCCESS - An applicaton by id + content: + application/json: + schema: + type: object + items: + $ref: 'apply_components.yml#/components/schemas/ReturnedApplication' + 404: + description: ERROR - Application cannot be found. + content: + application/json: + schema: + $ref: 'apply_components.yml#/components/schemas/Error' + example: + code: 404 + status: 'error' + message: 'Application corresponding to id not found.' + parameters: + - name: application_id + in: path + required: true + schema: + type: string + format: path + - in: query + name: with_questions_file + style: form + schema: + type: boolean + required: false + explode: false + + /applications/{application_id}/submit: + post: + tags: + - applications + summary: Submit an application + description: Application is submitted and cannot be changed from frontend + operationId: apply.api.ApplicationsView.submit + parameters: + - name: application_id + in: path + required: true + schema: + type: string + format: path + - name: dont_send_email + in: query + required: false + description: Whether to send an email notification for the submitted application + schema: + type: boolean + responses: + 201: + description: Application has been submitted successfully + content: + application/json: + schema: + type: object + items: + $ref: 'apply_components.yml#/components/schemas/SubmittedApplication' + 404: + description: ERROR - Application cannot be submitted. + content: + application/json: + schema: + $ref: 'apply_components.yml#/components/schemas/Error' + example: + code: 404 + status: 'error' + message: 'Application corresponding to id not found.' + + /application/feedback: + post: + tags: + - feedback + summary: Post a new feedback + description: Create a new feedback entry + operationId: apply.api.ApplicationsView.post_feedback + requestBody: + description: Feedback creation parameters + required: true + content: + application/json: + schema: + $ref: 'apply_components.yml#/components/schemas/PostFeedback' + example: + application_id: '47aef2f5-3fcb-4d45-acb5-f0152b5f03c4' + fund_id: '47aef2f5-3fcb-4d45-acb5-f0152b5f03c4' + round_id: 'c603d114-5364-4474-a0c4-c41cbf4d3bbd' + section_id: 'section_1' + feedback_json: {"comment": "Great work!", "rating": 5} + status: "NOT_STARTED" + responses: + 201: + description: SUCCESS - Feedback created + content: + application/json: + schema: + $ref: 'apply_components.yml#/components/schemas/Feedback' + 401: + description: ERROR - Could not create feedback + content: + application/json: + schema: + $ref: 'apply_components.yml#/components/schemas/Error' + example: + code: 401 + status: 'error' + message: 'Could not create feedback' + + get: + tags: + - feedback + summary: Search feedback entries + description: List all feedback entries + operationId: apply.api.ApplicationsView.get_feedback_for_section + responses: + 200: + description: SUCCESS - A list of feedback entries + content: + application/json: + schema: + type: object + items: + $ref: 'apply_components.yml#/components/schemas/Feedback' + parameters: + - in: query + name: application_id + style: form + schema: + type: string + required: true + explode: false + - in: query + name: section_id + style: form + schema: + type: string + required: true + explode: false + - in: query + name: fund_id + style: form + schema: + type: string + required: false + explode: false + - in: query + name: round_id + style: form + schema: + type: string + required: false + explode: false + + /application/end_of_application_survey_data: + post: + tags: + - survey + summary: Post end of application survey data + operationId: apply.api.ApplicationsView.post_end_of_application_survey_data + requestBody: + required: true + content: + application/json: + schema: + $ref: 'apply_components.yml#/components/schemas/PostEndOfApplicationSurveyDataRequest' + responses: + 201: + description: SUCCESS - Survey data saved + content: + application/json: + schema: + $ref: 'apply_components.yml#/components/schemas/GetEndOfApplicationSurveyDataResponse' + 400: + description: ERROR - Bad request + content: + application/json: + schema: + $ref: 'apply_components.yml#/components/schemas/Error' + + get: + tags: + - survey + summary: Get end of application survey data + operationId: apply.api.ApplicationsView.get_end_of_application_survey_data + parameters: + - in: query + name: application_id + required: true + schema: + type: string + description: ID of the application + - in: query + name: page_number + required: true + schema: + type: integer + description: Page number of the survey data + responses: + 200: + description: SUCCESS - Survey data retrieved + content: + application/json: + schema: + $ref: 'apply_components.yml#/components/schemas/GetEndOfApplicationSurveyDataResponse' + 404: + description: ERROR - Survey data not found + content: + application/json: + schema: + $ref: 'apply_components.yml#/components/schemas/Error' + + /application/research: + post: + tags: + - survey + summary: Post contact details for research survey + operationId: apply.api.ApplicationsView.post_research_survey_data + requestBody: + required: true + content: + application/json: + schema: + $ref: 'apply_components.yml#/components/schemas/PostResearchSurveyDataRequest' + responses: + 201: + description: SUCCESS - Survey data saved + content: + application/json: + schema: + $ref: 'apply_components.yml#/components/schemas/GetResearchSurveyDataResponse' + 400: + description: ERROR - Bad request + content: + application/json: + schema: + $ref: 'apply_components.yml#/components/schemas/Error' + get: + tags: + - survey + summary: Get contact details for research survey + operationId: apply.api.ApplicationsView.get_research_survey_data + parameters: + - in: query + name: application_id + required: true + schema: + type: string + description: ID of the application + responses: + 200: + description: SUCCESS - Survey data retrieved + content: + application/json: + schema: + $ref: 'apply_components.yml#/components/schemas/GetResearchSurveyDataResponse' + 404: + description: ERROR - Survey data not found + content: + application/json: + schema: + $ref: 'apply_components.yml#/components/schemas/Error' + + /queue_for_assessment/{application_id}: + post: + tags: + - SQS (queues) + summary: post an application (if it has the status submitted) to the assessment import queue + description: post an application (if it has the status submitted) to assessment import queue + operationId: apply.api.QueueView.post_submitted_application_to_assessment + parameters: + - name: application_id + in: path + required: true + schema: + type: string + format: path + responses: + 200: + description: SUCCESS - message sent + content: + application/json: + schema: + type: object + items: + type: object + properties: + sent: + type: boolean + 500: + description: ERROR - Application cannot be found. + content: + application/json: + schema: + $ref: 'apply_components.yml#/components/schemas/Error' + example: + code: 500 + status: 'error' + message: 'Message could not be staged' diff --git a/openapi/apply_components.yml b/openapi/apply_components.yml new file mode 100644 index 00000000..e0576348 --- /dev/null +++ b/openapi/apply_components.yml @@ -0,0 +1,267 @@ +components: + schemas: + StatusReport: + type: object + properties: + metrics: + type: array + items: + type: object + properties: + fund_id: + type: string + rounds: + type: array + items: + type: object + properties: + round_id: + type: string + application_statuses: + $ref: '#/components/schemas/ApplicationMetrics' + ApplicationMetrics: + type: object + properties: + NOT_STARTED: + type: integer + IN_PROGRESS: + type: integer + COMPLETED: + type: integer + SUBMITTED: + type: integer + Application: + type: object + properties: + id: + type: string + reference: + type: string + status: + type: string + account_id: + type: string + fund_id: + type: string + round_id: + type: string + project_name: + type: string + nullable: true + started_at: + type: string + last_edited: + type: string + round_name: + type: string + forms: + type: array + date_submitted: + type: string + PostApplication: + type: object + properties: + account_id: + type: string + fund_id: + type: string + round_id: + type: string + language: + type: string + CreatedApplication: + type: object + properties: + id: + type: string + account_id: + type: string + fund_id: + type: string + round_id: + type: string + PutForms: + type: object + properties: + name: + type: string + example: "funding round title" + questions: + type: array + items: + oneOf: + - type: object + example: [] + metadata: + type: object + properties: + form_name: + type: string + example: "Form name within application (declarations)" + application_id: + type: string + example: "Active Application ID (uuidv4)" + UpdatedForms: + type: object + properties: + name: + type: string + status: + type: string + questions: + type: array + items: + oneOf: + - type: object + metadata: + type: object + SubmittedApplication: + type: object + properties: + id: + type: string + reference: + type: string + email: + type: string + ReturnedApplication: + type: object + properties: + id: + type: string + ApplicationsStatuses: + type: object + properties: + applications_started: + type: integer + applications_submitted: + type: integer + KeyDataReport: + type: object + properties: + application_id: + type: string + organisation_type: + type: string + asset_type: + type: string + geography: + type: string + capital: + type: integer + revenue: + type: integer + KeyDataReports: + type: object + properties: + Report: + type: array + items: + oneOf: + - type: object + Feedback: + type: object + properties: + id: + type: string + format: uuid + application_id: + type: string + format: uuid + fund_id: + type: string + round_id: + type: string + section_id: + type: string + feedback_json: + type: object + status: + type: string + date_submitted: + type: string + PostFeedback: + type: object + properties: + application_id: + type: string + fund_id: + type: string + round_id: + type: string + section_id: + type: string + feedback_json: + type: object + status: + type: string + Error: + type: object + properties: + code: + type: integer + message: + type: string + required: + - code + - message + PostEndOfApplicationSurveyDataRequest: + type: object + properties: + application_id: + type: string + fund_id: + type: string + round_id: + type: string + page_number: + type: integer + data: + type: object + additionalProperties: true + GetEndOfApplicationSurveyDataResponse: + type: object + properties: + id: + type: integer + application_id: + type: string + fund_id: + type: string + round_id: + type: string + page_number: + type: integer + data: + type: object + additionalProperties: true + date_submitted: + type: string + PostResearchSurveyDataRequest: + type: object + properties: + application_id: + type: string + fund_id: + type: string + round_id: + type: string + data: + type: object + additionalProperties: true + GetResearchSurveyDataResponse: + type: object + properties: + id: + type: integer + application_id: + type: string + fund_id: + type: string + round_id: + type: string + data: + type: object + additionalProperties: true + date_submitted: + type: string diff --git a/services/apply/__init__.py b/services/apply/__init__.py new file mode 100644 index 00000000..cb364e4a --- /dev/null +++ b/services/apply/__init__.py @@ -0,0 +1,12 @@ +from .data import get_account # noqa +from .data import get_application_sections # noqa +from .data import get_data # noqa +from .data import get_fund # noqa +from .data import get_funds # noqa +from .data import get_local_data # noqa +from .data import get_remote_data # noqa +from .data import get_round # noqa +from .data import get_round_eoi_schema # noqa +from .data import get_round_name # noqa +from .data import get_rounds # noqa +from .http_methods import post_data # noqa diff --git a/services/apply/aws.py b/services/apply/aws.py new file mode 100644 index 00000000..cfdad814 --- /dev/null +++ b/services/apply/aws.py @@ -0,0 +1,38 @@ +from collections import namedtuple +from os import getenv + +import boto3 +from config import Config + +_KEY_PARTS = ("application_id", "form", "path", "component_id", "filename") + +if getenv("PRIMARY_QUEUE_URL", "Primary Queue URL Not Set") == "Primary Queue URL Not Set": + _S3_CLIENT = boto3.client( + "s3", + aws_access_key_id=Config.AWS_ACCESS_KEY_ID, + aws_secret_access_key=Config.AWS_SECRET_ACCESS_KEY, + region_name=Config.AWS_REGION, + endpoint_url=getenv("AWS_ENDPOINT_OVERRIDE", None), + ) +else: + _S3_CLIENT = boto3.client( + "s3", + region_name=Config.AWS_REGION, + endpoint_url=getenv("AWS_ENDPOINT_OVERRIDE", None), + ) + +FileData = namedtuple("FileData", _KEY_PARTS) + + +def list_files_by_prefix(prefix: str) -> list[FileData]: + objects_response = _S3_CLIENT.list_objects_v2( + Bucket=Config.AWS_BUCKET_NAME, + Prefix=prefix, + ) + + contents = objects_response.get("Contents") or [] + return [ + FileData(*key_parts) + for key in [file["Key"] for file in contents] + if len(key_parts := key.split("/")) == len(_KEY_PARTS) + ] diff --git a/services/apply/data.py b/services/apply/data.py new file mode 100644 index 00000000..1fe91c1c --- /dev/null +++ b/services/apply/data.py @@ -0,0 +1,163 @@ +import functools +import json +import os +from typing import Optional +from urllib.parse import urlencode + +import requests +from config import Config +from flask import abort +from flask import current_app + +from .models.account import Account +from .models.fund import Fund +from .models.round import Round + + +def get_data(endpoint: str, params: Optional[dict] = None): + """Queries the api endpoint provided and returns a data response in json + format. + + Args: + endpoint (str): an API get data address + + Returns: + data (json): data response in json format + + """ + + if Config.USE_LOCAL_DATA: + current_app.logger.info(f"Fetching local data from '{endpoint}'" + f" with params {params}.") + data = get_local_data(endpoint, params) + else: + current_app.logger.info(f"Fetching data from '{endpoint}'" + f" with params {params}.") + data = get_remote_data(endpoint, params) + if data is None: + current_app.logger.error(f"Data request failed, unable to recover: {endpoint}") + return abort(500) + return data + + +def get_remote_data(endpoint, params: Optional[dict] = None): + query_string = "" + if params: + params = {k: v for k, v in params.items() if v is not None} + query_string = urlencode(params) + + endpoint = endpoint + "?" + query_string + + response = requests.get(endpoint) + if response.status_code == 200: + data = response.json() + return data + else: + current_app.logger.warn(f"GET remote data call was unsuccessful with status code: {response.status_code}.") + return None + + +def get_local_data(endpoint: str, params: Optional[dict] = None): + query_string = "" + if params: + params = {k: v for k, v in params.items() if v is not None} + query_string = urlencode(params) + endpoint = endpoint + "?" + query_string + api_data_json = os.path.join(Config.FLASK_ROOT, "tests", "api_data", "get_endpoint_data.json") + with open(api_data_json) as json_file: + api_data = json.load(json_file) + if endpoint in api_data: + mocked_response = requests.models.Response() + mocked_response.status_code = 200 + content_str = json.dumps(api_data[endpoint]) + mocked_response._content = bytes(content_str, "utf-8") + return json.loads(mocked_response.text) + return None + + +def get_application_sections(fund_id, round_id, language): + endpoint = (Config.FUND_STORE_API_HOST + Config.FUND_ROUND_APPLICATION_SECTIONS_ENDPOINT).format( + fund_id=fund_id, round_id=round_id, language=language + ) + response = get_remote_data(endpoint) + return response + + +def get_funds() -> list[Fund] | None: + endpoint = Config.FUND_STORE_API_HOST + Config.FUNDS_ENDPOINT + response = get_data(endpoint) + if response and len(response) > 0: + funds = [] + for fund in response: + funds.append(Fund.from_json(fund)) + return funds + + +def get_fund(fund_id: str) -> Fund | None: + endpoint = Config.FUND_STORE_API_HOST + Config.FUND_ENDPOINT.format(fund_id=fund_id) + current_app.logger.info(f"Request made to {endpoint}") + response = get_data(endpoint) + if response is None: + current_app.logger.info("Request to fund store returned None") + fund = Fund.from_json(response) + return fund + + +def get_rounds(fund_id: str) -> Fund | list: + endpoint = Config.FUND_STORE_API_HOST + Config.FUND_ROUNDS_ENDPOINT.format(fund_id=fund_id) + response = get_data(endpoint) + rounds = [] + if response and len(response) > 0: + for round_data in response: + rounds.append(Round.from_json(round_data)) + return rounds + + +def get_round(fund_id: str, round_id: str) -> Round | None: + """Gets round from round store api using round_id if given.""" + round_endpoint = Config.FUND_STORE_API_HOST + Config.FUND_ROUND_ENDPOINT.format(fund_id=fund_id, round_id=round_id) + round_response = get_data(round_endpoint) + if round_response and "id" in round_response: + return Round.from_json(round_response) + + +def get_account(email: Optional[str] = None, account_id: Optional[str] = None) -> Account | None: + """Get an account from the account store using either an email address or + account_id. + + Args: + email (str, optional): The account email address + Defaults to None. + account_id (str, optional): The account id. Defaults to None. + + Raises: + TypeError: If both an email address or account id is given, + a TypeError is raised. + + Returns: + Account object or None + + """ + if email is account_id is None: + raise TypeError("Requires an email address or account_id") + + url = Config.ACCOUNT_STORE_API_HOST + Config.ACCOUNTS_ENDPOINT + params = {"email_address": email, "account_id": account_id} + response = get_data(url, params) + + if response and "account_id" in response: + return Account.from_json(response) + + +@functools.lru_cache(maxsize=1) +def get_round_name(fund_id, round_id): + response = get_data( + Config.FUND_STORE_API_HOST + Config.FUND_ROUND_ENDPOINT.format(fund_id=fund_id, round_id=round_id) + ) + if response: + return response.get("title") + + +def get_round_eoi_schema(fund_id, round_id, language=None): + language = {"language": language} + round_request_url = Config.FUND_ROUND_EOI_SCHEMA_ENDPOINT.format(fund_id=fund_id, round_id=round_id) + round_response = get_data(round_request_url, language) + return round_response diff --git a/services/apply/exceptions.py b/services/apply/exceptions.py new file mode 100644 index 00000000..9339c6dd --- /dev/null +++ b/services/apply/exceptions.py @@ -0,0 +1,30 @@ +class NotificationError(Exception): + """Exception raises an an error. + + Attributes: + message -- explanation of the error + + """ + + def __init__( + self, + message="Sorry, there was a problem posting to the notification service", # noqa + ): + self.message = message + super().__init__(self.message) + + +class SubmitError(Exception): + """Exception raises an an error. + + Attributes: + message -- explanation of the error + + """ + + def __init__( + self, + message="Sorry, there was a problem posting to the assessment service", # noqa + ): + self.message = message + super().__init__(self.message) diff --git a/services/apply/http_methods.py b/services/apply/http_methods.py new file mode 100644 index 00000000..63f23d82 --- /dev/null +++ b/services/apply/http_methods.py @@ -0,0 +1,46 @@ +import json +import os +from typing import Optional + +import requests +from config import Config +from flask import current_app +from services.apply.exceptions import NotificationError + + +def post_data(endpoint: str, json_payload: Optional[dict] = None) -> dict: + if Config.USE_LOCAL_DATA: + current_app.logger.info(f"Posting to local dummy endpoint: {endpoint}") + response = post_local_data(endpoint) + + else: + if json_payload: + json_payload = {k: v for k, v in json_payload.items() if v is not None} + current_app.logger.info(f"Attempting POST to the following endpoint: '{endpoint}'.") + response = requests.post(endpoint, json=json_payload) + + if response.status_code in [200, 201]: + current_app.logger.info(f"Post successfully sent to {endpoint} with response code: '{response.status_code}'.") + + return response.json() + + raise NotificationError( + message=( + "Sorry, the notification could not be sent for endpoint:" + f" '{endpoint}', params: '{json_payload}', response:" + f" '{response.json()}'" + ) + ) + + +def post_local_data(endpoint): + api_data_json = os.path.join(Config.FLASK_ROOT, "tests", "api_data", "post_endpoint_data.json") + with open(api_data_json) as json_file: + api_data = json.load(json_file) + if endpoint in api_data: + mocked_response = requests.models.Response() + mocked_response.status_code = 200 + content_str = json.dumps(api_data[endpoint]) + mocked_response._content = bytes(content_str, "utf-8") + return mocked_response + return None diff --git a/services/apply/models/account.py b/services/apply/models/account.py new file mode 100644 index 00000000..3dd59292 --- /dev/null +++ b/services/apply/models/account.py @@ -0,0 +1,12 @@ +from dataclasses import dataclass + + +@dataclass +class Account: + id: str + email: str + full_name: str + + @staticmethod + def from_json(data: dict): + return Account(id=data["account_id"], email=data["email_address"], full_name=data.get("full_name", "")) diff --git a/services/apply/models/fund.py b/services/apply/models/fund.py new file mode 100644 index 00000000..9bba2e6b --- /dev/null +++ b/services/apply/models/fund.py @@ -0,0 +1,36 @@ +from dataclasses import dataclass +from typing import Optional + +from flask import current_app +from services.apply.models.round import Round + + +@dataclass +class Fund: + name: str + identifier: str + short_name: str + description: str + welsh_available: bool + name_json: str + rounds: Optional[list[Round]] = None + + @staticmethod + def from_json(data: dict): + try: + return Fund( + name=data["name"], + identifier=data["id"], + short_name=data["short_name"], + description=data["description"], + welsh_available=data["welsh_available"], + name_json=data["name_json"], + ) + except AttributeError as e: + current_app.logger.error("Empty data passed to Fund.from_json") + raise e + + def add_round(self, fund_round: Round) -> None: + if not self.rounds: + self.rounds = [] + self.rounds.append(fund_round) diff --git a/services/apply/models/notification.py b/services/apply/models/notification.py new file mode 100644 index 00000000..025b8b34 --- /dev/null +++ b/services/apply/models/notification.py @@ -0,0 +1,61 @@ +import json +from uuid import uuid4 + +from config import Config +from flask import current_app +from services.apply.exceptions import NotificationError + +NOTIFICATION_CONST = "notification" +NOTIFICATION_S3_KEY_CONST = "application/notification" + + +class Notification: + """Class for holding Notification operations.""" + + @staticmethod + def send(template_type: str, to_email: str, full_name: str, content: dict): + """Sends a notification using the Gov.UK Notify Service. + + Args: + template_type: (str) A key of the template to use in the + DLUHC notifications service (which maps to a + Notify Service template key) + to_email: (str) The email to send the notification to + content: (dict) A dictionary of content to send to + fill out the notification template + + """ + json_payload = { + "type": template_type, + "to": to_email, + "full_name": full_name, + "content": content, + } + current_app.logger.info(f" json payload '{template_type}' to '{to_email}'.") + try: + sqs_extended_client = Notification._get_sqs_client() + message_id = sqs_extended_client.submit_single_message( + queue_url=Config.AWS_SQS_NOTIF_APP_PRIMARY_QUEUE_URL, + message=json.dumps(json_payload), + message_group_id=NOTIFICATION_CONST, + message_deduplication_id=str(uuid4()), # ensures message uniqueness + extra_attributes={ + "S3Key": { + "StringValue": NOTIFICATION_S3_KEY_CONST, + "DataType": "String", + }, + }, + ) + current_app.logger.info(f"Message sent to SQS queue and message id is [{message_id}]") + return message_id + except Exception as e: + current_app.logger.error("An error occurred while sending message") + current_app.logger.error(e) + raise NotificationError(message="Sorry, the notification could not be sent") + + @staticmethod + def _get_sqs_client(): + sqs_extended_client = current_app.extensions["sqs_extended_client"] + if sqs_extended_client is not None: + return sqs_extended_client + current_app.logger.error("An error occurred while sending message since client is not available") diff --git a/services/apply/models/round.py b/services/apply/models/round.py new file mode 100644 index 00000000..783c1150 --- /dev/null +++ b/services/apply/models/round.py @@ -0,0 +1,61 @@ +import inspect +from dataclasses import dataclass +from typing import Optional + + +@dataclass +class FeedbackSurveyConfig: + has_feedback_survey: bool = False + is_feedback_survey_optional: bool = True + has_section_feedback: bool = False + is_section_feedback_optional: bool = True + has_research_survey: bool = False + is_research_survey_optional: bool = True + + @staticmethod + def from_json(d: dict): + # Filter unknown fields from JSON dictionary + return FeedbackSurveyConfig( + **{k: v for k, v in d.items() if k in inspect.signature(FeedbackSurveyConfig).parameters} + ) + + +@dataclass +class Round: + id: str + assessment_deadline: str + deadline: str + fund_id: str + opens: str + title: str + short_name: str + contact_email: str + title_json: str + project_name_field_id: Optional[str] = None + mark_as_complete_enabled: bool = False + is_expression_of_interest: bool = False + feedback_survey_config: FeedbackSurveyConfig = None + + def __post_init__(self): + if isinstance(self.feedback_survey_config, dict): + self.feedback_survey_config = FeedbackSurveyConfig.from_json(self.feedback_survey_config) + elif self.feedback_survey_config is None: + self.feedback_survey_config = FeedbackSurveyConfig() + + @staticmethod + def from_json(data: dict): + return Round( + title=data["title"], + id=data["id"], + fund_id=data["fund_id"], + short_name=data["short_name"], + opens=data["opens"], + deadline=data["deadline"], + assessment_deadline=data["assessment_deadline"], + project_name_field_id=data.get("project_name_field_id", None), + contact_email=data.get("contact_email", None), + feedback_survey_config=data.get("feedback_survey_config") or FeedbackSurveyConfig(), + mark_as_complete_enabled=data.get("mark_as_complete_enabled") or False, + is_expression_of_interest=data.get("is_expression_of_interest") or False, + title_json=data["title_json"], + )