diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index a811277b4d9..707eceebf83 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -608,6 +608,7 @@ app/uploaders/validate_pdf.rb @department-of-veterans-affairs/Disability-Experie app/uploaders/vets_shrine.rb @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group app/validators/token_util.rb @department-of-veterans-affairs/backend-review-group app/sidekiq/account_login_statistics_job.rb @department-of-veterans-affairs/octo-identity +app/sidekiq/benefits_intake_remediation_status_job.rb @department-of-veterans-affairs/platform-va-product-forms @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group app/sidekiq/benefits_intake_status_job.rb @department-of-veterans-affairs/platform-va-product-forms @department-of-veterans-affairs/Disability-Experience @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group app/sidekiq/bgs @department-of-veterans-affairs/benefits-dependents-management @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group app/sidekiq/central_mail/delete_old_claims.rb @department-of-veterans-affairs/mbs-core-team @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group diff --git a/app/models/form_submission_attempt.rb b/app/models/form_submission_attempt.rb index ac65a57eb8f..93bda87e244 100644 --- a/app/models/form_submission_attempt.rb +++ b/app/models/form_submission_attempt.rb @@ -33,6 +33,10 @@ class FormSubmissionAttempt < ApplicationRecord transitions from: :pending, to: :vbms transitions from: :success, to: :vbms end + + event :remediate do + transitions from: :failure, to: :vbms + end end def log_status_change diff --git a/app/models/saved_claim/pension.rb b/app/models/saved_claim/pension.rb new file mode 100644 index 00000000000..a05f990a1ec --- /dev/null +++ b/app/models/saved_claim/pension.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +## +# Pension 21P-527EZ Active::Record +# proxy for backwards compatibility +# +# @see modules/pensions/app/models/pensions/saved_claim.rb +# +class SavedClaim::Pension < SavedClaim + # form_id, form_type + FORM = '21P-527EZ' +end diff --git a/app/sidekiq/benefits_intake_remediation_status_job.rb b/app/sidekiq/benefits_intake_remediation_status_job.rb new file mode 100644 index 00000000000..3d7c304ab54 --- /dev/null +++ b/app/sidekiq/benefits_intake_remediation_status_job.rb @@ -0,0 +1,143 @@ +# frozen_string_literal: true + +require 'lighthouse/benefits_intake/service' + +# Reporting job for Lighthouse Benefit Intake Failures +# @see https://vagov.ddog-gov.com/dashboard/8zk-ja2-xvm/benefits-intake-submission-remediation-report +class BenefitsIntakeRemediationStatusJob + include Sidekiq::Job + + sidekiq_options retry: false + + # metrics key + STATS_KEY = 'api.benefits_intake.remediation_status' + + # job batch size + BATCH_SIZE = Settings.lighthouse.benefits_intake.report.batch_size || 1000 + + # create an instance + def initialize(batch_size: BATCH_SIZE) + @batch_size = batch_size + @total_handled = 0 + end + + # search all submissions for outstanding failures + # poll LH endpoint to see if status has changed (case if endpoint had an error initially) + # report stats on submissions, grouped by form-type + def perform + Rails.logger.info('BenefitsIntakeRemediationStatusJob started') + + form_submissions = FormSubmission.includes(:form_submission_attempts) + failures = outstanding_failures(form_submissions.all) + + batch_process(failures) unless failures.empty? + + submission_audit + + Rails.logger.info('BenefitsIntakeRemediationStatusJob ended', total_handled:) + end + + private + + attr_reader :batch_size, :total_handled + + # determine if a claim has an outstanding failure + # each claim can have multiple FormSubmission, which can have multiple FormSubmissionAttempt + # conflate these and search for a non-failure, which rejects the claim from the list + # + # @param submissions [Array] + # + def outstanding_failures(submissions) + failures = submissions.group_by(&:saved_claim_id) + failures.map do |_claim_id, fs| + fs.sort_by!(&:created_at) + attempts = fs.map(&:form_submission_attempts).flatten.sort_by(&:created_at) + not_failure = attempts.find { |att| att.aasm_state != 'failure' } + not_failure ? nil : fs.last + end.compact + end + + # perform a bulk_status check in Lighthouse to retrieve current statuses + # a processing error will abort the job (no retries) + # + # @param failures [Array] submissions with only 'failure' statuses + # + def batch_process(failures) + intake_service = BenefitsIntake::Service.new + + failures.each_slice(batch_size) do |batch| + batch_uuids = batch.map(&:benefits_intake_uuid) + Rails.logger.info('BenefitsIntakeRemediationStatusJob processing batch', batch_uuids:) + + response = intake_service.bulk_status(uuids: batch_uuids) + raise response.body unless response.success? + + next unless (data = response.body['data']) + + handle_response(data, batch) + end + rescue => e + Rails.logger.error('BenefitsIntakeRemediationStatusJob ERROR processing batch', class: self.class.name, + message: e.message) + end + + # process response from Lighthouse to update outstanding failures + # + # @param response_date [Hash] Lighthouse Benefits Intake API response + # @param failure_batch [Array] current batch being processed + # + def handle_response(response_data, failure_batch) + response_data.each do |submission| + uuid = submission['id'] + form_submission = failure_batch.find do |submission_from_db| + submission_from_db.benefits_intake_uuid == uuid + end + form_submission.form_type + + form_submission_attempt = form_submission.form_submission_attempts.last + + # https://developer.va.gov/explore/api/benefits-intake/docs + status = submission.dig('attributes', 'status') + lighthouse_updated_at = submission.dig('attributes', 'updated_at') + if status == 'vbms' + # submission was successfully uploaded into a Veteran's eFolder within VBMS + form_submission_attempt.update(lighthouse_updated_at:) + form_submission_attempt.remediate! + end + + @total_handled = total_handled + 1 + end + end + + # gather metrics - grouped by form type + # - unsubmitted: list of SavedClaim ids that do not have a FormSubmission record + # - orphaned: list of saved_claim_ids with a FormSubmission, but no SavedClaim + # - failures: list of outstanding failures + def submission_audit + # requery form_submissions in case there was an update + form_submissions = FormSubmission.includes(:form_submission_attempts) + form_submission_groups = form_submissions.all.group_by(&:form_type) + + form_submission_groups.each do |form_id, submissions| + fs_saved_claim_ids = submissions.map(&:saved_claim_id).uniq + + claims = SavedClaim.where(form_id:).where('id >= ?', fs_saved_claim_ids.min) + claim_ids = claims.map(&:id).uniq + + unsubmitted = claim_ids - fs_saved_claim_ids + orphaned = fs_saved_claim_ids - claim_ids + + failures = outstanding_failures(submissions) + failures.map! do |fs| + last_attempt = fs.form_submission_attempts.max_by(&:created_at) + { claim_id: fs.saved_claim_id, uuid: fs.benefits_intake_uuid, error_message: last_attempt.error_message } + end + + StatsD.set("#{STATS_KEY}.#{form_id}.unsubmitted_claims", unsubmitted.length) + StatsD.set("#{STATS_KEY}.#{form_id}.orphaned_submissions", orphaned.length) + StatsD.set("#{STATS_KEY}.#{form_id}.outstanding_failures", failures.length) + Rails.logger.info("BenefitsIntakeRemediationStatusJob submission audit #{form_id}", form_id:, unsubmitted:, + orphaned:, failures:) + end + end +end diff --git a/lib/periodic_jobs.rb b/lib/periodic_jobs.rb index c3c58d9b4af..828491aa91c 100644 --- a/lib/periodic_jobs.rb +++ b/lib/periodic_jobs.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +# @see https://crontab.guru/ +# @see https://en.wikipedia.org/wiki/Cron PERIODIC_JOBS = lambda { |mgr| # rubocop:disable Metrics/BlockLength mgr.tz = ActiveSupport::TimeZone.new('America/New_York') @@ -49,9 +51,12 @@ # Update static data cache mgr.register('0 0 * * *', 'Crm::TopicsDataJob') - # Update static data cache for form 526 + # Update FormSubmissionAttempt status from Lighthouse Benefits Intake API mgr.register('0 0 * * *', 'BenefitsIntakeStatusJob') + # Generate FormSubmissionAttempt rememdiation statistics from Lighthouse Benefits Intake API + mgr.register('0 1 * * 1', 'BenefitsIntakeRemediationStatusJob') + # Update Lighthouse526DocumentUpload statuses according to Lighthouse Benefits Documents service tracking mgr.register('15 * * * *', 'Form526DocumentUploadPollingJob') diff --git a/modules/pensions/app/models/pensions/saved_claim.rb b/modules/pensions/app/models/pensions/saved_claim.rb index 0dcebc5a82e..33f3c9eeef0 100644 --- a/modules/pensions/app/models/pensions/saved_claim.rb +++ b/modules/pensions/app/models/pensions/saved_claim.rb @@ -5,6 +5,8 @@ module Pensions # Pension 21P-527EZ Active::Record # @see app/model/saved_claim # + # todo: migrate encryption to Pensions::SavedClaim, remove inheritance and encrytion shim + # class SavedClaim < ::SavedClaim # We want to use the `Type` behavior but we want to override it with our custom type default scope behaviors. self.inheritance_column = :_type_disabled @@ -12,6 +14,16 @@ class SavedClaim < ::SavedClaim # We want to override the `Type` behaviors for backwards compatability default_scope -> { where(type: 'SavedClaim::Pension') }, all_queries: true + ## + # The KMS Encryption Context is preserved from the saved claim model namespace we migrated from + # + def kms_encryption_context + { + model_name: 'SavedClaim::Pension', + model_id: id + } + end + # form_id, form_type FORM = '21P-527EZ' @@ -90,15 +102,5 @@ def upload_to_lighthouse(current_user = nil) Pensions::PensionBenefitIntakeJob.perform_async(id, current_user&.user_account_uuid) end - - ## - # The KMS Encryption Context is preserved from the saved claim model namespace we migrated from - # - def kms_encryption_context - { - model_name: 'SavedClaim::Pension', - model_id: id - } - end end end