Penison 91844 Failed Benefits Intake Remediation Reporting (#18339)

.github/CODEOWNERS

@@ -608,6 +608,7 @@ app/uploaders/validate_pdf.rb @department-of-veterans-affairs/Disability-Experie
 app/uploaders/vets_shrine.rb @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 app/validators/token_util.rb @department-of-veterans-affairs/backend-review-group
 app/sidekiq/account_login_statistics_job.rb @department-of-veterans-affairs/octo-identity
+app/sidekiq/benefits_intake_remediation_status_job.rb @department-of-veterans-affairs/platform-va-product-forms @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 app/sidekiq/benefits_intake_status_job.rb @department-of-veterans-affairs/platform-va-product-forms @department-of-veterans-affairs/Disability-Experience @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 app/sidekiq/bgs @department-of-veterans-affairs/benefits-dependents-management @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 app/sidekiq/central_mail/delete_old_claims.rb @department-of-veterans-affairs/mbs-core-team @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group

app/models/form_submission_attempt.rb

@@ -33,6 +33,10 @@ class FormSubmissionAttempt < ApplicationRecord
       transitions from: :pending, to: :vbms
       transitions from: :success, to: :vbms
     end
+
+    event :remediate do
+      transitions from: :failure, to: :vbms
+    end
   end
 
   def log_status_change

app/models/saved_claim/pension.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+##
+# Pension 21P-527EZ Active::Record
+# proxy for backwards compatibility
+#
+# @see modules/pensions/app/models/pensions/saved_claim.rb
+#
+class SavedClaim::Pension < SavedClaim
+  # form_id, form_type
+  FORM = '21P-527EZ'
+end

app/sidekiq/benefits_intake_remediation_status_job.rb

@@ -0,0 +1,143 @@
+# frozen_string_literal: true
+
+require 'lighthouse/benefits_intake/service'
+
+# Reporting job for Lighthouse Benefit Intake Failures
+# @see https://vagov.ddog-gov.com/dashboard/8zk-ja2-xvm/benefits-intake-submission-remediation-report
+class BenefitsIntakeRemediationStatusJob
+  include Sidekiq::Job
+
+  sidekiq_options retry: false
+
+  # metrics key
+  STATS_KEY = 'api.benefits_intake.remediation_status'
+
+  # job batch size
+  BATCH_SIZE = Settings.lighthouse.benefits_intake.report.batch_size || 1000
+
+  # create an instance
+  def initialize(batch_size: BATCH_SIZE)
+    @batch_size = batch_size
+    @total_handled = 0
+  end
+
+  # search all submissions for outstanding failures
+  # poll LH endpoint to see if status has changed (case if endpoint had an error initially)
+  # report stats on submissions, grouped by form-type
+  def perform
+    Rails.logger.info('BenefitsIntakeRemediationStatusJob started')
+
+    form_submissions = FormSubmission.includes(:form_submission_attempts)
+    failures = outstanding_failures(form_submissions.all)
+
+    batch_process(failures) unless failures.empty?
+
+    submission_audit
+
+    Rails.logger.info('BenefitsIntakeRemediationStatusJob ended', total_handled:)
+  end
+
+  private
+
+  attr_reader :batch_size, :total_handled
+
+  # determine if a claim has an outstanding failure
+  # each claim can have multiple FormSubmission, which can have multiple FormSubmissionAttempt
+  # conflate these and search for a non-failure, which rejects the claim from the list
+  #
+  # @param submissions [Array<FormSubmission>]
+  #
+  def outstanding_failures(submissions)
+    failures = submissions.group_by(&:saved_claim_id)
+    failures.map do |_claim_id, fs|
+      fs.sort_by!(&:created_at)
+      attempts = fs.map(&:form_submission_attempts).flatten.sort_by(&:created_at)
+      not_failure = attempts.find { |att| att.aasm_state != 'failure' }
+      not_failure ? nil : fs.last
+    end.compact
+  end
+
+  # perform a bulk_status check in Lighthouse to retrieve current statuses
+  # a processing error will abort the job (no retries)
+  #
+  # @param failures [Array<FormSubmission>] submissions with only 'failure' statuses
+  #
+  def batch_process(failures)
+    intake_service = BenefitsIntake::Service.new
+
+    failures.each_slice(batch_size) do |batch|
+      batch_uuids = batch.map(&:benefits_intake_uuid)
+      Rails.logger.info('BenefitsIntakeRemediationStatusJob processing batch', batch_uuids:)
+
+      response = intake_service.bulk_status(uuids: batch_uuids)
+      raise response.body unless response.success?
+
+      next unless (data = response.body['data'])
+
+      handle_response(data, batch)
+    end
+  rescue => e
+    Rails.logger.error('BenefitsIntakeRemediationStatusJob ERROR processing batch', class: self.class.name,
+                                                                                    message: e.message)
+  end
+
+  # process response from Lighthouse to update outstanding failures
+  #
+  # @param response_date [Hash] Lighthouse Benefits Intake API response
+  # @param failure_batch [Array<FormSubmission>] current batch being processed
+  #
+  def handle_response(response_data, failure_batch)
+    response_data.each do |submission|
+      uuid = submission['id']
+      form_submission = failure_batch.find do |submission_from_db|
+        submission_from_db.benefits_intake_uuid == uuid
+      end
+      form_submission.form_type
+
+      form_submission_attempt = form_submission.form_submission_attempts.last
+
+      # https://developer.va.gov/explore/api/benefits-intake/docs
+      status = submission.dig('attributes', 'status')
+      lighthouse_updated_at = submission.dig('attributes', 'updated_at')
+      if status == 'vbms'
+        # submission was successfully uploaded into a Veteran's eFolder within VBMS
+        form_submission_attempt.update(lighthouse_updated_at:)
+        form_submission_attempt.remediate!
+      end
+
+      @total_handled = total_handled + 1
+    end
+  end
+
+  # gather metrics - grouped by form type
+  # - unsubmitted: list of SavedClaim ids that do not have a FormSubmission record
+  # - orphaned: list of saved_claim_ids with a FormSubmission, but no SavedClaim
+  # - failures:  list of outstanding failures
+  def submission_audit
+    # requery form_submissions in case there was an update
+    form_submissions = FormSubmission.includes(:form_submission_attempts)
+    form_submission_groups = form_submissions.all.group_by(&:form_type)
+
+    form_submission_groups.each do |form_id, submissions|
+      fs_saved_claim_ids = submissions.map(&:saved_claim_id).uniq
+
+      claims = SavedClaim.where(form_id:).where('id >= ?', fs_saved_claim_ids.min)
+      claim_ids = claims.map(&:id).uniq
+
+      unsubmitted = claim_ids - fs_saved_claim_ids
+      orphaned = fs_saved_claim_ids - claim_ids
+
+      failures = outstanding_failures(submissions)
+      failures.map! do |fs|
+        last_attempt = fs.form_submission_attempts.max_by(&:created_at)
+        { claim_id: fs.saved_claim_id, uuid: fs.benefits_intake_uuid, error_message: last_attempt.error_message }
+      end
+
+      StatsD.set("#{STATS_KEY}.#{form_id}.unsubmitted_claims", unsubmitted.length)
+      StatsD.set("#{STATS_KEY}.#{form_id}.orphaned_submissions", orphaned.length)
+      StatsD.set("#{STATS_KEY}.#{form_id}.outstanding_failures", failures.length)
+      Rails.logger.info("BenefitsIntakeRemediationStatusJob submission audit #{form_id}", form_id:, unsubmitted:,
+                                                                                          orphaned:, failures:)
+    end
+  end
+end

lib/periodic_jobs.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+# @see https://crontab.guru/
+# @see https://en.wikipedia.org/wiki/Cron
 PERIODIC_JOBS = lambda { |mgr| # rubocop:disable Metrics/BlockLength
   mgr.tz = ActiveSupport::TimeZone.new('America/New_York')
 
@@ -49,9 +51,12 @@
   # Update static data cache
   mgr.register('0 0 * * *', 'Crm::TopicsDataJob')
 
-  # Update static data cache for form 526
+  # Update FormSubmissionAttempt status from Lighthouse Benefits Intake API
   mgr.register('0 0 * * *', 'BenefitsIntakeStatusJob')
 
+  # Generate FormSubmissionAttempt rememdiation statistics from Lighthouse Benefits Intake API
+  mgr.register('0 1 * * 1', 'BenefitsIntakeRemediationStatusJob')
+
   # Update Lighthouse526DocumentUpload statuses according to Lighthouse Benefits Documents service tracking
   mgr.register('15 * * * *', 'Form526DocumentUploadPollingJob')
 

modules/pensions/app/models/pensions/saved_claim.rb

@@ -5,13 +5,25 @@ module Pensions
   # Pension 21P-527EZ Active::Record
   # @see app/model/saved_claim
   #
+  # todo: migrate encryption to Pensions::SavedClaim, remove inheritance and encrytion shim
+  #
   class SavedClaim < ::SavedClaim
     # We want to use the `Type` behavior but we want to override it with our custom type default scope behaviors.
     self.inheritance_column = :_type_disabled
 
     # We want to override the `Type` behaviors for backwards compatability
     default_scope -> { where(type: 'SavedClaim::Pension') }, all_queries: true
 
+    ##
+    # The KMS Encryption Context is preserved from the saved claim model namespace we migrated from
+    #
+    def kms_encryption_context
+      {
+        model_name: 'SavedClaim::Pension',
+        model_id: id
+      }
+    end
+
     # form_id, form_type
     FORM = '21P-527EZ'
 
@@ -90,15 +102,5 @@ def upload_to_lighthouse(current_user = nil)
 
       Pensions::PensionBenefitIntakeJob.perform_async(id, current_user&.user_account_uuid)
     end
-
-    ##
-    # The KMS Encryption Context is preserved from the saved claim model namespace we migrated from
-    #
-    def kms_encryption_context
-      {
-        model_name: 'SavedClaim::Pension',
-        model_id: id
-      }
-    end
   end
 end