Merge remote-tracking branch 'origin/8145-appeal-fix' into 8145-appea…

…l-fix

.github/CODEOWNERS

@@ -642,10 +642,11 @@ app/sidekiq/feature_cleaner_job.rb @department-of-veterans-affairs/va-api-engine
 app/sidekiq/form1010cg @department-of-veterans-affairs/vfs-10-10 @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 app/sidekiq/form1095 @department-of-veterans-affairs/vfs-1095-b @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 app/sidekiq/form526_confirmation_email_job.rb @department-of-veterans-affairs/Disability-Experience @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
+app/sidekiq/form526_failure_state_snapshot_job.rb @department-of-veterans-affairs/disability-experience @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 app/sidekiq/form526_paranoid_success_polling_job.rb @department-of-veterans-affairs/Disability-Experience @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
-app/sidekiq/form526_state_logging_job.rb @department-of-veterans-affairs/disability-experience @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 app/sidekiq/form526_status_polling_job.rb @department-of-veterans-affairs/Disability-Experience @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 app/sidekiq/form526_submission_failed_email_job.rb @department-of-veterans-affairs/Disability-Experience @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
+app/sidekiq/form526_submission_processing_report_job.rb @department-of-veterans-affairs/disability-experience @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 app/sidekiq/gi_bill_feedback_submission_job.rb @department-of-veterans-affairs/my-education-benefits @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 app/sidekiq/hca @department-of-veterans-affairs/vfs-10-10 @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 app/sidekiq/identity @department-of-veterans-affairs/octo-identity
@@ -1343,9 +1344,10 @@ spec/sidekiq/facilities @department-of-veterans-affairs/vfs-facilities-frontend
 spec/sidekiq/form1010cg @department-of-veterans-affairs/vfs-10-10 @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 spec/sidekiq/form1095 @department-of-veterans-affairs/vfs-1095-b @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 spec/sidekiq/form526_confirmation_email_job_spec.rb @department-of-veterans-affairs/Disability-Experience @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
+spec/sidekiq/form526_failure_state_snapshot_job_spec.rb @department-of-veterans-affairs/disability-experience @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 spec/sidekiq/form526_paranoid_success_polling_job_spec.rb @department-of-veterans-affairs/Disability-Experience @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
-spec/sidekiq/form526_state_logging_job_spec.rb @department-of-veterans-affairs/disability-experience @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 spec/sidekiq/form526_status_polling_job_spec.rb @department-of-veterans-affairs/Disability-Experience @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
+spec/sidekiq/form526_submission_processing_report_job_spec.rb @department-of-veterans-affairs/disability-experience @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 spec/sidekiq/form5655 @department-of-veterans-affairs/vsa-debt-resolution @department-of-veterans-affairs/backend-review-group
 spec/sidekiq/gi_bill_feedback_submission_job_spec.rb @department-of-veterans-affairs/my-education-benefits @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 spec/sidekiq/hca @department-of-veterans-affairs/vfs-10-10 @department-of-veterans-affairs/backend-review-group @department-of-veterans-affairs/va-api-engineers
@@ -2106,15 +2108,10 @@ app/controllers/v0/profile/contacts_controller.rb @department-of-veterans-affair
 app/serializers/contact_serializer.rb @department-of-veterans-affairs/vfs-authenticated-experience-backend @department-of-veterans-affairs/vfs-mhv-integration @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 app/swagger/swagger/requests/profile.rb @department-of-veterans-affairs/vfs-authenticated-experience-backend @department-of-veterans-affairs/vfs-mhv-integration @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 app/swagger/swagger/schemas/contacts.rb @department-of-veterans-affairs/vfs-authenticated-experience-backend @department-of-veterans-affairs/vfs-mhv-integration @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
-lib/va_profile/health_benefit/associated_persons_response.rb @department-of-veterans-affairs/vfs-authenticated-experience-backend @department-of-veterans-affairs/vfs-mhv-integration @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
-lib/va_profile/health_benefit/configuration.rb @department-of-veterans-affairs/vfs-authenticated-experience-backend @department-of-veterans-affairs/vfs-mhv-integration @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
-lib/va_profile/health_benefit/service.rb @department-of-veterans-affairs/vfs-authenticated-experience-backend @department-of-veterans-affairs/vfs-mhv-integration @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 lib/va_profile/models/associated_person.rb @department-of-veterans-affairs/vfs-authenticated-experience-backend @department-of-veterans-affairs/vfs-mhv-integration @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 lib/va_profile/profile/v3/health_benefit_bio_response.rb @department-of-veterans-affairs/vfs-authenticated-experience-backend @department-of-veterans-affairs/vfs-mhv-integration @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 lib/va_profile/profile/v3/service.rb @department-of-veterans-affairs/vfs-authenticated-experience-backend @department-of-veterans-affairs/vfs-mhv-integration @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 spec/controllers/v0/profile/contacts_controller_spec.rb @department-of-veterans-affairs/vfs-authenticated-experience-backend @department-of-veterans-affairs/vfs-mhv-integration @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
-spec/fixtures/va_profile/health_benefit_v1_associated_persons.json @department-of-veterans-affairs/vfs-authenticated-experience-backend @department-of-veterans-affairs/vfs-mhv-integration @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
-spec/lib/va_profile/health_benefit/service_spec.rb @department-of-veterans-affairs/vfs-authenticated-experience-backend @department-of-veterans-affairs/vfs-mhv-integration @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 spec/lib/va_profile/profile/v3/service_spec.rb @department-of-veterans-affairs/vfs-authenticated-experience-backend @department-of-veterans-affairs/vfs-mhv-integration @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 lib/logging/third_party_transaction.rb @department-of-veterans-affairs/backend-review-group
 spec/lib/logging/third_party_transaction_spec.rb @department-of-veterans-affairs/backend-review-group
@@ -2131,4 +2128,5 @@ README.md @department-of-veterans-affairs/backend-review-group
 modules/accredited_representative_portal/spec/services/accredited_representative_portal/representative_user_loader_spec.rb @department-of-veterans-affairs/octo-identity
 modules/accredited_representative_portal/app/services/accredited_representative_portal/representative_user_loader.rb @department-of-veterans-affairs/octo-identity
 config/form_profile_mappings/FORM-MOCK-AE-DESIGN-PATTERNS.yml @department-of-veterans-affairs/tmf-auth-exp-design-patterns @department-of-veterans-affairs/backend-review-group
-vets-api.pm-collection.json @department-of-veterans-affairs/backend-review-group
+postman/vets-api.pm-collection.json @department-of-veterans-affairs/backend-review-group
+postman/Dockerfile @department-of-veterans-affairs/backend-review-group

.github/workflows/build-and-publish.yaml

@@ -33,7 +33,17 @@ jobs:
         uses: aws-actions/amazon-ecr-login@v2.0.1
         with:
           mask-password: true
-      - name: Build Docker Image
+      - name: Build Postman Image
+        uses: docker/build-push-action@v6
+        env:
+          DOCKER_BUILD_SUMMARY: false
+        with:
+          context: ./postman
+          file: ./postman/Dockerfile
+          push: true
+          tags: |
+            ${{ steps.ecr-login.outputs.registry }}/dsva/vets-api-postman:${{ steps.version.outputs.version }}
+      - name: Build vets-api Docker Image
         uses: docker/build-push-action@v6
         env:
           DOCKER_BUILD_SUMMARY: false

.github/workflows/build.yml

@@ -30,8 +30,17 @@ jobs:
       - name: Login to ECR
         id: login-ecr
         uses: aws-actions/amazon-ecr-login@v2
-
-      - name: Build Docker Image
+      - name: Build Postman Docker Image
+        uses: docker/build-push-action@v6
+        env:
+          DOCKER_BUILD_SUMMARY: false
+        with:
+          context: ./postman
+          file: ./postman/Dockerfile
+          push: true
+          tags: |
+            ${{ steps.login-ecr.outputs.registry }}/dsva/vets-api-postman:${{ github.sha }}
+      - name: Build vets-api Docker Image
         uses: docker/build-push-action@v6
         env:
           DOCKER_BUILD_SUMMARY: false

.github/workflows/code_checks.yml

@@ -135,14 +135,9 @@ jobs:
 
       - name: Run Specs
         timeout-minutes: 20
-        uses: nick-fields/retry@7152eba30c6575329ac0576536151aca5a72780e # v3.0.0
-        with:
-          timeout_minutes: 20
-          retry_wait_seconds: 3 # Seconds
-          max_attempts: 3
-          command: |
-            docker compose -f docker-compose.test.yml run web bash \
-              -c "CI=true DISABLE_BOOTSNAP=true bundle exec parallel_rspec spec/ modules/ -n 13 -o '--color --tty'"
+        run: |
+          docker compose -f docker-compose.test.yml run web bash \
+          -c "CI=true DISABLE_BOOTSNAP=true bundle exec parallel_rspec spec/ modules/ -n 13 -o '--color --tty'"
 
       - name: Upload Coverage Report
         uses: actions/upload-artifact@v4

Gemfile.lock

@@ -512,9 +512,14 @@ GEM
       thor (>= 0.20, < 2.a)
     google-cloud-env (2.1.1)
       faraday (>= 1.0, < 3.a)
-    google-protobuf (4.28.1)
+    google-protobuf (4.28.2)
       bigdecimal
       rake (>= 13)
+    google-protobuf (4.28.2-java)
+      bigdecimal
+      ffi (~> 1)
+      ffi-compiler (~> 1)
+      rake (>= 13)
     googleauth (1.11.0)
       faraday (>= 1.0, < 3.a)
       google-cloud-env (~> 2.1)
@@ -647,7 +652,7 @@ GEM
     mini_mime (1.1.5)
     mini_portile2 (2.8.7)
     minitest (5.25.1)
-    mock_redis (0.44.0)
+    mock_redis (0.45.0)
     msgpack (1.7.2)
     msgpack (1.7.2-java)
     multi_json (1.15.0)
@@ -764,9 +769,9 @@ GEM
     psych (5.1.2-java)
       jar-dependencies (>= 0.1.7)
     public_suffix (6.0.1)
-    puma (6.4.2)
+    puma (6.4.3)
       nio4r (~> 2.0)
-    puma (6.4.2-java)
+    puma (6.4.3-java)
       nio4r (~> 2.0)
     pundit (2.4.0)
       activesupport (>= 3.0.0)

app/models/form_submission_attempt.rb

@@ -74,7 +74,7 @@ def log_status_change
 
   def enqueue_result_email(notification_type)
     config = {
-      form_data: form_submission.form_data,
+      form_data: JSON.parse(form_submission.form_data),
       form_number: form_submission.form_type,
       confirmation_number: form_submission.benefits_intake_uuid,
       date_submitted: created_at.strftime('%B %d, %Y'),

app/services/mhv/user_account/creator.rb

@@ -5,11 +5,11 @@
 module MHV
   module UserAccount
     class Creator
-      attr_reader :user_verification, :cached
+      attr_reader :user_verification, :break_cache
 
-      def initialize(user_verification:, cached: true)
+      def initialize(user_verification:, break_cache: false)
         @user_verification = user_verification
-        @cached = cached
+        @break_cache = break_cache
       end
 
       def perform
@@ -33,8 +33,9 @@ def create_mhv_user_account!
       end
 
       def mhv_account_creation_response
-        MHV::AccountCreation::Service.new
-                                     .create_account(icn:, email:, tou_occurred_at: current_tou_agreement.created_at)
+        tou_occurred_at = current_tou_agreement.created_at
+
+        mhv_client.create_account(icn:, email:, tou_occurred_at:, break_cache:)
       end
 
       def icn
@@ -53,6 +54,10 @@ def user_account
         @user_account ||= user_verification.user_account
       end
 
+      def mhv_client
+        MHV::AccountCreation::Service.new
+      end
+
       def validate!
         errors = [
           ('ICN must be present' if icn.blank?),

app/sidekiq/benefits_intake_remediation_status_job.rb

@@ -24,12 +24,15 @@ def initialize(batch_size: BATCH_SIZE)
   # search all submissions for outstanding failures
   # poll LH endpoint to see if status has changed (case if endpoint had an error initially)
   # report stats on submissions, grouped by form-type
-  def perform
+  def perform(form_id = nil)
     Rails.logger.info('BenefitsIntakeRemediationStatusJob started')
 
     form_submissions = FormSubmission.includes(:form_submission_attempts)
     failures = outstanding_failures(form_submissions.all)
 
+    @form_id = form_id
+    failures.select! { |f| f.form_type == form_id } if form_id
+
     batch_process(failures) unless failures.empty?
 
     submission_audit
@@ -39,7 +42,7 @@ def perform
 
   private
 
-  attr_reader :batch_size, :total_handled
+  attr_reader :batch_size, :total_handled, :form_id
 
   # determine if a claim has an outstanding failure
   # each claim can have multiple FormSubmission, which can have multiple FormSubmissionAttempt
@@ -118,10 +121,12 @@ def submission_audit
     form_submissions = FormSubmission.includes(:form_submission_attempts)
     form_submission_groups = form_submissions.all.group_by(&:form_type)
 
-    form_submission_groups.each do |form_id, submissions|
+    form_submission_groups.each do |form_type, submissions|
+      next if form_id && form_id != form_type
+
       fs_saved_claim_ids = submissions.map(&:saved_claim_id).uniq
 
-      claims = SavedClaim.where(form_id:).where('id >= ?', fs_saved_claim_ids.min)
+      claims = SavedClaim.where(form_id: form_type).where('id >= ?', fs_saved_claim_ids.min)
       claim_ids = claims.map(&:id).uniq
 
       unsubmitted = claim_ids - fs_saved_claim_ids
@@ -133,9 +138,9 @@ def submission_audit
         { claim_id: fs.saved_claim_id, uuid: fs.benefits_intake_uuid, error_message: last_attempt.error_message }
       end
 
-      StatsD.set("#{STATS_KEY}.#{form_id}.unsubmitted_claims", unsubmitted.length)
-      StatsD.set("#{STATS_KEY}.#{form_id}.orphaned_submissions", orphaned.length)
-      StatsD.set("#{STATS_KEY}.#{form_id}.outstanding_failures", failures.length)
+      StatsD.gauge("#{STATS_KEY}.unsubmitted_claims", unsubmitted.length, tags: ["form_id:#{form_type}"])
+      StatsD.gauge("#{STATS_KEY}.orphaned_submissions", orphaned.length, tags: ["form_id:#{form_type}"])
+      StatsD.gauge("#{STATS_KEY}.outstanding_failures", failures.length, tags: ["form_id:#{form_type}"])
       Rails.logger.info("BenefitsIntakeRemediationStatusJob submission audit #{form_id}", form_id:, unsubmitted:,
                                                                                           orphaned:, failures:)
     end

app/sidekiq/form526_failure_state_snapshot_job.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+# Log information about Form526Submission state to populate an admin facing Datadog dashboard
+class Form526FailureStateSnapshotJob
+  include Sidekiq::Job
+  sidekiq_options retry: false
+
+  STATSD_PREFIX = 'form526.state.snapshot'
+
+  def perform
+    write_failure_snapshot
+  rescue => e
+    Rails.logger.error('Error logging 526 state snapshot',
+                       class: self.class.name,
+                       message: e.try(:message))
+  end
+
+  def write_failure_snapshot
+    state_as_counts.each do |description, count|
+      StatsD.gauge("#{STATSD_PREFIX}.#{description}", count)
+    end
+  end
+
+  def state_as_counts
+    @state_as_counts ||= {}.tap do |abbreviation|
+      snapshot_state.each do |dp, ids|
+        abbreviation[:"#{dp}_count"] = ids.count
+      end
+    end
+  end
+
+  def snapshot_state
+    @snapshot_state ||= load_snapshot_state
+  end
+
+  def load_snapshot_state
+    {
+      total_awaiting_backup_status: Form526Submission.pending_backup.pluck(:id).sort,
+      total_incomplete_type: Form526Submission.incomplete_type.pluck(:id).sort,
+      total_failure_type: Form526Submission.failure_type.pluck(:id).sort
+    }
+  end
+end

app/sidekiq/form526_state_logging_job.rb → app/sidekiq/form526_submission_processing_report_job.rb

@@ -1,23 +1,21 @@
 # frozen_string_literal: true
 
 # Log information about Form526Submission state to populate an admin facing Datadog dashboard
-class Form526StateLoggingJob
+class Form526SubmissionProcessingReportJob
   include Sidekiq::Job
   sidekiq_options retry: false
 
   attr_reader :start_date, :end_date
 
   END_DATE = Time.zone.today.beginning_of_day
   START_DATE = END_DATE - 1.week
-  STATSD_PREFIX = 'form526.state'
 
   def initialize(start_date: START_DATE, end_date: END_DATE)
     @start_date = start_date
     @end_date = end_date
   end
 
   def perform
-    write_as_gauges
     write_as_log
   rescue => e
     Rails.logger.error('Error logging 526 state data',
@@ -29,43 +27,23 @@ def perform
 
   def write_as_log
     Rails.logger.info('Form 526 State Data',
-                      state_log: counts_with_failures,
+                      state_log: state_as_counts,
                       start_date:,
                       end_date:)
   end
 
-  def counts_with_failures
-    counts = state_as_counts
-    counts[:total_failure_type_ids] = total_failure_type
-    counts
-  end
-
-  def base_state
-    @base_state ||= timeboxed_state.merge(all_time_state)
-  end
-
   def state_as_counts
     @state_as_counts ||= {}.tap do |abbreviation|
-      base_state.each do |dp, ids|
+      timeboxed_state.each do |dp, ids|
         abbreviation[:"#{dp}_count"] = ids.count
       end
     end
   end
 
-  def write_as_gauges
-    state_as_counts.each do |description, count|
-      StatsD.gauge("#{STATSD_PREFIX}.#{description}", count)
-    end
-  end
-
   def timeboxed_state
     @timeboxed_state ||= load_timeboxed_state
   end
 
-  def all_time_state
-    @all_time_state ||= load_all_time_state
-  end
-
   def load_timeboxed_state
     {
       timeboxed: timeboxed_submissions.pluck(:id).sort,
@@ -76,27 +54,10 @@ def load_timeboxed_state
     }
   end
 
-  def load_all_time_state
-    {
-      total_awaiting_backup_status: Form526Submission.pending_backup.pluck(:id).sort,
-      total_incomplete_type: Form526Submission.incomplete_type.pluck(:id).sort,
-      total_failure_type:
-    }
-  end
-
-  def total_failure_type
-    @total_failure_type ||= Form526Submission.failure_type.pluck(:id).sort
-  end
-
   def sub_arel
     @sub_arel ||= Form526Submission.arel_table
   end
 
-  def combined_pending_types_for(submissions)
-    submissions.incomplete.pluck(:id) +
-      submissions.in_process.pluck(:id)
-  end
-
   def backup_submissions
     @backup_submissions ||= timeboxed_submissions
                             .joins(:form526_job_statuses)