Merge branch 'master' into 93214-dr-saved-claim-error-filtering-and-logs

.github/CODEOWNERS

@@ -505,6 +505,7 @@ app/swagger/swagger/requests/mdot @department-of-veterans-affairs/va-cto-health-
 app/swagger/swagger/requests/medical_copays.rb @department-of-veterans-affairs/vsa-debt-resolution @department-of-veterans-affairs/backend-review-group
 app/swagger/swagger/requests/messages @department-of-veterans-affairs/vfs-mhv-secure-messaging @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 app/swagger/swagger/requests/mvi_users.rb @department-of-veterans-affairs/octo-identity
+app/swagger/swagger/requests/my_va/submission_statuses.rb @department-of-veterans-affairs/vfs-authenticated-experience-backend @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 app/swagger/swagger/requests/onsite_notifications.rb @department-of-veterans-affairs/vfs-authenticated-experience-backend @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 app/swagger/swagger/v1/requests/post911_gi_bill_statuses.rb @department-of-veterans-affairs/backend-review-group @department-of-veterans-affairs/va-iir
 app/swagger/swagger/requests/ppiu.rb @department-of-veterans-affairs/vfs-authenticated-experience-backend @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
@@ -642,10 +643,11 @@ app/sidekiq/feature_cleaner_job.rb @department-of-veterans-affairs/va-api-engine
 app/sidekiq/form1010cg @department-of-veterans-affairs/vfs-10-10 @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 app/sidekiq/form1095 @department-of-veterans-affairs/vfs-1095-b @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 app/sidekiq/form526_confirmation_email_job.rb @department-of-veterans-affairs/Disability-Experience @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
+app/sidekiq/form526_failure_state_snapshot_job.rb @department-of-veterans-affairs/disability-experience @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 app/sidekiq/form526_paranoid_success_polling_job.rb @department-of-veterans-affairs/Disability-Experience @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
-app/sidekiq/form526_state_logging_job.rb @department-of-veterans-affairs/disability-experience @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 app/sidekiq/form526_status_polling_job.rb @department-of-veterans-affairs/Disability-Experience @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 app/sidekiq/form526_submission_failed_email_job.rb @department-of-veterans-affairs/Disability-Experience @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
+app/sidekiq/form526_submission_processing_report_job.rb @department-of-veterans-affairs/disability-experience @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 app/sidekiq/gi_bill_feedback_submission_job.rb @department-of-veterans-affairs/my-education-benefits @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 app/sidekiq/hca @department-of-veterans-affairs/vfs-10-10 @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 app/sidekiq/identity @department-of-veterans-affairs/octo-identity
@@ -839,6 +841,7 @@ lib/benefits_intake_service @department-of-veterans-affairs/benefits-dependents-
 lib/bgs @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 lib/bid @department-of-veterans-affairs/Disability-Experience @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 lib/bip_claims @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
+lib/burials @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 lib/carma @department-of-veterans-affairs/vfs-10-10 @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 lib/caseflow @department-of-veterans-affairs/lighthouse-banana-peels @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 lib/central_mail @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
@@ -1343,9 +1346,10 @@ spec/sidekiq/facilities @department-of-veterans-affairs/vfs-facilities-frontend
 spec/sidekiq/form1010cg @department-of-veterans-affairs/vfs-10-10 @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 spec/sidekiq/form1095 @department-of-veterans-affairs/vfs-1095-b @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 spec/sidekiq/form526_confirmation_email_job_spec.rb @department-of-veterans-affairs/Disability-Experience @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
+spec/sidekiq/form526_failure_state_snapshot_job_spec.rb @department-of-veterans-affairs/disability-experience @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 spec/sidekiq/form526_paranoid_success_polling_job_spec.rb @department-of-veterans-affairs/Disability-Experience @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
-spec/sidekiq/form526_state_logging_job_spec.rb @department-of-veterans-affairs/disability-experience @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 spec/sidekiq/form526_status_polling_job_spec.rb @department-of-veterans-affairs/Disability-Experience @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
+spec/sidekiq/form526_submission_processing_report_job_spec.rb @department-of-veterans-affairs/disability-experience @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 spec/sidekiq/form5655 @department-of-veterans-affairs/vsa-debt-resolution @department-of-veterans-affairs/backend-review-group
 spec/sidekiq/gi_bill_feedback_submission_job_spec.rb @department-of-veterans-affairs/my-education-benefits @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 spec/sidekiq/hca @department-of-veterans-affairs/vfs-10-10 @department-of-veterans-affairs/backend-review-group @department-of-veterans-affairs/va-api-engineers
@@ -1379,6 +1383,7 @@ spec/lib/bgs @department-of-veterans-affairs/benefits-dependents-management @dep
 spec/lib/bid @department-of-veterans-affairs/Disability-Experience @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 spec/lib/bip_claims @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 spec/lib/breakers @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
+spec/lib/burials @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 spec/lib/carma @department-of-veterans-affairs/vfs-10-10 @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 spec/lib/caseflow @department-of-veterans-affairs/lighthouse-banana-peels @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 spec/lib/central_mail @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
@@ -2126,4 +2131,5 @@ README.md @department-of-veterans-affairs/backend-review-group
 modules/accredited_representative_portal/spec/services/accredited_representative_portal/representative_user_loader_spec.rb @department-of-veterans-affairs/octo-identity
 modules/accredited_representative_portal/app/services/accredited_representative_portal/representative_user_loader.rb @department-of-veterans-affairs/octo-identity
 config/form_profile_mappings/FORM-MOCK-AE-DESIGN-PATTERNS.yml @department-of-veterans-affairs/tmf-auth-exp-design-patterns @department-of-veterans-affairs/backend-review-group
-vets-api.pm-collection.json @department-of-veterans-affairs/backend-review-group
+postman/vets-api.pm-collection.json @department-of-veterans-affairs/backend-review-group
+postman/Dockerfile @department-of-veterans-affairs/backend-review-group

.github/workflows/build-and-publish.yaml

@@ -33,7 +33,17 @@ jobs:
         uses: aws-actions/amazon-ecr-login@v2.0.1
         with:
           mask-password: true
-      - name: Build Docker Image
+      - name: Build Postman Image
+        uses: docker/build-push-action@v6
+        env:
+          DOCKER_BUILD_SUMMARY: false
+        with:
+          context: ./postman
+          file: ./postman/Dockerfile
+          push: true
+          tags: |
+            ${{ steps.ecr-login.outputs.registry }}/dsva/vets-api-postman:${{ steps.version.outputs.version }}
+      - name: Build vets-api Docker Image
         uses: docker/build-push-action@v6
         env:
           DOCKER_BUILD_SUMMARY: false

.github/workflows/build.yml

@@ -30,8 +30,17 @@ jobs:
       - name: Login to ECR
         id: login-ecr
         uses: aws-actions/amazon-ecr-login@v2
-
-      - name: Build Docker Image
+      - name: Build Postman Docker Image
+        uses: docker/build-push-action@v6
+        env:
+          DOCKER_BUILD_SUMMARY: false
+        with:
+          context: ./postman
+          file: ./postman/Dockerfile
+          push: true
+          tags: |
+            ${{ steps.login-ecr.outputs.registry }}/dsva/vets-api-postman:${{ github.sha }}
+      - name: Build vets-api Docker Image
         uses: docker/build-push-action@v6
         env:
           DOCKER_BUILD_SUMMARY: false

Gemfile.lock

@@ -512,9 +512,14 @@ GEM
       thor (>= 0.20, < 2.a)
     google-cloud-env (2.1.1)
       faraday (>= 1.0, < 3.a)
-    google-protobuf (4.28.1)
+    google-protobuf (4.28.2)
       bigdecimal
       rake (>= 13)
+    google-protobuf (4.28.2-java)
+      bigdecimal
+      ffi (~> 1)
+      ffi-compiler (~> 1)
+      rake (>= 13)
     googleauth (1.11.0)
       faraday (>= 1.0, < 3.a)
       google-cloud-env (~> 2.1)
@@ -622,7 +627,7 @@ GEM
     llhttp-ffi (0.5.0)
       ffi-compiler (~> 1.0)
       rake (~> 13.0)
-    lockbox (1.4.0)
+    lockbox (1.4.1)
     logger (1.6.1)
     loofah (2.22.0)
       crass (~> 1.0.2)
@@ -647,7 +652,7 @@ GEM
     mini_mime (1.1.5)
     mini_portile2 (2.8.7)
     minitest (5.25.1)
-    mock_redis (0.44.0)
+    mock_redis (0.45.0)
     msgpack (1.7.2)
     msgpack (1.7.2-java)
     multi_json (1.15.0)
@@ -764,9 +769,9 @@ GEM
     psych (5.1.2-java)
       jar-dependencies (>= 0.1.7)
     public_suffix (6.0.1)
-    puma (6.4.2)
+    puma (6.4.3)
       nio4r (~> 2.0)
-    puma (6.4.2-java)
+    puma (6.4.3-java)
       nio4r (~> 2.0)
     pundit (2.4.0)
       activesupport (>= 3.0.0)
@@ -930,7 +935,7 @@ GEM
     rubocop-factory_bot (2.26.1)
       rubocop (~> 1.61)
     rubocop-junit-formatter (0.1.4)
-    rubocop-rails (2.26.1)
+    rubocop-rails (2.26.2)
       activesupport (>= 4.2.0)
       rack (>= 1.1)
       rubocop (>= 1.52.0, < 2.0)

app/controllers/v0/apidocs_controller.rb

@@ -164,6 +164,7 @@ class ApidocsController < ApplicationController
       Swagger::Requests::Messages::TriageTeams,
       Swagger::Requests::MviUsers,
       Swagger::Requests::OnsiteNotifications,
+      Swagger::Requests::MyVA::SubmissionStatuses,
       Swagger::Requests::IncomeAndAssetsClaims,
       Swagger::Requests::PPIU,
       Swagger::Requests::PreneedsClaims,

app/controllers/v0/backend_statuses_controller.rb

@@ -7,41 +7,48 @@ module V0
   class BackendStatusesController < ApplicationController
     service_tag 'maintenance-windows'
     skip_before_action :authenticate
+    before_action :validate_service, only: [:show]
 
-    # NOTE: this endpoint is somewhat misleading.  Index gets data from PagerDuty and
-    # show only looks at GI bill scheduled downtime (and gets no data from PagerDuty)
     def index
-      statuses = ExternalServicesRedis::Status.new.fetch_or_cache
-      maintenance_windows = MaintenanceWindow.end_after(Time.zone.now)
-
       options = { params: { maintenance_windows: } }
-      render json: BackendStatusesSerializer.new(statuses, options)
+      render json: BackendStatusesSerializer.new(backend_statuses, options)
     end
 
     # GET /v0/backend_statuses/:service
     def show
-      @backend_service = params[:service]
-      raise Common::Exceptions::RecordNotFound, @backend_service unless recognized_service?
+      render json: BackendStatusSerializer.new(backend_status)
+    end
 
-      # get status
-      be_status = BackendStatus.new(name: @backend_service)
-      case @backend_service
-      when BackendServices::GI_BILL_STATUS
-        be_status.is_available = BenefitsEducation::Service.within_scheduled_uptime?
-        be_status.uptime_remaining = BenefitsEducation::Service.seconds_until_downtime
-      else
-        # default service is up!
-        be_status.is_available = true
-        be_status.uptime_remaining = 0
-      end
+    private
 
-      render json: BackendStatusSerializer.new(be_status)
+    # NOTE: Data is from PagerDuty
+    def backend_statuses
+      @backend_statuses ||= ExternalServicesRedis::Status.new.fetch_or_cache
     end
 
-    private
+    def maintenance_windows
+      @maintenance_windows ||= MaintenanceWindow.end_after(Time.zone.now)
+    end
+
+    # NOTE: Data is GI bill scheduled downtime
+    def backend_status
+      @backend_status ||= BackendStatus.new(name: backend_service)
+    end
+
+    def backend_service
+      params[:service]
+    end
+
+    def validate_service
+      raise Common::Exceptions::RecordNotFound, backend_service unless recognized_service?
+    end
 
     def recognized_service?
-      BackendServices.all.include?(@backend_service)
+      BackendServices.all.include?(backend_service)
+    end
+
+    def backend_status_is_available
+      backend_service == BackendServices::GI_BILL_STATUS
     end
   end
 end

app/controllers/v0/burial_claims_controller.rb

@@ -1,52 +1,61 @@
 # frozen_string_literal: true
 
 require 'pension_burial/tag_sentry'
+require 'burials/monitor'
 
 module V0
   class BurialClaimsController < ClaimsBaseController
     service_tag 'burial-application'
 
     def show
-      submission_attempt = determine_submission_attempt
+      claim = claim_class.find_by!(guid: params[:id])
+      form_submission = claim&.form_submissions&.last
+      submission_attempt = form_submission&.form_submission_attempts&.last
       if submission_attempt
         state = submission_attempt.aasm_state == 'failure' ? 'failure' : 'success'
         render(json: { data: { attributes: { state: } } })
       elsif central_mail_submission
         render json: CentralMailSubmissionSerializer.new(central_mail_submission)
-      else
-        Rails.logger.error("ActiveRecord::RecordNotFound: Claim submission not found for claim_id: #{params[:id]}")
-        render(json: { data: { attributes: { state: 'not found' } } }, status: :not_found)
       end
+    rescue ActiveRecord::RecordNotFound => e
+      monitor.track_show404(params[:id], current_user, e)
+      render(json: { data: { attributes: { state: 'not found' } } }, status: :not_found)
     rescue => e
-      Rails.logger.error(e.to_s)
+      monitor.track_show_error(params[:id], current_user, e)
       render(json: { data: { attributes: { state: 'error processing request' } } }, status: :unprocessable_entity)
     end
 
     def create
       PensionBurial::TagSentry.tag_sentry
 
-      claim = if Flipper.enabled?(:va_burial_v2)
-                # cannot parse a nil form, to pass unit tests do a check for form presence
-                form = filtered_params[:form]
-                claim_class.new(form:, formV2: form.present? ? JSON.parse(form)['formV2'] : nil)
-              else
-                claim_class.new(form: filtered_params[:form])
-              end
-
-      unless claim.save
-        StatsD.increment("#{stats_key}.failure")
-        Sentry.set_tags(team: 'benefits-memorial-1') # tag sentry logs with team name
-        Rails.logger.error('Burial claim was not saved', {  error_messages: claim.errors,
-                                                            user_uuid: current_user&.uuid,
-                                                            in_progress_form_id: in_progress_form&.id })
-        raise Common::Exceptions::ValidationErrors, claim
-      end
+      claim = create_claim
+      monitor.track_create_attempt(claim, current_user)
+
+      track_claim_save_failure(claim) unless claim.save
+
       # this method also calls claim.process_attachments!
       claim.submit_to_structured_data_services!
 
       Rails.logger.info "ClaimID=#{claim.confirmation_number} Form=#{claim.form_id}"
+
+      in_progress_form = current_user ? InProgressForm.form_for_user(claim.form_id, current_user) : nil
+      claim.form_start_date = in_progress_form.created_at if in_progress_form
+      monitor.track_create_success(in_progress_form, claim, current_user)
+
       clear_saved_form(claim.form_id)
       render json: SavedClaimSerializer.new(claim)
+    rescue => e
+      monitor.track_create_error(in_progress_form, claim, current_user, e)
+      raise e
+    end
+
+    def create_claim
+      if Flipper.enabled?(:va_burial_v2)
+        form = filtered_params[:form]
+        claim_class.new(form:, formV2: form.present? ? JSON.parse(form)['formV2'] : nil)
+      else
+        claim_class.new(form: filtered_params[:form])
+      end
     end
 
     def short_name
@@ -59,18 +68,46 @@ def claim_class
 
     private
 
-    def determine_submission_attempt
-      claim = claim_class.find_by(guid: params[:id])
-      form_submission = claim&.form_submissions&.last
-      form_submission&.form_submission_attempts&.last
-    end
-
     def central_mail_submission
       CentralMailSubmission.joins(:central_mail_claim).find_by(saved_claims: { guid: params[:id] })
     end
 
     def in_progress_form
       current_user ? InProgressForm.form_for_user(claim.form_id, current_user) : nil
     end
+
+    def track_claim_save_failure(claim)
+      StatsD.increment("#{stats_key}.failure")
+      Sentry.set_tags(team: 'benefits-memorial-1') # tag sentry logs with team name
+      Rails.logger.error('Burial claim was not saved', {  error_messages: claim.errors,
+                                                          user_uuid: current_user&.uuid,
+                                                          in_progress_form_id: in_progress_form&.id })
+      log_validation_error_to_metadata(in_progress_form, claim)
+      raise Common::Exceptions::ValidationErrors, claim
+    end
+
+    ##
+    # include validation error on in_progress_form metadata.
+    # `noop` if in_progress_form is `blank?`
+    #
+    # @param in_progress_form [InProgressForm]
+    # @param claim [Pensions::SavedClaim]
+    #
+    def log_validation_error_to_metadata(in_progress_form, claim)
+      return if in_progress_form.blank?
+
+      metadata = in_progress_form.metadata
+      metadata['submission']['error_message'] = claim&.errors&.errors&.to_s
+      in_progress_form.update(metadata:)
+    end
+
+    ##
+    # retreive a monitor for tracking
+    #
+    # @return [Burials::Monitor]
+    #
+    def monitor
+      @monitor ||= Burials::Monitor.new
+    end
   end
 end