Merge pull request #11136 from 18F/stages/rc-2024-08-22

Deploy RC 408 to Production
18F · Aug 22, 2024 · 0dcaa38 · 0dcaa38
2 parents 7e4c865 + 97f9d7f
commit 0dcaa38
Show file tree

Hide file tree

Showing 109 changed files with 2,364 additions and 927 deletions.
diff --git a/.eslintignore b/.eslintignore
@@ -4,3 +4,4 @@ public
 vendor
 coverage
 doc
+app/javascript/packages/analytics/digital-analytics-program*.js
diff --git a/Gemfile.lock b/Gemfile.lock
@@ -257,7 +257,7 @@ GEM
     coderay (1.1.3)
     coercible (1.0.0)
       descendants_tracker (~> 0.0.1)
-    concurrent-ruby (1.3.3)
+    concurrent-ruby (1.3.4)
     connection_pool (2.4.1)
     cose (1.3.0)
       cbor (~> 0.5.9)
@@ -314,7 +314,7 @@ GEM
       smart_properties
     errbase (0.2.1)
     erubi (1.13.0)
-    et-orbi (1.2.7)
+    et-orbi (1.2.11)
       tzinfo
     factory_bot (6.4.6)
       activesupport (>= 5.0.0)
@@ -335,8 +335,8 @@ GEM
       ffi (>= 1.0.0)
       rake
     foundation_emails (2.2.1.0)
-    fugit (1.9.0)
-      et-orbi (~> 1, >= 1.2.7)
+    fugit (1.11.1)
+      et-orbi (~> 1, >= 1.2.11)
       raabro (~> 1.4)
     geocoder (1.7.0)
     get_process_mem (0.2.7)

diff --git a/app/assets/images/user-signup-ial2.svg b/app/assets/images/user-signup-ial2.svg
diff --git a/app/assets/images/user-signup-ial1.svg → app/assets/images/user-signup.svg b/app/assets/images/user-signup-ial1.svg → app/assets/images/user-signup.svg
diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb
@@ -7,6 +7,7 @@ class ApplicationController < ActionController::Base
   include VerifySpAttributesConcern
   include SecondMfaReminderConcern
   include TwoFactorAuthenticatableMethods
+  include AbTestingConcern
 
   # Prevent CSRF attacks by raising an exception.
   # For APIs, you may want to use :null_session instead.

diff --git a/app/controllers/concerns/ab_testing_concern.rb b/app/controllers/concerns/ab_testing_concern.rb
@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module AbTestingConcern
+  # @param [Symbol] test Name of the test, which should correspond to an A/B test defined in
+  # #                    config/initializer/ab_tests.rb.
+  # @return [Symbol,nil] Bucket to use for the given test, or nil if the test is not active.
+  def ab_test_bucket(test_name)
+    test = AbTests.all[test_name]
+    raise "Unknown A/B test: #{test_name}" unless test
+
+    test.bucket(
+      request:,
+      service_provider: current_sp&.issuer,
+      session:,
+      user: current_user,
+      user_session:,
+    )
+  end
+end
diff --git a/app/controllers/concerns/idv/ab_test_analytics_concern.rb b/app/controllers/concerns/idv/ab_test_analytics_concern.rb
@@ -13,7 +13,7 @@ def ab_test_analytics_buckets
         buckets = buckets.merge(opt_in_analytics_properties)
       end
 
-      buckets.merge(acuant_sdk_ab_test_analytics_args)
+      buckets
     end
   end
 end
diff --git a/app/controllers/concerns/idv/acuant_concern.rb b/app/controllers/concerns/idv/acuant_concern.rb
@@ -2,18 +2,12 @@
 
 module Idv
   module AcuantConcern
-    def acuant_sdk_ab_test_analytics_args
-      return {} if document_capture_session_uuid.blank?
-
-      {
-        acuant_sdk_upgrade_ab_test_bucket:
-          AbTests::ACUANT_SDK.bucket(document_capture_session_uuid),
-      }
-    end
+    include AbTestingConcern
 
     def acuant_sdk_upgrade_a_b_testing_variables
-      bucket = AbTests::ACUANT_SDK.bucket(document_capture_session_uuid)
-      testing_enabled = IdentityConfig.store.idv_acuant_sdk_upgrade_a_b_testing_enabled
+      bucket = ab_test_bucket(:ACUANT_SDK)
+      testing_enabled = IdentityConfig.store.idv_acuant_sdk_upgrade_a_b_testing_enabled &&
+                        bucket.present?
       use_alternate_sdk = (bucket == :use_alternate_sdk)
 
       if use_alternate_sdk

diff --git a/app/controllers/concerns/idv/doc_auth_vendor_concern.rb b/app/controllers/concerns/idv/doc_auth_vendor_concern.rb
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Idv
+  module DocAuthVendorConcern
+    include AbTestingConcern
+
+    # @returns[String] String identifying the vendor to use for doc auth.
+    def doc_auth_vendor
+      bucket = ab_test_bucket(:DOC_AUTH_VENDOR)
+      DocAuthRouter.doc_auth_vendor_for_bucket(bucket)
+    end
+  end
+end
diff --git a/app/controllers/concerns/idv/document_capture_concern.rb b/app/controllers/concerns/idv/document_capture_concern.rb
@@ -4,14 +4,11 @@ module Idv
   module DocumentCaptureConcern
     extend ActiveSupport::Concern
 
+    include DocAuthVendorConcern
+
     def save_proofing_components(user)
       return unless user
 
-      doc_auth_vendor = DocAuthRouter.doc_auth_vendor(
-        discriminator: document_capture_session_uuid,
-        analytics: analytics,
-      )
-
       component_attributes = {
         document_check: doc_auth_vendor,
         document_type: 'state_id',

diff --git a/app/controllers/concerns/two_factor_authenticatable_methods.rb b/app/controllers/concerns/two_factor_authenticatable_methods.rb
@@ -33,7 +33,7 @@ def handle_valid_verification_for_authentication_context(auth_method:)
     mark_user_session_authenticated(auth_method:, authentication_type: :valid_2fa)
     disavowal_event, disavowal_token = create_user_event_with_disavowal(:sign_in_after_2fa)
 
-    if IdentityConfig.store.feature_new_device_alert_aggregation_enabled && new_device?
+    if new_device?
       if current_user.sign_in_new_device_at.blank?
         if sign_in_notification_timeframe_expired_event.present?
           current_user.update(

diff --git a/app/controllers/idv/document_capture_controller.rb b/app/controllers/idv/document_capture_controller.rb
@@ -46,6 +46,7 @@ def update
     def extra_view_variables
       {
         document_capture_session_uuid: document_capture_session_uuid,
+        mock_client: doc_auth_vendor == 'mock',
         flow_path: 'standard',
         sp_name: decorated_sp_session.sp_name,
         failure_to_proof_url: return_to_sp_failure_to_proof_url(step: 'document_capture'),

diff --git a/app/controllers/idv/hybrid_mobile/document_capture_controller.rb b/app/controllers/idv/hybrid_mobile/document_capture_controller.rb
@@ -42,6 +42,7 @@ def update
       def extra_view_variables
         {
           flow_path: 'hybrid',
+          mock_client: doc_auth_vendor == 'mock',
           document_capture_session_uuid: document_capture_session_uuid,
           failure_to_proof_url: return_to_sp_failure_to_proof_url(step: 'document_capture'),
           doc_auth_selfie_capture: resolved_authn_context_result.biometric_comparison?,

diff --git a/app/controllers/idv/image_uploads_controller.rb b/app/controllers/idv/image_uploads_controller.rb
@@ -2,6 +2,8 @@
 
 module Idv
   class ImageUploadsController < ApplicationController
+    include DocAuthVendorConcern
+
     respond_to :json
 
     def create
@@ -20,6 +22,8 @@ def create
     def image_upload_form
       @image_upload_form ||= Idv::ApiImageUploadForm.new(
         params,
+        doc_auth_vendor:,
+        acuant_sdk_upgrade_ab_test_bucket: ab_test_bucket(:ACUANT_SDK),
         service_provider: current_sp,
         analytics: analytics,
         uuid_prefix: current_sp&.app_id,

diff --git a/app/controllers/idv/in_person/ssn_controller.rb b/app/controllers/idv/in_person/ssn_controller.rb
@@ -64,10 +64,7 @@ def self.step_info
           controller: self,
           next_steps: [:ipp_verify_info],
           preconditions: ->(idv_session:, user:) { idv_session.ipp_document_capture_complete? },
-          undo_step: ->(idv_session:, user:) do
-            idv_session.ssn = nil
-            idv_session.threatmetrix_session_id = nil
-          end,
+          undo_step: ->(idv_session:, user:) { idv_session.ssn = nil },
         )
       end
 

diff --git a/app/controllers/idv/ssn_controller.rb b/app/controllers/idv/ssn_controller.rb
@@ -62,10 +62,7 @@ def self.step_info
         controller: self,
         next_steps: [:verify_info],
         preconditions: ->(idv_session:, user:) { idv_session.remote_document_capture_complete? },
-        undo_step: ->(idv_session:, user:) do
-          idv_session.ssn = nil
-          idv_session.threatmetrix_session_id = nil
-        end,
+        undo_step: ->(idv_session:, user:) { idv_session.ssn = nil },
       )
     end
 

diff --git a/app/controllers/users/sessions_controller.rb b/app/controllers/users/sessions_controller.rb
@@ -245,7 +245,6 @@ def override_csp_for_google_analytics
       policy = current_content_security_policy
       policy.script_src(
         *policy.script_src,
-        'dap.digitalgov.gov',
         'www.google-analytics.com',
         'www.googletagmanager.com',
       )

diff --git a/app/forms/idv/api_image_upload_form.rb b/app/forms/idv/api_image_upload_form.rb
@@ -17,12 +17,16 @@ class ApiImageUploadForm
     def initialize(
       params,
       service_provider:,
+      doc_auth_vendor:,
+      acuant_sdk_upgrade_ab_test_bucket:,
       analytics: nil,
       uuid_prefix: nil,
       liveness_checking_required: false
     )
       @params = params
       @service_provider = service_provider
+      @doc_auth_vendor = doc_auth_vendor
+      @acuant_sdk_upgrade_ab_test_bucket = acuant_sdk_upgrade_ab_test_bucket
       @analytics = analytics
       @readable = {}
       @uuid_prefix = uuid_prefix
@@ -61,7 +65,7 @@ def submit
     private
 
     attr_reader :params, :analytics, :service_provider, :form_response, :uuid_prefix,
-                :liveness_checking_required
+                :liveness_checking_required, :acuant_sdk_upgrade_ab_test_bucket
 
     def increment_rate_limiter!
       return unless document_capture_session
@@ -315,7 +319,7 @@ def document_capture_session_uuid
 
     def doc_auth_client
       @doc_auth_client ||= DocAuthRouter.client(
-        vendor_discriminator: document_capture_session_uuid,
+        vendor: @doc_auth_vendor,
         warn_notifier: proc do |attrs|
           analytics&.doc_auth_warning(
             **attrs,
@@ -364,11 +368,9 @@ def update_analytics(client_response:, vendor_request_time_in_ms:)
     end
 
     def acuant_sdk_upgrade_ab_test_data
-      return {} unless IdentityConfig.store.idv_acuant_sdk_upgrade_a_b_testing_enabled
       {
-        acuant_sdk_upgrade_ab_test_bucket:
-          AbTests::ACUANT_SDK.bucket(document_capture_session.uuid),
-      }
+        acuant_sdk_upgrade_ab_test_bucket:,
+      }.compact
     end
 
     def acuant_sdk_captured?

diff --git a/app/helpers/script_helper.rb b/app/helpers/script_helper.rb
@@ -2,8 +2,8 @@
 
 # rubocop:disable Rails/HelperInstanceVariable
 module ScriptHelper
-  def javascript_packs_tag_once(*names, **attributes)
-    @scripts = @scripts.to_h.merge(names.index_with(attributes))
+  def javascript_packs_tag_once(*names, url_params: nil, **attributes)
+    @scripts = @scripts.to_h.merge(names.index_with([url_params, attributes]))
     nil
   end
 
@@ -14,10 +14,10 @@ def render_javascript_pack_once_tags(...)
       javascript_packs_tag_once(...)
       return if @scripts.blank?
       concat javascript_assets_tag
-      @scripts.each do |name, attributes|
+      @scripts.each do |name, (url_params, attributes)|
         asset_sources.get_sources(name).each do |source|
           concat javascript_include_tag(
-            source,
+            UriService.add_params(source, url_params),
             **attributes,
             crossorigin: local_crossorigin_sources? ? true : nil,
             integrity: asset_sources.get_integrity(source),

diff --git a/app/javascript/packages/analytics/.gitignore b/app/javascript/packages/analytics/.gitignore
@@ -0,0 +1 @@
+digital-analytics-program*.js
diff --git a/app/javascript/packages/analytics/Makefile b/app/javascript/packages/analytics/Makefile
@@ -0,0 +1,12 @@
+DAP_SHA ?= 7c14bb3
+
+digital-analytics-program.js: digital-analytics-program-$(DAP_SHA).js digital-analytics-program.patch
+	patch -p1 $^ --output $@
+
+digital-analytics-program-$(DAP_SHA).js:
+	curl https://raw.githubusercontent.com/digital-analytics-program/gov-wide-code/$(DAP_SHA)/Universal-Federated-Analytics.js --silent --output $@
+
+clean:
+	rm digital-analytics-program-$(DAP_SHA).js digital-analytics-program.js
+
+.PHONY: clean
diff --git a/app/javascript/packages/analytics/digital-analytics-program.patch b/app/javascript/packages/analytics/digital-analytics-program.patch
@@ -0,0 +1,8 @@
+73a74
+> GA4Object.defer = true;
+785d785
+< var piiRegex = [];
+900c900
+<         piiRegex.forEach(function (pii) {
+---
+>         window.piiRegex.forEach(function (pii) {
diff --git a/app/javascript/packages/analytics/digital-analytics-program.spec.ts b/app/javascript/packages/analytics/digital-analytics-program.spec.ts
@@ -0,0 +1,26 @@
+import { Worker } from 'node:worker_threads';
+import { join } from 'node:path';
+import { pathToFileURL } from 'node:url';
+
+describe('digital analytics program', () => {
+  it('parses without syntax error', async () => {
+    // Future: Replace with Promise.withResolvers once supported
+    // See: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise/withResolvers
+    let resolve;
+    const promise = new Promise((_resolve) => {
+      resolve = _resolve;
+    });
+
+    // Reference: https://github.com/nodejs/node/issues/30682
+    const toDataURL = (source: string) =>
+      new URL(`data:text/javascript,${encodeURIComponent(source)}`);
+    const url = pathToFileURL(join(__dirname, './digital-analytics-program.js'));
+    const code = `await import(${JSON.stringify(url)});`;
+    new Worker(toDataURL(code)).on('error', (error) => {
+      expect(error).not.to.be.instanceOf(SyntaxError);
+      resolve();
+    });
+
+    await promise;
+  });
+});
diff --git a/app/javascript/packages/analytics/package.json b/app/javascript/packages/analytics/package.json
@@ -2,7 +2,16 @@
   "name": "@18f/identity-analytics",
   "private": true,
   "version": "1.0.0",
+  "scripts": {
+    "postinstall": "make digital-analytics-program.js"
+  },
+  "exports": {
+    ".": "./index.ts",
+    "./click-observer-element": "./click-observer-element.ts",
+    "./digital-analytics-program": "./digital-analytics-program.js"
+  },
   "sideEffects": [
-    "./click-observer-element.ts"
+    "./click-observer-element.ts",
+    "./digital-analytics-program.js"
   ]
 }
diff --git a/app/javascript/packs/digital-analytics-program.ts b/app/javascript/packs/digital-analytics-program.ts
@@ -0,0 +1 @@
+import '@18f/identity-analytics/digital-analytics-program';
diff --git a/app/jobs/reports/combined_invoice_supplement_report_v2.rb b/app/jobs/reports/combined_invoice_supplement_report_v2.rb
@@ -142,6 +142,18 @@ def combine_by_iaa_month(
 
             year_months.each do |year_month|
               iaa_results = by_iaa_and_year_month[ [iaa_key, year_month] ]
+              if !iaa_results
+                logger.warn(
+                  {
+                    level: 'warning',
+                    name: 'missing iaa_results',
+                    iaa: iaa_key,
+                    year_month: year_month,
+                  }.to_json,
+                )
+                next
+              end
+
               issuer_results = year_months_data[year_month]
               year_month_start = Date.strptime(year_month, '%Y%m')
               iaa_start_date = Date.parse(iaa_results.first[:iaa_start_date])