Update lead status from salesforce (#1213)

* script to update lead verification status from salesforce

* add signup date to lead creation

* task to clean up forgotten leads

* confirm on lead creation task

* modify lead save/error handling

* can't check if string changed fix

* tweak lead error handling

* remove is_b_r_i user from contact sync

* save not working in transaction

* use prepend_before_action for privacy notice check

* record lead changes to security log

* add new roles and status to blazer

* slight modification to showing privacy notice updates

* revert changes to privacy for this PR

* remove bad before_action

* a couple more reverts

* add security logs back in

* use find_in_batches on lead update

* lead.id -> string

* only update leads that have been created in the last month

app/models/security_log.rb

@@ -106,6 +106,7 @@ class SecurityLog < ApplicationRecord
     user_contact_id_updated_from_salesforce
     attempted_to_add_school_not_cached_yet
     school_added_to_user_from_sheerid_webhook
+    user_lead_id_updated_from_salesforce
   ]
 
   json_serialize :event_data, Hash

app/models/user.rb

@@ -30,7 +30,6 @@ class User < ApplicationRecord
     PENDING_FACULTY = 'pending_faculty',
     CONFIRMED_FACULTY = 'confirmed_faculty',
     REJECTED_FACULTY = 'rejected_faculty',
-    # TODO: need to implement below this line - requires thorough application code checking
     PENDING_SHEERID = 'pending_sheerid',
     REJECTED_BY_SHEERID = 'rejected_by_sheerid',
     INCOMPLETE_SIGNUP = 'incomplete_signup'

app/routines/newflow/create_or_update_salesforce_lead.rb

@@ -82,6 +82,7 @@ def exec(user:)
         lead.sheerid_school_name = user.sheerid_reported_school
         lead.account_id = sf_school_id
         lead.school_id = sf_school_id
+        lead.signup_date = user.created_at.strftime("%Y-%m-%dT%T.%L%z")
 
       state = user.most_accurate_school_state
       unless state.blank?
@@ -103,41 +104,40 @@ def exec(user:)
       )
 
       if lead.save
-        store_salesforce_lead_id(user, lead.id)
-        transfer_errors_from(user, {type: :verbatim}, :fail_if_errors)
-
+        user.salesforce_lead_id = lead.id
+        if user.save
+          SecurityLog.create!(
+            user: user,
+            event_type: :created_salesforce_lead,
+            event_data: { lead_id: lead.id.to_s }
+          )
+        else
+          SecurityLog.create!(
+            user: user,
+            event_type: :educator_sign_up_failed,
+            event_data: {
+              message: "saving the user's lead id FAILED",
+              lead_id: lead.id
+            }
+          )
+          Sentry.capture_message("User #{user.id} was not successfully saved with lead #{lead.id}")
+        end
         outputs.lead = lead
       else
-        handle_lead_errors(lead, user)
-      end
-
-      outputs.user = user
-    end
+        message = "#{self.class.name} error creating SF lead! #{lead.inspect}; User: #{user.id}; Error: #{lead.errors.full_messages}"
 
-    def store_salesforce_lead_id(user, lead_id)
-      fatal_error(code: :lead_id_is_blank, message: :lead_id_is_blank.to_s.titleize) if lead_id.blank?
+        Sentry.capture_message(message)
 
-      user.salesforce_lead_id = lead_id
-
-      if user.save
         SecurityLog.create!(
           user: user,
-          event_type: :created_salesforce_lead,
-          event_data: { lead_id: lead_id }
-        )
-        return true
-      else
-        SecurityLog.create!(
-          user: user,
-          event_type: :educator_sign_up_failed,
+          event_type: :salesforce_error,
           event_data: {
-            message: 'saving the user\'s lead id FAILED',
-            lead_id: lead_id
+            message: message
           }
         )
-        transfer_errors_from(user, {type: :verbatim}, :fail_if_errors)
-        return
       end
+
+      outputs.user = user
     end
 
     def build_book_adoption_json_for_salesforce(user)
@@ -156,29 +156,5 @@ def build_book_adoption_json_for_salesforce(user)
       adoption_json['Books'] = books_json
       adoption_json.to_json
     end
-
-    def handle_lead_errors(lead, user)
-      SecurityLog.create!(
-        user: user,
-        event_type: :salesforce_error,
-        event_data: {
-          message: 'Error creating Salesforce lead!',
-        }
-      )
-
-      message = "#{self.class.name} error creating SF lead! #{lead.inspect}; User: #{user.id}; Error: #{lead.errors.full_messages}"
-
-      SecurityLog.create!(
-        user: user,
-        event_type: :salesforce_error,
-        event_data: {
-          message: message,
-        }
-      )
-
-      Rails.logger.warn(message)
-      fatal_error(code: :lead_error)
-    end
-
   end
 end

app/routines/update_user_contact_info.rb

@@ -44,13 +44,14 @@ def call
       sf_contact = contacts_by_uuid[user.uuid]
       school = schools_by_salesforce_id[sf_contact.school_id]
 
+      previous_contact_id = user.salesforce_contact_id
       user.salesforce_contact_id = sf_contact.id
 
-      if user.salesforce_contact_id.changed?
+      if sf_contact.id != previous_contact_id
         SecurityLog.create!(
           user: user,
           event_type: :user_contact_id_updated_from_salesforce,
-          event_data: { contact_id: sf_contact.id }
+          event_data: { previous_contact_id: previous_contact_id,new_contact_id: sf_contact.id }
         )
       end
 
@@ -117,7 +118,6 @@ def call
       user.adopter_status = sf_contact.adoption_status
       user.is_kip = sf_school&.is_kip || sf_school&.is_child_of_kip
       user.grant_tutor_access = sf_contact.grant_tutor_access
-      user.is_b_r_i_user = sf_contact.b_r_i_marketing
 
       if school.nil? && !sf_school.nil?
         SecurityLog.create!(

app/routines/update_user_lead_info.rb

@@ -0,0 +1,77 @@
+class UpdateUserLeadInfo
+
+  def self.call
+    new.call
+  end
+  def call
+    # we are only using this to check users created in the last month
+    start_date = Time.zone.now - 1.day
+    end_date = Time.zone.now - 30.day
+
+    users = User.where(salesforce_contact_id: nil).where("created_at <= ? AND created_at >= ?", start_date, end_date)
+                .where.not(salesforce_lead_id: nil, role: :student, faculty_status: :rejected_faculty)
+
+    if users.count > 300
+      Sentry.capture_message("Too many users #{users.count} to update in UpdateUserLeadInfo")
+      exit
+    end
+
+    leads = OpenStax::Salesforce::Remote::Lead.select(:id, :accounts_uuid, :verification_status)
+                                              .where(accounts_uuid: users.map(&:uuid))
+                                              .to_a
+                                              .index_by(&:accounts_uuid)
+
+    users.map do |user|
+      lead = leads[user.uuid]
+
+      unless lead.nil?
+        previous_lead_id = user.salesforce_lead_id
+        user.salesforce_lead_id = lead.id # it might change in SF lead merging
+
+        if lead.id != previous_lead_id
+          SecurityLog.create!(
+            user:       user,
+            event_type: :user_lead_id_updated_from_salesforce,
+            event_data: { previous_lead_id: previous_lead_id, new_lead_id: lead.id }
+          )
+        end
+
+        old_fv_status = user.faculty_status
+        user.faculty_status = case lead.verification_status
+                                when "confirmed_faculty"
+                                  :confirmed_faculty
+                                when "pending_faculty"
+                                  :pending_faculty
+                                when "rejected_faculty"
+                                  :rejected_faculty
+                                when "rejected_by_sheerid"
+                                  :rejected_by_sheerid
+                                when "incomplete_signup"
+                                  :incomplete_signup
+                                when "no_faculty_info"
+                                  :no_faculty_info
+                                when NilClass
+                                  :no_faculty_info
+                                else
+                                  Sentry.capture_message("Unknown faculty_verified field: '#{
+                                    lead.verification_status}'' on lead #{lead.id}")
+                              end
+
+        if user.faculty_status_changed?
+          SecurityLog.create!(
+            user: user,
+            event_type: :salesforce_updated_faculty_status,
+            event_data: {
+              user_id: user.id,
+              salesforce_lead_id: lead.id,
+              old_status: old_fv_status,
+              new_status: user.faculty_status
+            }
+          )
+        end
+
+        user.save if user.changed?
+      end
+    end
+  end
+end

config/blazer.yml

@@ -36,13 +36,17 @@ data_sources:
         5: "Designer",
         6: "Other",
         7: "Adjunct",
-        8: "Homeschool"
+        8: "Homeschool",
+        9: "Researcher"
       }
       faculty_status: {
         0: "No Info",
         1: "Pending",
         2: "Confirmed",
-        3: "Rejected"
+        3: "Rejected",
+        4: "Pending SheerID",
+        5: "Rejected by SheerID",
+        6: "Incomplete Signup"
       }
       school_type: {
         0: "Unknown",
@@ -77,13 +81,17 @@ data_sources:
         5: "Designer",
         6: "Other",
         7: "Adjunct",
-        8: "Homeschool"
+        8: "Homeschool",
+        9: "Researcher"
       }
       faculty_status: {
         0: "No Info",
         1: "Pending",
         2: "Confirmed",
-        3: "Rejected"
+        3: "Rejected",
+        4: "Pending SheerID",
+        5: "Rejected by SheerID",
+        6: "Incomplete Signup"
       }
       school_type: {
         0: "Unknown",
@@ -135,4 +143,3 @@ check_schedules:
 # note: with trend, time series are sent to https://trendapi.org
 # forecasting: prophet / trend
 # forecasting: prophet
-

lib/tasks/accounts/create_leads_for_instructors_not_sent_to_sf.rake

@@ -1,12 +1,30 @@
 namespace :accounts do
   desc 'Create leads for faculty verified by SheerID and never sent to Salesforce'
   # rake accounts:create_leads_for_instructors_not_sent_to_sf
-  # To be run to fix users that were not sent to SF during bug from 26Jan22 to xxFeb22 (hotfix release date)
   task :create_leads_for_instructors_not_sent_to_sf, [:day] => [:environment] do |t, args|
-    # get all the instructors that don't have lead or contact ids
-    users = User.where(salesforce_contact_id: nil, salesforce_lead_id: nil, role: :instructor, state: :activated).or(User.where.not(sheerid_verification_id: nil))
-    users.each { |user|
-      Newflow::CreateSalesforceLead.perform_later(user: user)
-    }
+    # get all the instructors that don't have lead or contact ids and have a complete profile
+    users = User.where(salesforce_contact_id: nil, salesforce_lead_id: nil, role: :instructor, is_profile_complete: true)
+    STDOUT.puts "This will process #{users.count} users. Do you want to continue? (y/n)"
+
+    begin
+      input = STDIN.gets.strip.downcase
+    end until %w(y n).include?(input)
+
+    if input !='y'
+      STDOUT.puts "Cancelling lead creation for abandoned users."
+      return
+    end
+
+    users.each do |user|
+      lead = OpenStax::Salesforce::Remote::Lead.select(:id, :verification_status).find_by(accounts_uuid: user.uuid)
+
+      if lead.nil?
+        Newflow::CreateOrUpdateSalesforceLead.call(user: user)
+      else
+        # set the lead id, this will update their status in UpdateUserLeadInfo
+        user.salesforce_lead_id = lead.id
+        user.save
+      end
+    end
   end
 end