Skip to content

Commit

Permalink
API-40310: Add counts to consumer expired details section of Slack alert
Browse files Browse the repository at this point in the history
  • Loading branch information
kristen-brown committed Sep 24, 2024
1 parent 1e93d8d commit 17a00c7
Show file tree
Hide file tree
Showing 2 changed files with 22 additions and 23 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ module VBADocuments
class SlackStatusNotifier
include Sidekiq::Job
include ActionView::Helpers::DateHelper
include ActiveSupport::NumberHelper

# Only retry for ~30 minutes since the job is run every hour
sidekiq_options retry: 5, unique_for: 1.hour
Expand Down Expand Up @@ -60,41 +61,39 @@ def report_expired
# allowed consumers to successfully request s3 upload urls and tracking GUIDs, but they not able to push their
# files to s3 resulting in a large number of expired upload submissions
created_at_range = EXPIRED_LOOKBACK_HOURS.hours.ago..DateTime.now
new_count = VBADocuments::UploadSubmission.where(created_at: created_at_range).count
expired_count = VBADocuments::UploadSubmission.where(status: 'expired',
created_at: created_at_range).count
percent_expired = (expired_count.to_f / new_count) * 100.0
submissions_in_range = VBADocuments::UploadSubmission.where(created_at: created_at_range).order(:consumer_name)
total_count = submissions_in_range.count
expired_count = submissions_in_range.where(status: 'expired').count
percent_expired = (expired_count.to_f / total_count) * 100.0

if percent_expired > EXPIRED_THRESHOLD
message_time = created_at_range.first.in_time_zone('America/New_York')
.strftime('%Y-%m-%d %I:%M:%S %p %Z')
fail_rate = ActiveSupport::NumberHelper.number_to_percentage(percent_expired, precision: 1)
message = "#{expired_count}(#{fail_rate}) " \
"out of #{new_count} Benefits Intake uploads created since #{message_time} " \
fail_rate = number_to_percentage(percent_expired, precision: 1)
message = "#{expired_count} (#{fail_rate}) " \
"out of #{total_count} Benefits Intake uploads created since #{message_time} " \
'have expired with no consumer uploads to S3' \
"\nThis could indicate an S3 issue impacting consumers."

notify_slack(message, expired_details_rate_by_consumer(created_at_range))
notify_slack(message, expired_details_by_consumer(submissions_in_range))
end
rescue => e
notify_slack("'Expired' status notifier exception: #{e.class}", e.message)
raise e
end

def expired_details_rate_by_consumer(created_at_range)
# break out expired rates by consumer
consumer_all_counts = VBADocuments::UploadSubmission.where(created_at: created_at_range)
.group(:consumer_name).count
consumer_exp_counts = VBADocuments::UploadSubmission.where(created_at: created_at_range, status: 'expired')
.group(:consumer_name).count
def expired_details_by_consumer(submissions_in_range)
counts_by_consumer = submissions_in_range.group(:consumer_name).count
expired_counts_by_consumer = submissions_in_range.where(status: 'expired').group(:consumer_name).count

# calc expired rate% by consumer
exp_rate = consumer_all_counts.map { |name, count| [name, (consumer_exp_counts[name].to_f / count) * 100] }

# sort by Consumer expired Rate, build slack reporting string
slack_details = "\n\t(Consumer, Expired Rate)\n"
exp_rate.sort_by { |e| -e[1] }.each do |cr|
slack_details += "\t#{cr[0]}: #{ActiveSupport::NumberHelper.number_to_percentage(cr[1], precision: 1)}\n"

counts_by_consumer.each do |consumer_detail|
consumer_name, total_count = consumer_detail
expired_count = expired_counts_by_consumer[consumer_name] || 0
expired_percentage = number_to_percentage((expired_count.to_f / total_count) * 100, precision: 1)

slack_details += "\t#{consumer_name}: #{expired_percentage}, #{expired_count}/#{total_count}\n"
end

slack_details
Expand Down Expand Up @@ -141,7 +140,7 @@ def uploaded_details(upload_submissions)
slack_details += "\t#{us[0]} " \
"#{us[1].to_i / 3600}:#{format('%02d', (us[1] / 60 % 60).to_i)} " \
"#{us[2] || 0} " \
"#{ActiveSupport::NumberHelper.number_to_delimited(us[3])} " \
"#{number_to_delimited(us[3])} " \
"#{us[4]}\n"
end
slack_details
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -57,10 +57,10 @@
expect(VBADocuments::Slack::Messenger).to have_received(:new).with(
{
class: 'VBADocuments::SlackStatusNotifier',
alert: "1(50.0%) out of 2 Benefits Intake uploads created since #{t} " \
alert: "1 (50.0%) out of 2 Benefits Intake uploads created since #{t} " \
'have expired with no consumer uploads to S3' \
"\nThis could indicate an S3 issue impacting consumers.",
details: "\n\t(Consumer, Expired Rate)\n\tvagov: 100.0%\n\tsometech: 0.0%\n"
details: "\n\t(Consumer, Expired Rate)\n\tsometech: 0.0%, 0/1\n\tvagov: 100.0%, 1/1\n"
}
)
end
Expand Down

0 comments on commit 17a00c7

Please sign in to comment.