Merge pull request #3675 from mlibrary/HELIO-4562/status-page-additions

HELIO-4562 Add more checks to the /status page

app/controllers/utilities_controller.rb

@@ -3,6 +3,8 @@
 include StatusPageService
 
 class UtilitiesController < ApplicationController
+  include Skylight::Helpers
+
   def whoami # rubocop:disable Metrics/CyclomaticComplexity
     headers = request.headers
     hash_tag_line = ''
@@ -59,6 +61,14 @@ def whoami # rubocop:disable Metrics/CyclomaticComplexity
     render plain: who_am_i
   end
 
+  instrument_method
+  def show_app_status?
+    # The "University of Michigan, Ann Arbor" institution has `identifier` == "1"
+    return true if Rails.env.eql?('development') || current_ability&.current_user&.platform_admin? || current_institutions&.map(&:identifier)&.include?('1')
+    false
+  end
+
+  instrument_method
   def status
     hash_tag_line = ''
     130.times { hash_tag_line += '#' }
@@ -71,80 +81,94 @@ def status
       #{hash_tag_line}
     STATUS_HEADER
 
-    # The "University of Michigan, Ann Arbor" institution has `identifier` == "1"
-    if Rails.env.eql?('development') || current_ability&.current_user&.platform_admin? || current_institutions&.map(&:identifier)&.include?('1')
-      output += <<~APP_STATUS
+    time = Benchmark.measure do # rubocop:disable Metrics/BlockLength:
+      if show_app_status?
+        output += <<~APP_STATUS
+
+        Application Checks
+
+          Config Files
+
+            database.yml .......... #{check_config_file('database.yml')}
+            fedora.yml ............ #{check_config_file('fedora.yml')}
+            secrets.yml ........... #{check_config_file('secrets.yml')}
+            solr.yml .............. #{check_config_file('solr.yml')}
+            analytics.yml ......... #{check_config_file('analytics.yml')}
+            aptrust.yml ........... #{check_config_file('aptrust.yml')}
+            blacklight.yml ........ #{check_config_file('blacklight.yml')}
+            box.yml ............... #{check_config_file('box.yml')}
+            crossref.yml .......... #{check_config_file('crossref.yml')}
+            redis.yml ............. #{check_config_file('redis.yml')}
+            resque-pool.yml ....... #{check_config_file('resque-pool.yml')}
+            role_map.yml .......... #{check_config_file('role_map.yml')}
+            skylight.yml .......... #{check_config_file('skylight.yml')}
+
+          Resque workers .......... #{resque_workers_count} registered, #{resque_workers_working_count} working
+
+
+        Environment Status
 
-      Application Checks
+            Shibboleth redirect ... #{shib_check_redirecting}
+            MySQL ................. #{check_active_record}
+            Fedora ................ #{fedora}
+            Solr .................. #{solr}
+            Redis ................. #{redis}
+            FITS .................. #{fits_version}
 
-        Config Files
 
-          database.yml .......... #{check_config_file('database.yml')}
-          fedora.yml ............ #{check_config_file('fedora.yml')}
-          secrets.yml ........... #{check_config_file('secrets.yml')}
-          solr.yml .............. #{check_config_file('solr.yml')}
-          analytics.yml ......... #{check_config_file('analytics.yml')}
-          aptrust.yml ........... #{check_config_file('aptrust.yml')}
-          blacklight.yml ........ #{check_config_file('blacklight.yml')}
-          box.yml ............... #{check_config_file('box.yml')}
-          crossref.yml .......... #{check_config_file('crossref.yml')}
-          redis.yml ............. #{check_config_file('redis.yml')}
-          resque-pool.yml ....... #{check_config_file('resque-pool.yml')}
-          role_map.yml .......... #{check_config_file('role_map.yml')}
-          skylight.yml .......... #{check_config_file('skylight.yml')}
+        Server
 
-        Resque workers .......... #{resque_workers_count} registered, #{resque_workers_working_count} working
+          #{Socket.gethostname}
 
 
-      Environment Status
+        Server Uptime
 
-          Shibboleth redirect ... #{shib_check_redirecting}
-          MySQL ................. #{check_active_record}
-          Fedora ................ #{fedora}
-          Solr .................. #{solr}
-          Redis ................. #{redis}
-          FITS .................. #{fits_version}
+          #{server_status}
 
 
-      Server
+        Processes - Puma workers
 
-        #{Socket.gethostname}
+          #{puma_workers}
 
 
-      Server Uptime
+        Processes - Resque workers
 
-        #{server_status}
+          #{resque_workers}
 
 
-      Processes - Puma workers
+        ActiveFedora - Connection config options
 
-        #{puma_workers}
+          #{ActiveFedora::Fedora.instance.authorized_connection.options}
 
 
-      Processes - Resque workers
+        Log Level
 
-        #{resque_workers}
+          #{Rails.logger.level}
 
 
-      ActiveFedora - Connection config options
+        File System (results of /bin/ls on directories in seconds)
 
-        #{ActiveFedora::Fedora.instance.authorized_connection.options}
+          derivatives ............. #{derivatives_path}
+          uploads ................. #{uploads_path}
+          riiif_network_files ..... #{riiif_network_files_path}
+          scratch_space ........... #{scratch_space_path}
 
 
-      Log Level
+        Sample Solr Query Time (in seconds): #{solr_sample_query}
 
-        #{Rails.logger.level}
 
-      APP_STATUS
-    else
-      output += <<~STATUS_LOGIN
+        APP_STATUS
+      else
+        output += <<~STATUS_LOGIN
 
-      Please connect from a University of Michigan IP to see more details!
-      note: The Shibboleth process is #{shib_process}.
-      STATUS_LOGIN
+        Please connect from a University of Michigan IP to see more details!
+        note: The Shibboleth process is #{shib_process}.
+        STATUS_LOGIN
+      end
     end
 
     output += "\n#{hash_tag_line}"
+    output += "\n\nStatus Page benchmark elapsed real time is #{time.real} seconds\n\n" if show_app_status?
 
     render plain: output
   end

app/services/status_page_service.rb

@@ -1,18 +1,26 @@
 # frozen_string_literal: true
 
+require 'benchmark'
+
 module StatusPageService
+  include Skylight::Helpers
+
+  instrument_method
   def resque_workers_count
     Resque.workers.count
   end
 
+  instrument_method
   def resque_workers_working_count
     Resque.workers.select { |w| w.working? }.count
   end
 
+  instrument_method
   def server_status
     `uptime`&.strip
   end
 
+  instrument_method
   def redis
     # Storing/retrieving an actual value might be better. This is good enough for now.
     redis = if Settings.host == "www.fulcrum.org" || Settings.host == "staging.fulcrum.org" # # HELIO-4477
@@ -26,13 +34,15 @@ def redis
   end
 
   # this is listed as a "MySQL" check in the output, which I think is fair enough
+  instrument_method
   def check_active_record
     ActiveRecord::Migrator.current_version
     'UP'
   rescue StandardError => _e
     'DOWN'
   end
 
+  instrument_method
   def check_config_file(filename)
     file = Rails.root.join('config', filename)
     return 'NOT FOUND' unless File.exist?(file)
@@ -43,43 +53,66 @@ def check_config_file(filename)
     'ERROR'
   end
 
+  instrument_method
   def helio_processes
     @helio_processes ||= `ps -f -u $USER`.split("\n")
   end
 
+  instrument_method
   def puma_workers
     helio_processes&.grep(/puma/)&.join("\n") || 'NONE FOUND'
   end
 
+  instrument_method
   def resque_workers
     helio_processes&.grep(/resque/)&.join("\n") || 'NONE FOUND'
   end
 
+  instrument_method
   def fedora
     file = Rails.root.join('config', 'fedora.yml')
     return 'YML CONFIG NOT FOUND' unless File.exist?(file)
     url = YAML.load(ERB.new(File.read(file)).result)[Rails.env]['url']
     `curl --max-time 5 -s -o /dev/null -w "%{http_code}" '#{url}'` == '200' ? 'UP' : 'DOWN'
   end
 
+  instrument_method
   def solr
     file = Rails.root.join('config', 'solr.yml')
     return 'YML CONFIG NOT FOUND' unless File.exist?(file)
-    url = YAML.load(ERB.new(File.read(file)).result)[Rails.env]['url'].sub('/solr/', '/solr/admin/cores?action=STATUS&core=')
-    # single quotes to prevent incorrect url parsing (maybe) but definitely the `&` results in a background job otherwise
-    response = `curl --max-time 5 -s -w "%{http_code}" '#{url}'`
-    output = response.ends_with?('200') ? 'UP' : 'DOWN'
-    output += response.include?('instanceDir') ? ' - core found' : ' - core not found'
+    parts = YAML.load(ERB.new(File.read(file)).result)[Rails.env]['url'].split("/solr/")
+    url = parts[0]
+    core = parts[1]
+    conn = RSolr.connect url: url
+
+    r = conn.get "/solr/admin/cores", params: { action: "STATUS", core: core }
+    output = r.response[:status] == 200 ? 'UP' : 'DOWN'
+    output += r["status"][core].include?("instanceDir") ? " - core (#{core}) found" : " - core (#{core}) NOT found"
+
+    r = conn.get "/solr/admin/info/system"
+    output += r["lucene"]["solr-spec-version"].present? ? " - version: #{r['lucene']['solr-spec-version']}" : " - version NOT found"
     output
-  rescue StandardError => _e
-    'ERROR'
+  rescue StandardError => e
+    Rails.logger.error("[StatusPageService Solr Error] #{e}")
+    'DOWN'
   end
 
+  # Not sure how useful this will be. This book has 2000 or so file sets so maybe a long response time?
+  # HELIO-4562
+  def solr_sample_query
+    time = Benchmark.measure do
+      ActiveFedora::SolrService.query("Brushed in Light Calligraphy in East Asian Cinema", df: 'title_tesim', rows: 1)
+    end
+    time.real
+  end
+
+  instrument_method
   def fits_version
     output = `fits.sh -v`&.strip
     output.presence || 'NOT FOUND'
   end
 
+  instrument_method
   def shib_process
     if `ps -ef | pgrep shibd`.include?("\n")
       'currently running'
@@ -88,6 +121,7 @@ def shib_process
     end
   end
 
+  instrument_method
   def shib_check_redirecting
     expected_redirect_location = 'https://www.fulcrum.org/Shibboleth.sso/Login?target=https%3A%2F%2Fwww.fulcrum.org%2Fshib_session&entityID=https%3A%2F%2Fshibboleth.umich.edu%2Fidp%2Fshibboleth'
     uri = URI('https://www.fulcrum.org/shib_login')
@@ -97,4 +131,52 @@ def shib_check_redirecting
     # https://docs.ruby-lang.org/en/2.5.0/Net/HTTP.html
     (response.kind_of? Net::HTTPFound) && response['location'] == expected_redirect_location ? 'UP' : 'DOWN'
   end
+
+  instrument_method
+  def derivatives_path
+    if Settings.derivatives_path.present?
+      time = Benchmark.measure do
+        `/bin/ls #{Settings.derivatives_path} > /dev/null 2> /dev/null`
+      end
+      time.real
+    else
+      "missing"
+    end
+  end
+
+  instrument_method
+  def uploads_path
+    if Settings.uploads_path.present?
+      time = Benchmark.measure do
+        `/bin/ls #{Settings.uploads_path} > /dev/null 2> /dev/null`
+      end
+      time.real
+    else
+      "missing"
+    end
+  end
+
+  instrument_method
+  def riiif_network_files_path
+    if Settings.riiif_network_files_path.present?
+      time = Benchmark.measure do
+        `/bin/ls #{Settings.riiif_network_files_path} > /dev/null 2> /dev/null`
+      end
+      time.real
+    else
+      "missing"
+    end
+  end
+
+  instrument_method
+  def scratch_space_path
+    if Settings.scratch_space_path.present?
+      time = Benchmark.measure do
+        `/bin/ls #{Settings.scratch_space_path} > /dev/null 2> /dev/null`
+      end
+      time.real
+    else
+      "missing"
+    end
+  end
 end