diff --git a/.lando.yml b/.lando.yml index 669e39f76..39487ce8e 100644 --- a/.lando.yml +++ b/.lando.yml @@ -22,7 +22,7 @@ services: type: postgres:15 portforward: true orangelight_redis: - type: redis:6.0.16 + type: redis:7 portforward: true proxy: orangelight_test_solr: diff --git a/Gemfile b/Gemfile index b345d36b7..9c6290d25 100644 --- a/Gemfile +++ b/Gemfile @@ -76,6 +76,7 @@ gem 'rsolr' # Should this be in the test, development group? gem 'rspec-rails' gem 'rubyzip', '>= 1.2.2' +gem 'sidekiq' gem 'simple_form' gem 'sneakers' gem 'sprockets-es6' diff --git a/Gemfile.lock b/Gemfile.lock index a7d6bb567..83998ba7d 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -615,6 +615,11 @@ GEM serverengine (2.0.7) sigdump (~> 0.2.2) set (1.1.1) + sidekiq (7.3.7) + connection_pool (>= 2.3.0) + logger + rack (>= 2.2.4) + redis-client (>= 0.22.2) sigdump (0.2.5) simple_form (5.3.1) actionpack (>= 5.2) @@ -826,6 +831,7 @@ DEPENDENCIES rubocop-rspec_rails rubyzip (>= 1.2.2) selenium-webdriver + sidekiq simple_form sneakers solargraph diff --git a/README.md b/README.md index 91496164c..461464a29 100644 --- a/README.md +++ b/README.md @@ -56,15 +56,17 @@ bundle exec cap staging deploy # deploys main branch to staging ``` ## Staging Mail Catcher - To see mail that has been sent on the staging server you must ssh tunnel into the server + To see mail that has been sent from the staging environment you must ssh tunnel into the staging indexer servers. ``` - ssh -L 1082:localhost:1080 pulsys@catalog-staging1 + ssh -L 1082:localhost:1080 pulsys@catalog-indexer-staging1 + ssh -L 1085:localhost:1080 pulsys@catalog-indexer-staging2 ``` To see mail that has been sent on the qa server you must ssh tunnel into the server ``` - ssh -L 1082:localhost:1080 pulsys@catalog-qa1 + ssh -L 1082:localhost:1080 pulsys@catalog-indexer-qa1 + ssh -L 1085:localhost:1080 pulsys@catalog-indexer-qa2 ``` - Once the tunnel is open [you can see the mail that has been sent on staging here]( http://localhost:1082/) + Once the tunnel is open [you can see the mail that has been sent on indexer1 here]( http://localhost:1082/) and [indexer2 here]( http://localhost:1085/) ## Testing @@ -245,6 +247,7 @@ Start the Bibdata server, and then set the ```bidata_base``` value in OrangeLigh ``` run every time ``` + bundle exec sidekiq -q default -q mailers mailcatcher ``` diff --git a/config/application.rb b/config/application.rb index 92fdb8afd..c312ff51e 100644 --- a/config/application.rb +++ b/config/application.rb @@ -50,6 +50,8 @@ class Application < Rails::Application # config.time_zone = 'Central Time (US & Canada)' config.time_zone = 'Eastern Time (US & Canada)' + config.active_job.queue_adapter = :sidekiq + # The default locale is :en and all translations from config/locales/*.rb,yml are auto loaded. # config.i18n.load_path += Dir[Rails.root.join('my', 'locales', '*.{rb,yml}').to_s] # config.i18n.default_locale = :de diff --git a/config/deploy.rb b/config/deploy.rb index 0652f5a78..d1b7bec62 100644 --- a/config/deploy.rb +++ b/config/deploy.rb @@ -8,8 +8,6 @@ set :branch, ENV.fetch('BRANCH', 'main') -# Default branch is :master -# ask :branch, proc { `git rev-parse --abbrev-ref HEAD`.chomp }.call # Default deploy_to directory is /var/www/my_app set :deploy_to, '/opt/orangelight' @@ -159,5 +157,15 @@ end end +namespace :sidekiq do + task :restart do + on roles(:indexer) do + execute :sudo, :service, :sidekiq, :restart + end + end +end + after 'deploy:reverted', 'sneakers:restart' after 'deploy:published', 'sneakers:restart' +after 'deploy:reverted', 'sidekiq:restart' +after 'deploy:published', 'sidekiq:restart' diff --git a/config/deploy/production.rb b/config/deploy/production.rb index 9853d3711..dd01e95d6 100644 --- a/config/deploy/production.rb +++ b/config/deploy/production.rb @@ -18,9 +18,9 @@ server 'catalog3', user: 'deploy', roles: %i[web app db worker] server 'catalog4', user: 'deploy', roles: %i[web app db worker] server 'catalog5', user: 'deploy', roles: %i[web app db worker] -server 'catalog-indexer1', user: 'deploy', roles: %i[cron_prod1 cron_db worker] -server 'catalog-indexer2', user: 'deploy', roles: %i[cron_prod2 worker] -server 'catalog-indexer3', user: 'deploy', roles: %i[cron_prod3 worker] +server 'catalog-indexer1', user: 'deploy', roles: %i[cron_prod1 cron_db worker indexer] +server 'catalog-indexer2', user: 'deploy', roles: %i[cron_prod2 worker indexer] +server 'catalog-indexer3', user: 'deploy', roles: %i[cron_prod3 worker indexer] set :deploy_to, '/opt/orangelight' set :log_level, :info diff --git a/config/deploy/qa.rb b/config/deploy/qa.rb index 2e82369cb..4a4339fe6 100644 --- a/config/deploy/qa.rb +++ b/config/deploy/qa.rb @@ -6,8 +6,8 @@ server 'catalog-qa1.princeton.edu', user: 'deploy', roles: %i[web app db worker mailcatcher] server 'catalog-qa2.princeton.edu', user: 'deploy', roles: %i[web app db worker mailcatcher cron_db] -server 'catalog-indexer-qa1', user: 'deploy', roles: %i[worker] -server 'catalog-indexer-qa2', user: 'deploy', roles: %i[cron_db worker] +server 'catalog-indexer-qa1', user: 'deploy', roles: %i[worker indexer] +server 'catalog-indexer-qa2', user: 'deploy', roles: %i[cron_db worker indexer] namespace :env do desc 'Set an Orangelight environment variable' diff --git a/config/deploy/staging.rb b/config/deploy/staging.rb index ba5cd5467..db7e175d9 100644 --- a/config/deploy/staging.rb +++ b/config/deploy/staging.rb @@ -6,8 +6,8 @@ server 'catalog-staging1', user: 'deploy', roles: %i[web app db worker mailcatcher cron_db] server 'catalog-staging2', user: 'deploy', roles: %i[web app worker mailcatcher] -server 'catalog-indexer-staging1', user: 'deploy', roles: %i[cron_db worker] -server 'catalog-indexer-staging2', user: 'deploy', roles: %i[cron_db worker] +server 'catalog-indexer-staging1', user: 'deploy', roles: %i[cron_db worker indexer] +server 'catalog-indexer-staging2', user: 'deploy', roles: %i[cron_db worker indexer] namespace :env do desc 'Set an Orangelight environment variable' diff --git a/config/initializers/sidekiq.rb b/config/initializers/sidekiq.rb new file mode 100644 index 000000000..3e76b3590 --- /dev/null +++ b/config/initializers/sidekiq.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +require "redis-client" + +# nosemgrep +redis_config_from_yml = YAML.safe_load(ERB.new(Rails.root.join("config/redis.yml").read).result, aliases: true)[Rails.env].with_indifferent_access +redis_config = RedisClient.config(host: redis_config_from_yml[:host], port: redis_config_from_yml[:port], db: redis_config_from_yml[:db]) + +# Sidekiq::Client.reliable_push! unless Rails.env.test? # sidekiq pro feature, we would need the key and sidekiq pro gem to enable this +Sidekiq.configure_server do |config| + config.redis = { url: redis_config.server_url } + # config.super_fetch! # sidekiq pro feature + # config.reliable_scheduler! # sidekiq pro feature +end + +Sidekiq.configure_client do |config| + config.redis = { url: redis_config.server_url } +end diff --git a/config/routes.rb b/config/routes.rb index d2df24455..225c5b8bd 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +require 'sidekiq/web' + Rails.application.routes.draw do scope module: 'orangelight' do get 'browse', to: 'browsables#browse' @@ -65,6 +67,10 @@ get "sign_out", to: "sessions#destroy" end + authenticate :user, ->(user) { user.admin? } do + mount Sidekiq::Web => '/sidekiq' + end + get '/catalog/oclc/:id', to: 'catalog#oclc' get '/catalog/isbn/:id', to: 'catalog#isbn' get '/catalog/lccn/:id', to: 'catalog#lccn' diff --git a/spec/requests/sidekiq_spec.rb b/spec/requests/sidekiq_spec.rb new file mode 100644 index 000000000..9662e90b0 --- /dev/null +++ b/spec/requests/sidekiq_spec.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true +require 'rails_helper' +RSpec.describe 'Sidekiq UI' do + it 'denies access to the sidekiq UI to non-admins' do + user = FactoryBot.create(:user) + login_as user + expect do + visit '/sidekiq' + end.to raise_error ActionController::RoutingError + end + it 'allows access to the sidekiq UI to admins' do + user = FactoryBot.create(:user) + allow(user).to receive(:admin?).and_return true + login_as user + visit '/sidekiq' + expect(page).to have_text 'Sidekiq' + end +end