Add Superconfig for Environment and Vault in the Backend (#273)

adds superconfig in the same way as the monolith

.env.development

@@ -0,0 +1,7 @@
+AWS_REGION=us-east-1
+AWS_ACCESS_KEY_ID=fake-key
+AWS_SECRET_ACCESS_KEY=fake-access-key
+DYNAMO_REGISTRATIONS_TABLE=registrations-development
+RAILS_LOG_TO_STDOUT=true
+JWT_SECRET=jwt-test-secret
+SECRET_KEY_BASE=a003fdc6f113ff7d295596a02192c7116a76724ba6d3071043eefdd16f05971be0dc58f244e67728757b2fb55ae7a41e1eb97c1fe247ddaeb6caa97cea32120c

.env.test

@@ -0,0 +1,6 @@
+AWS_REGION=us-east-1
+AWS_ACCESS_KEY_ID=fake-key
+AWS_SECRET_ACCESS_KEY=fake-access-key
+DYNAMO_REGISTRATIONS_TABLE=registrations-development
+JWT_SECRET=jwt-test-secret
+SECRET_KEY_BASE=a003fdc6f113ff7d295596a02192c7116a76724ba6d3071043eefdd16f05971be0dc58f244e67728757b2fb55ae7a41e1eb97c1fe247ddaeb6caa97cea32120c

Gemfile

@@ -53,7 +53,11 @@ gem 'tzinfo-data', platforms: %i[mingw mswin x64_mingw jruby]
 gem 'prometheus_exporter'
 
 # vault for secrets management
-gem 'vault-rails'
+gem 'vault'
+
+# for environment variable management
+gem 'dotenv-rails', require: 'dotenv/rails-now'
+gem 'superconfig'
 
 group :development, :test do
   # See https://guides.rubyonrails.org/debugging_rails_applications.html#debugging-with-the-debug-gem

Gemfile.lock

@@ -100,6 +100,10 @@ GEM
       irb (>= 1.5.0)
       reline (>= 0.3.1)
     diff-lcs (1.5.0)
+    dotenv (2.8.1)
+    dotenv-rails (2.8.1)
+      dotenv (= 2.8.1)
+      railties (>= 3.2)
     dynamoid (3.8.0)
       activemodel (>= 4)
       aws-sdk-dynamodb (~> 1.0)
@@ -273,16 +277,14 @@ GEM
     ruby-progressbar (1.13.0)
     sqlite3 (1.6.3-aarch64-linux)
     sqlite3 (1.6.3-x86_64-linux)
+    superconfig (2.1.1)
     thor (1.2.2)
     timeout (0.4.0)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
     unicode-display_width (2.5.0)
     vault (0.18.1)
       aws-sigv4
-    vault-rails (0.9.0)
-      activesupport (>= 5.0)
-      vault (~> 0.18)
     webmock (3.19.1)
       addressable (>= 2.8.0)
       crack (>= 0.3.2)
@@ -303,6 +305,7 @@ DEPENDENCIES
   bootsnap
   connection_pool
   debug
+  dotenv-rails
   dynamoid (= 3.8.0)
   factory_bot_rails
   hiredis
@@ -322,8 +325,9 @@ DEPENDENCIES
   rubocop
   ruby-prof
   sqlite3 (~> 1.4)
+  superconfig
   tzinfo-data
-  vault-rails
+  vault
   webmock
 
 RUBY VERSION

app/models/registration.rb

@@ -4,12 +4,8 @@
 class Registration
   include Dynamoid::Document
 
-  # We autoscale dynamodb in production
-  if ENV.fetch('CODE_ENVIRONMENT', 'development') == 'staging'
-    table name: 'registrations-staging', read_capacity: 5, write_capacity: 5, key: :attendee_id
-  else
-    table name: 'registrations', capacity_mode: nil, key: :attendee_id
-  end
+  # We autoscale dynamodb
+  table name: EnvConfig.DYNAMO_REGISTRATIONS_TABLE, capacity_mode: nil, key: :attendee_id
 
   REGISTRATION_STATES = %w[pending waiting_list accepted cancelled].freeze
   ADMIN_ONLY_STATES = %w[pending waiting_list accepted].freeze # Only admins are allowed to change registration state to one of these states

app/worker/env_config.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+require 'superconfig'
+
+EnvConfig = SuperConfig.new do
+  # We don't have RAILS_ENV in ruby
+  if ENV.fetch('CODE_ENVIRONMENT', 'development') == 'development'
+    mandatory :LOCALSTACK_ENDPOINT, :string
+    # Have to be the same as in localstack to simulate authentication
+    mandatory :AWS_ACCESS_KEY_ID, :string
+    mandatory :AWS_SECRET_ACCESS_KEY, :string
+    optional :WCA_HOST, :string, ''
+    optional :CODE_ENVIRONMENT, 'development'
+  else
+    mandatory :QUEUE_URL, :string
+    mandatory :WCA_HOST, :string
+    mandatory :CODE_ENVIRONMENT, :string
+  end
+  # We even need the AWS_REGION in dev because we fake authenticate with localstack
+  mandatory :AWS_REGION, :string
+  mandatory :PROMETHEUS_EXPORTER, :string
+  mandatory :DYNAMO_REGISTRATIONS_TABLE, :string
+end

app/worker/queue_poller.rb

@@ -6,6 +6,7 @@
 require 'prometheus_exporter/instrumentation'
 require 'prometheus_exporter/metric'
 require_relative 'registration_processor'
+require_relative 'env_config'
 
 class QueuePoller
   # Wait for 1 second so we can start work on 10 messages at at time
@@ -16,15 +17,9 @@ class QueuePoller
   def self.perform
     PrometheusExporter::Client.default = PrometheusExporter::Client.new(host: ENV.fetch('PROMETHEUS_EXPORTER'), port: 9091)
     # Instrumentation of the worker process is currently disabled per https://github.com/discourse/prometheus_exporter/issues/282
-    if ENV.fetch('CODE_ENVIRONMENT', 'dev') == 'staging'
-      # PrometheusExporter::Instrumentation::Process.start(type: "wca-registration-worker-staging", labels: { process: "1" })
-      @suffix = '-staging'
-    else
-      # PrometheusExporter::Instrumentation::Process.start(type: "wca-registration-worker", labels: { process: "1" })
-      @suffix = ''
-    end
-    registrations_counter = PrometheusExporter::Client.default.register('counter', "registrations_counter-#{@suffix}", 'The number of Registrations processed')
-    error_counter = PrometheusExporter::Client.default.register('counter', "worker_error_counter-#{@suffix}", 'The number of Errors in the worker')
+    suffix = "-#{ENV.fetch("CODE_ENVIRONMENT", "development")}"
+    registrations_counter = PrometheusExporter::Client.default.register('counter', "registrations_counter-#{suffix}", 'The number of Registrations processed')
+    error_counter = PrometheusExporter::Client.default.register('counter', "worker_error_counter-#{suffix}", 'The number of Errors in the worker')
 
     @sqs ||= if ENV['LOCALSTACK_ENDPOINT']
                Aws::SQS::Client.new(endpoint: ENV['LOCALSTACK_ENDPOINT'])

app_secrets.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require 'superconfig'
+
+require_relative 'env_config'
+
+SuperConfig::Base.class_eval do
+  # The skeleton is stolen from the source code of the `superconfig` gem, file lib/superconfig.rb:104
+  #   (method SuperConfig::Base#credential). The inner Vault fetching logic is custom-written :)
+  def vault(secret_name, &block)
+    define_singleton_method(secret_name) do
+      @__cache__["_vault_#{secret_name}".to_sym] ||= begin
+        value = self.vault_read(secret_name)[:value]
+        block ? block.call(value) : value
+      end
+    end
+  end
+
+  private def vault_read(secret_name)
+    Vault.with_retries(Vault::HTTPConnectionError, Vault::HTTPError) do |attempt, e|
+      puts "Received exception #{e} from Vault - attempt #{attempt}" if e.present?
+
+      secret = Vault.logical.read("kv/data/#{EnvConfig.VAULT_APPLICATION}/#{secret_name}")
+      raise "Tried to read #{secret_name}, but doesn't exist" unless secret.present?
+
+      secret.data[:data]
+    end
+  end
+end
+
+AppSecrets = SuperConfig.new do
+  if Rails.env.production?
+    require_relative 'vault_config'
+
+    vault :JWT_SECRET
+    vault :SECRET_KEY_BASE
+
+  else
+    mandatory :JWT_SECRET, :string
+    mandatory :SECRET_KEY_BASE, :string
+  end
+end

config/application.rb

@@ -7,6 +7,8 @@
 # Require the gems listed in Gemfile, including any gems
 # you've limited to :test, :development, or :production.
 Bundler.require(*Rails.groups)
+require_relative '../env_config'
+require_relative '../app_secrets'
 
 module WcaRegistration
   class Application < Rails::Application

config/database.yml

@@ -1,12 +1,7 @@
-# SQLite. Versions 3.8.0 and up are supported.
-#   gem install sqlite3
-#
-#   Ensure the SQLite 3 gem is defined in your Gemfile
-#   gem "sqlite3"
-#
+# We currently have no database configured
 default: &default
   adapter: sqlite3
-  pool: <%= ENV.fetch("RAILS_MAX_THREADS") { 5 } %>
+  pool: 5
   timeout: 5000
 
 development:

config/environments/development.rb

@@ -20,7 +20,7 @@
   config.server_timing = true
 
   # Caching with Redis even in Development to speed up calls to the external services
-  config.cache_store = :redis_cache_store, { url: ENV.fetch('REDIS_URL', nil) }
+  config.cache_store = :redis_cache_store, { url: EnvConfig.REDIS_URL }
 
   # Store uploaded files on the local file system (see config/storage.yml for options).
   config.active_storage.service = :local

config/environments/production.rb

@@ -4,12 +4,7 @@
 
 Rails.application.configure do
   # Settings specified here will take precedence over those in config/application.rb.
-  code_env = ENV.fetch('CODE_ENVIRONMENT', nil)
-  if code_env == 'production'
-    config.hosts << 'registration.worldcubeassociation.org'
-  elsif code_env == 'staging'
-    config.hosts << 'staging.registration.worldcubeassociation.org'
-  end
+  config.hosts = EnvConfig.HOST
 
   # Exclude requests for the /healthcheck/ path from host checking
   Rails.application.config.host_authorization = {
@@ -63,7 +58,7 @@
 
   # Use a different cache store in production.
   config.cache_store = :redis_cache_store, {
-    url: ENV.fetch('REDIS_URL', nil),
+    url: EnvConfig.REDIS_URL,
     connect_timeout: 30, # Defaults to 20 seconds
     read_timeout: 0.2, # Defaults to 1 second
     write_timeout: 0.2, # Defaults to 1 second

config/initializers/aws.rb

@@ -7,6 +7,6 @@
   $sqs = Aws::SQS::Client.new
 else
   # We are using localstack to emulate AWS in dev
-  $dynamodb = Aws::DynamoDB::Client.new(endpoint: ENV.fetch('LOCALSTACK_ENDPOINT', nil))
-  $sqs = Aws::SQS::Client.new(endpoint: ENV.fetch('LOCALSTACK_ENDPOINT', nil))
+  $dynamodb = Aws::DynamoDB::Client.new(endpoint: EnvConfig.LOCALSTACK_ENDPOINT)
+  $sqs = Aws::SQS::Client.new(endpoint: EnvConfig.LOCALSTACK_ENDPOINT)
 end

config/initializers/dynamoid.rb

@@ -3,11 +3,11 @@
 require 'dynamoid'
 
 Dynamoid.configure do |config|
-  config.region = ENV.fetch('AWS_REGION', 'us-west-2')
+  config.region = EnvConfig.AWS_REGION
   config.namespace = nil
   if Rails.env.production?
     config.credentials = Aws::ECSCredentials.new(retries: 3)
   else
-    config.endpoint = ENV.fetch('LOCALSTACK_ENDPOINT', nil)
+    config.endpoint = EnvConfig.LOCALSTACK_ENDPOINT
   end
 end

config/initializers/jwt.rb

@@ -2,7 +2,7 @@
 
 require_relative '../../app/helpers/jwt_options'
 
-JwtOptions.secret = read_secret('JWT_SECRET')
+JwtOptions.secret = AppSecrets.JWT_SECRET
 # Default algorithm for Devise-jwt
 JwtOptions.algorithm = 'HS256'
 # The expiry time we define in the monolith

config/initializers/prometheus.rb

@@ -8,12 +8,12 @@
 
 PrometheusExporter::Client.default = PrometheusExporter::Client.new(host: ENV.fetch('PROMETHEUS_EXPORTER'), port: 9091)
 
-if ENV.fetch('CODE_ENVIRONMENT', 'dev') == 'staging'
+if Rails.env.production? && !EnvConfig.REGISTRATION_LIVE_SITE
   PrometheusExporter::Instrumentation::Process.start(type: 'wca-registration-handler-staging', labels: { process: '1' })
-  @suffix = '-staging'
+  suffix = '-staging'
 else
   PrometheusExporter::Instrumentation::Process.start(type: 'wca-registration-handler', labels: { process: '1' })
-  @suffix = ''
+  suffix = ''
 end
 
 unless Rails.env.test?
@@ -24,9 +24,9 @@
 end
 
 # Create our Metric Counters
-Metrics.registration_dynamodb_errors_counter = PrometheusExporter::Client.default.register('counter', "registration_dynamodb_errors_counter#{@suffix}", 'The number of times interacting with dynamodb fails')
-Metrics.registration_competition_api_error_counter = PrometheusExporter::Client.default.register('counter', "registration_competition_api_error_counter#{@suffix}", 'The number of times interacting with the competition API failed')
-Metrics.registration_competitor_api_error_counter = PrometheusExporter::Client.default.register('counter', "registration_competitor_api_error_counter#{@suffix}", 'The number of times interacting with the user API failed')
-Metrics.registration_validation_errors_counter = PrometheusExporter::Client.default.register('counter', "registration_validation_errors_counter#{@suffix}", 'The number of times validation fails when an attendee tries to register')
-Metrics.jwt_verification_error_counter = PrometheusExporter::Client.default.register('counter', "jwt_verification_error_counter#{@suffix}", 'The number of times JWT verification failed')
-Metrics.registration_impersonation_attempt_counter = PrometheusExporter::Client.default.register('counter', "registration_impersonation_attempt_counter#{@suffix}", 'The number of times a Person tries to register as someone else')
+Metrics.registration_dynamodb_errors_counter = PrometheusExporter::Client.default.register('counter', "registration_dynamodb_errors_counter#{suffix}", 'The number of times interacting with dynamodb fails')
+Metrics.registration_competition_api_error_counter = PrometheusExporter::Client.default.register('counter', "registration_competition_api_error_counter#{suffix}", 'The number of times interacting with the competition API failed')
+Metrics.registration_competitor_api_error_counter = PrometheusExporter::Client.default.register('counter', "registration_competitor_api_error_counter#{suffix}", 'The number of times interacting with the user API failed')
+Metrics.registration_validation_errors_counter = PrometheusExporter::Client.default.register('counter', "registration_validation_errors_counter#{suffix}", 'The number of times validation fails when an attendee tries to register')
+Metrics.jwt_verification_error_counter = PrometheusExporter::Client.default.register('counter', "jwt_verification_error_counter#{suffix}", 'The number of times JWT verification failed')
+Metrics.registration_impersonation_attempt_counter = PrometheusExporter::Client.default.register('counter', "registration_impersonation_attempt_counter#{suffix}", 'The number of times a Person tries to register as someone else')