diff --git a/Gemfile b/Gemfile index 38f78459..fa1be7f6 100644 --- a/Gemfile +++ b/Gemfile @@ -40,6 +40,9 @@ gem 'kredis' # Windows does not include zoneinfo files, so bundle the tzinfo-data gem gem 'tzinfo-data', platforms: %i[mingw mswin x64_mingw jruby] +# Exposes Metrics +gem 'prometheus_exporter' + # vault for secrets management gem 'vault-rails' diff --git a/Gemfile.lock b/Gemfile.lock index 5d108240..b317cedb 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -138,6 +138,8 @@ GEM parallel (1.23.0) parser (3.2.2.1) ast (~> 2.4.1) + prometheus_exporter (2.0.8) + webrick puma (5.6.5) nio4r (~> 2.0) racc (1.6.2) @@ -204,6 +206,7 @@ GEM vault-rails (0.8.0) activesupport (>= 5.0) vault (~> 0.17) + webrick (1.8.1) websocket-driver (0.7.5) websocket-extensions (>= 0.1.0) websocket-extensions (0.1.5) @@ -221,6 +224,7 @@ DEPENDENCIES hiredis jbuilder kredis + prometheus_exporter puma (~> 5.0) rack-cors rails (~> 7.0.4, >= 7.0.4.3) diff --git a/app/controllers/metrics_controller.rb b/app/controllers/metrics_controller.rb deleted file mode 100644 index b3c5132d..00000000 --- a/app/controllers/metrics_controller.rb +++ /dev/null @@ -1,18 +0,0 @@ -# frozen_string_literal: true - -require 'securerandom' -class MetricsController < ApplicationController - def index - # Get the queue attributes - queue_url = $sqs.get_queue_url(queue_name: 'registrations.fifo').queue_url - response = $sqs.get_queue_attributes({ - queue_url: queue_url, - attribute_names: ['ApproximateNumberOfMessages'], - }) - - # Get the queue size - queue_size = response.attributes['ApproximateNumberOfMessages'].to_i - - render json: { queue_size: queue_size } - end -end diff --git a/app/controllers/registration_controller.rb b/app/controllers/registration_controller.rb index b2c7af39..38ea7c94 100644 --- a/app/controllers/registration_controller.rb +++ b/app/controllers/registration_controller.rb @@ -11,6 +11,7 @@ def create event_ids = params[:event_ids] unless validate_request(competitor_id, competition_id) && !event_ids.empty? + Metrics.registration_validation_errors_counter.increment return render json: { status: 'User cannot register for competition' }, status: :forbidden end @@ -42,6 +43,7 @@ def update status = params[:status] unless validate_request(competitor_id, competition_id, status) + Metrics.registration_validation_errors_counter.increment return render json: { status: 'User cannot register, wrong format' }, status: :forbidden end @@ -67,7 +69,8 @@ def update }) render json: { status: 'ok' } rescue Aws::DynamoDB::Errors::ServiceError => e - puts e # TODO: Expose this as a metric + puts e + Metrics.registration_dynamodb_errors_counter.increment render json: { status: 'Failed to update registration data' }, status: :internal_server_error end end @@ -77,6 +80,7 @@ def delete competition_id = params[:competition_id] unless validate_request(competitor_id, competition_id) + Metrics.registration_validation_errors_counter.increment return render json: { status: 'User cannot register, wrong format' }, status: :forbidden end @@ -97,7 +101,8 @@ def delete render json: { status: 'ok' } rescue Aws::DynamoDB::Errors::ServiceError => e # Render an error response - puts e # TODO: Expose this as a metric + puts e + Metrics.registration_dynamodb_errors_counter.increment render json: { status: "Error deleting item from DynamoDB: #{e.message}" }, status: :internal_server_error end @@ -106,8 +111,14 @@ def delete def list competition_id = params[:competition_id] registrations = get_registrations(competition_id) - + # Render a success response render json: registrations + rescue Aws::DynamoDB::Errors::ServiceError => e + # Render an error response + puts e + Metrics.registration_dynamodb_errors_counter.increment + render json: { status: "Error getting registrations" }, + status: :internal_server_error end private @@ -133,9 +144,7 @@ def can_user_register?(competitor_id, competition_id) def validate_request(competitor_id, competition_id, status = 'waiting') if competitor_id.present? && competitor_id =~ (/^\d{4}[a-zA-Z]{4}\d{2}$/) - puts 'correct competitor_id' if competition_id =~ (/^[a-zA-Z]+\d{4}$/) && REGISTRATION_STATUS.include?(status) - puts 'correct competition id and status' can_user_register?(competitor_id, competition_id) end else diff --git a/app/helpers/competition_api.rb b/app/helpers/competition_api.rb index 80b22e95..bbb108af 100644 --- a/app/helpers/competition_api.rb +++ b/app/helpers/competition_api.rb @@ -12,7 +12,7 @@ def self.check_competition(competition_id) body = JSON.parse res.body body['registration_open'].present? else - # The Competition Service is unreachable TODO We should track this as a metric + Metrics.registration_competition_api_error_counter.increment puts 'network request failed' false end diff --git a/app/helpers/competitor_api.rb b/app/helpers/competitor_api.rb index 312b44d3..99c2fb2a 100644 --- a/app/helpers/competitor_api.rb +++ b/app/helpers/competitor_api.rb @@ -7,13 +7,20 @@ class CompetitorApi def self.check_competitor(competitor_id) uri = URI("https://www.worldcubeassociation.org/api/v0/users/#{competitor_id}") - res = Net::HTTP.get_response(uri) - if res.is_a?(Net::HTTPSuccess) - body = JSON.parse res.body - body['user'].present? - else - # The Competitor Service is unreachable TODO We should track this as a metric - puts 'network request failed' + begin + res = Net::HTTP.get_response(uri) + if res.is_a?(Net::HTTPSuccess) + body = JSON.parse res.body + body['user'].present? + else + # The Competitor Service is unreachable + Metrics.registration_competitor_api_error_counter.increment + puts 'network request failed' + false + end + rescue StandardError => _e + puts 'The service does not have internet' + Metrics.registration_competitor_api_error_counter.increment false end end diff --git a/app/helpers/metrics.rb b/app/helpers/metrics.rb new file mode 100644 index 00000000..241d2d16 --- /dev/null +++ b/app/helpers/metrics.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +module Metrics + class << self + attr_accessor :registration_validation_errors_counter, :registration_dynamodb_errors_counter, :registrations_counter, :registration_competition_api_error_counter, :registration_competitor_api_error_counter + end +end diff --git a/app/worker/queue_poller.rb b/app/worker/queue_poller.rb index 89b69fac..6d0d17b1 100644 --- a/app/worker/queue_poller.rb +++ b/app/worker/queue_poller.rb @@ -2,6 +2,9 @@ require 'json' require 'aws-sdk-sqs' +require 'prometheus_exporter/client' +require 'prometheus_exporter/instrumentation' +require 'prometheus_exporter/metric' require_relative 'registration_processor' class QueuePoller @@ -11,6 +14,18 @@ class QueuePoller MAX_MESSAGES = 10 def self.perform + PrometheusExporter::Client.default = PrometheusExporter::Client.new(host: ENV.fetch("PROMETHEUS_EXPORTER"), port: 9091) + # Instrumentation of the worker process is currently disabled per https://github.com/discourse/prometheus_exporter/issues/282 + if ENV.fetch("ENVIRONMENT", "dev") == "staging" + PrometheusExporter::Instrumentation::Process.start(type: "wca-registration-worker-staging", labels: { process: "1" }) + @suffix = "-staging" + else + # PrometheusExporter::Instrumentation::Process.start(type: "wca-registration-worker", labels: { process: "1" }) + @suffix = "" + end + registrations_counter = PrometheusExporter::Client.default.register("counter", "registrations_counter-#{@suffix}", "The number of Registrations processed") + error_counter = PrometheusExporter::Client.default.register("counter", "worker_error_counter-#{@suffix}", "The number of Errors in the worker") + @sqs ||= if ENV['LOCALSTACK_ENDPOINT'] Aws::SQS::Client.new(endpoint: ENV['LOCALSTACK_ENDPOINT']) else @@ -27,10 +42,12 @@ def self.perform body = JSON.parse msg.body begin RegistrationProcessor.process_message(body) + registrations_counter.increment rescue StandardError => e # unexpected error occurred while processing messages, # log it, and skip delete so it can be re-processed later puts "Error #{e} when processing message with ID #{msg}" + error_counter.increment throw :skip_delete end end diff --git a/config/application.rb b/config/application.rb index bc1ef8ef..3dd2a2cd 100644 --- a/config/application.rb +++ b/config/application.rb @@ -12,7 +12,7 @@ module WcaRegistration class Application < Rails::Application # Initialize configuration defaults for originally generated Rails version. config.load_defaults 7.0 - + config.autoload_paths << "#{root}/lib/**" # Only loads a smaller set of middleware suitable for API only apps. # Middleware like session, flash, cookies can be added back manually. # Skip views, helpers and assets when generating a new resource. diff --git a/config/initializers/cors.rb b/config/initializers/cors.rb index 493e7448..4ef17dd2 100644 --- a/config/initializers/cors.rb +++ b/config/initializers/cors.rb @@ -21,11 +21,18 @@ else Rails.application.config.middleware.insert_before 0, Rack::Cors do allow do - origins '*.worldcubeassociation.org', 'http://test.registration.worldcubeassociation.org.s3-website-us-west-2.amazonaws.com' + origins 'https://www.worldcubeassociation.org', 'https://test-registration.worldcubeassociation.org/', 'http://test.registration.worldcubeassociation.org.s3-website-us-west-2.amazonaws.com' resource '*', headers: :any, methods: %i[get post put patch delete options head] end + allow do + origins '*' + + resource '/api/v1/*', + headers: :any, + methods: %i[get post put patch delete options head] + end end end diff --git a/config/initializers/prometheus.rb b/config/initializers/prometheus.rb new file mode 100644 index 00000000..17cc92ae --- /dev/null +++ b/config/initializers/prometheus.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +require 'prometheus_exporter/client' +require 'prometheus_exporter/instrumentation' +require 'prometheus_exporter/metric' + +require_relative '../../app/helpers/metrics' + +PrometheusExporter::Client.default = PrometheusExporter::Client.new(host: ENV.fetch("PROMETHEUS_EXPORTER"), port: 9091) + +if ENV.fetch("ENVIRONMENT", "dev") == "staging" + PrometheusExporter::Instrumentation::Process.start(type: "wca-registration-handler-staging", labels: { process: "1" }) + @suffix = "-staging" +else + PrometheusExporter::Instrumentation::Process.start(type: "wca-registration-handler", labels: { process: "1" }) + @suffix = "" +end + +unless Rails.env.test? + require 'prometheus_exporter/middleware' + + # This reports stats per request like HTTP status and timings + Rails.application.middleware.unshift PrometheusExporter::Middleware +end + +# Create our Metric Counters +Metrics.registration_dynamodb_errors_counter = PrometheusExporter::Client.default.register("counter", "registration_dynamodb_errors_counter#{@suffix}", "The number of times interacting with dynamodb fails") +Metrics.registration_competition_api_error_counter = PrometheusExporter::Client.default.register("counter", "registration_competition_api_error_counter#{@suffix}", "The number of times interacting with the competition API failed") +Metrics.registration_competitor_api_error_counter = PrometheusExporter::Client.default.register("counter", "registration_competitor_api_error_counter#{@suffix}", "The number of times interacting with the competitor API failed") +Metrics.registration_validation_errors_counter = PrometheusExporter::Client.default.register("counter", "registration_validation_errors_counter#{@suffix}", "The number of times validation fails when an attendee tries to register") diff --git a/config/routes.rb b/config/routes.rb index 72d61fe4..a9741582 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -6,5 +6,4 @@ patch '/api/v1/register', to: 'registration#update' delete '/api/v1/register', to: 'registration#delete' get '/api/v1/registrations', to: 'registration#list' - get '/metrics', to: 'metrics#index' end diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 9a3231ec..11fecbec 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -1,6 +1,7 @@ version: "3.8" services: wca_registration_handler: + container_name: wca_registration_handler build: context: . dockerfile: dockerfile.dev @@ -13,6 +14,7 @@ services: AWS_SECRET_ACCESS_KEY: "fake-access-key" VAULT_ADDR: "http://wca_vault:8200" VAULT_DEV_ROOT_TOKEN_ID: aDGdUASDGIUASGDKI + PROMETHEUS_EXPORTER: "prometheus_exporter" volumes: - .:/app - gems_volume_handler:/usr/local/bundle @@ -27,8 +29,22 @@ services: depends_on: - localstack - wca_vault + - prometheus_exporter + + prometheus_exporter: + container_name: prometheus_exporter + build: + context: . + dockerfile: dockerfile.metrics + tty: true + ports: + - "9091:9091" + networks: + - wca-registration + wca_registration_worker: + container_name: wca_registration_worker build: context: . dockerfile: dockerfile.dev @@ -37,6 +53,7 @@ services: AWS_REGION: "us-east-1" AWS_ACCESS_KEY_ID: "fake-key" AWS_SECRET_ACCESS_KEY: "fake-access-key" + PROMETHEUS_EXPORTER: "prometheus_exporter" volumes: - .:/app @@ -71,6 +88,7 @@ services: - wca-registration dynamodb-admin: + container_name: dynamodb-admin image: aaronshaf/dynamodb-admin ports: - "8001:8001" @@ -85,6 +103,7 @@ services: - wca-registration wca_vault: + container_name: vault image: hashicorp/vault environment: - VAULT_DEV_ROOT_TOKEN_ID=aDGdUASDGIUASGDKI @@ -95,6 +114,7 @@ services: # Frontend frontend: + container_name: frontend build: context: Frontend tty: @@ -106,7 +126,9 @@ services: - ./Frontend/src:/app/src depends_on: - nginx + nginx: + container_name: nginx image: nginx:latest ports: - "3002:80" @@ -114,7 +136,8 @@ services: - wca-registration volumes: - ./Frontend/frontend_local.conf:/etc/nginx/conf.d/default.conf - + depends_on: + - frontend volumes: gems_volume_handler: diff --git a/docker-compose.test.yml b/docker-compose.test.yml index 4b54fecd..bb99385e 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -13,6 +13,7 @@ services: AWS_SECRET_ACCESS_KEY: "fake-access-key" VAULT_ADDR: "http://wca_vault:8200" VAULT_DEV_ROOT_TOKEN_ID: aDGdUASDGIUASGDKI + PROMETHEUS_EXPORTER: "prometheus_exporter" volumes: - .:/app - gems_volume_handler:/usr/local/bundle @@ -25,6 +26,7 @@ services: depends_on: - localstack - wca_vault + - prometheus_exporter wca_vault: image: hashicorp/vault @@ -35,6 +37,16 @@ services: networks: - wca-registration + prometheus_exporter: + build: + context: . + dockerfile: dockerfile.metrics + tty: true + ports: + - "9090:9090" + networks: + - wca-registration + wca_registration_worker: build: context: . diff --git a/dockerfile.metrics b/dockerfile.metrics new file mode 100644 index 00000000..9b9fcbb1 --- /dev/null +++ b/dockerfile.metrics @@ -0,0 +1,8 @@ +FROM ruby:3.2.2-alpine + +RUN gem install prometheus_exporter + +CMD ["-v","-b", "0.0.0.0", "-p", "9091"] + + +ENTRYPOINT ["/usr/local/bundle/bin/prometheus_exporter"] diff --git a/infra/frontend/main.tf b/infra/frontend/main.tf index 7f136f5d..41443181 100644 --- a/infra/frontend/main.tf +++ b/infra/frontend/main.tf @@ -13,7 +13,9 @@ module "cdn" { origin_bucket = aws_s3_bucket.this.id s3_access_logging_enabled = false logging_enabled = false - + response_headers_policy_id = "5cc3b908-e619-4b99-88e5-2cf7f45965bd" + cached_methods = ["HEAD", "GET", "OPTIONS"] + default_ttl = "86400" name = "cdn" stage = "prod" namespace = "wca-registration" diff --git a/infra/handler/main.tf b/infra/handler/main.tf index 76407697..ba51af19 100644 --- a/infra/handler/main.tf +++ b/infra/handler/main.tf @@ -27,6 +27,10 @@ locals { { name = "ENVIRONMENT" value = "production" + }, + { + name = "PROMETHEUS_EXPORTER" + value = var.prometheus_address } ] } diff --git a/infra/handler/variables.tf b/infra/handler/variables.tf index aa6f583f..a28ea8e5 100644 --- a/infra/handler/variables.tf +++ b/infra/handler/variables.tf @@ -7,7 +7,13 @@ variable "env" { variable "vault_address" { type = string description = "The Address that vault is running at" - default = "http://35.85.244.104:8200" + default = "http://vault.worldcubeassociation.org:8200" +} + +variable "prometheus_address" { + type = string + description = "The Address that prometheus is running at" + default = "prometheus.worldcubeassociation.org" } variable "name_prefix" { diff --git a/infra/staging/main.tf b/infra/staging/main.tf index 42dd8f83..aeb03263 100644 --- a/infra/staging/main.tf +++ b/infra/staging/main.tf @@ -23,6 +23,10 @@ locals { { name = "ENVIRONMENT" value = "staging" + }, + { + name = "PROMETHEUS_EXPORTER" + value = var.prometheus_address } ] } @@ -178,4 +182,4 @@ resource "aws_ecs_task_definition" "this" { data "aws_ecs_task_definition" "this" { task_definition = aws_ecs_task_definition.this.family -} \ No newline at end of file +} diff --git a/infra/staging/variables.tf b/infra/staging/variables.tf index c2069d73..99876737 100644 --- a/infra/staging/variables.tf +++ b/infra/staging/variables.tf @@ -13,7 +13,13 @@ variable "name_prefix" { variable "vault_address" { type = string description = "The Address that vault is running at" - default = "http://35.85.244.104:8200" + default = "http://vault.worldcubeassociation.org:8200" +} + +variable "prometheus_address" { + type = string + description = "The Address that prometheus is running at" + default = "prometheus.worldcubeassociation.org" } variable "region" { @@ -42,4 +48,4 @@ variable "registration-handler-ecr-repository" { variable "registration-worker-ecr-repository" { type = string description = "The Repository for the Worker" -} \ No newline at end of file +} diff --git a/infra/variables.tf b/infra/variables.tf index 8a012a8f..25c33b27 100644 --- a/infra/variables.tf +++ b/infra/variables.tf @@ -22,31 +22,6 @@ variable "availability_zones" { default = ["us-west-2a", "us-west-2b"] } -#variable "secret_key_base" { -# type = string -# description = "The secret key base for the application" -# sensitive = true -#} - -# PoC will deploy without a external DB -#variable "db_username" { -# type = string -# description = "Username for the database" -# sensitive = true -#} -# -#variable "db_password" { -# type = string -# description = "Password for the database" -# sensitive = true -#} -# -#variable "db_name" { -# type = string -# description = "Name of the database" -# sensitive = true -#} - variable "host" { type = string description = "The host for generating absolute URLs in the application" diff --git a/infra/worker/main.tf b/infra/worker/main.tf index 35d805d8..0445ac51 100644 --- a/infra/worker/main.tf +++ b/infra/worker/main.tf @@ -19,6 +19,10 @@ locals { { name = "ENVIRONMENT" value = "production" + }, + { + name = "PROMETHEUS_EXPORTER" + value = var.prometheus_address } ] } diff --git a/infra/worker/variables.tf b/infra/worker/variables.tf index aeff59b4..8d5639fc 100644 --- a/infra/worker/variables.tf +++ b/infra/worker/variables.tf @@ -15,6 +15,11 @@ variable "region" { description = "The region to operate in" default = "us-west-2" } +variable "prometheus_address" { + type = string + description = "The Address that prometheus is running at" + default = "prometheus.worldcubeassociation.org" +} variable "availability_zones" { type = list(string)