From 673bc36ecf345783a6ab83bacd96f8f4a6fed105 Mon Sep 17 00:00:00 2001 From: Matt Almeida Date: Thu, 13 Feb 2025 13:15:35 -0500 Subject: [PATCH] Back to Postgres --- .github/workflows/ci.yml | 12 ++++----- Dockerfile | 4 +-- Gemfile | 6 +---- Gemfile.lock | 38 ++------------------------- README.md | 15 ++++++++--- app/models/database_dump/processed.rb | 21 ++++++++++----- app/models/lock.rb | 6 ++--- config/cable.yml | 7 ++--- config/database.yml | 27 ++++++++++--------- config/initializers/litestream.rb | 7 ----- config/litestream.yml | 8 ------ db/cable_schema.rb | 11 -------- db/schema.rb | 3 +++ docker-compose.production.yml | 26 ++++++++++++++++++ docker-compose.yml | 32 +++++++--------------- 15 files changed, 95 insertions(+), 128 deletions(-) delete mode 100644 config/initializers/litestream.rb delete mode 100644 config/litestream.yml delete mode 100644 db/cable_schema.rb create mode 100644 docker-compose.production.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6f4d8147..1a9ded07 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,22 +22,22 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 15 steps: - - name: Install system dependencies - env: - DEBIAN_FRONTEND: noninteractive + - name: Install dependencies run: | + sudo apt-get install postgresql-common + sudo /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh -y sudo apt-get update - sudo apt-get install sqlite3 libvips + sudo apt-get install libvips postgresql-client - name: Checkout code uses: actions/checkout@v4 - name: Setup Ruby uses: ruby/setup-ruby@v1 with: bundler-cache: true + - name: Setup Postgres + run: docker compose up --detach - name: Run tests run: bin/rails test; bin/rails test:system - env: - RAILS_ENV: test - name: Keep screenshots from failed system tests uses: actions/upload-artifact@v4 if: failure() diff --git a/Dockerfile b/Dockerfile index ac8b4986..2d389649 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,7 +11,7 @@ ENV RAILS_ENV="production" \ FROM base AS build RUN apt-get update -qq && \ - apt-get install --no-install-recommends -y build-essential git pkg-config libyaml-dev sqlite3 libvips + apt-get install --no-install-recommends -y build-essential git pkg-config libyaml-dev libpq-dev libvips COPY .ruby-version Gemfile Gemfile.lock ./ RUN bundle install && \ @@ -29,7 +29,7 @@ RUN SECRET_KEY_BASE_DUMMY=1 ./bin/rails assets:precompile FROM base RUN apt-get update -qq && \ - apt-get install --no-install-recommends -y sqlite3 libvips curl libjemalloc2 && \ + apt-get install --no-install-recommends -y postgresql-client libvips curl libjemalloc2 && \ rm -rf /var/lib/apt/lists /var/cache/apt/archives ENV LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.2 diff --git a/Gemfile b/Gemfile index 99b95115..f1060dbd 100644 --- a/Gemfile +++ b/Gemfile @@ -8,11 +8,9 @@ gem "rails", github: "rails/rails" gem "dotenv-rails", require: "dotenv/load" # Drivers -gem "sqlite3" +gem "pg" gem "puma" -gem "solid_cable" - # Assets gem "sprockets-rails" gem "dartsass-rails" @@ -54,8 +52,6 @@ gem "hashid-rails" # Non-sequential IDs gem "tzinfo-data", platforms: %i[mingw mswin x64_mingw jruby] # Windows doesn't include zoneinfo files gem "bootsnap", require: false # reduces boot times through caching; required in config/boot.rb -gem "litestream" - gem "appsignal" gem "lograge" diff --git a/Gemfile.lock b/Gemfile.lock index 036bfc33..a28e2f86 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -260,32 +260,7 @@ GEM railties (>= 6.1) rexml lint_roller (1.1.0) - litestream (0.12.0) - actionpack (>= 7.0) - actionview (>= 7.0) - activejob (>= 7.0) - activesupport (>= 7.0) - logfmt (>= 0.0.10) - railties (>= 7.0) - sqlite3 - litestream (0.12.0-arm64-darwin) - actionpack (>= 7.0) - actionview (>= 7.0) - activejob (>= 7.0) - activesupport (>= 7.0) - logfmt (>= 0.0.10) - railties (>= 7.0) - sqlite3 - litestream (0.12.0-x86_64-linux) - actionpack (>= 7.0) - actionview (>= 7.0) - activejob (>= 7.0) - activesupport (>= 7.0) - logfmt (>= 0.0.10) - railties (>= 7.0) - sqlite3 local_time (3.0.2) - logfmt (0.0.10) logger (1.6.5) lograge (0.14.0) actionpack (>= 4) @@ -338,6 +313,7 @@ GEM parser (3.3.7.0) ast (~> 2.4.1) racc + pg (1.5.9) pp (0.6.2) prettyprint premailer (1.21.0) @@ -417,11 +393,6 @@ GEM simple_form (5.3.1) actionpack (>= 5.2) activemodel (>= 5.2) - solid_cable (3.0.7) - actioncable (>= 7.2) - activejob (>= 7.2) - activerecord (>= 7.2) - railties (>= 7.2) solid_queue (1.1.3) activejob (>= 7.1) activerecord (>= 7.1) @@ -436,9 +407,6 @@ GEM actionpack (>= 6.1) activesupport (>= 6.1) sprockets (>= 3.0.0) - sqlite3 (2.5.0-aarch64-linux-gnu) - sqlite3 (2.5.0-arm64-darwin) - sqlite3 (2.5.0-x86_64-linux-gnu) standard (1.44.0) language_server-protocol (~> 3.17.0.2) lint_roller (~> 1.0) @@ -521,20 +489,18 @@ DEPENDENCIES importmap-rails jbuilder letter_opener_web - litestream local_time lograge mission_control-jobs + pg premailer-rails puma rack-cors rack-mini-profiler rails! simple_form - solid_cable solid_queue sprockets-rails - sqlite3 standard stimulus-rails turbo-rails diff --git a/README.md b/README.md index a0c795fa..24c238d6 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ _The thing that powers [hackathons.hackclub.com](https://hackathons.hackclub.com ## Contributing This app is built with 🛤️ [Ruby on Rails](https://rubyonrails.org) (running [on the edge](https://shopify.engineering/living-on-the-edge-of-rails)) -and uses 🥋 [Solid Queue](https://github.com/rails/solid_queue) for running background jobs. +using 🐘 [PostgreSQL](https://www.postgresql.org) for the database and 💾 [Solid Queue](https://github.com/rails/solid_queue) for running background jobs. ### Getting Started @@ -39,6 +39,12 @@ and uses 🥋 [Solid Queue](https://github.com/rails/solid_queue) for running ba ```sh bundle install ``` + +4. Start Postgres + + ```sh + docker compose up --detach + ``` 4. Setup the database and run the server @@ -51,8 +57,11 @@ The application will now be running at [localhost:3000](http://localhost:3000)! ### Additional Dependencies -Rails 7 (Active Storage) depends on [vips](https://libvips.github.io/libvips/) to process images. You'll want this -dependency installed on your machine. For macs, run: +[Postgres will need to be installed](https://www.postgresql.org/download) +in order for the `pg` gem to be compiled. + +Additionally, Active Storage depends on [vips](https://libvips.github.io/libvips/) +to process images. You'll want this dependency installed on your machine. For Macs, run: ```sh brew install vips diff --git a/app/models/database_dump/processed.rb b/app/models/database_dump/processed.rb index 94002071..bdb959fb 100644 --- a/app/models/database_dump/processed.rb +++ b/app/models/database_dump/processed.rb @@ -4,7 +4,7 @@ module DatabaseDump::Processed included do has_one_attached :file - after_create_commit :process_later + after_create_commit { DatabaseDumpJob.perform_later(self) } end def processed? @@ -14,7 +14,7 @@ def processed? def process return if processed? - raise "sqlite3 not found" unless `which sqlite3`.present? + raise "pg_dump not found" unless `which pg_dump`.present? transaction do Tempfile.create do |io| @@ -28,12 +28,19 @@ def process private - def process_later - DatabaseDumpJob.perform_later(self) + def dump(tables, to:) + system postgres_env, "pg_dump --table '#{tables.join("|")}' --file #{to}", exception: true end - def dump(tables, to:) - db = self.class.connection.raw_connection.filename - system "sqlite3 #{db} \".dump '#{tables.join("' '")}'\" > #{to}", exception: true + def postgres_env + connection = ApplicationRecord.connection_db_config.configuration_hash + + {}.tap do |env| + env["PGHOST"] = connection[:host] + env["PGPORT"] = connection[:port] + env["PGUSER"] = connection[:username] + env["PGPASSWORD"] = connection[:password] + env["PGDATABASE"] = connection[:database] + end.transform_values!(&:to_s) end end diff --git a/app/models/lock.rb b/app/models/lock.rb index 1d5e15de..d94ed1fe 100644 --- a/app/models/lock.rb +++ b/app/models/lock.rb @@ -6,12 +6,13 @@ class << self def acquire(key, limit: 1, duration: nil) lock = nil transaction do - lock = active.find_by(key:) || create!(key:, expiration: duration&.from_now) + lock = active.lock.find_or_create_by!(key:) if lock.capacity >= limit return false else lock.acquire + lock.update! expiration: duration&.from_now end end @@ -29,8 +30,7 @@ def acquire(capacity = 1) end def release(quantity = 1) - transaction do - reload + with_lock do if capacity <= 1 destroy! else diff --git a/config/cable.yml b/config/cable.yml index fd924c75..cc73650e 100644 --- a/config/cable.yml +++ b/config/cable.yml @@ -1,11 +1,8 @@ development: - adapter: async + adapter: postgresql test: adapter: test production: - adapter: solid_cable - connects_to: - database: - writing: cable + adapter: postgresql diff --git a/config/database.yml b/config/database.yml index a5326d38..169e04f7 100644 --- a/config/database.yml +++ b/config/database.yml @@ -1,21 +1,22 @@ default: &default - adapter: sqlite3 + adapter: postgresql + encoding: unicode pool: <%= ENV.fetch("RAILS_MAX_THREADS") { 5 } %> - timeout: 5000 -development: +local: &local <<: *default - database: storage/development.sqlite3 + username: hackathons + host: 127.0.0.1 + gssencmode: disable # https://github.com/ged/ruby-pg/issues/311#issuecomment-561927000 + +development: + <<: *local + database: hackathons_development test: - <<: *default - database: storage/test.sqlite3 + <<: *local + database: hackathons_test production: - primary: - <<: *default - database: storage/production.sqlite3 - cable: - <<: *default - database: storage/production_cable.sqlite3 - migrations_paths: db/cable_migrate + <<: *default + database: hackathons_production diff --git a/config/initializers/litestream.rb b/config/initializers/litestream.rb deleted file mode 100644 index dac5a018..00000000 --- a/config/initializers/litestream.rb +++ /dev/null @@ -1,7 +0,0 @@ -Rails.application.configure do - aws = Rails.application.credentials.aws - - config.litestream.replica_bucket = aws&.bucket - config.litestream.replica_key_id = aws&.access_key_id - config.litestream.replica_access_key = aws&.secret_access_key -end diff --git a/config/litestream.yml b/config/litestream.yml deleted file mode 100644 index 6ad961c8..00000000 --- a/config/litestream.yml +++ /dev/null @@ -1,8 +0,0 @@ -dbs: - - path: storage/production.sqlite3 - replicas: - - type: s3 - bucket: $LITESTREAM_REPLICA_BUCKET - path: production.sqlite3 - access-key-id: $LITESTREAM_ACCESS_KEY_ID - secret-access-key: $LITESTREAM_SECRET_ACCESS_KEY diff --git a/db/cable_schema.rb b/db/cable_schema.rb deleted file mode 100644 index 23666604..00000000 --- a/db/cable_schema.rb +++ /dev/null @@ -1,11 +0,0 @@ -ActiveRecord::Schema[7.1].define(version: 1) do - create_table "solid_cable_messages", force: :cascade do |t| - t.binary "channel", limit: 1024, null: false - t.binary "payload", limit: 536870912, null: false - t.datetime "created_at", null: false - t.integer "channel_hash", limit: 8, null: false - t.index ["channel"], name: "index_solid_cable_messages_on_channel" - t.index ["channel_hash"], name: "index_solid_cable_messages_on_channel_hash" - t.index ["created_at"], name: "index_solid_cable_messages_on_created_at" - end -end diff --git a/db/schema.rb b/db/schema.rb index 06b97da1..7d265afe 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -11,6 +11,9 @@ # It's strongly recommended that you check this file into your version control system. ActiveRecord::Schema[8.1].define(version: 2025_01_28_210851) do + # These are extensions that must be enabled in order to support this database + enable_extension "pg_catalog.plpgsql" + create_table "active_storage_attachments", force: :cascade do |t| t.bigint "blob_id", null: false t.datetime "created_at", null: false diff --git a/docker-compose.production.yml b/docker-compose.production.yml new file mode 100644 index 00000000..07179896 --- /dev/null +++ b/docker-compose.production.yml @@ -0,0 +1,26 @@ +services: + web: + build: . + deploy: + resources: + limits: + cpus: 1 + memory: 2G + environment: + WEB_CONCURRENCY: 2 + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:3000/up"] + volumes: ["storage:/hackathons/storage"] + jobs: + volumes: ["storage:/hackathons/storage"] + build: . + depends_on: [web] + command: bin/rails solid_queue:start + deploy: + resources: + limits: + cpus: 1 + memory: 1G + +volumes: + storage: {} diff --git a/docker-compose.yml b/docker-compose.yml index 07179896..c0179be1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,26 +1,14 @@ services: - web: - build: . - deploy: - resources: - limits: - cpus: 1 - memory: 2G + postgres: + image: postgres:17.2 + ports: + - ${POSTGRES_PORT-:5432}:5432 environment: - WEB_CONCURRENCY: 2 - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:3000/up"] - volumes: ["storage:/hackathons/storage"] - jobs: - volumes: ["storage:/hackathons/storage"] - build: . - depends_on: [web] - command: bin/rails solid_queue:start - deploy: - resources: - limits: - cpus: 1 - memory: 1G + POSTGRES_USER: hackathons + POSTGRES_DB: hackathons_test + POSTGRES_HOST_AUTH_METHOD: trust + volumes: + - postgres-data:/var/lib/postgresql/data volumes: - storage: {} + postgres-data: {}