Skip to content

Commit

Permalink
Initial project commit
Browse files Browse the repository at this point in the history
`bundle lock --add-platform x86_64-linux`
Rubocop tidy up
Exclude test files from block size cop
Fix CI setup
Run rspec not rake
  • Loading branch information
lloydwatkin committed Mar 27, 2024
0 parents commit d70a6b3
Show file tree
Hide file tree
Showing 16 changed files with 644 additions and 0 deletions.
51 changes: 51 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
name: CI

on:
push:
branches:
- main
pull_request:

jobs:
build:
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
ruby:
- '3.2'
name: Ruby ${{ matrix.ruby }}
services:
mysql:
image: mysql:5.7
env:
MYSQL_ALLOW_EMPTY_PASSWORD: yes
MYSQL_DATABASE: mysql2_split_test
ports:
- 3306:3306
options: >-
--health-cmd "mysqladmin ping"
--health-interval 10s
--health-timeout 5s
--health-retries 5
steps:
- uses: actions/checkout@v2
- uses: ruby/setup-ruby@v1
with:
ruby-version: ${{ matrix.ruby }}
bundler-cache: true
- run: |
bundle exec rspec
env:
MYSQL_HOST: 127.0.0.1
RAILS_ENV: test
RuboCop:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- uses: ruby/setup-ruby@v1
with:
ruby-version: '3.2'
bundler-cache: true
- run: |
bundle exec rubocop --parallel --color
3 changes: 3 additions & 0 deletions .rspec
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
--force-color
--format documentation
--require ./spec/spec_helper.rb
12 changes: 12 additions & 0 deletions .rubocop.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
require:
- rubocop-rails

Style/Documentation:
Enabled: false

Rails/Delegate:
Enabled: false

Metrics/BlockLength:
Exclude:
- spec/**/*
4 changes: 4 additions & 0 deletions Gemfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
# frozen_string_literal: true

source 'http://rubygems.org'
gemspec
103 changes: 103 additions & 0 deletions Gemfile.lock
Original file line number Diff line number Diff line change
@@ -0,0 +1,103 @@
PATH
remote: .
specs:
mysql2-split (0.1.0)
forwardable (~> 1)

GEM
remote: http://rubygems.org/
specs:
activemodel (7.1.3.2)
activesupport (= 7.1.3.2)
activerecord (7.1.3.2)
activemodel (= 7.1.3.2)
activesupport (= 7.1.3.2)
timeout (>= 0.4.0)
activesupport (7.1.3.2)
base64
bigdecimal
concurrent-ruby (~> 1.0, >= 1.0.2)
connection_pool (>= 2.2.5)
drb
i18n (>= 1.6, < 2)
minitest (>= 5.1)
mutex_m
tzinfo (~> 2.0)
ast (2.4.2)
base64 (0.2.0)
bigdecimal (3.1.7)
concurrent-ruby (1.2.3)
connection_pool (2.4.1)
diff-lcs (1.5.1)
drb (2.2.1)
forwardable (1.3.3)
i18n (1.14.4)
concurrent-ruby (~> 1.0)
json (2.7.1)
language_server-protocol (3.17.0.3)
minitest (5.22.3)
mutex_m (0.2.0)
mysql2 (0.5.6)
parallel (1.24.0)
parser (3.3.0.5)
ast (~> 2.4.1)
racc
racc (1.7.3)
rack (3.0.10)
rainbow (3.1.1)
rake (13.1.0)
regexp_parser (2.9.0)
rexml (3.2.6)
rspec (3.13.0)
rspec-core (~> 3.13.0)
rspec-expectations (~> 3.13.0)
rspec-mocks (~> 3.13.0)
rspec-core (3.13.0)
rspec-support (~> 3.13.0)
rspec-expectations (3.13.0)
diff-lcs (>= 1.2.0, < 2.0)
rspec-support (~> 3.13.0)
rspec-mocks (3.13.0)
diff-lcs (>= 1.2.0, < 2.0)
rspec-support (~> 3.13.0)
rspec-support (3.13.1)
rubocop (1.62.1)
json (~> 2.3)
language_server-protocol (>= 3.17.0)
parallel (~> 1.10)
parser (>= 3.3.0.2)
rainbow (>= 2.2.2, < 4.0)
regexp_parser (>= 1.8, < 3.0)
rexml (>= 3.2.5, < 4.0)
rubocop-ast (>= 1.31.1, < 2.0)
ruby-progressbar (~> 1.7)
unicode-display_width (>= 2.4.0, < 3.0)
rubocop-ast (1.31.2)
parser (>= 3.3.0.4)
rubocop-rails (2.24.1)
activesupport (>= 4.2.0)
rack (>= 1.1)
rubocop (>= 1.33.0, < 2.0)
rubocop-ast (>= 1.31.1, < 2.0)
ruby-progressbar (1.13.0)
timeout (0.4.1)
tzinfo (2.0.6)
concurrent-ruby (~> 1.0)
unicode-display_width (2.5.0)

PLATFORMS
arm64-darwin-23
x86_64-linux

DEPENDENCIES
activerecord (>= 7.1.0)
activesupport (>= 7.1.0)
mysql2
mysql2-split!
rake
rspec (~> 3)
rubocop (~> 1.62.0)
rubocop-rails (~> 2.24.0)

BUNDLED WITH
2.4.22
76 changes: 76 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
# MySQL2 Split

![Build Status](https://github.com/olioex/mysql2-split/workflows/ci/badge.svg)

MySQL2Split is generic primary/replica proxy for ActiveRecord 7.1+ and MySQL. It handles the switching of connections between primary and replica database servers. It comes with an ActiveRecord database adapter implementation.

Mysql2Split is heavily inspired by [Makara](https://github.com/instacart/makara) from TaskRabbit and then Instacart. Unfortunately this project is unmaintained and broke for us with Rails 7.1. This is an attempt to start afresh on the project. It is definitely not as fully featured as Makara at this stage.

## Installation

Use the current version of the gem from [rubygems](https://rubygems.org/gems/makara) in your `Gemfile`.

```ruby
gem 'mysql2-split'
```

This project assumes that your read/write endpoints are handled by a separate system (e.g. DNS).

## Usage

After a write request during a thread the adapter will continue using the `primary` server, unless the context is specifically released.

### Configuration

Update your **database.yml** as follows:

```yml
development:
adapter: mysql2_split
mysql2_split:
primary:
<<: *default
database: database_name
host: primary-host.local
replica:
<<: *default
password: ithappenstobedifferent
host: replica-host.local
```
### Forcing connections
A context is local to the curent thread of execution. This will allow you to stick to the primary safely in a single thread
in systems such as sidekiq, for instance.
#### Releasing stuck connections (clearing context)
If you need to clear the current context, releasing any stuck connections, all you have to do is:
```ruby
Mysql2Split::Context.release_all
```

#### Forcing connection to primary server

```ruby
Mysql2Split::Context.stick_to_primary
```

### Logging

You can set a logger instance to ::Mysql2Split::Logging::Logger.logger and Mysql2Split.

```ruby
Mysql2Split::Logging::Logger.logger = ::Logger.new(STDOUT)
```

### What queries goes where?

In general: Any `SELECT` statements will execute against your replica(s), anything else will go to the primary.

There are some edge cases:
* `SET` operations will be sent to all connections
* Execution of specific methods such as `connect!`, `disconnect!`, and `clear_cache!` are invoked on all underlying connections
* Calls inside a transaction will always be sent to the primary (otherwise changes from within the transaction could not be read back on most transaction isolation levels)
* Locking reads (e.g. `SELECT ... FOR UPDATE`) will always be sent to the primary
124 changes: 124 additions & 0 deletions lib/active_record/connection_adapters/mysql2_split_adapter.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,124 @@
# frozen_string_literal: true

require 'active_record/connection_adapters/abstract_adapter'
require 'active_record/connection_adapters/mysql2_adapter'
require_relative '../../mysql2_split'

module ActiveRecord
module ConnectionHandling
def mysql2_split_connection(config)
ActiveRecord::ConnectionAdapters::Mysql2SplitAdapter.new(config)
end
end
end

module ActiveRecord
module ConnectionAdapters
class Mysql2SplitAdapter < ActiveRecord::ConnectionAdapters::Mysql2Adapter
SQL_PRIMARY_MATCHERS = [
/\A\s*select.+for update\Z/i, /select.+lock in share mode\Z/i,
/\A\s*select.+(nextval|currval|lastval|get_lock|release_lock|pg_advisory_lock|pg_advisory_unlock)\(/i,
/\A\s*show/i
].freeze
SQL_REPLICA_MATCHERS = [/\A\s*(select|with.+\)\s*select)\s/i].freeze
SQL_ALL_MATCHERS = [/\A\s*set\s/i].freeze
SQL_SKIP_ALL_MATCHERS = [/\A\s*set\s+local\s/i].freeze

def initialize(*args)
@replica_config = args[0][:mysql2_split]['replica']
args[0] = args[0][:mysql2_split]['primary']

super(*args)
@connection_parameters ||= args[0]
update_config
end

def execute(sql)
if should_send_to_all?(sql)
send_to_replica(sql, connection: :all, method: :execute)
return super(sql)
end
return send_to_replica(sql, connection: :replica, method: :execute) if can_go_to_replica?(sql)

Mysql2Split::Context.stick_to_primary if write_query?(sql)
Mysql2Split::Context.used_connection(:primary)

super(sql)
end

def execute_and_free(sql, name = nil, async: false) # :nodoc:#
if should_send_to_all?(sql)
send_to_replica(sql, name, connection: :all)
return super(sql, name, async:)
end
return send_to_replica(sql, connection: :replica) if can_go_to_replica?(sql)

Mysql2Split::Context.stick_to_primary if write_query?(sql)
Mysql2Split::Context.used_connection(:primary)

super(sql, name, async:)
end

def connect!(...)
replica_connection.connect!(...)
super
end

def reconnect!(...)
replica_connection.reconnect!(...)
super
end

def disconnect!(...)
replica_connection.disconnect!(...)
super
end

def clear_cache!(...)
replica_connection.clear_cache!(...)
super
end

private

def should_send_to_all?(sql)
SQL_ALL_MATCHERS.any? { |matcher| sql =~ matcher } && SQL_SKIP_ALL_MATCHERS.none? { |matcher| sql =~ matcher }
end

def can_go_to_replica?(sql)
return false if Mysql2Split::Context.use_primary? ||
open_transactions.positive? ||
SQL_PRIMARY_MATCHERS.any? { |matcher| sql =~ matcher }

true
end

def send_to_replica(sql, connection: nil, method: :exec_query)
Mysql2Split::Context.used_connection(connection) if connection
if method == :execute
replica_connection.execute(sql)
else
replica_connection.exec_query(sql)
end
end

def write_query?(sql)
%w[INSERT UPDATE DELETE LOCK].include?(sql.split(' ').first)
end

def replica_connection
@replica_connection ||= ActiveRecord::ConnectionAdapters::Mysql2Adapter.new(@replica_config)
end

def update_config
@config[:flags] ||= 0

if @config[:flags].is_a? Array
@config[:flags].push 'FOUND_ROWS'
else
@config[:flags] |= ::Mysql2::Client::FOUND_ROWS
end
end
end
end
end
Loading

0 comments on commit d70a6b3

Please sign in to comment.