diff --git a/.gitignore b/.gitignore index 73efbbe0..5bbcaa15 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,5 @@ *.gem .bundle -Gemfile.lock gemfiles/*.lock pkg/* .rvmrc @@ -12,3 +11,7 @@ Gemfile gemfiles/vendor omg.ponies *~ +coverage +bin/dbdeployer +/dbdeployer/sandboxes +/dbdeployer/binaries diff --git a/.shopify-build/VERSION b/.shopify-build/VERSION new file mode 100644 index 00000000..626799f0 --- /dev/null +++ b/.shopify-build/VERSION @@ -0,0 +1 @@ +v1 diff --git a/.shopify-build/lhm.yml b/.shopify-build/lhm.yml new file mode 100644 index 00000000..cdcd91da --- /dev/null +++ b/.shopify-build/lhm.yml @@ -0,0 +1,18 @@ +containers: + default: + docker: circleci/ruby:2.5.1 + +steps: +- label: Tests + timeout: 15m + cache: + - /usr/local/bundle + - /app/dbdeployer/binaries + run: + - bundle check || bundle install + - sudo apt-get update + - sudo apt-get install numactl libaio-dev + - ./dbdeployer/install.sh + - bundle exec rake integration + - bundle exec rake unit + diff --git a/Gemfile b/Gemfile index 6c657870..877e2d41 100644 --- a/Gemfile +++ b/Gemfile @@ -3,7 +3,3 @@ source 'https://rubygems.org' # Specify your gem's dependencies in shopify_lhm.gemspec gemspec -group :deployment do - gem 'package_cloud' - gem 'rake' -end diff --git a/Gemfile.lock b/Gemfile.lock new file mode 100644 index 00000000..d18f5048 --- /dev/null +++ b/Gemfile.lock @@ -0,0 +1,73 @@ +PATH + remote: . + specs: + lhm (3.0.0.alpha) + +GEM + remote: https://rubygems.org/ + specs: + activemodel (5.2.0) + activesupport (= 5.2.0) + activerecord (5.2.0) + activemodel (= 5.2.0) + activesupport (= 5.2.0) + arel (>= 9.0) + activesupport (5.2.0) + concurrent-ruby (~> 1.0, >= 1.0.2) + i18n (>= 0.7, < 2) + minitest (~> 5.1) + tzinfo (~> 1.1) + arel (9.0.0) + concurrent-ruby (1.0.5) + domain_name (0.5.20180417) + unf (>= 0.0.5, < 1.0.0) + highline (1.6.20) + http-cookie (1.0.3) + domain_name (~> 0.5) + i18n (1.0.1) + concurrent-ruby (~> 1.0) + json_pure (1.8.1) + metaclass (0.0.4) + mime-types (3.1) + mime-types-data (~> 3.2015) + mime-types-data (3.2016.0521) + minitest (5.11.3) + mocha (1.5.0) + metaclass (~> 0.0.1) + mysql2 (0.5.2) + netrc (0.11.0) + package_cloud (0.3.05) + highline (= 1.6.20) + json_pure (= 1.8.1) + rainbow (= 2.2.2) + rest-client (~> 2.0) + thor (~> 0.18) + rainbow (2.2.2) + rake + rake (12.3.1) + rest-client (2.0.2) + http-cookie (>= 1.0.2, < 2.0) + mime-types (>= 1.16, < 4.0) + netrc (~> 0.8) + thor (0.20.0) + thread_safe (0.3.6) + tzinfo (1.2.5) + thread_safe (~> 0.1) + unf (0.1.4) + unf_ext + unf_ext (0.0.7.5) + +PLATFORMS + ruby + +DEPENDENCIES + activerecord + lhm! + minitest + mocha + mysql2 + package_cloud + rake + +BUNDLED WITH + 1.16.1 diff --git a/README.md b/README.md index 3eb3ba66..91b6e56b 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,6 @@ -# Large Hadron Migrator [![Build Status][5]][4] +# Large Hadron Migrator +[![Build status](https://badge.buildkite.com/6ed04595f04c5cf6f9f1453afd3705046f6c83088bd29cecf7.svg)](https://buildkite.com/shopify/lhm) +This is the Shopify downstream fork of https://github.com/soundcloud/lhm. Rails style database migrations are a useful way to evolve your data schema in an agile manner. Most Rails projects start like this, and at first, making @@ -215,18 +217,24 @@ Lhm.cleanup(true, until: Time.now - 86400) First, get set up for local development: - git clone git://github.com/soundcloud/lhm.git - cd lhm - -To run the tests, follow the instructions on [spec/README](https://github.com/soundcloud/lhm/blob/master/spec/README.md). - -We'll check out your contribution if you: +``` +dev clone lhm +dev up +``` - * Provide a comprehensive suite of tests for your fork. - * Have a clear and documented rationale for your changes. - * Package these up in a pull request. +To run the tests: +``` +dev unit # unit tests +dev int # integration tests +dev test # all tests +``` -We'll do our best to help you out with any contribution issues you may have. +### dbdeployer +The integration tests rely on a master/slave replication setup of MySQL. +We're using [dbdeployer](https://github.com/datacharmer/dbdeployer) to set this up via `./dbdeployer/install.sh`. +`dbdeployer` provides scripts for operating and accessing the nodes in `./dbdeployer/sandboxes/rsandbox_5_7_22`. +There is a lot in there, and most of time you shouldn't need to work with the nodes directly, but it's good +to know where to go! ## License diff --git a/dbdeployer/config.json b/dbdeployer/config.json new file mode 100644 index 00000000..611d98eb --- /dev/null +++ b/dbdeployer/config.json @@ -0,0 +1,32 @@ +{ + "version": "1.8.0", + "sandbox-home": "./dbdeployer/sandboxes", + "sandbox-binary": "./dbdeployer/binaries", + "use-sandbox-catalog": true, + "master-slave-base-port": 11000, + "group-replication-base-port": 12000, + "group-replication-sp-base-port": 13000, + "fan-in-replication-base-port": 14000, + "all-masters-replication-base-port": 15000, + "multiple-base-port": 16000, + "group-port-delta": 125, + "mysqlx-port-delta": 10000, + "master-name": "master", + "master-abbr": "m", + "node-prefix": "node", + "slave-prefix": "slave", + "slave-abbr": "s", + "sandbox-prefix": "msb_", + "master-slave-prefix": "rsandbox_", + "group-prefix": "group_msb_", + "group-sp-prefix": "group_sp_msb_", + "multiple-prefix": "multi_msb_", + "fan-in-prefix": "fan_in_msb_", + "all-masters-prefix": "all_masters_msb_", + "reserved-ports": [ + 1186, + 3306, + 33060 + ], + "timestamp": "Mon Jul 16 17:36:55 AST 2018" + } diff --git a/dbdeployer/install.sh b/dbdeployer/install.sh new file mode 100755 index 00000000..46de21b4 --- /dev/null +++ b/dbdeployer/install.sh @@ -0,0 +1,64 @@ +set -e +mkdir -p ./dbdeployer/sandboxes +mkdir -p ./dbdeployer/binaries + +if [ -z "$(uname | grep Darwin)" ]; then + OS=linux + set -x +else + OS=osx +fi + +echo "Checking if dbdeployer is installed" +if ! [ -x "$(command -v ./bin/dbdeployer)" ]; then + echo "Not installed...starting install" + VERSION=1.8.0 + origin=https://github.com/datacharmer/dbdeployer/releases/download/$VERSION + filename=dbdeployer-$VERSION.$OS + wget -q $origin/$filename.tar.gz + tar -xzf $filename.tar.gz + chmod +x $filename + sudo mv $filename ./bin/dbdeployer + rm $filename.tar.gz +else + echo "Installation found!" +fi + + +echo "Checking if mysql 5.7.22 is available for dbdeployer" +if [ -z "$(./bin/dbdeployer --config ./dbdeployer/config.json --sandbox-binary "./dbdeployer/binaries" available | grep 5.7.22)" ]; then + echo "Not found..." + + if [ "$OS" = "linux" ]; then + MYSQL_FILE=mysql-5.7.22-linux-glibc2.12-x86_64.tar.gz + else + MYSQL_FILE=mysql-5.7.22-macos10.13-x86_64.tar.gz + fi + + if [ ! -f $MYSQL_FILE ]; then + echo "Downloading $MYSQL_FILE...(this may take a while)" + wget -q "https://dev.mysql.com/get/Downloads/MySQL-5.7/$MYSQL_FILE" + fi + + echo "Setting up..." + ./bin/dbdeployer unpack $MYSQL_FILE --verbosity 0 --config ./dbdeployer/config.json --sandbox-binary "./dbdeployer/binaries" + rm $MYSQL_FILE +else + echo "mysql 5.7.22 found!" +fi + +echo "Forcing new replication setup..." +./bin/dbdeployer deploy replication 5.7.22 --nodes 2 --force --config ./dbdeployer/config.json --sandbox-binary "./dbdeployer/binaries" --sandbox-home "./dbdeployer/sandboxes" +./bin/dbdeployer global status --config ./dbdeployer/config.json --sandbox-binary "./dbdeployer/binaries" --sandbox-home "./dbdeployer/sandboxes" + +echo "Setting up database.yml" +DATABASE_YML=spec/integration/database.yml +echo "master:" > $DATABASE_YML +cat ./dbdeployer/sandboxes/rsandbox_5_7_22/master/my.sandbox.cnf | grep -A 4 client | tail -n 4 | sed -e 's/ * = /: /' -e 's/^/ /' >> $DATABASE_YML + +echo "slave:" >> $DATABASE_YML +cat ./dbdeployer/sandboxes/rsandbox_5_7_22/node1/my.sandbox.cnf | grep -A 4 client | tail -n 4 | sed -e 's/ * = /: /' -e 's/^/ /' >> $DATABASE_YML + +cat $DATABASE_YML + +echo "You are ready to run the integration test suite..." diff --git a/dev.yml b/dev.yml new file mode 100644 index 00000000..bdf00288 --- /dev/null +++ b/dev.yml @@ -0,0 +1,18 @@ +name: lhm +up: + - homebrew: + - openssl + - shopify/shopify/mysql-client + - wget + - ruby: 2.5.0 + - bundler + - custom: + name: Database + met?: test -f spec/integration/database.yml && test "$(./dbdeployer/sandboxes/rsandbox_5_7_22/status_all | grep on | wc -l | xargs echo)" = "2" + meet: ./dbdeployer/install.sh + down: ./dbdeployer/sandboxes/rsandbox_5_7_22/stop_all + +commands: + unit: bundle exec rake unit + int: bundle exec rake integration + test: bundle exec rake unit && bundle exec rake integration diff --git a/lhm.gemspec b/lhm.gemspec index d5c4c811..4c3119aa 100644 --- a/lhm.gemspec +++ b/lhm.gemspec @@ -24,5 +24,6 @@ Gem::Specification.new do |s| s.add_development_dependency 'mocha' s.add_development_dependency 'rake' s.add_development_dependency 'activerecord' - s.add_development_dependency 'mysql' + s.add_development_dependency 'mysql2' + s.add_development_dependency 'package_cloud' end diff --git a/lib/lhm/throttler/slave_lag.rb b/lib/lhm/throttler/slave_lag.rb index 9361bd11..4a239a60 100644 --- a/lib/lhm/throttler/slave_lag.rb +++ b/lib/lhm/throttler/slave_lag.rb @@ -62,7 +62,7 @@ def get_slaves while slave_hosts.any? do host = slave_hosts.pop slave = Slave.new(host, @get_config) - if slaves.map(&:host).exclude?(host) && slave.connection + if !slaves.map(&:host).include?(host) && slave.connection slaves << slave slave_hosts.concat(slave.slave_hosts) end diff --git a/spec/integration/integration_helper.rb b/spec/integration/integration_helper.rb index 1c9fd274..83b11f0c 100644 --- a/spec/integration/integration_helper.rb +++ b/spec/integration/integration_helper.rb @@ -4,13 +4,14 @@ require 'yaml' require 'active_support' -config = YAML.load_file(File.expand_path(File.dirname(__FILE__)) + '/database.yml') rescue {} -$lhm_user = config['user'] ||= 'root' -$password = config['password'] ||= '1234' -$master_host = config['master_host'] ||= '127.0.0.1' -$master_port = config['master_port'] ||= 3306 -$slave_host = config['slave_host'] ||= '127.0.0.1' -$slave_port = config['slave_port'] ||= 3307 +begin + $db_config = YAML.load_file(File.expand_path(File.dirname(__FILE__)) + '/database.yml') +rescue StandardError => e + puts "Run install.sh to setup database" + raise e +end + +$db_name = 'test' require 'lhm/table' require 'lhm/sql_helper' @@ -24,15 +25,27 @@ def connection end def connect_master! - connect!($master_host, $master_port) + connect!( + '127.0.0.1', + $db_config['master']['port'], + $db_config['master']['user'], + $db_config['master']['password'], + $db_config['master']['socket'] + ) end def connect_slave! - connect!($slave_host, $slave_host) + connect!( + '127.0.0.1', + $db_config['slave']['port'], + $db_config['slave']['user'], + $db_config['slave']['password'], + $db_config['slave']['socket'] + ) end - def connect!(hostname, port) - adapter = ar_conn(hostname, port) + def connect!(hostname, port, user, password, socket) + adapter = ar_conn(hostname, port, user, password, socket) Lhm.setup(adapter) unless defined?(@@cleaned_up) Lhm.cleanup(true) @@ -41,14 +54,15 @@ def connect!(hostname, port) @connection = adapter end - def ar_conn(host, port) + def ar_conn(host, port, user, password, socket) ActiveRecord::Base.establish_connection( - :adapter => defined?(Mysql2) ? 'mysql2' : 'mysql', + :adapter => 'mysql2', :host => host, - :database => 'lhm', - :username => $lhm_user, + :username => user, :port => port, - :password => $password + :password => password, + :socket => socket, + :database => $db_name ) ActiveRecord::Base.connection end @@ -97,7 +111,7 @@ def slave(&block) # Helps testing behaviour when another client locks the db def start_locking_thread(lock_for, queue, locking_query) Thread.new do - conn = Mysql2::Client.new(host: '127.0.0.1', database: 'lhm', user: 'root', port: 3306) + conn = Mysql2::Client.new(host: '127.0.0.1', database: $db_name, user: 'root', port: 3306) conn.query('BEGIN') conn.query(locking_query) queue.push(true)