Run tests for LHM with dev and shopify-build

.gitignore

@@ -1,6 +1,5 @@
 *.gem
 .bundle
-Gemfile.lock
 gemfiles/*.lock
 pkg/*
 .rvmrc
@@ -12,3 +11,7 @@ Gemfile
 gemfiles/vendor
 omg.ponies
 *~
+coverage
+bin/dbdeployer
+/dbdeployer/sandboxes
+/dbdeployer/binaries

.shopify-build/VERSION

@@ -0,0 +1 @@
+v1

.shopify-build/lhm.yml

@@ -0,0 +1,18 @@
+containers:
+  default:
+    docker: circleci/ruby:2.5.1
+
+steps:
+- label: Tests
+  timeout: 15m
+  cache:
+    - /usr/local/bundle
+    - /app/dbdeployer/binaries
+  run:
+  - bundle check || bundle install
+  - sudo apt-get update
+  - sudo apt-get install numactl libaio-dev
+  - ./dbdeployer/install.sh
+  - bundle exec rake integration
+  - bundle exec rake unit
+

Gemfile

@@ -3,7 +3,3 @@ source 'https://rubygems.org'
 # Specify your gem's dependencies in shopify_lhm.gemspec
 gemspec
 
-group :deployment do
-  gem 'package_cloud'
-  gem 'rake'
-end

Gemfile.lock

@@ -0,0 +1,73 @@
+PATH
+  remote: .
+  specs:
+    lhm (3.0.0.alpha)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activemodel (5.2.0)
+      activesupport (= 5.2.0)
+    activerecord (5.2.0)
+      activemodel (= 5.2.0)
+      activesupport (= 5.2.0)
+      arel (>= 9.0)
+    activesupport (5.2.0)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 0.7, < 2)
+      minitest (~> 5.1)
+      tzinfo (~> 1.1)
+    arel (9.0.0)
+    concurrent-ruby (1.0.5)
+    domain_name (0.5.20180417)
+      unf (>= 0.0.5, < 1.0.0)
+    highline (1.6.20)
+    http-cookie (1.0.3)
+      domain_name (~> 0.5)
+    i18n (1.0.1)
+      concurrent-ruby (~> 1.0)
+    json_pure (1.8.1)
+    metaclass (0.0.4)
+    mime-types (3.1)
+      mime-types-data (~> 3.2015)
+    mime-types-data (3.2016.0521)
+    minitest (5.11.3)
+    mocha (1.5.0)
+      metaclass (~> 0.0.1)
+    mysql2 (0.5.2)
+    netrc (0.11.0)
+    package_cloud (0.3.05)
+      highline (= 1.6.20)
+      json_pure (= 1.8.1)
+      rainbow (= 2.2.2)
+      rest-client (~> 2.0)
+      thor (~> 0.18)
+    rainbow (2.2.2)
+      rake
+    rake (12.3.1)
+    rest-client (2.0.2)
+      http-cookie (>= 1.0.2, < 2.0)
+      mime-types (>= 1.16, < 4.0)
+      netrc (~> 0.8)
+    thor (0.20.0)
+    thread_safe (0.3.6)
+    tzinfo (1.2.5)
+      thread_safe (~> 0.1)
+    unf (0.1.4)
+      unf_ext
+    unf_ext (0.0.7.5)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  activerecord
+  lhm!
+  minitest
+  mocha
+  mysql2
+  package_cloud
+  rake
+
+BUNDLED WITH
+   1.16.1

README.md

@@ -1,4 +1,6 @@
-# Large Hadron Migrator [![Build Status][5]][4]
+# Large Hadron Migrator
+[![Build status](https://badge.buildkite.com/6ed04595f04c5cf6f9f1453afd3705046f6c83088bd29cecf7.svg)](https://buildkite.com/shopify/lhm)
+This is the Shopify downstream fork of https://github.com/soundcloud/lhm.
 
 Rails style database migrations are a useful way to evolve your data schema in
 an agile manner. Most Rails projects start like this, and at first, making
@@ -215,18 +217,24 @@ Lhm.cleanup(true, until: Time.now - 86400)
 
 First, get set up for local development:
 
-    git clone git://github.com/soundcloud/lhm.git
-    cd lhm
-
-To run the tests, follow the instructions on [spec/README](https://github.com/soundcloud/lhm/blob/master/spec/README.md).
-
-We'll check out your contribution if you:
+```
+dev clone lhm
+dev up
+```
 
-  * Provide a comprehensive suite of tests for your fork.
-  * Have a clear and documented rationale for your changes.
-  * Package these up in a pull request.
+To run the tests:
+```
+dev unit # unit tests
+dev int # integration tests
+dev test # all tests
+```
 
-We'll do our best to help you out with any contribution issues you may have.
+### dbdeployer
+The integration tests rely on a master/slave replication setup of MySQL.
+We're using [dbdeployer](https://github.com/datacharmer/dbdeployer) to set this up via `./dbdeployer/install.sh`.
+`dbdeployer` provides scripts for operating and accessing the nodes in `./dbdeployer/sandboxes/rsandbox_5_7_22`.
+There is a lot in there, and most of time you shouldn't need to work with the nodes directly, but it's good
+to know where to go!
 
 ## License
 

dbdeployer/config.json

@@ -0,0 +1,32 @@
+{
+ 	"version": "1.8.0",
+ 	"sandbox-home": "./dbdeployer/sandboxes",
+ 	"sandbox-binary": "./dbdeployer/binaries",
+ 	"use-sandbox-catalog": true,
+ 	"master-slave-base-port": 11000,
+ 	"group-replication-base-port": 12000,
+ 	"group-replication-sp-base-port": 13000,
+ 	"fan-in-replication-base-port": 14000,
+ 	"all-masters-replication-base-port": 15000,
+ 	"multiple-base-port": 16000,
+ 	"group-port-delta": 125,
+ 	"mysqlx-port-delta": 10000,
+ 	"master-name": "master",
+ 	"master-abbr": "m",
+ 	"node-prefix": "node",
+ 	"slave-prefix": "slave",
+ 	"slave-abbr": "s",
+ 	"sandbox-prefix": "msb_",
+ 	"master-slave-prefix": "rsandbox_",
+ 	"group-prefix": "group_msb_",
+ 	"group-sp-prefix": "group_sp_msb_",
+ 	"multiple-prefix": "multi_msb_",
+ 	"fan-in-prefix": "fan_in_msb_",
+ 	"all-masters-prefix": "all_masters_msb_",
+ 	"reserved-ports": [
+ 		1186,
+ 		3306,
+ 		33060
+ 	],
+ 	"timestamp": "Mon Jul 16 17:36:55 AST 2018"
+ }

dbdeployer/install.sh

@@ -0,0 +1,64 @@
+set -e
+mkdir -p ./dbdeployer/sandboxes
+mkdir -p ./dbdeployer/binaries
+
+if [ -z "$(uname | grep Darwin)" ]; then
+  OS=linux
+  set -x
+else
+  OS=osx
+fi
+
+echo "Checking if dbdeployer is installed"
+if ! [ -x "$(command -v ./bin/dbdeployer)" ]; then
+  echo "Not installed...starting install"
+  VERSION=1.8.0
+  origin=https://github.com/datacharmer/dbdeployer/releases/download/$VERSION
+  filename=dbdeployer-$VERSION.$OS
+  wget -q $origin/$filename.tar.gz
+  tar -xzf $filename.tar.gz
+  chmod +x $filename
+  sudo mv $filename ./bin/dbdeployer
+  rm $filename.tar.gz
+else
+  echo "Installation found!"
+fi
+
+
+echo "Checking if mysql 5.7.22 is available for dbdeployer"
+if [ -z "$(./bin/dbdeployer --config ./dbdeployer/config.json --sandbox-binary "./dbdeployer/binaries" available | grep 5.7.22)" ]; then
+  echo "Not found..."
+
+  if [ "$OS" = "linux" ]; then
+    MYSQL_FILE=mysql-5.7.22-linux-glibc2.12-x86_64.tar.gz
+  else
+    MYSQL_FILE=mysql-5.7.22-macos10.13-x86_64.tar.gz
+  fi
+
+  if [ ! -f $MYSQL_FILE ]; then
+    echo "Downloading $MYSQL_FILE...(this may take a while)"
+    wget -q "https://dev.mysql.com/get/Downloads/MySQL-5.7/$MYSQL_FILE"
+  fi
+
+  echo "Setting up..."
+  ./bin/dbdeployer unpack $MYSQL_FILE --verbosity 0 --config ./dbdeployer/config.json --sandbox-binary "./dbdeployer/binaries"
+  rm $MYSQL_FILE
+else
+  echo "mysql 5.7.22 found!"
+fi
+
+echo "Forcing new replication setup..."
+./bin/dbdeployer deploy replication 5.7.22 --nodes 2 --force --config ./dbdeployer/config.json --sandbox-binary "./dbdeployer/binaries" --sandbox-home "./dbdeployer/sandboxes"
+./bin/dbdeployer global status --config ./dbdeployer/config.json --sandbox-binary "./dbdeployer/binaries" --sandbox-home "./dbdeployer/sandboxes"
+
+echo "Setting up database.yml"
+DATABASE_YML=spec/integration/database.yml
+echo "master:" > $DATABASE_YML
+cat ./dbdeployer/sandboxes/rsandbox_5_7_22/master/my.sandbox.cnf | grep -A 4 client | tail -n 4 | sed -e 's/  * = /: /' -e 's/^/  /' >> $DATABASE_YML
+
+echo "slave:" >> $DATABASE_YML
+cat ./dbdeployer/sandboxes/rsandbox_5_7_22/node1/my.sandbox.cnf | grep -A 4 client | tail -n 4 | sed -e 's/  * = /: /' -e 's/^/  /' >> $DATABASE_YML
+
+cat $DATABASE_YML
+
+echo "You are ready to run the integration test suite..."

dev.yml

@@ -0,0 +1,18 @@
+name: lhm
+up:
+  - homebrew:
+      - openssl
+      - shopify/shopify/mysql-client
+      - wget
+  - ruby: 2.5.0
+  - bundler
+  - custom:
+      name: Database
+      met?: test -f spec/integration/database.yml && test "$(./dbdeployer/sandboxes/rsandbox_5_7_22/status_all | grep on  | wc -l | xargs echo)" = "2"
+      meet: ./dbdeployer/install.sh
+      down: ./dbdeployer/sandboxes/rsandbox_5_7_22/stop_all
+
+commands:
+  unit: bundle exec rake unit
+  int: bundle exec rake integration
+  test: bundle exec rake unit && bundle exec rake integration

lhm.gemspec

@@ -24,5 +24,6 @@ Gem::Specification.new do |s|
   s.add_development_dependency 'mocha'
   s.add_development_dependency 'rake'
   s.add_development_dependency 'activerecord'
-  s.add_development_dependency 'mysql'
+  s.add_development_dependency 'mysql2'
+  s.add_development_dependency 'package_cloud'
 end

lib/lhm/throttler/slave_lag.rb

@@ -62,7 +62,7 @@ def get_slaves
           while slave_hosts.any? do
             host = slave_hosts.pop
             slave = Slave.new(host, @get_config)
-            if slaves.map(&:host).exclude?(host) && slave.connection
+            if !slaves.map(&:host).include?(host) && slave.connection
               slaves << slave
               slave_hosts.concat(slave.slave_hosts)
             end

spec/integration/integration_helper.rb

@@ -4,13 +4,14 @@
 require 'yaml'
 require 'active_support'
 
-config = YAML.load_file(File.expand_path(File.dirname(__FILE__)) + '/database.yml') rescue {}
-$lhm_user = config['user'] ||= 'root'
-$password = config['password'] ||= '1234'
-$master_host = config['master_host'] ||= '127.0.0.1'
-$master_port = config['master_port'] ||= 3306
-$slave_host = config['slave_host'] ||= '127.0.0.1'
-$slave_port = config['slave_port'] ||= 3307
+begin
+  $db_config = YAML.load_file(File.expand_path(File.dirname(__FILE__)) + '/database.yml')
+rescue StandardError => e
+  puts "Run install.sh to setup database"
+  raise e
+end
+
+$db_name = 'test'
 
 require 'lhm/table'
 require 'lhm/sql_helper'
@@ -24,15 +25,27 @@ def connection
   end
 
   def connect_master!
-    connect!($master_host, $master_port)
+    connect!(
+      '127.0.0.1',
+      $db_config['master']['port'],
+      $db_config['master']['user'],
+      $db_config['master']['password'],
+      $db_config['master']['socket']
+    )
   end
 
   def connect_slave!
-    connect!($slave_host, $slave_host)
+    connect!(
+      '127.0.0.1',
+      $db_config['slave']['port'],
+      $db_config['slave']['user'],
+      $db_config['slave']['password'],
+      $db_config['slave']['socket']
+    )
   end
 
-  def connect!(hostname, port)
-    adapter = ar_conn(hostname, port)
+  def connect!(hostname, port, user, password, socket)
+    adapter = ar_conn(hostname, port, user, password, socket)
     Lhm.setup(adapter)
     unless defined?(@@cleaned_up)
       Lhm.cleanup(true)
@@ -41,14 +54,15 @@ def connect!(hostname, port)
     @connection = adapter
   end
 
-  def ar_conn(host, port)
+  def ar_conn(host, port, user, password, socket)
     ActiveRecord::Base.establish_connection(
-      :adapter  => defined?(Mysql2) ? 'mysql2' : 'mysql',
+      :adapter  => 'mysql2',
       :host     => host,
-      :database => 'lhm',
-      :username => $lhm_user,
+      :username => user,
       :port     => port,
-      :password => $password
+      :password => password,
+      :socket   => socket,
+      :database => $db_name
     )
     ActiveRecord::Base.connection
   end
@@ -97,7 +111,7 @@ def slave(&block)
   # Helps testing behaviour when another client locks the db
   def start_locking_thread(lock_for, queue, locking_query)
     Thread.new do
-      conn = Mysql2::Client.new(host: '127.0.0.1', database: 'lhm', user: 'root', port: 3306)
+      conn = Mysql2::Client.new(host: '127.0.0.1', database: $db_name, user: 'root', port: 3306)
       conn.query('BEGIN')
       conn.query(locking_query)
       queue.push(true)