Setup databases for integration tests with dev

.gitignore

@@ -1,6 +1,5 @@
 *.gem
 .bundle
-Gemfile.lock
 gemfiles/*.lock
 pkg/*
 .rvmrc

Gemfile

@@ -3,7 +3,3 @@ source 'https://rubygems.org'
 # Specify your gem's dependencies in shopify_lhm.gemspec
 gemspec
 
-group :deployment do
-  gem 'package_cloud'
-  gem 'rake'
-end

Gemfile.lock

@@ -0,0 +1,73 @@
+PATH
+  remote: .
+  specs:
+    lhm (3.0.0.alpha)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activemodel (5.2.0)
+      activesupport (= 5.2.0)
+    activerecord (5.2.0)
+      activemodel (= 5.2.0)
+      activesupport (= 5.2.0)
+      arel (>= 9.0)
+    activesupport (5.2.0)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 0.7, < 2)
+      minitest (~> 5.1)
+      tzinfo (~> 1.1)
+    arel (9.0.0)
+    concurrent-ruby (1.0.5)
+    domain_name (0.5.20180417)
+      unf (>= 0.0.5, < 1.0.0)
+    highline (1.6.20)
+    http-cookie (1.0.3)
+      domain_name (~> 0.5)
+    i18n (1.0.1)
+      concurrent-ruby (~> 1.0)
+    json_pure (1.8.1)
+    metaclass (0.0.4)
+    mime-types (3.1)
+      mime-types-data (~> 3.2015)
+    mime-types-data (3.2016.0521)
+    minitest (5.11.3)
+    mocha (1.5.0)
+      metaclass (~> 0.0.1)
+    mysql2 (0.5.2)
+    netrc (0.11.0)
+    package_cloud (0.3.05)
+      highline (= 1.6.20)
+      json_pure (= 1.8.1)
+      rainbow (= 2.2.2)
+      rest-client (~> 2.0)
+      thor (~> 0.18)
+    rainbow (2.2.2)
+      rake
+    rake (12.3.1)
+    rest-client (2.0.2)
+      http-cookie (>= 1.0.2, < 2.0)
+      mime-types (>= 1.16, < 4.0)
+      netrc (~> 0.8)
+    thor (0.20.0)
+    thread_safe (0.3.6)
+    tzinfo (1.2.5)
+      thread_safe (~> 0.1)
+    unf (0.1.4)
+      unf_ext
+    unf_ext (0.0.7.5)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  activerecord
+  lhm!
+  minitest
+  mocha
+  mysql2
+  package_cloud
+  rake
+
+BUNDLED WITH
+   1.16.1

README.md

@@ -215,18 +215,24 @@ Lhm.cleanup(true, until: Time.now - 86400)
 
 First, get set up for local development:
 
-    git clone git://github.com/soundcloud/lhm.git
-    cd lhm
-
-To run the tests, follow the instructions on [spec/README](https://github.com/soundcloud/lhm/blob/master/spec/README.md).
-
-We'll check out your contribution if you:
+```
+dev clone lhm
+dev up
+```
 
-  * Provide a comprehensive suite of tests for your fork.
-  * Have a clear and documented rationale for your changes.
-  * Package these up in a pull request.
+To run the tests:
+```
+dev unit # unit tests
+dev int # integration tests
+dev test # all tests
+```
 
-We'll do our best to help you out with any contribution issues you may have.
+### dbdeployer
+The integration tests rely on a master/slave replication setup of MySQL.
+We're using [dbdeployer](https://github.com/datacharmer/dbdeployer) to set this up via `./install.sh`.
+`dbdeployer` provides scripts for operating and accessing the nodes in `$HOME/sandboxes/rsandbox_5_7_22`.
+There is a lot in there, and most of time you shouldn't need to work with the nodes directly, but it's good
+to know where to go!
 
 ## License
 

dev.yml

@@ -0,0 +1,17 @@
+name: lhm
+up:
+  - homebrew:
+      - openssl
+      - shopify/shopify/mysql-client
+      - wget
+  - ruby: 2.5.0
+  - bundler
+  - custom:
+      name: Database
+      met?: test -f spec/integration/database.yml && test "$(~/sandboxes/rsandbox_5_7_22/status_all | grep on  | wc -l | xargs echo)" = "2"
+      meet: ./install.sh
+
+commands:
+  unit: bundle exec rake unit
+  int: bundle exec rake integration
+  test: bundle exec rake unit && bundle exec rake integration

install.sh

@@ -0,0 +1,51 @@
+set -e
+
+echo "Checking if dbdeployer is installed"
+if ! [ -x "$(command -v dbdeployer)" ]; then
+  echo "Not installed...starting install"
+  VERSION=1.8.0
+  OS=osx
+  origin=https://github.com/datacharmer/dbdeployer/releases/download/$VERSION
+  filename=dbdeployer-$VERSION.$OS
+  wget -q $origin/$filename.tar.gz
+  tar -xzf $filename.tar.gz
+  chmod +x $filename
+  sudo mv $filename /usr/local/bin/dbdeployer
+  rm $filename.tar.gz
+else
+  echo "Installation found!"
+fi
+
+
+echo "Checking if mysql 5.7.22 is available for dbdeployer"
+if [ -z "$(dbdeployer available | grep 5.7.22)" ]; then
+  echo "Not found..."
+  mkdir -p $HOME/opt/mysql
+
+  MYSQL_FILE=mysql-5.7.22-macos10.13-x86_64.tar.gz
+  rm -f $MYSQL_FILE*
+  echo "Downloading $MYSQL_FILE...(this may take a while)"
+  wget -q "https://dev.mysql.com/get/Downloads/MySQL-5.7/$MYSQL_FILE"
+
+  echo "Setting up..."
+  dbdeployer unpack $MYSQL_FILE --verbosity 0
+  rm $MYSQL_FILE
+else
+  echo "mysql 5.7.22 found!"
+fi
+
+echo "Forcing new replication setup..."
+dbdeployer deploy replication 5.7.22 --nodes 2 --force
+dbdeployer global status
+
+echo "Setting up database.yml"
+DATABASE_YML=spec/integration/database.yml
+echo "master:" > $DATABASE_YML
+cat ~/sandboxes/rsandbox_5_7_22/master/my.sandbox.cnf | grep -A 4 client | tail -n 4 | awk $'{print "  " $1 ": " $3}' >> $DATABASE_YML
+
+echo "slave:" >> $DATABASE_YML
+cat ~/sandboxes/rsandbox_5_7_22/node1/my.sandbox.cnf | grep -A 4 client | tail -n 4 | awk $'{print "  " $1 ": " $3}' >> $DATABASE_YML
+
+cat $DATABASE_YML
+
+echo "You are ready to run the integration test suite..."

lhm.gemspec

@@ -24,5 +24,6 @@ Gem::Specification.new do |s|
   s.add_development_dependency 'mocha'
   s.add_development_dependency 'rake'
   s.add_development_dependency 'activerecord'
-  s.add_development_dependency 'mysql'
+  s.add_development_dependency 'mysql2'
+  s.add_development_dependency 'package_cloud'
 end

spec/integration/integration_helper.rb

@@ -4,13 +4,14 @@
 require 'yaml'
 require 'active_support'
 
-config = YAML.load_file(File.expand_path(File.dirname(__FILE__)) + '/database.yml') rescue {}
-$lhm_user = config['user'] ||= 'root'
-$password = config['password'] ||= '1234'
-$master_host = config['master_host'] ||= '127.0.0.1'
-$master_port = config['master_port'] ||= 3306
-$slave_host = config['slave_host'] ||= '127.0.0.1'
-$slave_port = config['slave_port'] ||= 3307
+begin
+  $db_config = YAML.load_file(File.expand_path(File.dirname(__FILE__)) + '/database.yml')
+rescue StandardError => e
+  puts "Run install.sh to setup database"
+  raise e
+end
+
+$db_name = 'test'
 
 require 'lhm/table'
 require 'lhm/sql_helper'
@@ -24,15 +25,27 @@ def connection
   end
 
   def connect_master!
-    connect!($master_host, $master_port)
+    connect!(
+      '127.0.0.1',
+      $db_config['master']['port'],
+      $db_config['master']['user'],
+      $db_config['master']['password'],
+      $db_config['master']['socket']
+    )
   end
 
   def connect_slave!
-    connect!($slave_host, $slave_host)
+    connect!(
+      '127.0.0.1',
+      $db_config['slave']['port'],
+      $db_config['slave']['user'],
+      $db_config['slave']['password'],
+      $db_config['slave']['socket']
+    )
   end
 
-  def connect!(hostname, port)
-    adapter = ar_conn(hostname, port)
+  def connect!(hostname, port, user, password, socket)
+    adapter = ar_conn(hostname, port, user, password, socket)
     Lhm.setup(adapter)
     unless defined?(@@cleaned_up)
       Lhm.cleanup(true)
@@ -41,14 +54,15 @@ def connect!(hostname, port)
     @connection = adapter
   end
 
-  def ar_conn(host, port)
+  def ar_conn(host, port, user, password, socket)
     ActiveRecord::Base.establish_connection(
-      :adapter  => defined?(Mysql2) ? 'mysql2' : 'mysql',
+      :adapter  => 'mysql2',
       :host     => host,
-      :database => 'lhm',
-      :username => $lhm_user,
+      :username => user,
       :port     => port,
-      :password => $password
+      :password => password,
+      :socket   => socket,
+      :database => $db_name
     )
     ActiveRecord::Base.connection
   end
@@ -97,7 +111,7 @@ def slave(&block)
   # Helps testing behaviour when another client locks the db
   def start_locking_thread(lock_for, queue, locking_query)
     Thread.new do
-      conn = Mysql2::Client.new(host: '127.0.0.1', database: 'lhm', user: 'root', port: 3306)
+      conn = Mysql2::Client.new(host: '127.0.0.1', database: $db_name, user: 'root', port: 3306)
       conn.query('BEGIN')
       conn.query(locking_query)
       queue.push(true)