diff --git a/.gitignore b/.gitignore index 73efbbe0..645dbf9c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,5 @@ *.gem .bundle -Gemfile.lock gemfiles/*.lock pkg/* .rvmrc diff --git a/Gemfile b/Gemfile index 6c657870..877e2d41 100644 --- a/Gemfile +++ b/Gemfile @@ -3,7 +3,3 @@ source 'https://rubygems.org' # Specify your gem's dependencies in shopify_lhm.gemspec gemspec -group :deployment do - gem 'package_cloud' - gem 'rake' -end diff --git a/Gemfile.lock b/Gemfile.lock new file mode 100644 index 00000000..d18f5048 --- /dev/null +++ b/Gemfile.lock @@ -0,0 +1,73 @@ +PATH + remote: . + specs: + lhm (3.0.0.alpha) + +GEM + remote: https://rubygems.org/ + specs: + activemodel (5.2.0) + activesupport (= 5.2.0) + activerecord (5.2.0) + activemodel (= 5.2.0) + activesupport (= 5.2.0) + arel (>= 9.0) + activesupport (5.2.0) + concurrent-ruby (~> 1.0, >= 1.0.2) + i18n (>= 0.7, < 2) + minitest (~> 5.1) + tzinfo (~> 1.1) + arel (9.0.0) + concurrent-ruby (1.0.5) + domain_name (0.5.20180417) + unf (>= 0.0.5, < 1.0.0) + highline (1.6.20) + http-cookie (1.0.3) + domain_name (~> 0.5) + i18n (1.0.1) + concurrent-ruby (~> 1.0) + json_pure (1.8.1) + metaclass (0.0.4) + mime-types (3.1) + mime-types-data (~> 3.2015) + mime-types-data (3.2016.0521) + minitest (5.11.3) + mocha (1.5.0) + metaclass (~> 0.0.1) + mysql2 (0.5.2) + netrc (0.11.0) + package_cloud (0.3.05) + highline (= 1.6.20) + json_pure (= 1.8.1) + rainbow (= 2.2.2) + rest-client (~> 2.0) + thor (~> 0.18) + rainbow (2.2.2) + rake + rake (12.3.1) + rest-client (2.0.2) + http-cookie (>= 1.0.2, < 2.0) + mime-types (>= 1.16, < 4.0) + netrc (~> 0.8) + thor (0.20.0) + thread_safe (0.3.6) + tzinfo (1.2.5) + thread_safe (~> 0.1) + unf (0.1.4) + unf_ext + unf_ext (0.0.7.5) + +PLATFORMS + ruby + +DEPENDENCIES + activerecord + lhm! + minitest + mocha + mysql2 + package_cloud + rake + +BUNDLED WITH + 1.16.1 diff --git a/README.md b/README.md index 3eb3ba66..33bdeddc 100644 --- a/README.md +++ b/README.md @@ -215,18 +215,24 @@ Lhm.cleanup(true, until: Time.now - 86400) First, get set up for local development: - git clone git://github.com/soundcloud/lhm.git - cd lhm - -To run the tests, follow the instructions on [spec/README](https://github.com/soundcloud/lhm/blob/master/spec/README.md). - -We'll check out your contribution if you: +``` +dev clone lhm +dev up +``` - * Provide a comprehensive suite of tests for your fork. - * Have a clear and documented rationale for your changes. - * Package these up in a pull request. +To run the tests: +``` +dev unit # unit tests +dev int # integration tests +dev test # all tests +``` -We'll do our best to help you out with any contribution issues you may have. +### dbdeployer +The integration tests rely on a master/slave replication setup of MySQL. +We're using [dbdeployer](https://github.com/datacharmer/dbdeployer) to set this up via `./install.sh`. +`dbdeployer` provides scripts for operating and accessing the nodes in `$HOME/sandboxes/rsandbox_5_7_22`. +There is a lot in there, and most of time you shouldn't need to work with the nodes directly, but it's good +to know where to go! ## License diff --git a/dev.yml b/dev.yml new file mode 100644 index 00000000..6307f3ec --- /dev/null +++ b/dev.yml @@ -0,0 +1,17 @@ +name: lhm +up: + - homebrew: + - openssl + - shopify/shopify/mysql-client + - wget + - ruby: 2.5.0 + - bundler + - custom: + name: Database + met?: test -f spec/integration/database.yml && test "$(~/sandboxes/rsandbox_5_7_22/status_all | grep on | wc -l | xargs echo)" = "2" + meet: ./install.sh + +commands: + unit: bundle exec rake unit + int: bundle exec rake integration + test: bundle exec rake unit && bundle exec rake integration diff --git a/install.sh b/install.sh new file mode 100755 index 00000000..b94fb090 --- /dev/null +++ b/install.sh @@ -0,0 +1,51 @@ +set -e + +echo "Checking if dbdeployer is installed" +if ! [ -x "$(command -v dbdeployer)" ]; then + echo "Not installed...starting install" + VERSION=1.8.0 + OS=osx + origin=https://github.com/datacharmer/dbdeployer/releases/download/$VERSION + filename=dbdeployer-$VERSION.$OS + wget -q $origin/$filename.tar.gz + tar -xzf $filename.tar.gz + chmod +x $filename + sudo mv $filename /usr/local/bin/dbdeployer + rm $filename.tar.gz +else + echo "Installation found!" +fi + + +echo "Checking if mysql 5.7.22 is available for dbdeployer" +if [ -z "$(dbdeployer available | grep 5.7.22)" ]; then + echo "Not found..." + mkdir -p $HOME/opt/mysql + + MYSQL_FILE=mysql-5.7.22-macos10.13-x86_64.tar.gz + rm -f $MYSQL_FILE* + echo "Downloading $MYSQL_FILE...(this may take a while)" + wget -q "https://dev.mysql.com/get/Downloads/MySQL-5.7/$MYSQL_FILE" + + echo "Setting up..." + dbdeployer unpack $MYSQL_FILE --verbosity 0 + rm $MYSQL_FILE +else + echo "mysql 5.7.22 found!" +fi + +echo "Forcing new replication setup..." +dbdeployer deploy replication 5.7.22 --nodes 2 --force +dbdeployer global status + +echo "Setting up database.yml" +DATABASE_YML=spec/integration/database.yml +echo "master:" > $DATABASE_YML +cat ~/sandboxes/rsandbox_5_7_22/master/my.sandbox.cnf | grep -A 4 client | tail -n 4 | awk $'{print " " $1 ": " $3}' >> $DATABASE_YML + +echo "slave:" >> $DATABASE_YML +cat ~/sandboxes/rsandbox_5_7_22/node1/my.sandbox.cnf | grep -A 4 client | tail -n 4 | awk $'{print " " $1 ": " $3}' >> $DATABASE_YML + +cat $DATABASE_YML + +echo "You are ready to run the integration test suite..." diff --git a/lhm.gemspec b/lhm.gemspec index d5c4c811..4c3119aa 100644 --- a/lhm.gemspec +++ b/lhm.gemspec @@ -24,5 +24,6 @@ Gem::Specification.new do |s| s.add_development_dependency 'mocha' s.add_development_dependency 'rake' s.add_development_dependency 'activerecord' - s.add_development_dependency 'mysql' + s.add_development_dependency 'mysql2' + s.add_development_dependency 'package_cloud' end diff --git a/spec/integration/integration_helper.rb b/spec/integration/integration_helper.rb index 1c9fd274..83b11f0c 100644 --- a/spec/integration/integration_helper.rb +++ b/spec/integration/integration_helper.rb @@ -4,13 +4,14 @@ require 'yaml' require 'active_support' -config = YAML.load_file(File.expand_path(File.dirname(__FILE__)) + '/database.yml') rescue {} -$lhm_user = config['user'] ||= 'root' -$password = config['password'] ||= '1234' -$master_host = config['master_host'] ||= '127.0.0.1' -$master_port = config['master_port'] ||= 3306 -$slave_host = config['slave_host'] ||= '127.0.0.1' -$slave_port = config['slave_port'] ||= 3307 +begin + $db_config = YAML.load_file(File.expand_path(File.dirname(__FILE__)) + '/database.yml') +rescue StandardError => e + puts "Run install.sh to setup database" + raise e +end + +$db_name = 'test' require 'lhm/table' require 'lhm/sql_helper' @@ -24,15 +25,27 @@ def connection end def connect_master! - connect!($master_host, $master_port) + connect!( + '127.0.0.1', + $db_config['master']['port'], + $db_config['master']['user'], + $db_config['master']['password'], + $db_config['master']['socket'] + ) end def connect_slave! - connect!($slave_host, $slave_host) + connect!( + '127.0.0.1', + $db_config['slave']['port'], + $db_config['slave']['user'], + $db_config['slave']['password'], + $db_config['slave']['socket'] + ) end - def connect!(hostname, port) - adapter = ar_conn(hostname, port) + def connect!(hostname, port, user, password, socket) + adapter = ar_conn(hostname, port, user, password, socket) Lhm.setup(adapter) unless defined?(@@cleaned_up) Lhm.cleanup(true) @@ -41,14 +54,15 @@ def connect!(hostname, port) @connection = adapter end - def ar_conn(host, port) + def ar_conn(host, port, user, password, socket) ActiveRecord::Base.establish_connection( - :adapter => defined?(Mysql2) ? 'mysql2' : 'mysql', + :adapter => 'mysql2', :host => host, - :database => 'lhm', - :username => $lhm_user, + :username => user, :port => port, - :password => $password + :password => password, + :socket => socket, + :database => $db_name ) ActiveRecord::Base.connection end @@ -97,7 +111,7 @@ def slave(&block) # Helps testing behaviour when another client locks the db def start_locking_thread(lock_for, queue, locking_query) Thread.new do - conn = Mysql2::Client.new(host: '127.0.0.1', database: 'lhm', user: 'root', port: 3306) + conn = Mysql2::Client.new(host: '127.0.0.1', database: $db_name, user: 'root', port: 3306) conn.query('BEGIN') conn.query(locking_query) queue.push(true)