From 89d80d4dbc001ba0322f6be0f6083a0d8a288354 Mon Sep 17 00:00:00 2001 From: Brian Buchalter Date: Thu, 12 Jul 2018 12:17:29 -0400 Subject: [PATCH 1/2] Fix dependencies * Do not specify dependencies in Gemfile (this is a gem) * rake was already in gemspec causing bundler to complain * added package_cloud to gemspec * Change `mysql` to `mysql2` because `mysql` is so ancient it won't even compile * Add Gemfile.lock - this is now considered best practice: https://github.com/bundler/bundler/pull/6184 --- .gitignore | 1 - Gemfile | 4 --- Gemfile.lock | 73 ++++++++++++++++++++++++++++++++++++++++++++++++++++ lhm.gemspec | 3 ++- 4 files changed, 75 insertions(+), 6 deletions(-) create mode 100644 Gemfile.lock diff --git a/.gitignore b/.gitignore index 73efbbe0..645dbf9c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,5 @@ *.gem .bundle -Gemfile.lock gemfiles/*.lock pkg/* .rvmrc diff --git a/Gemfile b/Gemfile index 6c657870..877e2d41 100644 --- a/Gemfile +++ b/Gemfile @@ -3,7 +3,3 @@ source 'https://rubygems.org' # Specify your gem's dependencies in shopify_lhm.gemspec gemspec -group :deployment do - gem 'package_cloud' - gem 'rake' -end diff --git a/Gemfile.lock b/Gemfile.lock new file mode 100644 index 00000000..d18f5048 --- /dev/null +++ b/Gemfile.lock @@ -0,0 +1,73 @@ +PATH + remote: . + specs: + lhm (3.0.0.alpha) + +GEM + remote: https://rubygems.org/ + specs: + activemodel (5.2.0) + activesupport (= 5.2.0) + activerecord (5.2.0) + activemodel (= 5.2.0) + activesupport (= 5.2.0) + arel (>= 9.0) + activesupport (5.2.0) + concurrent-ruby (~> 1.0, >= 1.0.2) + i18n (>= 0.7, < 2) + minitest (~> 5.1) + tzinfo (~> 1.1) + arel (9.0.0) + concurrent-ruby (1.0.5) + domain_name (0.5.20180417) + unf (>= 0.0.5, < 1.0.0) + highline (1.6.20) + http-cookie (1.0.3) + domain_name (~> 0.5) + i18n (1.0.1) + concurrent-ruby (~> 1.0) + json_pure (1.8.1) + metaclass (0.0.4) + mime-types (3.1) + mime-types-data (~> 3.2015) + mime-types-data (3.2016.0521) + minitest (5.11.3) + mocha (1.5.0) + metaclass (~> 0.0.1) + mysql2 (0.5.2) + netrc (0.11.0) + package_cloud (0.3.05) + highline (= 1.6.20) + json_pure (= 1.8.1) + rainbow (= 2.2.2) + rest-client (~> 2.0) + thor (~> 0.18) + rainbow (2.2.2) + rake + rake (12.3.1) + rest-client (2.0.2) + http-cookie (>= 1.0.2, < 2.0) + mime-types (>= 1.16, < 4.0) + netrc (~> 0.8) + thor (0.20.0) + thread_safe (0.3.6) + tzinfo (1.2.5) + thread_safe (~> 0.1) + unf (0.1.4) + unf_ext + unf_ext (0.0.7.5) + +PLATFORMS + ruby + +DEPENDENCIES + activerecord + lhm! + minitest + mocha + mysql2 + package_cloud + rake + +BUNDLED WITH + 1.16.1 diff --git a/lhm.gemspec b/lhm.gemspec index d5c4c811..4c3119aa 100644 --- a/lhm.gemspec +++ b/lhm.gemspec @@ -24,5 +24,6 @@ Gem::Specification.new do |s| s.add_development_dependency 'mocha' s.add_development_dependency 'rake' s.add_development_dependency 'activerecord' - s.add_development_dependency 'mysql' + s.add_development_dependency 'mysql2' + s.add_development_dependency 'package_cloud' end From 98d5ffd574ac777284b4b150ade9edca66de74f5 Mon Sep 17 00:00:00 2001 From: Brian Buchalter Date: Fri, 13 Jul 2018 09:32:16 -0400 Subject: [PATCH 2/2] Setup databases for integration tests with dev These tests are critical to exercising LHM. We need this to be as easy as possible for developers to run, so let's automate it. --- README.md | 26 ++++++++----- dev.yml | 17 +++++++++ install.sh | 51 ++++++++++++++++++++++++++ spec/integration/integration_helper.rb | 48 +++++++++++++++--------- 4 files changed, 115 insertions(+), 27 deletions(-) create mode 100644 dev.yml create mode 100755 install.sh diff --git a/README.md b/README.md index 3eb3ba66..33bdeddc 100644 --- a/README.md +++ b/README.md @@ -215,18 +215,24 @@ Lhm.cleanup(true, until: Time.now - 86400) First, get set up for local development: - git clone git://github.com/soundcloud/lhm.git - cd lhm - -To run the tests, follow the instructions on [spec/README](https://github.com/soundcloud/lhm/blob/master/spec/README.md). - -We'll check out your contribution if you: +``` +dev clone lhm +dev up +``` - * Provide a comprehensive suite of tests for your fork. - * Have a clear and documented rationale for your changes. - * Package these up in a pull request. +To run the tests: +``` +dev unit # unit tests +dev int # integration tests +dev test # all tests +``` -We'll do our best to help you out with any contribution issues you may have. +### dbdeployer +The integration tests rely on a master/slave replication setup of MySQL. +We're using [dbdeployer](https://github.com/datacharmer/dbdeployer) to set this up via `./install.sh`. +`dbdeployer` provides scripts for operating and accessing the nodes in `$HOME/sandboxes/rsandbox_5_7_22`. +There is a lot in there, and most of time you shouldn't need to work with the nodes directly, but it's good +to know where to go! ## License diff --git a/dev.yml b/dev.yml new file mode 100644 index 00000000..6307f3ec --- /dev/null +++ b/dev.yml @@ -0,0 +1,17 @@ +name: lhm +up: + - homebrew: + - openssl + - shopify/shopify/mysql-client + - wget + - ruby: 2.5.0 + - bundler + - custom: + name: Database + met?: test -f spec/integration/database.yml && test "$(~/sandboxes/rsandbox_5_7_22/status_all | grep on | wc -l | xargs echo)" = "2" + meet: ./install.sh + +commands: + unit: bundle exec rake unit + int: bundle exec rake integration + test: bundle exec rake unit && bundle exec rake integration diff --git a/install.sh b/install.sh new file mode 100755 index 00000000..b94fb090 --- /dev/null +++ b/install.sh @@ -0,0 +1,51 @@ +set -e + +echo "Checking if dbdeployer is installed" +if ! [ -x "$(command -v dbdeployer)" ]; then + echo "Not installed...starting install" + VERSION=1.8.0 + OS=osx + origin=https://github.com/datacharmer/dbdeployer/releases/download/$VERSION + filename=dbdeployer-$VERSION.$OS + wget -q $origin/$filename.tar.gz + tar -xzf $filename.tar.gz + chmod +x $filename + sudo mv $filename /usr/local/bin/dbdeployer + rm $filename.tar.gz +else + echo "Installation found!" +fi + + +echo "Checking if mysql 5.7.22 is available for dbdeployer" +if [ -z "$(dbdeployer available | grep 5.7.22)" ]; then + echo "Not found..." + mkdir -p $HOME/opt/mysql + + MYSQL_FILE=mysql-5.7.22-macos10.13-x86_64.tar.gz + rm -f $MYSQL_FILE* + echo "Downloading $MYSQL_FILE...(this may take a while)" + wget -q "https://dev.mysql.com/get/Downloads/MySQL-5.7/$MYSQL_FILE" + + echo "Setting up..." + dbdeployer unpack $MYSQL_FILE --verbosity 0 + rm $MYSQL_FILE +else + echo "mysql 5.7.22 found!" +fi + +echo "Forcing new replication setup..." +dbdeployer deploy replication 5.7.22 --nodes 2 --force +dbdeployer global status + +echo "Setting up database.yml" +DATABASE_YML=spec/integration/database.yml +echo "master:" > $DATABASE_YML +cat ~/sandboxes/rsandbox_5_7_22/master/my.sandbox.cnf | grep -A 4 client | tail -n 4 | awk $'{print " " $1 ": " $3}' >> $DATABASE_YML + +echo "slave:" >> $DATABASE_YML +cat ~/sandboxes/rsandbox_5_7_22/node1/my.sandbox.cnf | grep -A 4 client | tail -n 4 | awk $'{print " " $1 ": " $3}' >> $DATABASE_YML + +cat $DATABASE_YML + +echo "You are ready to run the integration test suite..." diff --git a/spec/integration/integration_helper.rb b/spec/integration/integration_helper.rb index 1c9fd274..83b11f0c 100644 --- a/spec/integration/integration_helper.rb +++ b/spec/integration/integration_helper.rb @@ -4,13 +4,14 @@ require 'yaml' require 'active_support' -config = YAML.load_file(File.expand_path(File.dirname(__FILE__)) + '/database.yml') rescue {} -$lhm_user = config['user'] ||= 'root' -$password = config['password'] ||= '1234' -$master_host = config['master_host'] ||= '127.0.0.1' -$master_port = config['master_port'] ||= 3306 -$slave_host = config['slave_host'] ||= '127.0.0.1' -$slave_port = config['slave_port'] ||= 3307 +begin + $db_config = YAML.load_file(File.expand_path(File.dirname(__FILE__)) + '/database.yml') +rescue StandardError => e + puts "Run install.sh to setup database" + raise e +end + +$db_name = 'test' require 'lhm/table' require 'lhm/sql_helper' @@ -24,15 +25,27 @@ def connection end def connect_master! - connect!($master_host, $master_port) + connect!( + '127.0.0.1', + $db_config['master']['port'], + $db_config['master']['user'], + $db_config['master']['password'], + $db_config['master']['socket'] + ) end def connect_slave! - connect!($slave_host, $slave_host) + connect!( + '127.0.0.1', + $db_config['slave']['port'], + $db_config['slave']['user'], + $db_config['slave']['password'], + $db_config['slave']['socket'] + ) end - def connect!(hostname, port) - adapter = ar_conn(hostname, port) + def connect!(hostname, port, user, password, socket) + adapter = ar_conn(hostname, port, user, password, socket) Lhm.setup(adapter) unless defined?(@@cleaned_up) Lhm.cleanup(true) @@ -41,14 +54,15 @@ def connect!(hostname, port) @connection = adapter end - def ar_conn(host, port) + def ar_conn(host, port, user, password, socket) ActiveRecord::Base.establish_connection( - :adapter => defined?(Mysql2) ? 'mysql2' : 'mysql', + :adapter => 'mysql2', :host => host, - :database => 'lhm', - :username => $lhm_user, + :username => user, :port => port, - :password => $password + :password => password, + :socket => socket, + :database => $db_name ) ActiveRecord::Base.connection end @@ -97,7 +111,7 @@ def slave(&block) # Helps testing behaviour when another client locks the db def start_locking_thread(lock_for, queue, locking_query) Thread.new do - conn = Mysql2::Client.new(host: '127.0.0.1', database: 'lhm', user: 'root', port: 3306) + conn = Mysql2::Client.new(host: '127.0.0.1', database: $db_name, user: 'root', port: 3306) conn.query('BEGIN') conn.query(locking_query) queue.push(true)