From 051937453076e18ade5948d9546106144fc2d41b Mon Sep 17 00:00:00 2001 From: Tom Christ Date: Fri, 3 Apr 2015 13:10:49 -0400 Subject: [PATCH 01/86] Add index create/drop methods on table and hook into import/export workflow --- lib/jetpants/db/import_export.rb | 32 +++++++++++++++++++++++++++++++- lib/jetpants/table.rb | 29 +++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 1 deletion(-) diff --git a/lib/jetpants/db/import_export.rb b/lib/jetpants/db/import_export.rb index 23d6194..e45fb3c 100644 --- a/lib/jetpants/db/import_export.rb +++ b/lib/jetpants/db/import_export.rb @@ -307,8 +307,38 @@ def rebuild!(tables=false, min_id=false, max_id=false) export_schemata tables export_data tables, min_id, max_id import_schemata! - alter_schemata if respond_to? :alter_schemata + if respond_to? :alter_schemata + alter_schemata + # re-retrieve table metadata in the case that we alter the tables + pool.probe_tables! + tables = pool.tables.select{|t| pool.tables.map(&:name).include?(table.name)} + end + + index_list = {} + + if Jetpants.import_without_indices + tables.each do |t| + index_list[t] ||= [] + + t.indexes.each do |i| + cmd = table.drop_index_query + output "Dropping index #{i} prior to import" + root_cmd(cmd) + end + end + end + import_data tables, min_id, max_id + + if Jetpants.import_without_indices + index_list.each do |table, indexes| + indexes.each do |i| + create_idx_cmd = table.create_index_query(i) + output "Recreating index #{i} after import" + root_cmd(create_idx_cmd) + end + end + end restart_mysql catch_up_to_master if is_slave? diff --git a/lib/jetpants/table.rb b/lib/jetpants/table.rb index 15f9b18..c492a5b 100644 --- a/lib/jetpants/table.rb +++ b/lib/jetpants/table.rb @@ -106,6 +106,35 @@ def max_pk_val_query end return sql end + + # generates a query to drop a specified index named by + # the symbol passed in to the method + def drop_index_query(index_name) + index_info = indexes.select{|idx_name,idx_info| idx_name == index_name}.first + raise "Unable to find index #{index_name}!" if index_info.nil? + + "ALTER TABLE #{name} DROP INDEX #{index_name}" + end + + # generates a query to create a specified index, given + # a hash of columns, a name, and a unique flag as show below: + # {:index_name=> + # {:columns=>[:column_one, :column_two], :unique=>false}}, + # + def create_index_query(index_spec) + index_name = index_spec.keys.first + throw "Cannot determine index name!" if index_name.nil? + + index_opts = index_spec[index_name] + throw "Cannot determine index metadata!" if index_opts[:columns].nil? + + unique = "" + if index_opts[:unique] + unique = "UNIQUE" + end + + "ALTER TABLE #{name} ADD #{unique} INDEX #{index_name} (#{index_opts[:columns].join(',')})" + end # Returns the first column of the primary key, or nil if there isn't one def first_pk_col From 3b75eabfe96dede70d6111caee228e9333368441 Mon Sep 17 00:00:00 2001 From: Tom Christ Date: Fri, 3 Apr 2015 13:15:40 -0400 Subject: [PATCH 02/86] Update calling info --- lib/jetpants/db/import_export.rb | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/jetpants/db/import_export.rb b/lib/jetpants/db/import_export.rb index e45fb3c..a2c1487 100644 --- a/lib/jetpants/db/import_export.rb +++ b/lib/jetpants/db/import_export.rb @@ -321,9 +321,10 @@ def rebuild!(tables=false, min_id=false, max_id=false) index_list[t] ||= [] t.indexes.each do |i| - cmd = table.drop_index_query - output "Dropping index #{i} prior to import" - root_cmd(cmd) + drop_idx_cmd = table.drop_index_query(i) + index_list[t] << i + output "Dropping index #{i.keys.first} prior to import" + root_cmd(drop_idx_cmd) end end end @@ -334,7 +335,7 @@ def rebuild!(tables=false, min_id=false, max_id=false) index_list.each do |table, indexes| indexes.each do |i| create_idx_cmd = table.create_index_query(i) - output "Recreating index #{i} after import" + output "Recreating index #{i.keys.first} after import" root_cmd(create_idx_cmd) end end From 65ad0935584f6a9e4c0c4d8122c667155eff2149 Mon Sep 17 00:00:00 2001 From: Tom Christ Date: Fri, 3 Apr 2015 14:09:35 -0400 Subject: [PATCH 03/86] Fix naming --- lib/jetpants/db/import_export.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/jetpants/db/import_export.rb b/lib/jetpants/db/import_export.rb index a2c1487..8fc620d 100644 --- a/lib/jetpants/db/import_export.rb +++ b/lib/jetpants/db/import_export.rb @@ -311,7 +311,7 @@ def rebuild!(tables=false, min_id=false, max_id=false) alter_schemata # re-retrieve table metadata in the case that we alter the tables pool.probe_tables! - tables = pool.tables.select{|t| pool.tables.map(&:name).include?(table.name)} + tables = pool.tables.select{|t| pool.tables.map(&:name).include?(t.name)} end index_list = {} @@ -321,7 +321,7 @@ def rebuild!(tables=false, min_id=false, max_id=false) index_list[t] ||= [] t.indexes.each do |i| - drop_idx_cmd = table.drop_index_query(i) + drop_idx_cmd = t.drop_index_query(i) index_list[t] << i output "Dropping index #{i.keys.first} prior to import" root_cmd(drop_idx_cmd) From 331a77e89a3115560fdba8f64062187b2b2edc60 Mon Sep 17 00:00:00 2001 From: Tom Christ Date: Mon, 6 Apr 2015 16:30:40 -0400 Subject: [PATCH 04/86] Add column validation and the ability to generate multiple indexes at once --- lib/jetpants/table.rb | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/lib/jetpants/table.rb b/lib/jetpants/table.rb index c492a5b..f2bdaec 100644 --- a/lib/jetpants/table.rb +++ b/lib/jetpants/table.rb @@ -121,19 +121,29 @@ def drop_index_query(index_name) # {:index_name=> # {:columns=>[:column_one, :column_two], :unique=>false}}, # - def create_index_query(index_spec) - index_name = index_spec.keys.first - throw "Cannot determine index name!" if index_name.nil? + def create_index_query(index_specs*) + index_defs = [] - index_opts = index_spec[index_name] - throw "Cannot determine index metadata!" if index_opts[:columns].nil? + index_specs.each do |index_spec| + index_name = index_spec.keys.first + throw "Cannot determine index name!" if index_name.nil? - unique = "" - if index_opts[:unique] - unique = "UNIQUE" + index_opts = index_spec[index_name] + throw "Cannot determine index metadata for new index #{index_name}!" if index_opts[:columns].nil? + + index_spec[:columns].each do |col| + throw "Table #{name} does not have column #{col}" unless columns.include?(col) + end + + unique = "" + if index_opts[:unique] + unique = "UNIQUE" + end + + index_defs << "ADD #{unique} INDEX #{index_name} (#{index_opts[:columns].join(',')})" end - "ALTER TABLE #{name} ADD #{unique} INDEX #{index_name} (#{index_opts[:columns].join(',')})" + "ALTER TABLE #{name} #{index_defs.join(", ")}" end # Returns the first column of the primary key, or nil if there isn't one From 467c68683e8b6769f3dc39f49e90c5eb062b6f10 Mon Sep 17 00:00:00 2001 From: Tom Christ Date: Thu, 23 Apr 2015 14:49:47 -0400 Subject: [PATCH 05/86] Remove extraneous bang --- lib/jetpants/db/import_export.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/jetpants/db/import_export.rb b/lib/jetpants/db/import_export.rb index 8fc620d..0671a83 100644 --- a/lib/jetpants/db/import_export.rb +++ b/lib/jetpants/db/import_export.rb @@ -310,7 +310,7 @@ def rebuild!(tables=false, min_id=false, max_id=false) if respond_to? :alter_schemata alter_schemata # re-retrieve table metadata in the case that we alter the tables - pool.probe_tables! + pool.probe_tables tables = pool.tables.select{|t| pool.tables.map(&:name).include?(t.name)} end From 5296fd086bd90ee4981714eca4709dd21b6bb213 Mon Sep 17 00:00:00 2001 From: Tom Christ Date: Thu, 23 Apr 2015 15:22:19 -0400 Subject: [PATCH 06/86] Fix selector for add_pool command --- plugins/jetpants_collins/commandsuite.rb | 2 +- plugins/jetpants_collins/topology.rb | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/plugins/jetpants_collins/commandsuite.rb b/plugins/jetpants_collins/commandsuite.rb index 26fb34d..4497120 100644 --- a/plugins/jetpants_collins/commandsuite.rb +++ b/plugins/jetpants_collins/commandsuite.rb @@ -30,7 +30,7 @@ def self.jetpants_collins_before_dispatch(task) method_option :master, :master => 'ip of pre-configured master for new pool' def create_pool name = options[:name] || ask('Please enter the name of the new pool.') - if configuration_assets('MYSQL_POOL').map(&:pool).include? name.upcase + if Jetpants.topology.configuration_assets('MYSQL_POOL').map(&:pool).include? name.upcase error "Pool #{name} already exists" end master = options[:master] || ask("Please enter the ip of the master, or 'none' if one does not yet exist.") diff --git a/plugins/jetpants_collins/topology.rb b/plugins/jetpants_collins/topology.rb index e654a2e..5c1bc46 100644 --- a/plugins/jetpants_collins/topology.rb +++ b/plugins/jetpants_collins/topology.rb @@ -209,19 +209,18 @@ def configuration_assets(*primary_roles) operation: 'and', details: true, size: per_page, - query: 'status != ^DECOMMISSIONED$', + query: 'status != ^DECOMMISSIONED$ AND type = ^CONFIGURATION$', } if primary_roles.count == 1 - selector[:type] = '^CONFIGURATION$' selector[:primary_role] = primary_roles.first else values = primary_roles.map {|r| "primary_role = ^#{r}$"} - selector[:query] += ' AND type = ^CONFIGURATION$ AND (' + values.join(' OR ') + ')' + selector[:query] += ' AND (' + values.join(' OR ') + ')' end selector[:remoteLookup] = true if Jetpants.plugins['jetpants_collins']['remote_lookup'] - + done = false page = 0 assets = [] From 26f3224da4d46c1a64d671205c1b6be6b72e85a2 Mon Sep 17 00:00:00 2001 From: Tom Christ Date: Thu, 23 Apr 2015 17:00:46 -0400 Subject: [PATCH 07/86] First pass at adding support for multiple sharding pools --- bin/jetpants | 17 +++++++++--- lib/jetpants/shard.rb | 9 ++++--- lib/jetpants/topology.rb | 27 +++++++++++++++---- plugins/jetpants_collins/shard.rb | 2 +- plugins/jetpants_collins/topology.rb | 6 ++--- plugins/merge_helper/lib/commandsuite.rb | 7 +++-- plugins/merge_helper/lib/shard.rb | 2 +- .../online_schema_change/lib/commandsuite.rb | 10 +++++-- plugins/online_schema_change/lib/topology.rb | 20 +++++++------- plugins/upgrade_helper/commandsuite.rb | 16 ++++++----- 10 files changed, 79 insertions(+), 37 deletions(-) diff --git a/bin/jetpants b/bin/jetpants index 5c4fadc..abe6b2a 100755 --- a/bin/jetpants +++ b/bin/jetpants @@ -407,10 +407,15 @@ module Jetpants method_option :max_id, :desc => 'Maximum ID of parent shard to split' method_option :ranges, :desc => 'Optional comma-separated list of ranges per child ie "1000-1999,2000-2499" (default if omitted: split evenly)' method_option :count, :desc => 'How many child shards to split the parent into (only necessary if the ranges option is omitted)' + method_option :shard_pool, :desc => 'The sharding pool for which to perform the split' def shard_split shard_min = options[:min_id] || ask('Please enter min ID of the parent shard: ') shard_max = options[:max_id] || ask('Please enter max ID of the parent shard: ') - s = Jetpants.topology.shard shard_min, shard_max + + shard_pool = options[:shard_pool] || ask('Please enter the sharding pool for which to perform the split (enter for default pool): ') + shard_pool = default_shard_pool if shard_pool.empty? + + s = Jetpants.topology.shard(shard_min, shard_max, shard_pool) raise "Shard not found" unless s raise "Shard isn't in ready state" unless s.state == :ready @@ -503,10 +508,14 @@ module Jetpants desc 'shard_cutover', 'truncate the current last shard range, and add a new shard after it' method_option :cutover_id, :desc => 'Minimum ID of new last shard being created' + method_option :shard_pool, :desc => 'The sharding pool for which to perform the cutover' def shard_cutover cutover_id = options[:cutover_id] || ask('Please enter min ID of the new shard to be created: ') cutover_id = cutover_id.to_i - last_shard = Jetpants.topology.shards.select {|s| s.max_id == 'INFINITY' && s.in_config?}.first + shard_pool = options[:shard_pool] || ask('Please enter the sharding pool for which to perform the split (enter for default pool): ') + shard_pool = default_shard_pool if shard_pool.empty? + + last_shard = Jetpants.topology.shards(shard_pool).select {|s| s.max_id == 'INFINITY' && s.in_config?}.first last_shard_master = last_shard.master # Simple sanity-check that the cutover ID is greater than the current last shard's MIN id. @@ -794,7 +803,9 @@ module Jetpants output 'Which shard would you like to perform this action on?' shard_min = ask('Please enter min ID of the shard: ') shard_max = ask('Please enter max ID of the shard: ') - s = Jetpants.topology.shard shard_min, shard_max + shard_pool = ask('Please enter the sharding pool which to perform the action on (enter for default pool): ') + shard_pool = default_shard_pool if shard_pool.empty? + s = Jetpants.topology.shard(shard_min, shard_max, shard_pool) raise 'Shard not found' unless s s end diff --git a/lib/jetpants/shard.rb b/lib/jetpants/shard.rb index 4a889b6..ce4b520 100644 --- a/lib/jetpants/shard.rb +++ b/lib/jetpants/shard.rb @@ -107,20 +107,20 @@ def db(mode=:read) # Override the probe_tables method to accommodate shard topology - # delegate everything to the first shard. def probe_tables - if Jetpants.topology.shards.first == self + if Jetpants.topology.shards(self.shard_pool).first == self super else - Jetpants.topology.shards.first.probe_tables + Jetpants.topology.shards(self.shard_pool).first.probe_tables end end # Override the tables accessor to accommodate shard topology - delegate # everything to the first shard def tables - if Jetpants.topology.shards.first == self + if Jetpants.topology.shards(self.shard_pool).first == self super else - Jetpants.topology.shards.first.tables + Jetpants.topology.shards(self.shard_pool).first.tables end end @@ -414,6 +414,7 @@ def init_child_shard_masters(id_ranges) spare.disable_read_only! if (spare.running? && spare.read_only?) spare.output "Will be master for new shard with ID range of #{my_range.first} to #{my_range.last} (inclusive)" child_shard = Shard.new(my_range.first, my_range.last, spare, :initializing) + child_shard.shard_pool = self.shard_pool child_shard.sync_configuration add_child(child_shard) Jetpants.topology.add_pool child_shard diff --git a/lib/jetpants/topology.rb b/lib/jetpants/topology.rb index c59f999..0d79ae5 100644 --- a/lib/jetpants/topology.rb +++ b/lib/jetpants/topology.rb @@ -127,8 +127,9 @@ def slave_roles ###### Instance Methods #################################################### # Returns array of this topology's Jetpants::Pool objects of type Jetpants::Shard - def shards - pools.select {|p| p.is_a? Shard} + def shards(shard_pool = nil) + shard_pool = default_shard_pool if shard_pool.nil + pools.select {|p| p.is_a? Shard}.select { |p| p.shard_pool = shard_pool } end # Returns array of this topology's Jetpants::Pool objects that are NOT of type Jetpants::Shard @@ -151,13 +152,28 @@ def pool(target) # * just a min ID # * a Range object def shard(*args) - if args.count == 2 || args[0].is_a?(Array) + if args.count >= 2 || args[0].is_a?(Array) args.flatten! + if(args.last.to_i == 0 && args.last.upcase != 'INFINITY') + shard_pool = args.last + else + shard_pool = default_shard_pool + end args.map! {|x| x.to_s.upcase == 'INFINITY' ? 'INFINITY' : x.to_i} shards.select {|s| s.min_id == args[0] && s.max_id == args[1]}.first elsif args[0].is_a?(Range) - shards.select {|s| s.min_id == args[0].min && s.max_id == args[0].max}.first + if(args[1].nil?) + shard_pool = default_shard_pool + else + shard_pool = args[1] + end + shards.select {|s| s.min_id == args[0].min && s.max_id == args[0].max && s.shard_pool = shard_pool}.first else + if(args[1].nil?) + shard_pool = default_shard_pool + else + shard_pool = args[1] + end result = shards.select {|s| s.min_id == args[0].to_i} raise "Multiple shards found with that min_id!" if result.count > 1 result.first @@ -170,7 +186,8 @@ def shard(*args) # child is fully built / in production, this method will always return # the child shard. However, Shard#db(:write) will correctly delegate writes # to the parent shard when appropriate in this case. (see also: Topology#shard_db_for_id) - def shard_for_id(id) + def shard_for_id(id, shard_pool = nil) + shard_pool = default_shard_pool if shard_pool.nil? choices = shards.select {|s| s.min_id <= id && (s.max_id == 'INFINITY' || s.max_id >= id)} choices.reject! {|s| s.parent && ! s.in_config?} # filter out child shards that are still being built diff --git a/plugins/jetpants_collins/shard.rb b/plugins/jetpants_collins/shard.rb index da7418e..67fb295 100644 --- a/plugins/jetpants_collins/shard.rb +++ b/plugins/jetpants_collins/shard.rb @@ -7,7 +7,7 @@ class Shard < Pool include Plugin::JetCollins - collins_attr_accessor :shard_min_id, :shard_max_id, :shard_state, :shard_parent + collins_attr_accessor :shard_min_id, :shard_max_id, :shard_state, :shard_parent, :shard_pool # Returns a Collins::Asset for this pool def collins_asset(create_if_missing=false) diff --git a/plugins/jetpants_collins/topology.rb b/plugins/jetpants_collins/topology.rb index e654a2e..1780814 100644 --- a/plugins/jetpants_collins/topology.rb +++ b/plugins/jetpants_collins/topology.rb @@ -117,9 +117,9 @@ def spares(options={}) ##### NEW METHODS ########################################################## - def db_location_report(shards_only = false) - if shards_only - pools_to_consider = shards + def db_location_report(shards_only = nil) + unless shards_only.nil? + pools_to_consider = shards(shards_only) else pools_to_consider = pools end diff --git a/plugins/merge_helper/lib/commandsuite.rb b/plugins/merge_helper/lib/commandsuite.rb index e5b6f3b..f9e9854 100644 --- a/plugins/merge_helper/lib/commandsuite.rb +++ b/plugins/merge_helper/lib/commandsuite.rb @@ -295,11 +295,14 @@ def ask_merge_shard_ranges min_id = ask("Please provide the min ID of the shard range to merge:") max_id = ask("Please provide the max ID of the shard range to merge:") + shard_pool = ask('Please enter the sharding pool which to perform the action on (enter for default pool): ') + shard_pool = default_shard_pool if shard_pool.empty? + # for now we assume we'll never merge the shard at the head of the list - shards_to_merge = Jetpants.shards.select do |shard| + shards_to_merge = Jetpants.shards(shard_pool).select do |shard| shard.min_id.to_i >= min_id.to_i && shard.max_id.to_i <= max_id.to_i && - shard.max_id != 'INFINITY' + shard.max_id != 'INFINITY' && end shard_str = shards_to_merge.join(', ') diff --git a/plugins/merge_helper/lib/shard.rb b/plugins/merge_helper/lib/shard.rb index 4769212..3a7cae3 100644 --- a/plugins/merge_helper/lib/shard.rb +++ b/plugins/merge_helper/lib/shard.rb @@ -4,7 +4,7 @@ module Jetpants class Shard # Runs queries against a slave in the pool to verify sharding key values def validate_shard_data - tables = Table.from_config 'sharded_tables' + tables = Table.from_config('sharded_tables', self.shard_pool) table_statuses = {} tables.limited_concurrent_map(8) { |table| table.sharding_keys.each do |col| diff --git a/plugins/online_schema_change/lib/commandsuite.rb b/plugins/online_schema_change/lib/commandsuite.rb index 4022c6e..b7c5f28 100644 --- a/plugins/online_schema_change/lib/commandsuite.rb +++ b/plugins/online_schema_change/lib/commandsuite.rb @@ -11,6 +11,7 @@ class CommandSuite < Thor method_option :table, :desc => 'Table to run the alter table on' method_option :all_shards, :desc => 'To run on all the shards', :type => :boolean method_option :no_check_plan, :desc => 'Do not check the query execution plan', :type => :boolean + method_option :shard_pool, :desc => 'The sharding pool for which to perform the alter' def alter_table unless options[:all_shards] pool_name = options[:pool] || ask('Please enter a name of a pool: ') @@ -23,7 +24,9 @@ def alter_table alter = options[:alter] || ask('Please enter a alter table statement (eg ADD COLUMN c1 INT): ') if options[:all_shards] - Jetpants.topology.alter_table_shards(database, table, alter, options[:dry_run], options[:no_check_plan]) + shard_pool = options[:shard_pool] || ask('Please enter the sharding pool for which to perform the split (enter for default pool): ') + shard_pool = default_shard_pool if shard_pool.empty? + Jetpants.topology.alter_table_shards(database, table, alter, options[:dry_run], options[:no_check_plan], shard_pool) else unless pool.alter_table(database, table, alter, options[:dry_run], false, options[:no_check_plan]) output "Check for errors during online schema change".red, :error @@ -36,6 +39,7 @@ def alter_table method_option :table, :desc => 'Table you ran the alter table on' method_option :database, :desc => 'Database you ran the alter table on' method_option :all_shards, :desc => 'To run on all the shards', :type => :boolean + method_option :shard_pool, :desc => 'The sharding pool for which to drop the old table' def alter_table_drop unless options[:all_shards] pool_name = options[:pool] || ask('Please enter a name of a pool: ') @@ -47,7 +51,9 @@ def alter_table_drop table = options[:table] || ask('Please enter a name of a table: ') if options[:all_shards] - Jetpants.topology.drop_old_alter_table_shards(database, table) + shard_pool = options[:shard_pool] || ask('Please enter the sharding pool for which to perform the split (enter for default pool): ') + shard_pool = default_shard_pool if shard_pool.empty? + Jetpants.topology.drop_old_alter_table_shards(database, table, shard_pool) else pool.drop_old_alter_table(database, table) end diff --git a/plugins/online_schema_change/lib/topology.rb b/plugins/online_schema_change/lib/topology.rb index 904d802..967486a 100644 --- a/plugins/online_schema_change/lib/topology.rb +++ b/plugins/online_schema_change/lib/topology.rb @@ -5,13 +5,13 @@ class Topology # if you specify dry run it will run a dry run on all the shards # otherwise it will run on the first shard and ask if you want to # continue on the rest of the shards, 10 shards at a time - def alter_table_shards(database, table, alter, dry_run=true, no_check_plan=false) - my_shards = shards.dup + def alter_table_shards(database, table, alter, dry_run=true, no_check_plan=false, shard_pool) + my_shards = shards(shard_pool).dup first_shard = my_shards.shift - print "Will run on first shard and prompt for going past the dry run only on the first shard\n\n" - print "[#{Time.now.to_s.blue}] #{first_shard.pool.to_s}\n" + output "Will run on first shard and prompt for going past the dry run only on the first shard\n\n" + output "[#{Time.now.to_s.blue}] #{first_shard.pool.to_s}\n" unless first_shard.alter_table(database, table, alter, dry_run, false) - print "First shard had an error, please check output\n" + output "First shard had an error, please check output\n" return end @@ -20,12 +20,12 @@ def alter_table_shards(database, table, alter, dry_run=true, no_check_plan=false errors = [] my_shards.limited_concurrent_map(10) do |shard| - print "[#{Time.now.to_s.blue}] #{shard.pool.to_s}\n" + output "[#{Time.now.to_s.blue}] #{shard.pool.to_s}\n" errors << shard unless shard.alter_table(database, table, alter, dry_run, true, no_check_plan) end errors.each do |shard| - print "check #{shard.name} for errors during online schema change\n" + output "check #{shard.name} for errors during online schema change\n" end end end @@ -34,11 +34,11 @@ def alter_table_shards(database, table, alter, dry_run=true, no_check_plan=false # this is because we do not drop the old table in the osc # also I will do the first shard and ask if you want to # continue, after that it will do each table serially - def drop_old_alter_table_shards(database, table) + def drop_old_alter_table_shards(database, table, shard_pool) my_shards = shards.dup first_shard = my_shards.shift - print "Will run on first shard and prompt before going on to the rest\n\n" - print "[#{Time.now.to_s.blue}] #{first_shard.pool.to_s}\n" + output "Will run on first shard and prompt before going on to the rest\n\n" + output "[#{Time.now.to_s.blue}] #{first_shard.pool.to_s}\n" first_shard.drop_old_alter_table(database, table) continue = ask('First shard complete would you like to continue with the rest of the shards?: (YES/no) - YES has to be in all caps and fully typed') diff --git a/plugins/upgrade_helper/commandsuite.rb b/plugins/upgrade_helper/commandsuite.rb index f8e46ec..92acbe2 100644 --- a/plugins/upgrade_helper/commandsuite.rb +++ b/plugins/upgrade_helper/commandsuite.rb @@ -122,10 +122,14 @@ def self.after_upgrade_promotion method_option :reads, :desc => 'Move reads to the new master', :type => :boolean method_option :writes, :desc => 'Move writes to new master', :type => :boolean method_option :cleanup, :desc => 'Tear down the old-version nodes', :type => :boolean + method_option :shard_pool, :desc => 'The sharding pool for which to perform the upgrade' def shard_upgrade + shard_pool = options[:shard_pool] || ask('Please enter the sharding pool which to perform the action on (enter for default pool): ') + shard_pool = default_shard_pool if shard_pool.empty? + if options[:reads] raise 'The --reads, --writes, and --cleanup options are mutually exclusive' if options[:writes] || options[:cleanup] - s = ask_shard_being_upgraded :reads + s = ask_shard_being_upgraded(:reads, shard_pool) s.branched_upgrade_move_reads Jetpants.topology.write_config self.class.reminders( @@ -136,7 +140,7 @@ def shard_upgrade ) elsif options[:writes] raise 'The --reads, --writes, and --cleanup options are mutually exclusive' if options[:reads] || options[:cleanup] - s = ask_shard_being_upgraded :writes + s = ask_shard_being_upgraded(:writes, shard_pool) s.branched_upgrade_move_writes Jetpants.topology.write_config self.class.reminders( @@ -148,7 +152,7 @@ def shard_upgrade elsif options[:cleanup] raise 'The --reads, --writes, and --cleanup options are mutually exclusive' if options[:reads] || options[:writes] - s = ask_shard_being_upgraded :cleanup + s = ask_shard_being_upgraded(:cleanup, shard_pool) s.cleanup! else @@ -156,7 +160,7 @@ def shard_upgrade 'This process may take an hour or two. You probably want to run this from a screen session.', 'Be especially careful if you are relying on SSH Agent Forwarding for your root key, since this is not screen-friendly.' ) - s = ask_shard_being_upgraded :prep + s = ask_shard_being_upgraded(:prep, shard_pool) s.branched_upgrade_prep self.class.reminders( 'Proceed to next step: jetpants shard_upgrade --reads' @@ -208,8 +212,8 @@ def check_pool_queries end no_tasks do - def ask_shard_being_upgraded(stage=:prep) - shards_being_upgraded = Jetpants.shards.select {|s| [:child, :needs_cleanup].include?(s.state) && !s.parent && s.master.master} + def ask_shard_being_upgraded(stage = :prep, shard_pool = nil) + shards_being_upgraded = Jetpants.shards(shard_pool).select {|s| [:child, :needs_cleanup].include?(s.state) && !s.parent && s.master.master} if stage == :writes || stage == :cleanup if shards_being_upgraded.size == 0 raise 'No shards are currently being upgraded. You can only use this task after running "jetpants shard_upgrade".' From ded761a2cb66dc563b3009d5224b07710fb3a0a7 Mon Sep 17 00:00:00 2001 From: Tom Christ Date: Mon, 27 Apr 2015 15:41:16 -0400 Subject: [PATCH 08/86] Add pool after syncing to collins --- lib/jetpants/pool.rb | 2 +- plugins/jetpants_collins/commandsuite.rb | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/jetpants/pool.rb b/lib/jetpants/pool.rb index d210534..139990b 100644 --- a/lib/jetpants/pool.rb +++ b/lib/jetpants/pool.rb @@ -354,7 +354,7 @@ def sync_configuration end # Callback to ensure that a sync'ed pool is already in Topology.pools - def before_sync_configuration + def after_sync_configuration unless Jetpants.topology.pools.include? self Jetpants.topology.add_pool self end diff --git a/plugins/jetpants_collins/commandsuite.rb b/plugins/jetpants_collins/commandsuite.rb index 4497120..8eaf68e 100644 --- a/plugins/jetpants_collins/commandsuite.rb +++ b/plugins/jetpants_collins/commandsuite.rb @@ -29,11 +29,11 @@ def self.jetpants_collins_before_dispatch(task) method_option :name, :name => 'unique name of new pool to be created' method_option :master, :master => 'ip of pre-configured master for new pool' def create_pool - name = options[:name] || ask('Please enter the name of the new pool.') + name = options[:name] || ask('Please enter the name of the new pool: ') if Jetpants.topology.configuration_assets('MYSQL_POOL').map(&:pool).include? name.upcase error "Pool #{name} already exists" end - master = options[:master] || ask("Please enter the ip of the master, or 'none' if one does not yet exist.") + master = options[:master] || ask("Please enter the ip of the master, or 'none' if one does not yet exist: ") if (master.downcase != 'none') && ! (is_ip? master) error "Master must either be 'none' or a valid ip." end From c8e8676215a7c2279aab30f0ccdd26c5e6318a7e Mon Sep 17 00:00:00 2001 From: Tom Christ Date: Mon, 27 Apr 2015 17:00:34 -0400 Subject: [PATCH 09/86] Move splat operator to correct position --- lib/jetpants/table.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/jetpants/table.rb b/lib/jetpants/table.rb index f2bdaec..6b35aa6 100644 --- a/lib/jetpants/table.rb +++ b/lib/jetpants/table.rb @@ -121,7 +121,7 @@ def drop_index_query(index_name) # {:index_name=> # {:columns=>[:column_one, :column_two], :unique=>false}}, # - def create_index_query(index_specs*) + def create_index_query(*index_specs) index_defs = [] index_specs.each do |index_spec| From f2f06bddf4f2709f65a05daf8f75d17b7b7b6b73 Mon Sep 17 00:00:00 2001 From: Tom Christ Date: Tue, 28 Apr 2015 17:33:28 -0400 Subject: [PATCH 10/86] Fix index name/metadata references --- lib/jetpants/db/import_export.rb | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/lib/jetpants/db/import_export.rb b/lib/jetpants/db/import_export.rb index 0671a83..707d22f 100644 --- a/lib/jetpants/db/import_export.rb +++ b/lib/jetpants/db/import_export.rb @@ -318,12 +318,11 @@ def rebuild!(tables=false, min_id=false, max_id=false) if Jetpants.import_without_indices tables.each do |t| - index_list[t] ||= [] + index_list[t] = t.indexes - t.indexes.each do |i| - drop_idx_cmd = t.drop_index_query(i) - index_list[t] << i - output "Dropping index #{i.keys.first} prior to import" + t.indexes.each do |index_name, index_info| + drop_idx_cmd = t.drop_index_query(index_name) + output "Dropping index #{index_name} prior to import" root_cmd(drop_idx_cmd) end end From 568f31c8902f31e5726b8d418a0d3d772a3dcfee Mon Sep 17 00:00:00 2001 From: Tom Christ Date: Wed, 29 Apr 2015 12:04:40 -0400 Subject: [PATCH 11/86] Fix root cmd invocation --- lib/jetpants/db/import_export.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/jetpants/db/import_export.rb b/lib/jetpants/db/import_export.rb index 707d22f..d3958b2 100644 --- a/lib/jetpants/db/import_export.rb +++ b/lib/jetpants/db/import_export.rb @@ -323,7 +323,7 @@ def rebuild!(tables=false, min_id=false, max_id=false) t.indexes.each do |index_name, index_info| drop_idx_cmd = t.drop_index_query(index_name) output "Dropping index #{index_name} prior to import" - root_cmd(drop_idx_cmd) + mysql_root_cmd(drop_idx_cmd) end end end @@ -335,7 +335,7 @@ def rebuild!(tables=false, min_id=false, max_id=false) indexes.each do |i| create_idx_cmd = table.create_index_query(i) output "Recreating index #{i.keys.first} after import" - root_cmd(create_idx_cmd) + mysql_root_cmd(create_idx_cmd) end end end From 54ed9076c4f95605ca27fb98d276f3434f50f8eb Mon Sep 17 00:00:00 2001 From: Tom Christ Date: Wed, 29 Apr 2015 19:01:14 -0400 Subject: [PATCH 12/86] Load table metadata from config when probing for shard --- lib/jetpants/db/import_export.rb | 5 +++-- lib/jetpants/shard.rb | 5 +++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/lib/jetpants/db/import_export.rb b/lib/jetpants/db/import_export.rb index d3958b2..776ee04 100644 --- a/lib/jetpants/db/import_export.rb +++ b/lib/jetpants/db/import_export.rb @@ -315,6 +315,7 @@ def rebuild!(tables=false, min_id=false, max_id=false) end index_list = {} + db_prefix = "USE #{app_schema};" if Jetpants.import_without_indices tables.each do |t| @@ -323,7 +324,7 @@ def rebuild!(tables=false, min_id=false, max_id=false) t.indexes.each do |index_name, index_info| drop_idx_cmd = t.drop_index_query(index_name) output "Dropping index #{index_name} prior to import" - mysql_root_cmd(drop_idx_cmd) + mysql_root_cmd("#{db_prefix}#{drop_idx_cmd}") end end end @@ -335,7 +336,7 @@ def rebuild!(tables=false, min_id=false, max_id=false) indexes.each do |i| create_idx_cmd = table.create_index_query(i) output "Recreating index #{i.keys.first} after import" - mysql_root_cmd(create_idx_cmd) + mysql_root_cmd("#{db_prefix}#{create_idx_cmd}") end end end diff --git a/lib/jetpants/shard.rb b/lib/jetpants/shard.rb index 4a889b6..369f02f 100644 --- a/lib/jetpants/shard.rb +++ b/lib/jetpants/shard.rb @@ -109,6 +109,11 @@ def db(mode=:read) def probe_tables if Jetpants.topology.shards.first == self super + config = Jetpants.send('sharded_tables') + @tables.map! do |table| + table.parse_params(config[table.name]) if config[table.name] + table + end else Jetpants.topology.shards.first.probe_tables end From 5dfcb2779e0687baa7eba41aaf4b3a9894484264 Mon Sep 17 00:00:00 2001 From: Tom Christ Date: Wed, 29 Apr 2015 21:48:17 -0400 Subject: [PATCH 13/86] Add default config value --- lib/jetpants.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/jetpants.rb b/lib/jetpants.rb index 2c835d4..fc66d11 100644 --- a/lib/jetpants.rb +++ b/lib/jetpants.rb @@ -48,6 +48,7 @@ module Jetpants 'log_file' => '/var/log/jetpants.log', # where to log all output from the jetpants commands 'local_private_interface' => nil, # local network interface corresponding to private IP of the machine jetpants is running on 'free_mem_min_mb' => 0, # Minimum amount of free memory in MB to be maintained on the node while performing the task (eg. network copy) + 'import_without_indices' => false, } config_paths = ["/etc/jetpants.yaml", "~/.jetpants.yml", "~/.jetpants.yaml"] From c3832076a7f816a36a9214e06a7903147fd26290 Mon Sep 17 00:00:00 2001 From: Tom Christ Date: Thu, 30 Apr 2015 14:13:25 -0400 Subject: [PATCH 14/86] Move jetpants.yaml parsing up in the table probe hierarchy --- lib/jetpants/db/schema.rb | 6 ++++++ lib/jetpants/shard.rb | 5 ----- lib/jetpants/table.rb | 4 +--- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/lib/jetpants/db/schema.rb b/lib/jetpants/db/schema.rb index 7db532c..54c780d 100644 --- a/lib/jetpants/db/schema.rb +++ b/lib/jetpants/db/schema.rb @@ -24,6 +24,12 @@ def detect_table_schema(table_name) 'columns' => connection.schema(table_name).map{|schema| schema[0]} } + config_params = Jetpants.send('sharded_tables') + + unless(config_params[table_name].nil?) + params.merge!(config_params[table_name]) + end + Table.new(table_name, params) end diff --git a/lib/jetpants/shard.rb b/lib/jetpants/shard.rb index 369f02f..4a889b6 100644 --- a/lib/jetpants/shard.rb +++ b/lib/jetpants/shard.rb @@ -109,11 +109,6 @@ def db(mode=:read) def probe_tables if Jetpants.topology.shards.first == self super - config = Jetpants.send('sharded_tables') - @tables.map! do |table| - table.parse_params(config[table.name]) if config[table.name] - table - end else Jetpants.topology.shards.first.probe_tables end diff --git a/lib/jetpants/table.rb b/lib/jetpants/table.rb index 6b35aa6..3adee21 100644 --- a/lib/jetpants/table.rb +++ b/lib/jetpants/table.rb @@ -124,11 +124,9 @@ def drop_index_query(index_name) def create_index_query(*index_specs) index_defs = [] - index_specs.each do |index_spec| - index_name = index_spec.keys.first + index_specs.each do |index_name, index_opts| throw "Cannot determine index name!" if index_name.nil? - index_opts = index_spec[index_name] throw "Cannot determine index metadata for new index #{index_name}!" if index_opts[:columns].nil? index_spec[:columns].each do |col| From da83bf918d783496f16847ec90aa22e691f5a4ce Mon Sep 17 00:00:00 2001 From: Tom Christ Date: Thu, 30 Apr 2015 14:20:18 -0400 Subject: [PATCH 15/86] Bound port for running service grep --- lib/jetpants/db/server.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/jetpants/db/server.rb b/lib/jetpants/db/server.rb index 375e0f8..284837b 100644 --- a/lib/jetpants/db/server.rb +++ b/lib/jetpants/db/server.rb @@ -11,7 +11,7 @@ def stop_mysql output "Attempting to shutdown MySQL" disconnect if @db output service(:stop, 'mysql') - running = ssh_cmd "netstat -ln | grep ':#{@port}' | wc -l" + running = ssh_cmd "netstat -ln | grep \"\s:#{@port}\s\" | wc -l" raise "[#{@ip}] Failed to shut down MySQL: Something is still listening on port #{@port}" unless running.chomp == '0' @options = [] @running = false From 067a54613db7c0f4628e349b895ebc09f7560bf0 Mon Sep 17 00:00:00 2001 From: Tom Christ Date: Thu, 30 Apr 2015 14:47:06 -0400 Subject: [PATCH 16/86] Remove prepended whitespace --- lib/jetpants/db/server.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/jetpants/db/server.rb b/lib/jetpants/db/server.rb index 284837b..4db3cf7 100644 --- a/lib/jetpants/db/server.rb +++ b/lib/jetpants/db/server.rb @@ -11,7 +11,7 @@ def stop_mysql output "Attempting to shutdown MySQL" disconnect if @db output service(:stop, 'mysql') - running = ssh_cmd "netstat -ln | grep \"\s:#{@port}\s\" | wc -l" + running = ssh_cmd "netstat -ln | grep \":#{@port}\s\" | wc -l" raise "[#{@ip}] Failed to shut down MySQL: Something is still listening on port #{@port}" unless running.chomp == '0' @options = [] @running = false From a796cf009310c99ca66f2ca8325cd4169f9e46a3 Mon Sep 17 00:00:00 2001 From: Tom Christ Date: Thu, 30 Apr 2015 15:02:34 -0400 Subject: [PATCH 17/86] Properly escape grep regex --- lib/jetpants/db/server.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/jetpants/db/server.rb b/lib/jetpants/db/server.rb index 4db3cf7..593b9d0 100644 --- a/lib/jetpants/db/server.rb +++ b/lib/jetpants/db/server.rb @@ -11,7 +11,7 @@ def stop_mysql output "Attempting to shutdown MySQL" disconnect if @db output service(:stop, 'mysql') - running = ssh_cmd "netstat -ln | grep \":#{@port}\s\" | wc -l" + running = ssh_cmd "netstat -ln | grep \":#{@port}\\s\" | wc -l" raise "[#{@ip}] Failed to shut down MySQL: Something is still listening on port #{@port}" unless running.chomp == '0' @options = [] @running = false From d7eb59f2f8a43cf061736b25b0c0557b1f941b81 Mon Sep 17 00:00:00 2001 From: Tom Christ Date: Thu, 30 Apr 2015 20:21:30 -0400 Subject: [PATCH 18/86] Fix var ref --- lib/jetpants/table.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/jetpants/table.rb b/lib/jetpants/table.rb index 3adee21..2772a1f 100644 --- a/lib/jetpants/table.rb +++ b/lib/jetpants/table.rb @@ -129,7 +129,7 @@ def create_index_query(*index_specs) throw "Cannot determine index metadata for new index #{index_name}!" if index_opts[:columns].nil? - index_spec[:columns].each do |col| + index_opts[:columns].each do |col| throw "Table #{name} does not have column #{col}" unless columns.include?(col) end From c9636dea358c26f17c21bcc6ccaf5949302e271e Mon Sep 17 00:00:00 2001 From: Tom Christ Date: Fri, 1 May 2015 14:01:26 -0400 Subject: [PATCH 19/86] Add table name to index transformation --- lib/jetpants/db/import_export.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/jetpants/db/import_export.rb b/lib/jetpants/db/import_export.rb index 776ee04..dca4f02 100644 --- a/lib/jetpants/db/import_export.rb +++ b/lib/jetpants/db/import_export.rb @@ -323,7 +323,7 @@ def rebuild!(tables=false, min_id=false, max_id=false) t.indexes.each do |index_name, index_info| drop_idx_cmd = t.drop_index_query(index_name) - output "Dropping index #{index_name} prior to import" + output "Dropping index #{index_name} from #{t.name} prior to import" mysql_root_cmd("#{db_prefix}#{drop_idx_cmd}") end end @@ -335,7 +335,7 @@ def rebuild!(tables=false, min_id=false, max_id=false) index_list.each do |table, indexes| indexes.each do |i| create_idx_cmd = table.create_index_query(i) - output "Recreating index #{i.keys.first} after import" + output "Recreating index #{i.keys.first} for #{table.name} after import" mysql_root_cmd("#{db_prefix}#{create_idx_cmd}") end end From a80a20d813e58047a1ae8b833737cfbb0ee7e8c8 Mon Sep 17 00:00:00 2001 From: Tom Christ Date: Fri, 1 May 2015 14:57:37 -0400 Subject: [PATCH 20/86] First cut at adding collins hooks --- lib/jetpants/shard.rb | 7 ++++++- lib/jetpants/shardingpool.rb | 15 +++++++++++++++ lib/jetpants/topology.rb | 13 +++++++++++++ plugins/jetpants_collins/topology.rb | 8 ++++++++ 4 files changed, 42 insertions(+), 1 deletion(-) create mode 100644 lib/jetpants/shardingpool.rb diff --git a/lib/jetpants/shard.rb b/lib/jetpants/shard.rb index ce4b520..eb986d3 100644 --- a/lib/jetpants/shard.rb +++ b/lib/jetpants/shard.rb @@ -37,19 +37,24 @@ class Shard < Pool # :deprecated -- Parent shard that has been split but children are still in :child or :needs_cleanup state. Shard may still be in production for writes / replication not torn down yet. # :recycle -- Parent shard that has been split and children are now in the :ready state. Shard no longer in production, replication to children has been torn down. attr_accessor :state + + # the sharding pool to which this shard belongs + attr_accessor :shard_pool # Constructor for Shard -- # * min_id: int # * max_id: int or the string "INFINITY" # * master: string (IP address) or a Jetpants::DB object # * state: one of the above state symbols - def initialize(min_id, max_id, master, state=:ready) + def initialize(min_id, max_id, master, state=:ready, shard_pool=nil) @min_id = min_id.to_i @max_id = (max_id.to_s.upcase == 'INFINITY' ? 'INFINITY' : max_id.to_i) @state = state @children = [] # array of shards being initialized by splitting this one @parent = nil + shard_pool = Jetpants.topology.default_sharding_pool if shard_pool.nil? + @shard_pool = shard_pool super(generate_name, master) end diff --git a/lib/jetpants/shardingpool.rb b/lib/jetpants/shardingpool.rb new file mode 100644 index 0000000..ec211ef --- /dev/null +++ b/lib/jetpants/shardingpool.rb @@ -0,0 +1,15 @@ +module Jetpants + # A ShardingPool is a sharding keyspace in Jetpants that contains + # man Shards. All shards within the pool partition a logically coherent + # keyspace + + class ShardingPool + def initialize(name) + @name = name + end + + def shards + Jetpants.topology.shards(@name) + end + end +end diff --git a/lib/jetpants/topology.rb b/lib/jetpants/topology.rb index 0d79ae5..108eb6c 100644 --- a/lib/jetpants/topology.rb +++ b/lib/jetpants/topology.rb @@ -12,6 +12,9 @@ def initialize # initialize @pools to an empty state @pools = nil + # initialize sharding pools to empty + @sharding_pools = nil + # We intentionally don't call load_pools here. The caller must do that. # This allows Jetpants module to create Jetpants.topology object, and THEN # invoke load_pools, which might then refer back to Jetpants.topology. @@ -22,6 +25,11 @@ def pools @pools end + def sharding_pools + load_sharding_pools if @sharding_pools.nil? + @sharding_pools + end + ###### Class methods ####################################################### # Metaprogramming hackery to create a "synchronized" method decorator @@ -69,6 +77,11 @@ def load_pools output "Notice: no plugin has overridden Topology#load_pools, so *no* pools are imported automatically" end + synchronized + def load_sharding_pools + output "Notice: no plugin has overriddent Topology#load_sharding_pools, so *no* sharding pools are imported automaticaly" + end + synchronized # Plugin should override so that this adds the given pool to the current topology (@pools) def add_pool(pool) diff --git a/plugins/jetpants_collins/topology.rb b/plugins/jetpants_collins/topology.rb index 069dbf6..ab7120a 100644 --- a/plugins/jetpants_collins/topology.rb +++ b/plugins/jetpants_collins/topology.rb @@ -38,6 +38,14 @@ def process_spare_selector_options(selector, options) ##### METHOD OVERRIDES ##################################################### + def load_sharding_pools + @sharding_pools = configuration_assets('MYSQL_SHARDING_POOL').map(&:to_sharding_pool) + @sharding_pools.compact! # remove nils from pools that had no master + @sharding_pools.sort_by! { |p| p.name } + + true + end + # Initializes list of pools + shards from Collins def load_pools # We keep a cache of Collins::Asset objects, organized as pool_name => role => [asset, asset, ...] From ccec7887399bb63eed42f4deb7a90ec908b75a40 Mon Sep 17 00:00:00 2001 From: Tom Christ Date: Mon, 4 May 2015 14:16:16 -0400 Subject: [PATCH 21/86] Fix up batch index creation --- lib/jetpants/db/import_export.rb | 9 ++++----- lib/jetpants/table.rb | 2 +- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/lib/jetpants/db/import_export.rb b/lib/jetpants/db/import_export.rb index dca4f02..0f45c79 100644 --- a/lib/jetpants/db/import_export.rb +++ b/lib/jetpants/db/import_export.rb @@ -333,11 +333,10 @@ def rebuild!(tables=false, min_id=false, max_id=false) if Jetpants.import_without_indices index_list.each do |table, indexes| - indexes.each do |i| - create_idx_cmd = table.create_index_query(i) - output "Recreating index #{i.keys.first} for #{table.name} after import" - mysql_root_cmd("#{db_prefix}#{create_idx_cmd}") - end + create_idx_cmd = table.create_index_query(indexes) + index_names = indexes.keys.join(", ") + output "Recreating indexes #{index_names} for #{table.name} after import" + mysql_root_cmd("#{db_prefix}#{create_idx_cmd}") end end diff --git a/lib/jetpants/table.rb b/lib/jetpants/table.rb index 2772a1f..b16bb9b 100644 --- a/lib/jetpants/table.rb +++ b/lib/jetpants/table.rb @@ -121,7 +121,7 @@ def drop_index_query(index_name) # {:index_name=> # {:columns=>[:column_one, :column_two], :unique=>false}}, # - def create_index_query(*index_specs) + def create_index_query(index_specs) index_defs = [] index_specs.each do |index_name, index_opts| From d15f5e428a3cf495cee683c8ded4d19cfc08c3cb Mon Sep 17 00:00:00 2001 From: Tom Christ Date: Tue, 5 May 2015 11:20:26 -0400 Subject: [PATCH 22/86] Skip tables without indexes --- lib/jetpants/db/import_export.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/jetpants/db/import_export.rb b/lib/jetpants/db/import_export.rb index 0f45c79..89a35ee 100644 --- a/lib/jetpants/db/import_export.rb +++ b/lib/jetpants/db/import_export.rb @@ -333,6 +333,8 @@ def rebuild!(tables=false, min_id=false, max_id=false) if Jetpants.import_without_indices index_list.each do |table, indexes| + next if indexes.keys.empty? + create_idx_cmd = table.create_index_query(indexes) index_names = indexes.keys.join(", ") output "Recreating indexes #{index_names} for #{table.name} after import" From 6dc52758fe9cc06c515d3a6570422a24c4e8e500 Mon Sep 17 00:00:00 2001 From: Tom Christ Date: Wed, 6 May 2015 17:54:39 -0400 Subject: [PATCH 23/86] Use raise instead of throw --- lib/jetpants/table.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/jetpants/table.rb b/lib/jetpants/table.rb index b16bb9b..0081442 100644 --- a/lib/jetpants/table.rb +++ b/lib/jetpants/table.rb @@ -125,12 +125,12 @@ def create_index_query(index_specs) index_defs = [] index_specs.each do |index_name, index_opts| - throw "Cannot determine index name!" if index_name.nil? + raise "Cannot determine index name!" if index_name.nil? - throw "Cannot determine index metadata for new index #{index_name}!" if index_opts[:columns].nil? + raise "Cannot determine index metadata for new index #{index_name}!" if index_opts[:columns].nil? index_opts[:columns].each do |col| - throw "Table #{name} does not have column #{col}" unless columns.include?(col) + raise "Table #{name} does not have column #{col}" unless columns.include?(col) end unique = "" From e63b1b7ece61869abc43dc981d16f96f7fc8b834 Mon Sep 17 00:00:00 2001 From: Tom Christ Date: Wed, 6 May 2015 20:41:21 -0400 Subject: [PATCH 24/86] Clarify and tighten contraints --- lib/jetpants/table.rb | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/jetpants/table.rb b/lib/jetpants/table.rb index 0081442..508b016 100644 --- a/lib/jetpants/table.rb +++ b/lib/jetpants/table.rb @@ -110,8 +110,7 @@ def max_pk_val_query # generates a query to drop a specified index named by # the symbol passed in to the method def drop_index_query(index_name) - index_info = indexes.select{|idx_name,idx_info| idx_name == index_name}.first - raise "Unable to find index #{index_name}!" if index_info.nil? + raise "Unable to find index #{index_name}!" unless indexes.has_key? index_name "ALTER TABLE #{name} DROP INDEX #{index_name}" end @@ -127,7 +126,7 @@ def create_index_query(index_specs) index_specs.each do |index_name, index_opts| raise "Cannot determine index name!" if index_name.nil? - raise "Cannot determine index metadata for new index #{index_name}!" if index_opts[:columns].nil? + raise "Cannot determine index metadata for new index #{index_name}!" unless index_opts[:columns].kind_of?(Array) index_opts[:columns].each do |col| raise "Table #{name} does not have column #{col}" unless columns.include?(col) From f3072b4e6f777f4253c3cbf33a012a4aedf42d53 Mon Sep 17 00:00:00 2001 From: Tom Christ Date: Thu, 7 May 2015 14:12:06 -0400 Subject: [PATCH 25/86] Add config for default shard pool and accessor --- lib/jetpants.rb | 1 + lib/jetpants/shard.rb | 2 +- lib/jetpants/topology.rb | 5 +++++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/jetpants.rb b/lib/jetpants.rb index 2c835d4..8a95700 100644 --- a/lib/jetpants.rb +++ b/lib/jetpants.rb @@ -48,6 +48,7 @@ module Jetpants 'log_file' => '/var/log/jetpants.log', # where to log all output from the jetpants commands 'local_private_interface' => nil, # local network interface corresponding to private IP of the machine jetpants is running on 'free_mem_min_mb' => 0, # Minimum amount of free memory in MB to be maintained on the node while performing the task (eg. network copy) + 'default_shard_pool' => nil, # default pool for sharding operations } config_paths = ["/etc/jetpants.yaml", "~/.jetpants.yml", "~/.jetpants.yaml"] diff --git a/lib/jetpants/shard.rb b/lib/jetpants/shard.rb index eb986d3..8f299b2 100644 --- a/lib/jetpants/shard.rb +++ b/lib/jetpants/shard.rb @@ -53,7 +53,7 @@ def initialize(min_id, max_id, master, state=:ready, shard_pool=nil) @children = [] # array of shards being initialized by splitting this one @parent = nil - shard_pool = Jetpants.topology.default_sharding_pool if shard_pool.nil? + shard_pool = Jetpants.topology.default_shard_pool if shard_pool.nil? @shard_pool = shard_pool super(generate_name, master) diff --git a/lib/jetpants/topology.rb b/lib/jetpants/topology.rb index 108eb6c..afe650c 100644 --- a/lib/jetpants/topology.rb +++ b/lib/jetpants/topology.rb @@ -30,6 +30,11 @@ def sharding_pools @sharding_pools end + def default_shard_pool + raise "Default shard pool not defined!" if @config['default_shard_pool'].nil? + @config['default_shard_pool'] + end + ###### Class methods ####################################################### # Metaprogramming hackery to create a "synchronized" method decorator From c1ab031b30626e791de09d72d641b00833850bcd Mon Sep 17 00:00:00 2001 From: Tom Christ Date: Thu, 7 May 2015 14:41:08 -0400 Subject: [PATCH 26/86] Add asset to shard pool conversion --- lib/jetpants/{shardingpool.rb => shardpool.rb} | 2 +- plugins/jetpants_collins/asset.rb | 12 ++++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) rename lib/jetpants/{shardingpool.rb => shardpool.rb} (93%) diff --git a/lib/jetpants/shardingpool.rb b/lib/jetpants/shardpool.rb similarity index 93% rename from lib/jetpants/shardingpool.rb rename to lib/jetpants/shardpool.rb index ec211ef..caefe75 100644 --- a/lib/jetpants/shardingpool.rb +++ b/lib/jetpants/shardpool.rb @@ -3,7 +3,7 @@ module Jetpants # man Shards. All shards within the pool partition a logically coherent # keyspace - class ShardingPool + class ShardPool def initialize(name) @name = name end diff --git a/plugins/jetpants_collins/asset.rb b/plugins/jetpants_collins/asset.rb index af586d4..659b54b 100644 --- a/plugins/jetpants_collins/asset.rb +++ b/plugins/jetpants_collins/asset.rb @@ -15,8 +15,16 @@ def to_host raise "Can only call to_host on SERVER_NODE assets, but #{self} has type #{type}" unless type.upcase == 'SERVER_NODE' backend_ip_address.to_host end - - + + # Convert a Collins:Asset to a Jetpants::Shard_pool + def to_shard_pool + raise "Can only call to_shard_pool on CONFIGURATION assets, but #{self} has type #{type}" unless type.upcase == 'CONFIGURATION' + raise "Unknown primary role #{primary_role} for configuration asset #{self}" unless ['MYSQL_SHARD_POOL'].include?(primary_role.upcase) + raise "No shard_pool attribute set on asset #{self}" unless shard_pool && shard_pool.length > 0 + + Jetpants::ShardPool.new(shard_pool) + end + # Convert a Collins::Asset to either a Jetpants::Pool or a Jetpants::Shard, depending # on the value of PRIMARY_ROLE. Requires asset TYPE to be CONFIGURATION. def to_pool From 57cfd5e36a74223f994000a51fca2f485879cd63 Mon Sep 17 00:00:00 2001 From: Tom Christ Date: Thu, 7 May 2015 15:46:01 -0400 Subject: [PATCH 27/86] Consistent naming --- lib/jetpants/topology.rb | 14 +++++++------- plugins/jetpants_collins/topology.rb | 8 ++++---- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/lib/jetpants/topology.rb b/lib/jetpants/topology.rb index afe650c..5764376 100644 --- a/lib/jetpants/topology.rb +++ b/lib/jetpants/topology.rb @@ -12,8 +12,8 @@ def initialize # initialize @pools to an empty state @pools = nil - # initialize sharding pools to empty - @sharding_pools = nil + # initialize shard pools to empty + @shard_pools = nil # We intentionally don't call load_pools here. The caller must do that. # This allows Jetpants module to create Jetpants.topology object, and THEN @@ -25,9 +25,9 @@ def pools @pools end - def sharding_pools - load_sharding_pools if @sharding_pools.nil? - @sharding_pools + def shard_pools + load_shard_pools if @shard_pools.nil? + @shard_pools end def default_shard_pool @@ -83,8 +83,8 @@ def load_pools end synchronized - def load_sharding_pools - output "Notice: no plugin has overriddent Topology#load_sharding_pools, so *no* sharding pools are imported automaticaly" + def load_shard_pools + output "Notice: no plugin has overriddent Topology#load_shard_pools, so *no* shard pools are imported automaticaly" end synchronized diff --git a/plugins/jetpants_collins/topology.rb b/plugins/jetpants_collins/topology.rb index ab7120a..53aa606 100644 --- a/plugins/jetpants_collins/topology.rb +++ b/plugins/jetpants_collins/topology.rb @@ -38,10 +38,10 @@ def process_spare_selector_options(selector, options) ##### METHOD OVERRIDES ##################################################### - def load_sharding_pools - @sharding_pools = configuration_assets('MYSQL_SHARDING_POOL').map(&:to_sharding_pool) - @sharding_pools.compact! # remove nils from pools that had no master - @sharding_pools.sort_by! { |p| p.name } + def load_shard_pools + @shard_pools = configuration_assets('MYSQL_SHARD_POOL').map(&:to_shard_pool) + @shard_pools.compact! # remove nils from pools that had no master + @shard_pools.sort_by! { |p| p.name } true end From 23d65136f23839a5a3d62c733c90f80e503506f3 Mon Sep 17 00:00:00 2001 From: Tom Christ Date: Thu, 7 May 2015 16:39:25 -0400 Subject: [PATCH 28/86] Add shardpool collins asset retrieval --- lib/jetpants/shardpool.rb | 2 + plugins/jetpants_collins/shardpool.rb | 56 +++++++++++++++++++++++++++ 2 files changed, 58 insertions(+) create mode 100644 plugins/jetpants_collins/shardpool.rb diff --git a/lib/jetpants/shardpool.rb b/lib/jetpants/shardpool.rb index caefe75..26be3ec 100644 --- a/lib/jetpants/shardpool.rb +++ b/lib/jetpants/shardpool.rb @@ -3,6 +3,8 @@ module Jetpants # man Shards. All shards within the pool partition a logically coherent # keyspace + attr_accessor :name + class ShardPool def initialize(name) @name = name diff --git a/plugins/jetpants_collins/shardpool.rb b/plugins/jetpants_collins/shardpool.rb new file mode 100644 index 0000000..c9e8782 --- /dev/null +++ b/plugins/jetpants_collins/shardpool.rb @@ -0,0 +1,56 @@ +module Jetpants + # A ShardingPool is a sharding keyspace in Jetpants that contains + # man Shards. All shards within the pool partition a logically coherent + # keyspace + + class ShardPool + + ##### JETCOLLINS MIX-IN #################################################### + + include Plugin::JetCollins + + collins_attr_accessor :shard_pool + + def collins_asset(create_if_missing=false) + selector = { + operation: 'and', + details: true, + type: 'CONFIGURATION', + primary_role: 'MYSQL_SHARD_POOL', + shard_pool: "^#{@name.upcase}$", + status: 'Allocated', + } + selector[:remoteLookup] = true if Jetpants.plugins['jetpants_collins']['remote_lookup'] + + results = Plugin::JetCollins.find selector, !create_if_missing + + # If we got back multiple results, try ignoring the remote datacenter ones + if results.count > 1 + filtered_results = results.select {|a| a.location.nil? || a.location.upcase == Plugin::JetCollins.datacenter} + results = filtered_results if filtered_results.count > 0 + end + + if results.count > 1 + raise "Multiple configuration assets found for pool #{@name}" + elsif results.count == 0 && create_if_missing + output "Could not find configuration asset for pool; creating now" + new_tag = 'mysql-shard-pool-' + @name + asset = Collins::Asset.new type: 'CONFIGURATION', tag: new_tag, status: 'Allocated' + begin + Plugin::JetCollins.create!(asset) + rescue + collins_set asset: asset, + status: 'Allocated' + end + collins_set asset: asset, + primary_role: 'MYSQL_SHARD_POOL', + shard_pool: @name.upcase, + Plugin::JetCollins.get new_tag + elsif results.count == 0 && !create_if_missing + raise "Could not find configuration asset for pool #{name}" + else + results.first + end + end + end +end From f93d71f5bde2a335ae1e32d5cf68a705dae7acfa Mon Sep 17 00:00:00 2001 From: Tom Christ Date: Mon, 11 May 2015 16:58:04 -0400 Subject: [PATCH 29/86] Clean up includes and syntax --- lib/jetpants.rb | 2 +- lib/jetpants/topology.rb | 6 +++--- plugins/jetpants_collins/jetpants_collins.rb | 2 +- plugins/jetpants_collins/shardpool.rb | 2 +- plugins/merge_helper/lib/commandsuite.rb | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/jetpants.rb b/lib/jetpants.rb index 0ce4653..3377de5 100644 --- a/lib/jetpants.rb +++ b/lib/jetpants.rb @@ -9,7 +9,7 @@ module Jetpants; end $LOAD_PATH.unshift File.join(File.dirname(__FILE__), 'jetpants'), File.join(File.dirname(__FILE__), '..', 'plugins') -%w(output callback table host db pool topology shard monkeypatch commandsuite).each {|g| require g} +%w(output callback table host db pool topology shard monkeypatch shardpool commandsuite).each {|g| require g} # Since Jetpants is extremely multi-threaded, we need to force uncaught exceptions to # kill all threads in order to have any kind of sane error handling. diff --git a/lib/jetpants/topology.rb b/lib/jetpants/topology.rb index 5764376..b513109 100644 --- a/lib/jetpants/topology.rb +++ b/lib/jetpants/topology.rb @@ -31,8 +31,8 @@ def shard_pools end def default_shard_pool - raise "Default shard pool not defined!" if @config['default_shard_pool'].nil? - @config['default_shard_pool'] + raise "Default shard pool not defined!" if Jetpants.default_shard_pool.nil? + Jetpants.default_shard_pool end ###### Class methods ####################################################### @@ -146,7 +146,7 @@ def slave_roles # Returns array of this topology's Jetpants::Pool objects of type Jetpants::Shard def shards(shard_pool = nil) - shard_pool = default_shard_pool if shard_pool.nil + shard_pool = default_shard_pool if shard_pool.nil? pools.select {|p| p.is_a? Shard}.select { |p| p.shard_pool = shard_pool } end diff --git a/plugins/jetpants_collins/jetpants_collins.rb b/plugins/jetpants_collins/jetpants_collins.rb index b8a3f5d..49a1402 100644 --- a/plugins/jetpants_collins/jetpants_collins.rb +++ b/plugins/jetpants_collins/jetpants_collins.rb @@ -303,4 +303,4 @@ def collins_status_state # load all the monkeypatches for other Jetpants classes -%w(monkeypatch asset host db pool shard topology commandsuite).each {|mod| require "jetpants_collins/#{mod}"} +%w(monkeypatch asset host db pool shard topology shardpool commandsuite).each {|mod| require "jetpants_collins/#{mod}"} diff --git a/plugins/jetpants_collins/shardpool.rb b/plugins/jetpants_collins/shardpool.rb index c9e8782..a82eb29 100644 --- a/plugins/jetpants_collins/shardpool.rb +++ b/plugins/jetpants_collins/shardpool.rb @@ -44,7 +44,7 @@ def collins_asset(create_if_missing=false) end collins_set asset: asset, primary_role: 'MYSQL_SHARD_POOL', - shard_pool: @name.upcase, + shard_pool: @name.upcase Plugin::JetCollins.get new_tag elsif results.count == 0 && !create_if_missing raise "Could not find configuration asset for pool #{name}" diff --git a/plugins/merge_helper/lib/commandsuite.rb b/plugins/merge_helper/lib/commandsuite.rb index f9e9854..89c589e 100644 --- a/plugins/merge_helper/lib/commandsuite.rb +++ b/plugins/merge_helper/lib/commandsuite.rb @@ -302,7 +302,7 @@ def ask_merge_shard_ranges shards_to_merge = Jetpants.shards(shard_pool).select do |shard| shard.min_id.to_i >= min_id.to_i && shard.max_id.to_i <= max_id.to_i && - shard.max_id != 'INFINITY' && + shard.max_id != 'INFINITY' end shard_str = shards_to_merge.join(', ') From 7f04d0e0786605f80975b9a306bbf4ba903acb95 Mon Sep 17 00:00:00 2001 From: Tom Christ Date: Mon, 11 May 2015 19:15:20 -0400 Subject: [PATCH 30/86] Pull in ShardPool mixins --- lib/jetpants/shardpool.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/jetpants/shardpool.rb b/lib/jetpants/shardpool.rb index 26be3ec..0176642 100644 --- a/lib/jetpants/shardpool.rb +++ b/lib/jetpants/shardpool.rb @@ -6,6 +6,9 @@ module Jetpants attr_accessor :name class ShardPool + include CallbackHandler + include Output + def initialize(name) @name = name end From d5e975ef0056c486062e8f5473a46fa1eff3643e Mon Sep 17 00:00:00 2001 From: Tom Christ Date: Mon, 11 May 2015 19:29:33 -0400 Subject: [PATCH 31/86] First cut at simple tracker integration --- plugins/simple_tracker/lib/shard.rb | 3 +- plugins/simple_tracker/lib/shardpool.rb | 36 ++++++++++++++++++++++++ plugins/simple_tracker/simple_tracker.rb | 5 +++- 3 files changed, 42 insertions(+), 2 deletions(-) create mode 100644 plugins/simple_tracker/lib/shardpool.rb diff --git a/plugins/simple_tracker/lib/shard.rb b/plugins/simple_tracker/lib/shard.rb index 7971762..d632f91 100644 --- a/plugins/simple_tracker/lib/shard.rb +++ b/plugins/simple_tracker/lib/shard.rb @@ -26,7 +26,7 @@ def self.from_hash(h) # we just return the shard for now... we have to wait until later to # set up children + parents, since it's easier to grab the corresponding # objects once all pools have been initialized. - Shard.new(h['min_id'], h['max_id'], h['master'], h['state'].to_sym) + Shard.new(h['min_id'], h['max_id'], h['master'], h['state'].to_sym, h['shard_pool']) end # Sets up parent/child relationships for the shard represented by the @@ -82,6 +82,7 @@ def to_hash(for_app_config=false) 'state' => state, 'master' => master, 'slaves' => slave_data, + 'shard_pool' => shard_pool, } end end diff --git a/plugins/simple_tracker/lib/shardpool.rb b/plugins/simple_tracker/lib/shardpool.rb new file mode 100644 index 0000000..1645396 --- /dev/null +++ b/plugins/simple_tracker/lib/shardpool.rb @@ -0,0 +1,36 @@ +module Jetpants + class Shard < Pool + + ##### CALLBACKS ############################################################ + + # After changing the state of a shard, sync config back to the asset tracker json + def after_state=(value) + sync_configuration + end + + ##### NEW CLASS-LEVEL METHODS ############################################## + + # Converts a hash (from asset tracker json file) into a Shard. + def self.from_hash(h) + # we just return the shard for now... we have to wait until later to + # set up children + parents, since it's easier to grab the corresponding + # objects once all pools have been initialized. + Shard.new(h['shard_pool']) + end + + def shards + Jetpants.topology.shards(@name) + end + + ##### NEW METHODS ########################################################## + + # Converts a Shard to a hash, for use in either the internal asset tracker + # json (for_app_config=false) or for use in the application config file yaml + # (for_app_config=true) + def to_hash + { + shard_pool: @name + } + end + end +end diff --git a/plugins/simple_tracker/simple_tracker.rb b/plugins/simple_tracker/simple_tracker.rb index 678b6ed..dbbd4fb 100644 --- a/plugins/simple_tracker/simple_tracker.rb +++ b/plugins/simple_tracker/simple_tracker.rb @@ -17,6 +17,9 @@ class SimpleTracker # Array of hashes, each containing info from Shard#to_hash attr_accessor :shards + + # Array of hashes, each containing info from ShardPool#to_hash + attr_accessor :shard_pools # Clean state DB nodes that are ready for use. Array of any of the following: # * hashes each containing key 'node'. could expand to include 'role' or other metadata as well, @@ -37,7 +40,7 @@ def initialize def save File.open(@tracker_data_file_path, 'w') do |f| - data = {'pools' => @global_pools, 'shards' => @shards, 'spares' => @spares} + data = {'pools' => @global_pools, 'shards' => @shards, 'spares' => @spares, 'shard_pools' => @shard_pools} f.puts JSON.pretty_generate(data) f.close end From 2d97fabb6e99ea6343df3dae29db16c749c5fb91 Mon Sep 17 00:00:00 2001 From: Tom Christ Date: Mon, 11 May 2015 20:20:01 -0400 Subject: [PATCH 32/86] Override load_shard_pools in simpletracker --- plugins/simple_tracker/lib/topology.rb | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/plugins/simple_tracker/lib/topology.rb b/plugins/simple_tracker/lib/topology.rb index fcdf2a7..5b02d64 100644 --- a/plugins/simple_tracker/lib/topology.rb +++ b/plugins/simple_tracker/lib/topology.rb @@ -18,6 +18,12 @@ def load_pools @tracker.shards.each {|h| Shard.assign_relationships(h, all_shards)} end + # Populate @shard_pools by reading asset tracker data + def load_shard_pools + @tracker = Jetpants::Plugin::SimpleTracker.new + @shard_pools = @tracker.shard_pools.map{|h| ShardPool.from_hash(h) }.compact + end + def add_pool(pool) @pools << pool unless pools.include? pool end From 1de0410d9b70963203d4064566ce0ece4b43f7e3 Mon Sep 17 00:00:00 2001 From: Tom Christ Date: Tue, 12 May 2015 10:49:11 -0400 Subject: [PATCH 33/86] Add add_shard_pool functionality to base and trackers --- lib/jetpants/topology.rb | 6 ++++++ plugins/jetpants_collins/topology.rb | 9 +++++++++ plugins/simple_tracker/lib/topology.rb | 4 ++++ plugins/simple_tracker/simple_tracker.rb | 3 ++- 4 files changed, 21 insertions(+), 1 deletion(-) diff --git a/lib/jetpants/topology.rb b/lib/jetpants/topology.rb index b513109..3dfbc64 100644 --- a/lib/jetpants/topology.rb +++ b/lib/jetpants/topology.rb @@ -93,6 +93,12 @@ def add_pool(pool) output "Notice: no plugin has overridden Topology#add_pool, so the pool was *not* added to the topology" end + synchronized + # Plugin should override so that this adds the given shard pool to the current topology (@shard_pools) + def add_shard_pool(shard_pool) + output "Notice: no plugin has overridden Topology#add_shard_pool, so the shard pool was *not* added to the topology" + end + synchronized # Plugin should override so that it writes a configuration file or commits a # configuration change to a config service. diff --git a/plugins/jetpants_collins/topology.rb b/plugins/jetpants_collins/topology.rb index 53aa606..7a8451f 100644 --- a/plugins/jetpants_collins/topology.rb +++ b/plugins/jetpants_collins/topology.rb @@ -79,6 +79,15 @@ def add_pool(pool) true end + def add_shard_pool(shard_pool) + raise 'Attempt to add a non shard pool to the sharding pools topology' unless shard_pool.is_a?(ShardPool) + + unless shard_pools.include? shard_pool + @shard_pools << shard_pool + @shard_pools.sort_by! { |sp| sp.name } + end + end + # Returns (count) DB objects. Pulls from machines in the spare state # and converts them to the Allocated status. # You can pass in :role to request spares with a particular secondary_role diff --git a/plugins/simple_tracker/lib/topology.rb b/plugins/simple_tracker/lib/topology.rb index 5b02d64..f48e5e8 100644 --- a/plugins/simple_tracker/lib/topology.rb +++ b/plugins/simple_tracker/lib/topology.rb @@ -28,6 +28,10 @@ def add_pool(pool) @pools << pool unless pools.include? pool end + def add_shard_pool(shard_pool) + @shard_pools << shard_pool unless shard_pools.include? shard_pool + end + # Generates a database configuration file for a hypothetical web application def write_config config_file_path = @tracker.app_config_file_path diff --git a/plugins/simple_tracker/simple_tracker.rb b/plugins/simple_tracker/simple_tracker.rb index dbbd4fb..3459fe3 100644 --- a/plugins/simple_tracker/simple_tracker.rb +++ b/plugins/simple_tracker/simple_tracker.rb @@ -32,10 +32,11 @@ class SimpleTracker def initialize @tracker_data_file_path = Jetpants.plugins['simple_tracker']['tracker_data_file_path'] || '/etc/jetpants_tracker.json' @app_config_file_path = Jetpants.plugins['simple_tracker']['app_config_file_path'] || '/var/lib/mysite/config/databases.yaml' - data = JSON.parse(File.read(@tracker_data_file_path)) rescue {'pools' => {}, 'shards' => [], 'spares' => []} + data = JSON.parse(File.read(@tracker_data_file_path)) rescue {'pools' => {}, 'shards' => [], 'spares' => [], 'shard_pools' => []} @global_pools = data['pools'] @shards = data['shards'] @spares = data['spares'] + @shard_pools = data['shard_pools'] end def save From 94e87812b6442b97ce0f441832484b9c9195da62 Mon Sep 17 00:00:00 2001 From: Tom Christ Date: Tue, 12 May 2015 14:47:30 -0400 Subject: [PATCH 34/86] Better management around shardpool inclusion --- lib/jetpants/topology.rb | 2 ++ plugins/simple_tracker/lib/shardpool.rb | 10 +++++----- plugins/simple_tracker/lib/topology.rb | 5 +++-- plugins/simple_tracker/simple_tracker.rb | 2 +- 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/lib/jetpants/topology.rb b/lib/jetpants/topology.rb index 3dfbc64..65335bc 100644 --- a/lib/jetpants/topology.rb +++ b/lib/jetpants/topology.rb @@ -252,6 +252,7 @@ def normalize_roles(*roles) # Clears the pool list and nukes cached DB and Host object lookup tables def clear @pools = [] + @shard_pools = [] DB.clear Host.clear end @@ -259,6 +260,7 @@ def clear # Empties and then reloads the pool list def refresh clear + load_shard_pools load_pools true end diff --git a/plugins/simple_tracker/lib/shardpool.rb b/plugins/simple_tracker/lib/shardpool.rb index 1645396..9fb1423 100644 --- a/plugins/simple_tracker/lib/shardpool.rb +++ b/plugins/simple_tracker/lib/shardpool.rb @@ -8,6 +8,10 @@ def after_state=(value) sync_configuration end + def shards + Jetpants.topology.shards(@name) + end + ##### NEW CLASS-LEVEL METHODS ############################################## # Converts a hash (from asset tracker json file) into a Shard. @@ -18,16 +22,12 @@ def self.from_hash(h) Shard.new(h['shard_pool']) end - def shards - Jetpants.topology.shards(@name) - end - ##### NEW METHODS ########################################################## # Converts a Shard to a hash, for use in either the internal asset tracker # json (for_app_config=false) or for use in the application config file yaml # (for_app_config=true) - def to_hash + def to_hash(for_app_config = true) { shard_pool: @name } diff --git a/plugins/simple_tracker/lib/topology.rb b/plugins/simple_tracker/lib/topology.rb index f48e5e8..2b34cbb 100644 --- a/plugins/simple_tracker/lib/topology.rb +++ b/plugins/simple_tracker/lib/topology.rb @@ -7,7 +7,7 @@ class Topology # Populates @pools by reading asset tracker data def load_pools - @tracker = Jetpants::Plugin::SimpleTracker.new + @tracker ||= Jetpants::Plugin::SimpleTracker.new # Create Pool and Shard objects @pools = @tracker.global_pools.map {|h| Pool.from_hash(h)}.compact @@ -20,7 +20,7 @@ def load_pools # Populate @shard_pools by reading asset tracker data def load_shard_pools - @tracker = Jetpants::Plugin::SimpleTracker.new + @tracker ||= Jetpants::Plugin::SimpleTracker.new @shard_pools = @tracker.shard_pools.map{|h| ShardPool.from_hash(h) }.compact end @@ -41,6 +41,7 @@ def write_config 'database' => { 'pools' => functional_partitions.map {|p| p.to_hash(true)}, 'shards' => shards.select {|s| s.in_config?}.map {|s| s.to_hash(true)}, + 'shard_pools' => shard_pools.map {|sp| sp.to_hash(true)}, } } diff --git a/plugins/simple_tracker/simple_tracker.rb b/plugins/simple_tracker/simple_tracker.rb index 3459fe3..e37da0a 100644 --- a/plugins/simple_tracker/simple_tracker.rb +++ b/plugins/simple_tracker/simple_tracker.rb @@ -75,4 +75,4 @@ def determine_slaves(ip, port=3306) end # load all the monkeypatches for other Jetpants classes -%w(pool shard topology db commandsuite).each { |mod| require "simple_tracker/lib/#{mod}" } +%w(pool shard topology db shardpool commandsuite).each { |mod| require "simple_tracker/lib/#{mod}" } From e45837e01b0093d826f86395aeb23fbc9b153f07 Mon Sep 17 00:00:00 2001 From: Tom Christ Date: Tue, 12 May 2015 15:30:06 -0400 Subject: [PATCH 35/86] Better state persistence --- lib/jetpants/shardpool.rb | 2 +- plugins/simple_tracker/lib/shardpool.rb | 13 +++---------- plugins/simple_tracker/lib/topology.rb | 7 +++++-- 3 files changed, 9 insertions(+), 13 deletions(-) diff --git a/lib/jetpants/shardpool.rb b/lib/jetpants/shardpool.rb index 0176642..045fc99 100644 --- a/lib/jetpants/shardpool.rb +++ b/lib/jetpants/shardpool.rb @@ -3,7 +3,7 @@ module Jetpants # man Shards. All shards within the pool partition a logically coherent # keyspace - attr_accessor :name + attr_reader :name class ShardPool include CallbackHandler diff --git a/plugins/simple_tracker/lib/shardpool.rb b/plugins/simple_tracker/lib/shardpool.rb index 9fb1423..b88e1fa 100644 --- a/plugins/simple_tracker/lib/shardpool.rb +++ b/plugins/simple_tracker/lib/shardpool.rb @@ -1,15 +1,8 @@ module Jetpants - class Shard < Pool + class ShardPool - ##### CALLBACKS ############################################################ - - # After changing the state of a shard, sync config back to the asset tracker json - def after_state=(value) - sync_configuration - end - - def shards - Jetpants.topology.shards(@name) + def sync_configuration + Jetpants.topology.update_tracker_data end ##### NEW CLASS-LEVEL METHODS ############################################## diff --git a/plugins/simple_tracker/lib/topology.rb b/plugins/simple_tracker/lib/topology.rb index 2b34cbb..1c51dda 100644 --- a/plugins/simple_tracker/lib/topology.rb +++ b/plugins/simple_tracker/lib/topology.rb @@ -40,11 +40,13 @@ def write_config db_data = { 'database' => { 'pools' => functional_partitions.map {|p| p.to_hash(true)}, - 'shards' => shards.select {|s| s.in_config?}.map {|s| s.to_hash(true)}, - 'shard_pools' => shard_pools.map {|sp| sp.to_hash(true)}, } } + shard_pools.each do |shard_pool| + db_data[shard_pool.name] = shard_pool.shards.select {|s| s.in_config?}.map {|s| s.to_hash(true)} + end + # Convert that hash to YAML and write it to a file File.open(config_file_path, 'w') do |f| f.write db_data.to_yaml @@ -89,6 +91,7 @@ def spares(options={}) def update_tracker_data @tracker.global_pools = functional_partitions.map &:to_hash @tracker.shards = shards.reject {|s| s.state == :recycle}.map &:to_hash + @tracker.shard_pools = shards_pools.map(&:to_hash) @tracker.save end From 4bc85e8e9d44cac9c9c0d5d63e08cff9a627ca81 Mon Sep 17 00:00:00 2001 From: Tom Christ Date: Tue, 12 May 2015 17:49:45 -0400 Subject: [PATCH 36/86] Better class singleton access and fixed initializer --- plugins/simple_tracker/lib/shardpool.rb | 2 +- plugins/simple_tracker/lib/topology.rb | 32 +++++++++++++------------ 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/plugins/simple_tracker/lib/shardpool.rb b/plugins/simple_tracker/lib/shardpool.rb index b88e1fa..f46dcc1 100644 --- a/plugins/simple_tracker/lib/shardpool.rb +++ b/plugins/simple_tracker/lib/shardpool.rb @@ -12,7 +12,7 @@ def self.from_hash(h) # we just return the shard for now... we have to wait until later to # set up children + parents, since it's easier to grab the corresponding # objects once all pools have been initialized. - Shard.new(h['shard_pool']) + ShardPool.new(h['shard_pool']) end ##### NEW METHODS ########################################################## diff --git a/plugins/simple_tracker/lib/topology.rb b/plugins/simple_tracker/lib/topology.rb index 1c51dda..f3ec56e 100644 --- a/plugins/simple_tracker/lib/topology.rb +++ b/plugins/simple_tracker/lib/topology.rb @@ -1,27 +1,29 @@ module Jetpants class Topology - attr_accessor :tracker + def self.tracker + @tracker ||= Jetpants::Plugin::SimpleTracker.new + + @tracker + end ##### METHOD OVERRIDES ##################################################### # Populates @pools by reading asset tracker data def load_pools - @tracker ||= Jetpants::Plugin::SimpleTracker.new # Create Pool and Shard objects - @pools = @tracker.global_pools.map {|h| Pool.from_hash(h)}.compact - all_shards = @tracker.shards.map {|h| Shard.from_hash(h)}.reject {|s| s.state == :recycle} + @pools = self.class.tracker.global_pools.map {|h| Pool.from_hash(h)}.compact + all_shards = self.class.tracker.shards.map {|h| Shard.from_hash(h)}.reject {|s| s.state == :recycle} @pools.concat all_shards # Now that all shards exist, we can safely assign parent/child relationships - @tracker.shards.each {|h| Shard.assign_relationships(h, all_shards)} + self.class.tracker.shards.each {|h| Shard.assign_relationships(h, all_shards)} end # Populate @shard_pools by reading asset tracker data def load_shard_pools - @tracker ||= Jetpants::Plugin::SimpleTracker.new - @shard_pools = @tracker.shard_pools.map{|h| ShardPool.from_hash(h) }.compact + @shard_pools = self.class.tracker.shard_pools.map{|h| ShardPool.from_hash(h) }.compact end def add_pool(pool) @@ -57,8 +59,8 @@ def write_config # simple_tracker completely ignores any options like :role or :like def claim_spares(count, options={}) - raise "Not enough spare machines -- requested #{count}, only have #{@tracker.spares.count}" if @tracker.spares.count < count - hashes = @tracker.spares.shift(count) + raise "Not enough spare machines -- requested #{count}, only have #{self.class.tracker.spares.count}" if self.class.tracker.spares.count < count + hashes = self.class.tracker.spares.shift(count) update_tracker_data dbs = hashes.map {|h| h.is_a?(Hash) && h['node'] ? h['node'].to_db : h.to_db} @@ -73,11 +75,11 @@ def claim_spares(count, options={}) end def count_spares(options={}) - @tracker.spares.count + self.class.tracker.spares.count end def spares(options={}) - @tracker.spares.map(&:to_db) + self.class.tracker.spares.map(&:to_db) end @@ -89,10 +91,10 @@ def spares(options={}) # instead Pool#sync_configuration could just update the info for that pool # only. def update_tracker_data - @tracker.global_pools = functional_partitions.map &:to_hash - @tracker.shards = shards.reject {|s| s.state == :recycle}.map &:to_hash - @tracker.shard_pools = shards_pools.map(&:to_hash) - @tracker.save + self.class.tracker.global_pools = functional_partitions.map &:to_hash + self.class.tracker.shards = shards.reject {|s| s.state == :recycle}.map &:to_hash + self.class.tracker.shard_pools = shard_pools.map(&:to_hash) + self.class.tracker.save end end From 6505a01b0c9ff8cb7a31c8ceeb55921b4f51f742 Mon Sep 17 00:00:00 2001 From: Tom Christ Date: Wed, 13 May 2015 11:41:17 -0400 Subject: [PATCH 37/86] Move shards to subkey --- lib/jetpants/shardpool.rb | 4 ++-- plugins/simple_tracker/lib/topology.rb | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/lib/jetpants/shardpool.rb b/lib/jetpants/shardpool.rb index 045fc99..53a01c6 100644 --- a/lib/jetpants/shardpool.rb +++ b/lib/jetpants/shardpool.rb @@ -3,12 +3,12 @@ module Jetpants # man Shards. All shards within the pool partition a logically coherent # keyspace - attr_reader :name - class ShardPool include CallbackHandler include Output + attr_reader :name + def initialize(name) @name = name end diff --git a/plugins/simple_tracker/lib/topology.rb b/plugins/simple_tracker/lib/topology.rb index f3ec56e..b8447c0 100644 --- a/plugins/simple_tracker/lib/topology.rb +++ b/plugins/simple_tracker/lib/topology.rb @@ -36,17 +36,18 @@ def add_shard_pool(shard_pool) # Generates a database configuration file for a hypothetical web application def write_config - config_file_path = @tracker.app_config_file_path + config_file_path = self.class.tracker.app_config_file_path # Convert the pool list into a hash db_data = { 'database' => { 'pools' => functional_partitions.map {|p| p.to_hash(true)}, - } + }, + 'shard_pools' => [] } shard_pools.each do |shard_pool| - db_data[shard_pool.name] = shard_pool.shards.select {|s| s.in_config?}.map {|s| s.to_hash(true)} + db_data['shard_pools'][shard_pool.name] = shard_pool.shards.select {|s| s.in_config?}.map {|s| s.to_hash(true)} end # Convert that hash to YAML and write it to a file From 52cc28d65c6a9173a2fd4bc77657400ec0ce3891 Mon Sep 17 00:00:00 2001 From: Tom Christ Date: Wed, 13 May 2015 11:47:39 -0400 Subject: [PATCH 38/86] Specify shard pool in shard_for_key --- lib/jetpants/topology.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/jetpants/topology.rb b/lib/jetpants/topology.rb index 65335bc..73d1dd6 100644 --- a/lib/jetpants/topology.rb +++ b/lib/jetpants/topology.rb @@ -212,7 +212,7 @@ def shard(*args) # to the parent shard when appropriate in this case. (see also: Topology#shard_db_for_id) def shard_for_id(id, shard_pool = nil) shard_pool = default_shard_pool if shard_pool.nil? - choices = shards.select {|s| s.min_id <= id && (s.max_id == 'INFINITY' || s.max_id >= id)} + choices = shards(shard_pool).select {|s| s.min_id <= id && (s.max_id == 'INFINITY' || s.max_id >= id)} choices.reject! {|s| s.parent && ! s.in_config?} # filter out child shards that are still being built # Preferentially return child shards at this point From 6b2f8f8839e70356f816f59704519940279e08a0 Mon Sep 17 00:00:00 2001 From: Tom Christ Date: Wed, 13 May 2015 14:23:02 -0400 Subject: [PATCH 39/86] Load shard pool state at initialization --- lib/jetpants.rb | 1 + lib/jetpants/topology.rb | 1 + 2 files changed, 2 insertions(+) diff --git a/lib/jetpants.rb b/lib/jetpants.rb index 3377de5..c3f366c 100644 --- a/lib/jetpants.rb +++ b/lib/jetpants.rb @@ -159,4 +159,5 @@ def with_retries(retries = nil, max_retry_backoff = nil) # Finally, initialize topology object @topology = Topology.new @topology.load_pools unless @config['lazy_load_pools'] + @topology.load_shard_pools unless @config['lazy_load_pools'] end diff --git a/lib/jetpants/topology.rb b/lib/jetpants/topology.rb index 73d1dd6..0f6fd87 100644 --- a/lib/jetpants/topology.rb +++ b/lib/jetpants/topology.rb @@ -83,6 +83,7 @@ def load_pools end synchronized + # Plugin should override this to initialize @shard_pools def load_shard_pools output "Notice: no plugin has overriddent Topology#load_shard_pools, so *no* shard pools are imported automaticaly" end From 7a9f063389d421e2e6eaf7e4e680eeae16b4e430 Mon Sep 17 00:00:00 2001 From: Tom Christ Date: Wed, 13 May 2015 14:35:27 -0400 Subject: [PATCH 40/86] Pass in shard pool for shard_db_for_id --- lib/jetpants/topology.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/jetpants/topology.rb b/lib/jetpants/topology.rb index 0f6fd87..f8f4f1b 100644 --- a/lib/jetpants/topology.rb +++ b/lib/jetpants/topology.rb @@ -226,8 +226,8 @@ def shard_for_id(id, shard_pool = nil) # Returns the Jetpants::DB that handles the given ID with the specified # mode (either :read or :write) - def shard_db_for_id(id, mode=:read) - shard_for_id(id).db(mode) + def shard_db_for_id(id, mode=:read, shard_pool = nil) + shard_for_id(id, shard_pool).db(mode) end # Nicer inteface into claim_spares when only one DB is desired -- returns From 17a1f904999ba4cacd01923615d5062a56b70a8e Mon Sep 17 00:00:00 2001 From: Tom Christ Date: Wed, 13 May 2015 15:51:45 -0400 Subject: [PATCH 41/86] Solidify shard pool naming and access --- lib/jetpants.rb | 2 +- lib/jetpants/shard.rb | 10 +++++----- lib/jetpants/shardpool.rb | 4 ++++ lib/jetpants/topology.rb | 5 +++++ 4 files changed, 15 insertions(+), 6 deletions(-) diff --git a/lib/jetpants.rb b/lib/jetpants.rb index c3f366c..b7bcade 100644 --- a/lib/jetpants.rb +++ b/lib/jetpants.rb @@ -158,6 +158,6 @@ def with_retries(retries = nil, max_retry_backoff = nil) # Finally, initialize topology object @topology = Topology.new - @topology.load_pools unless @config['lazy_load_pools'] @topology.load_shard_pools unless @config['lazy_load_pools'] + @topology.load_pools unless @config['lazy_load_pools'] end diff --git a/lib/jetpants/shard.rb b/lib/jetpants/shard.rb index 8f299b2..86a38bb 100644 --- a/lib/jetpants/shard.rb +++ b/lib/jetpants/shard.rb @@ -39,29 +39,29 @@ class Shard < Pool attr_accessor :state # the sharding pool to which this shard belongs - attr_accessor :shard_pool + attr_reader :shard_pool # Constructor for Shard -- # * min_id: int # * max_id: int or the string "INFINITY" # * master: string (IP address) or a Jetpants::DB object # * state: one of the above state symbols - def initialize(min_id, max_id, master, state=:ready, shard_pool=nil) + def initialize(min_id, max_id, master, state=:ready, shard_pool_name=nil) @min_id = min_id.to_i @max_id = (max_id.to_s.upcase == 'INFINITY' ? 'INFINITY' : max_id.to_i) @state = state @children = [] # array of shards being initialized by splitting this one @parent = nil - shard_pool = Jetpants.topology.default_shard_pool if shard_pool.nil? - @shard_pool = shard_pool + shard_pool_name = Jetpants.topology.default_shard_pool if shard_pool_name.nil? + @shard_pool = Jetpants.topology.shard_pool(shard_pool_name) super(generate_name, master) end # Generates a string containing the shard's min and max IDs. Plugin may want to override. def generate_name - "shard-#{min_id}-#{max_id.to_s.downcase}" + "#{@shard_pool.name}-#{min_id}-#{max_id.to_s.downcase}" end # Returns true if the shard state is one of the values that indicates it's diff --git a/lib/jetpants/shardpool.rb b/lib/jetpants/shardpool.rb index 53a01c6..e31f71d 100644 --- a/lib/jetpants/shardpool.rb +++ b/lib/jetpants/shardpool.rb @@ -16,5 +16,9 @@ def initialize(name) def shards Jetpants.topology.shards(@name) end + + def to_s + @name + end end end diff --git a/lib/jetpants/topology.rb b/lib/jetpants/topology.rb index f8f4f1b..eec52e2 100644 --- a/lib/jetpants/topology.rb +++ b/lib/jetpants/topology.rb @@ -204,6 +204,11 @@ def shard(*args) result.first end end + + # Finds a ShardPool object by name + def shard_pool(name) + shard_pools.select{|sp| sp.name == name}.first + end # Returns the Jetpants::Shard that handles the given ID. # During a shard split, if the child isn't "in production" yet (ie, it's From 5fedc0142a581cdd5010b8c146d576e1eea9ecde Mon Sep 17 00:00:00 2001 From: Tom Christ Date: Wed, 13 May 2015 16:08:48 -0400 Subject: [PATCH 42/86] Modify persistence to include all shards, modify shard pool selection --- lib/jetpants/topology.rb | 6 +++--- plugins/simple_tracker/lib/topology.rb | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/jetpants/topology.rb b/lib/jetpants/topology.rb index eec52e2..13dbb81 100644 --- a/lib/jetpants/topology.rb +++ b/lib/jetpants/topology.rb @@ -152,9 +152,9 @@ def slave_roles ###### Instance Methods #################################################### # Returns array of this topology's Jetpants::Pool objects of type Jetpants::Shard - def shards(shard_pool = nil) - shard_pool = default_shard_pool if shard_pool.nil? - pools.select {|p| p.is_a? Shard}.select { |p| p.shard_pool = shard_pool } + def shards(shard_pool_name = nil) + shard_pool_name = default_shard_pool if shard_pool_name.nil? + pools.select {|p| p.is_a? Shard}.select { |p| p.shard_pool.name == shard_pool_name } end # Returns array of this topology's Jetpants::Pool objects that are NOT of type Jetpants::Shard diff --git a/plugins/simple_tracker/lib/topology.rb b/plugins/simple_tracker/lib/topology.rb index b8447c0..4933106 100644 --- a/plugins/simple_tracker/lib/topology.rb +++ b/plugins/simple_tracker/lib/topology.rb @@ -93,7 +93,7 @@ def spares(options={}) # only. def update_tracker_data self.class.tracker.global_pools = functional_partitions.map &:to_hash - self.class.tracker.shards = shards.reject {|s| s.state == :recycle}.map &:to_hash + self.class.tracker.shards = pools.select{|p| p.is_a? Shard}.reject {|s| s.state == :recycle}.map &:to_hash self.class.tracker.shard_pools = shard_pools.map(&:to_hash) self.class.tracker.save end From 130f3c985b8bc5d8f6e9ef4247f37262e3091458 Mon Sep 17 00:00:00 2001 From: Tom Christ Date: Wed, 13 May 2015 16:20:41 -0400 Subject: [PATCH 43/86] Pass shard pool name --- lib/jetpants/shard.rb | 8 ++++---- plugins/simple_tracker/lib/topology.rb | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/jetpants/shard.rb b/lib/jetpants/shard.rb index 86a38bb..103186e 100644 --- a/lib/jetpants/shard.rb +++ b/lib/jetpants/shard.rb @@ -112,20 +112,20 @@ def db(mode=:read) # Override the probe_tables method to accommodate shard topology - # delegate everything to the first shard. def probe_tables - if Jetpants.topology.shards(self.shard_pool).first == self + if Jetpants.topology.shards(self.shard_pool.name).first == self super else - Jetpants.topology.shards(self.shard_pool).first.probe_tables + Jetpants.topology.shards(self.shard_pool.name).first.probe_tables end end # Override the tables accessor to accommodate shard topology - delegate # everything to the first shard def tables - if Jetpants.topology.shards(self.shard_pool).first == self + if Jetpants.topology.shards(self.shard_pool.name).first == self super else - Jetpants.topology.shards(self.shard_pool).first.tables + Jetpants.topology.shards(self.shard_pool.name).first.tables end end diff --git a/plugins/simple_tracker/lib/topology.rb b/plugins/simple_tracker/lib/topology.rb index 4933106..be89b91 100644 --- a/plugins/simple_tracker/lib/topology.rb +++ b/plugins/simple_tracker/lib/topology.rb @@ -43,7 +43,7 @@ def write_config 'database' => { 'pools' => functional_partitions.map {|p| p.to_hash(true)}, }, - 'shard_pools' => [] + 'shard_pools' => {} } shard_pools.each do |shard_pool| From e7a6a63f743d87cbf3ff48dee72056447e5d0b6c Mon Sep 17 00:00:00 2001 From: Tom Christ Date: Wed, 13 May 2015 17:09:32 -0400 Subject: [PATCH 44/86] Correctly use shard pool in shards method --- lib/jetpants/topology.rb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/jetpants/topology.rb b/lib/jetpants/topology.rb index 13dbb81..3f10bb4 100644 --- a/lib/jetpants/topology.rb +++ b/lib/jetpants/topology.rb @@ -173,19 +173,19 @@ def pool(target) end # Finds and returns a single Jetpants::Shard. Pass in one of these: - # * a min ID and a max ID - # * just a min ID - # * a Range object + # * a min ID and a max ID, shard pool + # * just a min ID, shard pool + # * a Range object, shard pool def shard(*args) if args.count >= 2 || args[0].is_a?(Array) args.flatten! - if(args.last.to_i == 0 && args.last.upcase != 'INFINITY') + if(args.last.is_a?(String) && args.last.upcase != 'INFINITY') shard_pool = args.last else shard_pool = default_shard_pool end args.map! {|x| x.to_s.upcase == 'INFINITY' ? 'INFINITY' : x.to_i} - shards.select {|s| s.min_id == args[0] && s.max_id == args[1]}.first + shards(shard_pool).select {|s| s.min_id == args[0] && s.max_id == args[1]}.first elsif args[0].is_a?(Range) if(args[1].nil?) shard_pool = default_shard_pool From 51aa975b8dff3925cd1a392086687b7882c740e5 Mon Sep 17 00:00:00 2001 From: Tom Christ Date: Thu, 14 May 2015 10:18:29 -0400 Subject: [PATCH 45/86] Add appropriate prompts for shard pool --- bin/jetpants | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/bin/jetpants b/bin/jetpants index abe6b2a..d8f7dd0 100755 --- a/bin/jetpants +++ b/bin/jetpants @@ -590,9 +590,12 @@ module Jetpants method_option :min_id, :desc => 'Minimum ID of shard involved in master promotion' method_option :max_id, :desc => 'Maximum ID of shard involved in master promotion' method_option :new_master, :desc => 'New node to become master of the shard' + method_option :shard_pool, :desc => 'The sharding pool for which to perform the cutover' def shard_promote_master + shard_pool = options[:shard_pool] || ask('Please enter the sharding pool for which to perform the split (enter for default pool): ') + shard_pool = default_shard_pool if shard_pool.empty? # find the shard we are going to do master promotion on - s = ask_shard_being_promoted(:prep, options[:max_id], options[:max_id]) + s = ask_shard_being_promoted(:prep, options[:max_id], options[:max_id], shard_pool) new_master = ask_node "Please enter the IP of the new master for #{s}: ", options[:new_master] raise "New master node #{new_master} is not currently a slave in shard #{s}" unless s.slaves && s.slaves.include?(new_master) @@ -619,9 +622,12 @@ module Jetpants desc 'shard_promote_master_reads', 'Lockless shard master promotion (step 2 of 4): move reads to new master' method_option :min_id, :desc => 'Minimum ID of shard involved in master promotion' method_option :max_id, :desc => 'Maximum ID of shard involved in master promotion' + method_option :shard_pool, :desc => 'The sharding pool for which to perform the cutover' def shard_promote_master_reads + shard_pool = options[:shard_pool] || ask('Please enter the sharding pool for which to perform the split (enter for default pool): ') + shard_pool = default_shard_pool if shard_pool.empty? # find the shard we are going to do master promotion on - s = ask_shard_being_promoted(:prep, options[:max_id], options[:max_id]) + s = ask_shard_being_promoted(:prep, options[:max_id], options[:max_id], shard_pool) # at this point we only have one slave, which is the new master new_master = s.master.slaves.last @@ -646,8 +652,11 @@ module Jetpants end desc 'shard_promote_master_writes', 'Lockless shard master promotion (step 3 of 4): move writes to new master' + method_option :shard_pool, :desc => 'The sharding pool for which to perform the cutover' def shard_promote_master_writes - s = ask_shard_being_promoted :writes + shard_pool = options[:shard_pool] || ask('Please enter the sharding pool for which to perform the split (enter for default pool): ') + shard_pool = default_shard_pool if shard_pool.empty? + s = ask_shard_being_promoted(:writes, nil, nil, shard_pool) if s.state != :child raise "Shard #{s} is in wrong state to perform this action! Expected :child, found #{s.state}" end @@ -668,8 +677,11 @@ module Jetpants end desc 'shard_promote_master_cleanup', 'Lockless shard master promotion (step 4 of 4): clean up shard and eject old master' + method_option :shard_pool, :desc => 'The sharding pool for which to perform the cutover' def shard_promote_master_cleanup - s = ask_shard_being_promoted :cleanup + shard_pool = options[:shard_pool] || ask('Please enter the sharding pool for which to perform the split (enter for default pool): ') + shard_pool = default_shard_pool if shard_pool.empty? + s = ask_shard_being_promoted(:cleanup, nil, nil, shard_pool) if s.state != :needs_cleanup raise "Shard #{s} is in wrong state to perform this action! Expected :needs_cleanup, found #{s.state}" end @@ -825,7 +837,7 @@ module Jetpants s end - def ask_shard_being_promoted(stage = :prep, min_id = nil, max_id = nil) + def ask_shard_being_promoted(stage = :prep, min_id = nil, max_id = nil, shard_pool) if stage == :writes || stage == :cleanup shards_being_promoted = Jetpants.shards.select do |s| [:reads, :child, :needs_cleanup].include?(s.state) && !s.parent && s.master.master From 24359ae28fa260a3cd953a4fe3b3dbdfcb0ca5ff Mon Sep 17 00:00:00 2001 From: Tom Christ Date: Thu, 14 May 2015 10:44:43 -0400 Subject: [PATCH 46/86] Simplify shards method and reinitialize to nil on clear --- lib/jetpants/topology.rb | 35 ++++++----------------------------- 1 file changed, 6 insertions(+), 29 deletions(-) diff --git a/lib/jetpants/topology.rb b/lib/jetpants/topology.rb index 3f10bb4..82278ca 100644 --- a/lib/jetpants/topology.rb +++ b/lib/jetpants/topology.rb @@ -176,33 +176,10 @@ def pool(target) # * a min ID and a max ID, shard pool # * just a min ID, shard pool # * a Range object, shard pool - def shard(*args) - if args.count >= 2 || args[0].is_a?(Array) - args.flatten! - if(args.last.is_a?(String) && args.last.upcase != 'INFINITY') - shard_pool = args.last - else - shard_pool = default_shard_pool - end - args.map! {|x| x.to_s.upcase == 'INFINITY' ? 'INFINITY' : x.to_i} - shards(shard_pool).select {|s| s.min_id == args[0] && s.max_id == args[1]}.first - elsif args[0].is_a?(Range) - if(args[1].nil?) - shard_pool = default_shard_pool - else - shard_pool = args[1] - end - shards.select {|s| s.min_id == args[0].min && s.max_id == args[0].max && s.shard_pool = shard_pool}.first - else - if(args[1].nil?) - shard_pool = default_shard_pool - else - shard_pool = args[1] - end - result = shards.select {|s| s.min_id == args[0].to_i} - raise "Multiple shards found with that min_id!" if result.count > 1 - result.first - end + def shard(min_id, max_id, shard_pool = nil) + shard_pool = default_shard_pool if shard_pool.nil? + max_id.upcase! if max_id.is_a?(String) + shards(shard_pool).select {|s| s.min_id == min_id.to_i && s.max_id == max_id}.first end # Finds a ShardPool object by name @@ -257,8 +234,8 @@ def normalize_roles(*roles) synchronized # Clears the pool list and nukes cached DB and Host object lookup tables def clear - @pools = [] - @shard_pools = [] + @pools = nil + @shard_pools = nil DB.clear Host.clear end From 83de47b106e557fddc8091e313e3cb3c7ebf5bf5 Mon Sep 17 00:00:00 2001 From: Tom Christ Date: Thu, 14 May 2015 15:59:58 -0400 Subject: [PATCH 47/86] Equality in primary role comparison --- plugins/jetpants_collins/asset.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/jetpants_collins/asset.rb b/plugins/jetpants_collins/asset.rb index 659b54b..0e6cfd9 100644 --- a/plugins/jetpants_collins/asset.rb +++ b/plugins/jetpants_collins/asset.rb @@ -19,7 +19,7 @@ def to_host # Convert a Collins:Asset to a Jetpants::Shard_pool def to_shard_pool raise "Can only call to_shard_pool on CONFIGURATION assets, but #{self} has type #{type}" unless type.upcase == 'CONFIGURATION' - raise "Unknown primary role #{primary_role} for configuration asset #{self}" unless ['MYSQL_SHARD_POOL'].include?(primary_role.upcase) + raise "Unknown primary role #{primary_role} for configuration asset #{self}" unless primary_role.upcase == 'MYSQL_SHARD_POOL' raise "No shard_pool attribute set on asset #{self}" unless shard_pool && shard_pool.length > 0 Jetpants::ShardPool.new(shard_pool) From 85c9a36cc6b30f7d01c68044c187bdf143be349d Mon Sep 17 00:00:00 2001 From: Tom Christ Date: Wed, 20 May 2015 16:21:42 -0400 Subject: [PATCH 48/86] Add default shard pool to ask output --- bin/jetpants | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/bin/jetpants b/bin/jetpants index d8f7dd0..a050a51 100755 --- a/bin/jetpants +++ b/bin/jetpants @@ -412,8 +412,8 @@ module Jetpants shard_min = options[:min_id] || ask('Please enter min ID of the parent shard: ') shard_max = options[:max_id] || ask('Please enter max ID of the parent shard: ') - shard_pool = options[:shard_pool] || ask('Please enter the sharding pool for which to perform the split (enter for default pool): ') - shard_pool = default_shard_pool if shard_pool.empty? + shard_pool = options[:shard_pool] || ask("Please enter the sharding pool for which to perform the split (enter for default pool, #{Jetpants.topology.default_shard_pool}): ") + shard_pool = Jetpants.topology.default_shard_pool if shard_pool.empty? s = Jetpants.topology.shard(shard_min, shard_max, shard_pool) @@ -512,8 +512,8 @@ module Jetpants def shard_cutover cutover_id = options[:cutover_id] || ask('Please enter min ID of the new shard to be created: ') cutover_id = cutover_id.to_i - shard_pool = options[:shard_pool] || ask('Please enter the sharding pool for which to perform the split (enter for default pool): ') - shard_pool = default_shard_pool if shard_pool.empty? + shard_pool = options[:shard_pool] || ask("Please enter the sharding pool for which to perform the split (enter for default pool, #{Jetpants.topology.default_shard_pool}): ") + shard_pool = Jetpants.topology.default_shard_pool if shard_pool.empty? last_shard = Jetpants.topology.shards(shard_pool).select {|s| s.max_id == 'INFINITY' && s.in_config?}.first last_shard_master = last_shard.master @@ -592,8 +592,8 @@ module Jetpants method_option :new_master, :desc => 'New node to become master of the shard' method_option :shard_pool, :desc => 'The sharding pool for which to perform the cutover' def shard_promote_master - shard_pool = options[:shard_pool] || ask('Please enter the sharding pool for which to perform the split (enter for default pool): ') - shard_pool = default_shard_pool if shard_pool.empty? + shard_pool = options[:shard_pool] || ask("Please enter the sharding pool for which to perform the split (enter for default, #{Jetpants.topology.default_shard_pool}): ") + shard_pool = Jetpants.topology.default_shard_pool if shard_pool.empty? # find the shard we are going to do master promotion on s = ask_shard_being_promoted(:prep, options[:max_id], options[:max_id], shard_pool) @@ -624,8 +624,8 @@ module Jetpants method_option :max_id, :desc => 'Maximum ID of shard involved in master promotion' method_option :shard_pool, :desc => 'The sharding pool for which to perform the cutover' def shard_promote_master_reads - shard_pool = options[:shard_pool] || ask('Please enter the sharding pool for which to perform the split (enter for default pool): ') - shard_pool = default_shard_pool if shard_pool.empty? + shard_pool = options[:shard_pool] || ask("Please enter the sharding pool for which to perform the split (enter for default pool, #{Jetpants.topology.default_shard_pool}): ") + shard_pool = Jetpants.topology.default_shard_pool if shard_pool.empty? # find the shard we are going to do master promotion on s = ask_shard_being_promoted(:prep, options[:max_id], options[:max_id], shard_pool) @@ -654,8 +654,8 @@ module Jetpants desc 'shard_promote_master_writes', 'Lockless shard master promotion (step 3 of 4): move writes to new master' method_option :shard_pool, :desc => 'The sharding pool for which to perform the cutover' def shard_promote_master_writes - shard_pool = options[:shard_pool] || ask('Please enter the sharding pool for which to perform the split (enter for default pool): ') - shard_pool = default_shard_pool if shard_pool.empty? + shard_pool = options[:shard_pool] || ask("Please enter the sharding pool for which to perform the split (enter for default pool, #{Jetpants.topology.default_shard_pool}): ") + shard_pool = Jetpants.topology.default_shard_pool if shard_pool.empty? s = ask_shard_being_promoted(:writes, nil, nil, shard_pool) if s.state != :child raise "Shard #{s} is in wrong state to perform this action! Expected :child, found #{s.state}" @@ -679,8 +679,8 @@ module Jetpants desc 'shard_promote_master_cleanup', 'Lockless shard master promotion (step 4 of 4): clean up shard and eject old master' method_option :shard_pool, :desc => 'The sharding pool for which to perform the cutover' def shard_promote_master_cleanup - shard_pool = options[:shard_pool] || ask('Please enter the sharding pool for which to perform the split (enter for default pool): ') - shard_pool = default_shard_pool if shard_pool.empty? + shard_pool = options[:shard_pool] || ask("Please enter the sharding pool for which to perform the split (enter for default pool, #{Jetpants.topology.default_shard_pool}): ") + shard_pool = Jetpants.topology.default_shard_pool if shard_pool.empty? s = ask_shard_being_promoted(:cleanup, nil, nil, shard_pool) if s.state != :needs_cleanup raise "Shard #{s} is in wrong state to perform this action! Expected :needs_cleanup, found #{s.state}" @@ -815,8 +815,8 @@ module Jetpants output 'Which shard would you like to perform this action on?' shard_min = ask('Please enter min ID of the shard: ') shard_max = ask('Please enter max ID of the shard: ') - shard_pool = ask('Please enter the sharding pool which to perform the action on (enter for default pool): ') - shard_pool = default_shard_pool if shard_pool.empty? + shard_pool = ask("Please enter the sharding pool which to perform the action on (enter for default pool, #{Jetpants.topology.default_shard_pool}): ") + shard_pool = Jetpants.topology.default_shard_pool if shard_pool.empty? s = Jetpants.topology.shard(shard_min, shard_max, shard_pool) raise 'Shard not found' unless s s From b7a78c759192ed21a07823b5660b1b4d1a13ab86 Mon Sep 17 00:00:00 2001 From: Tom Christ Date: Thu, 21 May 2015 14:54:43 -0400 Subject: [PATCH 49/86] Order includes --- lib/jetpants.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/jetpants.rb b/lib/jetpants.rb index b7bcade..59b05e3 100644 --- a/lib/jetpants.rb +++ b/lib/jetpants.rb @@ -9,7 +9,7 @@ module Jetpants; end $LOAD_PATH.unshift File.join(File.dirname(__FILE__), 'jetpants'), File.join(File.dirname(__FILE__), '..', 'plugins') -%w(output callback table host db pool topology shard monkeypatch shardpool commandsuite).each {|g| require g} +%w(output callback table host db pool topology shard shardpool monkeypatch commandsuite).each {|g| require g} # Since Jetpants is extremely multi-threaded, we need to force uncaught exceptions to # kill all threads in order to have any kind of sane error handling. From 640e6d96b979f27d0de237b1f9022b34530d1c7b Mon Sep 17 00:00:00 2001 From: Tom Christ Date: Wed, 27 May 2015 17:06:02 -0400 Subject: [PATCH 50/86] Add shard pool awareness to table definitions --- lib/jetpants.rb | 2 +- lib/jetpants/db/import_export.rb | 2 +- lib/jetpants/db/schema.rb | 2 +- lib/jetpants/shard.rb | 4 ++-- lib/jetpants/table.rb | 5 +++-- 5 files changed, 8 insertions(+), 7 deletions(-) diff --git a/lib/jetpants.rb b/lib/jetpants.rb index 59b05e3..e774e45 100644 --- a/lib/jetpants.rb +++ b/lib/jetpants.rb @@ -37,7 +37,7 @@ module Jetpants 'verify_replication' => true, # raise exception if the 2 repl threads are in different states, or if actual repl topology differs from Jetpants' understanding of it 'plugins' => {}, # hash of plugin name => arbitrary plugin data (usually a nested hash of settings) 'ssh_keys' => nil, # array of SSH key file locations - 'sharded_tables' => [], # array of name => {sharding_key=>X, chunks=>Y} hashes + 'sharded_tables' => [], # hash of {shard_pool => {name => {sharding_key=>X, chunks=>Y}} hashes 'compress_with' => false, # command line to use for compression in large file transfers 'decompress_with' => false, # command line to use for decompression in large file transfers 'private_interface' => 'bond0', # network interface corresponding to private IP diff --git a/lib/jetpants/db/import_export.rb b/lib/jetpants/db/import_export.rb index 89a35ee..3b8752d 100644 --- a/lib/jetpants/db/import_export.rb +++ b/lib/jetpants/db/import_export.rb @@ -274,7 +274,7 @@ def rebuild!(tables=false, min_id=false, max_id=false) p = pool if p.is_a?(Shard) - tables ||= Table.from_config 'sharded_tables' + tables ||= Table.from_config('sharded_tables', pool.shard_pool) min_id ||= p.min_id max_id ||= p.max_id if p.max_id != 'INFINITY' end diff --git a/lib/jetpants/db/schema.rb b/lib/jetpants/db/schema.rb index 54c780d..281afe8 100644 --- a/lib/jetpants/db/schema.rb +++ b/lib/jetpants/db/schema.rb @@ -24,7 +24,7 @@ def detect_table_schema(table_name) 'columns' => connection.schema(table_name).map{|schema| schema[0]} } - config_params = Jetpants.send('sharded_tables') + config_params = Jetpants.send('sharded_tables')[pool.shard_pool] unless(config_params[table_name].nil?) params.merge!(config_params[table_name]) diff --git a/lib/jetpants/shard.rb b/lib/jetpants/shard.rb index 103186e..b2562bd 100644 --- a/lib/jetpants/shard.rb +++ b/lib/jetpants/shard.rb @@ -226,7 +226,7 @@ def prune_data! raise "Shard #{self} is not in a state compatible with calling prune_data! (current state=#{@state})" end - tables = Table.from_config 'sharded_tables' + tables = Table.from_config('sharded_tables', pool.shard_pool) if @state == :initializing @state = :exporting @@ -320,7 +320,7 @@ def cleanup! # situation A - clean up after a shard split if @state == :deprecated && @children.size > 0 - tables = Table.from_config 'sharded_tables' + tables = Table.from_config('sharded_tables', pool.shard_pool) @master.revoke_all_access! @children.concurrent_each do |child_shard| raise "Child state does not indicate cleanup is needed" unless child_shard.state == :needs_cleanup diff --git a/lib/jetpants/table.rb b/lib/jetpants/table.rb index 508b016..45a4c0e 100644 --- a/lib/jetpants/table.rb +++ b/lib/jetpants/table.rb @@ -161,9 +161,10 @@ def belongs_to?(pool) # of the given label. # TODO: integrate better with table schema detection code. Consider auto-detecting chunk # count based on file size and row count estimate. - def Table.from_config(label) + def Table.from_config(label, shard_pool = nil) + shard_pool = Jetpants.topology.default_shard_pool if shard_pool.nil result = [] - Jetpants.send(label).map {|name, attributes| Table.new name, attributes} + Jetpants.send(label)[shard_pool].map {|name, attributes| Table.new name, attributes} end def to_s From e02b94cd796eb8cac07a9bdcc45ee7e732380268 Mon Sep 17 00:00:00 2001 From: Kiril Angov Date: Thu, 28 May 2015 18:30:06 -0400 Subject: [PATCH 51/86] Add all tokudb.* files to the import/export logic --- lib/jetpants/db/import_export.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/jetpants/db/import_export.rb b/lib/jetpants/db/import_export.rb index 89a35ee..61fc764 100644 --- a/lib/jetpants/db/import_export.rb +++ b/lib/jetpants/db/import_export.rb @@ -376,9 +376,9 @@ def clone_to!(*targets) [self, targets].flatten.concurrent_each {|t| t.stop_query_killer; t.stop_mysql} targets.concurrent_each {|t| t.ssh_cmd "rm -rf #{t.mysql_directory}/ib_logfile*"} - files = (databases + ['ibdata1', app_schema]).uniq + files = (databases + ['ibdata1', 'tokudb.*', app_schema]).uniq files << 'ib_lru_dump' if ssh_cmd("test -f #{mysql_directory}/ib_lru_dump 2>/dev/null; echo $?").chomp.to_i == 0 - + fast_copy_chain(mysql_directory, destinations, :port => 3306, :files => files, :overwrite => true) clone_settings_to!(*targets) From bd7f93f105e494f9ce0796afdecc4be7f9d85af5 Mon Sep 17 00:00:00 2001 From: Kiril Angov Date: Thu, 28 May 2015 18:42:51 -0400 Subject: [PATCH 52/86] Adding also `*.tokudb` to the list of files to copy on import/export --- lib/jetpants/db/import_export.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/jetpants/db/import_export.rb b/lib/jetpants/db/import_export.rb index 61fc764..33060b2 100644 --- a/lib/jetpants/db/import_export.rb +++ b/lib/jetpants/db/import_export.rb @@ -376,7 +376,7 @@ def clone_to!(*targets) [self, targets].flatten.concurrent_each {|t| t.stop_query_killer; t.stop_mysql} targets.concurrent_each {|t| t.ssh_cmd "rm -rf #{t.mysql_directory}/ib_logfile*"} - files = (databases + ['ibdata1', 'tokudb.*', app_schema]).uniq + files = (databases + ['ibdata1', 'tokudb.*', '*.tokudb', app_schema]).uniq files << 'ib_lru_dump' if ssh_cmd("test -f #{mysql_directory}/ib_lru_dump 2>/dev/null; echo $?").chomp.to_i == 0 fast_copy_chain(mysql_directory, destinations, :port => 3306, :files => files, :overwrite => true) From dad3f648fcbcbb7acba65860a28da4e7e6a85cdb Mon Sep 17 00:00:00 2001 From: Kiril Angov Date: Thu, 28 May 2015 20:15:34 -0400 Subject: [PATCH 53/86] Adding also `log*.tokulog*` to the list of files to copy on import/export --- lib/jetpants/db/import_export.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/jetpants/db/import_export.rb b/lib/jetpants/db/import_export.rb index 33060b2..05fe0a4 100644 --- a/lib/jetpants/db/import_export.rb +++ b/lib/jetpants/db/import_export.rb @@ -376,7 +376,8 @@ def clone_to!(*targets) [self, targets].flatten.concurrent_each {|t| t.stop_query_killer; t.stop_mysql} targets.concurrent_each {|t| t.ssh_cmd "rm -rf #{t.mysql_directory}/ib_logfile*"} - files = (databases + ['ibdata1', 'tokudb.*', '*.tokudb', app_schema]).uniq + files = (databases + ['ibdata1', app_schema]).uniq + files += ['*.tokudb', 'tokudb.*', 'log*.tokulog*'] if mysql_root_cmd("SHOW ENGINES").match /Engine:\s+TokuDB\n/ files << 'ib_lru_dump' if ssh_cmd("test -f #{mysql_directory}/ib_lru_dump 2>/dev/null; echo $?").chomp.to_i == 0 fast_copy_chain(mysql_directory, destinations, :port => 3306, :files => files, :overwrite => true) From c65261253c991fb31c5b80b8a6dc4bb2df14bc0a Mon Sep 17 00:00:00 2001 From: Kiril Angov Date: Thu, 28 May 2015 20:21:16 -0400 Subject: [PATCH 54/86] MySQL is not running so we cannot check with SHOW ENGINES --- lib/jetpants/db/import_export.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/jetpants/db/import_export.rb b/lib/jetpants/db/import_export.rb index 05fe0a4..8c1f8fa 100644 --- a/lib/jetpants/db/import_export.rb +++ b/lib/jetpants/db/import_export.rb @@ -377,7 +377,7 @@ def clone_to!(*targets) targets.concurrent_each {|t| t.ssh_cmd "rm -rf #{t.mysql_directory}/ib_logfile*"} files = (databases + ['ibdata1', app_schema]).uniq - files += ['*.tokudb', 'tokudb.*', 'log*.tokulog*'] if mysql_root_cmd("SHOW ENGINES").match /Engine:\s+TokuDB\n/ + files += ['*.tokudb', 'tokudb.*', 'log*.tokulog*'] if ssh_cmd("test -f #{mysql_directory}/tokudb.environment 2>/dev/null; echo $?").chomp.to_i == 0 files << 'ib_lru_dump' if ssh_cmd("test -f #{mysql_directory}/ib_lru_dump 2>/dev/null; echo $?").chomp.to_i == 0 fast_copy_chain(mysql_directory, destinations, :port => 3306, :files => files, :overwrite => true) From 2bee32e46e2fc603e899b38829c2d91eb4071abd Mon Sep 17 00:00:00 2001 From: Tom Christ Date: Fri, 29 May 2015 15:06:05 -0400 Subject: [PATCH 55/86] Pass in shard pool when initializing shard --- lib/jetpants/topology.rb | 2 +- plugins/jetpants_collins/asset.rb | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/jetpants/topology.rb b/lib/jetpants/topology.rb index 82278ca..d4d27d4 100644 --- a/lib/jetpants/topology.rb +++ b/lib/jetpants/topology.rb @@ -154,7 +154,7 @@ def slave_roles # Returns array of this topology's Jetpants::Pool objects of type Jetpants::Shard def shards(shard_pool_name = nil) shard_pool_name = default_shard_pool if shard_pool_name.nil? - pools.select {|p| p.is_a? Shard}.select { |p| p.shard_pool.name == shard_pool_name } + pools.select {|p| p.is_a? Shard}.select { |p| p.shard_pool && p.shard_pool.name.downcase == shard_pool_name.downcase } end # Returns array of this topology's Jetpants::Pool objects that are NOT of type Jetpants::Shard diff --git a/plugins/jetpants_collins/asset.rb b/plugins/jetpants_collins/asset.rb index 0e6cfd9..da636cc 100644 --- a/plugins/jetpants_collins/asset.rb +++ b/plugins/jetpants_collins/asset.rb @@ -73,7 +73,8 @@ def to_pool result = Jetpants::Shard.new(shard_min_id.to_i, shard_max_id == 'INFINITY' ? 'INFINITY' : shard_max_id.to_i, master_assets.first.to_db, - shard_state.downcase.to_sym) + shard_state.downcase.to_sym, + shard_pool) # We'll need to set up the parent/child relationship if a shard split is in progress, # BUT we need to wait to do that later since the shards may have been returned by From 6e55290b6bd1cbbfbc3dae6b950ed2d82b2cc0f5 Mon Sep 17 00:00:00 2001 From: Tom Christ Date: Fri, 29 May 2015 16:23:59 -0400 Subject: [PATCH 56/86] Add debug output and formatting --- lib/jetpants/shard.rb | 2 +- lib/jetpants/shardpool.rb | 2 +- lib/jetpants/topology.rb | 9 ++++++++- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/lib/jetpants/shard.rb b/lib/jetpants/shard.rb index b2562bd..fdd7caa 100644 --- a/lib/jetpants/shard.rb +++ b/lib/jetpants/shard.rb @@ -61,7 +61,7 @@ def initialize(min_id, max_id, master, state=:ready, shard_pool_name=nil) # Generates a string containing the shard's min and max IDs. Plugin may want to override. def generate_name - "#{@shard_pool.name}-#{min_id}-#{max_id.to_s.downcase}" + "#{@shard_pool.name.downcase}-#{min_id}-#{max_id.to_s.downcase}" end # Returns true if the shard state is one of the values that indicates it's diff --git a/lib/jetpants/shardpool.rb b/lib/jetpants/shardpool.rb index e31f71d..8e7a8c1 100644 --- a/lib/jetpants/shardpool.rb +++ b/lib/jetpants/shardpool.rb @@ -18,7 +18,7 @@ def shards end def to_s - @name + @name.downcase end end end diff --git a/lib/jetpants/topology.rb b/lib/jetpants/topology.rb index d4d27d4..5c5e46f 100644 --- a/lib/jetpants/topology.rb +++ b/lib/jetpants/topology.rb @@ -20,6 +20,10 @@ def initialize # invoke load_pools, which might then refer back to Jetpants.topology. end + def to_s + "Jetpants.topology" + end + def pools load_pools if @pools.nil? @pools @@ -153,7 +157,10 @@ def slave_roles # Returns array of this topology's Jetpants::Pool objects of type Jetpants::Shard def shards(shard_pool_name = nil) - shard_pool_name = default_shard_pool if shard_pool_name.nil? + if shard_pool_name.nil? + shard_pool_name = default_shard_pool + output "Using default shard pool #{default_shard_pool}" + end pools.select {|p| p.is_a? Shard}.select { |p| p.shard_pool && p.shard_pool.name.downcase == shard_pool_name.downcase } end From 2bab6c01ba417edcfbcf06651526ff4f8411ce7e Mon Sep 17 00:00:00 2001 From: Tom Christ Date: Mon, 1 Jun 2015 15:05:28 -0400 Subject: [PATCH 57/86] Fix type conversion in shard method --- bin/jetpants | 2 ++ lib/jetpants/topology.rb | 14 +++++++++----- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/bin/jetpants b/bin/jetpants index a050a51..f9bf81d 100755 --- a/bin/jetpants +++ b/bin/jetpants @@ -415,6 +415,8 @@ module Jetpants shard_pool = options[:shard_pool] || ask("Please enter the sharding pool for which to perform the split (enter for default pool, #{Jetpants.topology.default_shard_pool}): ") shard_pool = Jetpants.topology.default_shard_pool if shard_pool.empty? + output "Using shard pool `#{shard_pool}`" + s = Jetpants.topology.shard(shard_min, shard_max, shard_pool) raise "Shard not found" unless s diff --git a/lib/jetpants/topology.rb b/lib/jetpants/topology.rb index 5c5e46f..ef20de3 100644 --- a/lib/jetpants/topology.rb +++ b/lib/jetpants/topology.rb @@ -179,13 +179,17 @@ def pool(target) end end - # Finds and returns a single Jetpants::Shard. Pass in one of these: - # * a min ID and a max ID, shard pool - # * just a min ID, shard pool - # * a Range object, shard pool + # Finds and returns a single Jetpants::Shard def shard(min_id, max_id, shard_pool = nil) shard_pool = default_shard_pool if shard_pool.nil? - max_id.upcase! if max_id.is_a?(String) + if max_id.upcase == 'INFINITY' + max_id.upcase! + else + max_id = max_id.to_i + end + + min_id = min_id.to_i + shards(shard_pool).select {|s| s.min_id == min_id.to_i && s.max_id == max_id}.first end From 1ab25a9a779fee54030836341ba7f2574f5e9714 Mon Sep 17 00:00:00 2001 From: Tom Christ Date: Mon, 1 Jun 2015 15:11:20 -0400 Subject: [PATCH 58/86] Add shard pool when creating new shards in split/merge/cutover --- bin/jetpants | 2 +- lib/jetpants/shard.rb | 2 +- plugins/merge_helper/lib/commandsuite.rb | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/jetpants b/bin/jetpants index f9bf81d..0ce17a1 100755 --- a/bin/jetpants +++ b/bin/jetpants @@ -539,7 +539,7 @@ module Jetpants # has the same master/slaves but now has a non-infinity max ID. last_shard.state = :recycle last_shard.sync_configuration - last_shard_replace = Shard.new(last_shard.min_id, cutover_id - 1, last_shard_master) + last_shard_replace = Shard.new(last_shard.min_id, cutover_id - 1, last_shard_master, last_shard.shard_pool.name) last_shard_replace.sync_configuration Jetpants.topology.add_pool last_shard_replace diff --git a/lib/jetpants/shard.rb b/lib/jetpants/shard.rb index fdd7caa..e0889ee 100644 --- a/lib/jetpants/shard.rb +++ b/lib/jetpants/shard.rb @@ -418,7 +418,7 @@ def init_child_shard_masters(id_ranges) spare = Jetpants.topology.claim_spare(role: :master, like: master) spare.disable_read_only! if (spare.running? && spare.read_only?) spare.output "Will be master for new shard with ID range of #{my_range.first} to #{my_range.last} (inclusive)" - child_shard = Shard.new(my_range.first, my_range.last, spare, :initializing) + child_shard = Shard.new(my_range.first, my_range.last, spare, :initializing, shard_pool.name) child_shard.shard_pool = self.shard_pool child_shard.sync_configuration add_child(child_shard) diff --git a/plugins/merge_helper/lib/commandsuite.rb b/plugins/merge_helper/lib/commandsuite.rb index 89c589e..6b6835f 100644 --- a/plugins/merge_helper/lib/commandsuite.rb +++ b/plugins/merge_helper/lib/commandsuite.rb @@ -118,7 +118,7 @@ def merge_shards aggregate_shard_master.catch_up_to_master - aggregate_shard = Shard.new(shards_to_merge.first.min_id, shards_to_merge.last.max_id, aggregate_shard_master, :initializing) + aggregate_shard = Shard.new(shards_to_merge.first.min_id, shards_to_merge.last.max_id, aggregate_shard_master, :initializing, shards_to_merge.first.shard_pool.name) # ensure a record is present in collins aggregate_shard.sync_configuration Jetpants.topology.add_pool aggregate_shard From 55eacbe32ac7ac295c5764545c7530369cd2bd6a Mon Sep 17 00:00:00 2001 From: Tom Christ Date: Mon, 1 Jun 2015 15:27:39 -0400 Subject: [PATCH 59/86] Pass in shard pool in constructor --- lib/jetpants/shard.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/jetpants/shard.rb b/lib/jetpants/shard.rb index e0889ee..84d9fc9 100644 --- a/lib/jetpants/shard.rb +++ b/lib/jetpants/shard.rb @@ -419,7 +419,6 @@ def init_child_shard_masters(id_ranges) spare.disable_read_only! if (spare.running? && spare.read_only?) spare.output "Will be master for new shard with ID range of #{my_range.first} to #{my_range.last} (inclusive)" child_shard = Shard.new(my_range.first, my_range.last, spare, :initializing, shard_pool.name) - child_shard.shard_pool = self.shard_pool child_shard.sync_configuration add_child(child_shard) Jetpants.topology.add_pool child_shard From b8a5527101f985b6443fd401edd445829cd432ca Mon Sep 17 00:00:00 2001 From: Tom Christ Date: Mon, 1 Jun 2015 17:44:54 -0400 Subject: [PATCH 60/86] nil? --- lib/jetpants/table.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/jetpants/table.rb b/lib/jetpants/table.rb index 45a4c0e..81dd329 100644 --- a/lib/jetpants/table.rb +++ b/lib/jetpants/table.rb @@ -162,7 +162,7 @@ def belongs_to?(pool) # TODO: integrate better with table schema detection code. Consider auto-detecting chunk # count based on file size and row count estimate. def Table.from_config(label, shard_pool = nil) - shard_pool = Jetpants.topology.default_shard_pool if shard_pool.nil + shard_pool = Jetpants.topology.default_shard_pool if shard_pool.nil? result = [] Jetpants.send(label)[shard_pool].map {|name, attributes| Table.new name, attributes} end From ff37a2fdaa9811e5c16e7436715056efb2930262 Mon Sep 17 00:00:00 2001 From: Tom Christ Date: Tue, 2 Jun 2015 11:26:19 -0400 Subject: [PATCH 61/86] Use shard pool name correctly when retrieving tables --- lib/jetpants/db/import_export.rb | 2 +- lib/jetpants/shard.rb | 4 ++-- lib/jetpants/table.rb | 4 ++-- plugins/merge_helper/lib/aggregator.rb | 2 +- plugins/merge_helper/lib/shard.rb | 6 +++--- plugins/merge_helper/merge_helper.rb | 4 ++-- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/lib/jetpants/db/import_export.rb b/lib/jetpants/db/import_export.rb index 3b8752d..087587c 100644 --- a/lib/jetpants/db/import_export.rb +++ b/lib/jetpants/db/import_export.rb @@ -274,7 +274,7 @@ def rebuild!(tables=false, min_id=false, max_id=false) p = pool if p.is_a?(Shard) - tables ||= Table.from_config('sharded_tables', pool.shard_pool) + tables ||= Table.from_config('sharded_tables', pool.shard_pool.name) min_id ||= p.min_id max_id ||= p.max_id if p.max_id != 'INFINITY' end diff --git a/lib/jetpants/shard.rb b/lib/jetpants/shard.rb index 84d9fc9..a1f5168 100644 --- a/lib/jetpants/shard.rb +++ b/lib/jetpants/shard.rb @@ -226,7 +226,7 @@ def prune_data! raise "Shard #{self} is not in a state compatible with calling prune_data! (current state=#{@state})" end - tables = Table.from_config('sharded_tables', pool.shard_pool) + tables = Table.from_config('sharded_tables', shard_pool.name) if @state == :initializing @state = :exporting @@ -320,7 +320,7 @@ def cleanup! # situation A - clean up after a shard split if @state == :deprecated && @children.size > 0 - tables = Table.from_config('sharded_tables', pool.shard_pool) + tables = Table.from_config('sharded_tables', pool.shard_pool.name) @master.revoke_all_access! @children.concurrent_each do |child_shard| raise "Child state does not indicate cleanup is needed" unless child_shard.state == :needs_cleanup diff --git a/lib/jetpants/table.rb b/lib/jetpants/table.rb index 81dd329..7c0244e 100644 --- a/lib/jetpants/table.rb +++ b/lib/jetpants/table.rb @@ -163,8 +163,8 @@ def belongs_to?(pool) # count based on file size and row count estimate. def Table.from_config(label, shard_pool = nil) shard_pool = Jetpants.topology.default_shard_pool if shard_pool.nil? - result = [] - Jetpants.send(label)[shard_pool].map {|name, attributes| Table.new name, attributes} + raise "Unable to find sharded tables for shard pool `#{shard_pool.downcase}`" if Jetpants.send(label)[shard_pool.downcase].nil? + Jetpants.send(label)[shard_pool.downcase].map {|name, attributes| Table.new name, attributes} end def to_s diff --git a/plugins/merge_helper/lib/aggregator.rb b/plugins/merge_helper/lib/aggregator.rb index 19011cd..b0bba51 100644 --- a/plugins/merge_helper/lib/aggregator.rb +++ b/plugins/merge_helper/lib/aggregator.rb @@ -347,7 +347,7 @@ def cleanup! # WARNING! This will pause replication on the nodes this machine aggregates from # And perform expensive row count operations on them def validate_aggregate_row_counts(restart_monitoring = true, tables = false) - tables = Table.from_config 'sharded_tables' unless tables + tables = Table.from_config('sharded_tables', pool.shard_pool.name) unless tables query_nodes = [ slaves, aggregating_nodes ].flatten aggregating_nodes.concurrent_each do |node| node.disable_monitoring diff --git a/plugins/merge_helper/lib/shard.rb b/plugins/merge_helper/lib/shard.rb index 3a7cae3..2b19fce 100644 --- a/plugins/merge_helper/lib/shard.rb +++ b/plugins/merge_helper/lib/shard.rb @@ -4,7 +4,7 @@ module Jetpants class Shard # Runs queries against a slave in the pool to verify sharding key values def validate_shard_data - tables = Table.from_config('sharded_tables', self.shard_pool) + tables = Table.from_config('sharded_tables', shard_pool.name) table_statuses = {} tables.limited_concurrent_map(8) { |table| table.sharding_keys.each do |col| @@ -125,7 +125,7 @@ def self.find_duplicate_keys(shards, table, key, min_key_val = nil, max_key_val # Generate a list of filenames for exported data def table_export_filenames(full_path = true, tables = false) export_filenames = [] - tables = Table.from_config 'sharded_tables' unless tables + tables = Table.from_config('sharded_tables', shard_pool.name) unless tables export_filenames = tables.map { |table| table.export_filenames(@min_id, @max_id) }.flatten export_filenames.map!{ |filename| File.basename filename } unless full_path @@ -159,7 +159,7 @@ def self.set_up_aggregate_node(shards_to_merge, aggregate_node, new_shard_master slaves_to_replicate = shards_to_merge.map { |shard| shard.standby_slaves.last } # sharded table list to ship - tables = Plugin::MergeHelper.tables_to_merge + tables = Plugin::MergeHelper.tables_to_merge(shards_to_merge.first.shard_pool.name) # data export counts for validation later export_counts = {} diff --git a/plugins/merge_helper/merge_helper.rb b/plugins/merge_helper/merge_helper.rb index 47cde4f..34ab732 100644 --- a/plugins/merge_helper/merge_helper.rb +++ b/plugins/merge_helper/merge_helper.rb @@ -3,8 +3,8 @@ module Plugin module MergeHelper class << self # Provide a config hook to specify a list of tables to merge, overriding the sharded_tables list - def tables_to_merge - tables = Table.from_config 'sharded_tables' + def tables_to_merge(shard_pool) + tables = Table.from_config('sharded_tables', shard_pool) table_list = [] if (!Jetpants.plugins['merge_helper'].nil? && Jetpants.plugins['merge_helper'].has_key?('table_list')) table_list = Jetpants.plugins['merge_helper']['table_list'] From 80c0b0acb400bdb09202da029bdea3e7967165df Mon Sep 17 00:00:00 2001 From: Tom Christ Date: Tue, 2 Jun 2015 15:32:54 -0400 Subject: [PATCH 62/86] Allow for anon pool naming --- lib/jetpants/shard.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/jetpants/shard.rb b/lib/jetpants/shard.rb index a1f5168..5f490c9 100644 --- a/lib/jetpants/shard.rb +++ b/lib/jetpants/shard.rb @@ -61,7 +61,8 @@ def initialize(min_id, max_id, master, state=:ready, shard_pool_name=nil) # Generates a string containing the shard's min and max IDs. Plugin may want to override. def generate_name - "#{@shard_pool.name.downcase}-#{min_id}-#{max_id.to_s.downcase}" + prefix = (@shard_pool.nil?) ? 'anon' : @shard_pool.name.downcase + "#{prefix}-#{min_id}-#{max_id.to_s.downcase}" end # Returns true if the shard state is one of the values that indicates it's From d5c5a3e0a1fa16ce041d574a9454adc4bbeb81ae Mon Sep 17 00:00:00 2001 From: Tom Christ Date: Wed, 3 Jun 2015 01:26:18 -0400 Subject: [PATCH 63/86] Use shard pool names correctly and various tweaks --- bin/jetpants | 3 ++- lib/jetpants/db/schema.rb | 2 +- plugins/jetpants_collins/shard.rb | 4 +++- plugins/jetpants_collins/topology.rb | 2 +- plugins/merge_helper/lib/aggregator.rb | 2 +- plugins/merge_helper/lib/commandsuite.rb | 15 +++++++++------ plugins/merge_helper/lib/shard.rb | 2 +- 7 files changed, 18 insertions(+), 12 deletions(-) diff --git a/bin/jetpants b/bin/jetpants index 0ce17a1..6be377f 100755 --- a/bin/jetpants +++ b/bin/jetpants @@ -825,7 +825,8 @@ module Jetpants end def ask_shard_being_split - shards_being_split = Jetpants.shards.select {|s| s.children.count > 0} + shard_pool = ask("Enter shard pool to take action on:") + shards_being_split = Jetpants.shards(shard_pool).select {|s| s.children.count > 0} if shards_being_split.count == 0 raise 'No shards are currently being split. You can only use this task after running "jetpants shard_split".' elsif shards_being_split.count == 1 diff --git a/lib/jetpants/db/schema.rb b/lib/jetpants/db/schema.rb index 281afe8..da9df24 100644 --- a/lib/jetpants/db/schema.rb +++ b/lib/jetpants/db/schema.rb @@ -24,7 +24,7 @@ def detect_table_schema(table_name) 'columns' => connection.schema(table_name).map{|schema| schema[0]} } - config_params = Jetpants.send('sharded_tables')[pool.shard_pool] + config_params = Jetpants.send('sharded_tables')[pool.shard_pool.name.downcase] unless(config_params[table_name].nil?) params.merge!(config_params[table_name]) diff --git a/plugins/jetpants_collins/shard.rb b/plugins/jetpants_collins/shard.rb index 67fb295..77863ee 100644 --- a/plugins/jetpants_collins/shard.rb +++ b/plugins/jetpants_collins/shard.rb @@ -18,6 +18,7 @@ def collins_asset(create_if_missing=false) primary_role: 'MYSQL_SHARD', shard_min_id: "^#{@min_id}$", shard_max_id: "^#{@max_id}$", + shard_pool: "^#{@shard_pool.name}$" } selector[:remoteLookup] = true if Jetpants.plugins['jetpants_collins']['remote_lookup'] @@ -45,7 +46,8 @@ def collins_asset(create_if_missing=false) primary_role: 'MYSQL_SHARD', pool: @name.upcase, shard_min_id: @min_id, - shard_max_id: @max_id + shard_max_id: @max_id, + shard_pool: @shard_pool.name.upcase Plugin::JetCollins.get new_tag elsif results.count == 0 && !create_if_missing raise "Could not find configuration asset for pool #{name}" diff --git a/plugins/jetpants_collins/topology.rb b/plugins/jetpants_collins/topology.rb index 7a8451f..f37c962 100644 --- a/plugins/jetpants_collins/topology.rb +++ b/plugins/jetpants_collins/topology.rb @@ -327,7 +327,7 @@ def query_spare_assets(count, options={}) nodes.map(&:to_db).concurrent_each {|db| db.probe rescue nil} # Now iterate in a single-threaded way for simplicity - nodes.each do |node| + nodes.concurrent_each do |node| db = node.to_db if(db.usable_spare? && ( diff --git a/plugins/merge_helper/lib/aggregator.rb b/plugins/merge_helper/lib/aggregator.rb index b0bba51..81f618f 100644 --- a/plugins/merge_helper/lib/aggregator.rb +++ b/plugins/merge_helper/lib/aggregator.rb @@ -347,7 +347,7 @@ def cleanup! # WARNING! This will pause replication on the nodes this machine aggregates from # And perform expensive row count operations on them def validate_aggregate_row_counts(restart_monitoring = true, tables = false) - tables = Table.from_config('sharded_tables', pool.shard_pool.name) unless tables + tables = Table.from_config('sharded_tables', aggregating_nodes.first.pool.shard_pool.name) unless tables query_nodes = [ slaves, aggregating_nodes ].flatten aggregating_nodes.concurrent_each do |node| node.disable_monitoring diff --git a/plugins/merge_helper/lib/commandsuite.rb b/plugins/merge_helper/lib/commandsuite.rb index 6b6835f..e4480d1 100644 --- a/plugins/merge_helper/lib/commandsuite.rb +++ b/plugins/merge_helper/lib/commandsuite.rb @@ -78,16 +78,17 @@ def merge_shards raise "Invalid aggregate node!" unless aggregate_node.aggregator? # claim the slaves further along in the process - aggregate_shard_master = ask_node("Enter the IP address of the new master or press enter to select a spare:") + aggregate_shard_master_ip = ask("Enter the IP address of the new master or press enter to select a spare:") - if aggregate_shard_master + unless aggregate_shard_master_ip.empty? + aggregate_shard_master = aggregate_shard_master.to_db aggregate_shard_master.claim! if aggregate_shard_master.is_spare? else aggregate_shard_master = Jetpants.topology.claim_spare(role: :master, like: shards_to_merge.first.master) end # claim node for the new shard master - spare_count = shards_to_merge.first.slaves_layout[:standby_slave] + 1; + spare_count = shards_to_merge.first.slaves_layout[:standby_slave]; raise "Not enough spares available!" unless Jetpants.count_spares(like: aggregate_shard_master) >= spare_count raise "Not enough backup_slave role spare machines!" unless Jetpants.topology.count_spares(role: :backup_slave) >= shards_to_merge.first.slaves_layout[:backup_slave] @@ -283,7 +284,9 @@ def validate_merge_replication no_tasks do def ask_merge_shards - shards_to_merge = Jetpants.shards.select{ |shard| !shard.combined_shard.nil? } + shard_pool_name = ask("Enter shard pool name performing a merge operation (enter for default #{Jetpants.topology.default_shard_pool}):") + shard_pool_name = Jetpants.topology.default_shard_pool if shard_pool_name.empty? + shards_to_merge = Jetpants.shards(shard_pool_name).select{ |shard| !shard.combined_shard.nil? } shards_str = shards_to_merge.join(', ') answer = ask "Detected shards to merge as #{shards_str}, proceed (enter YES in all caps if so)?" raise "Aborting on user input" unless answer == "YES" @@ -295,8 +298,8 @@ def ask_merge_shard_ranges min_id = ask("Please provide the min ID of the shard range to merge:") max_id = ask("Please provide the max ID of the shard range to merge:") - shard_pool = ask('Please enter the sharding pool which to perform the action on (enter for default pool): ') - shard_pool = default_shard_pool if shard_pool.empty? + shard_pool = ask("Please enter the sharding pool which to perform the action on (enter for default pool #{Jetpants.topology.default_shard_pool}): ") + shard_pool = Jetpants.topology.default_shard_pool if shard_pool.empty? # for now we assume we'll never merge the shard at the head of the list shards_to_merge = Jetpants.shards(shard_pool).select do |shard| diff --git a/plugins/merge_helper/lib/shard.rb b/plugins/merge_helper/lib/shard.rb index 2b19fce..99ccc79 100644 --- a/plugins/merge_helper/lib/shard.rb +++ b/plugins/merge_helper/lib/shard.rb @@ -235,7 +235,7 @@ def self.set_up_aggregate_node(shards_to_merge, aggregate_node, new_shard_master end def combined_shard - Jetpants.shards.select { |shard| ( + Jetpants.shards(shard_pool.name).select { |shard| ( shard.min_id.to_i <= @min_id.to_i \ && shard.max_id.to_i >= @max_id.to_i \ && shard.max_id != 'INFINITY' \ From 5f15da49996f19d72e21530b0f2ba844f8311669 Mon Sep 17 00:00:00 2001 From: Tom Christ Date: Wed, 3 Jun 2015 01:45:19 -0400 Subject: [PATCH 64/86] Raise error if not shards detected merging --- plugins/merge_helper/lib/commandsuite.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/merge_helper/lib/commandsuite.rb b/plugins/merge_helper/lib/commandsuite.rb index e4480d1..9b42397 100644 --- a/plugins/merge_helper/lib/commandsuite.rb +++ b/plugins/merge_helper/lib/commandsuite.rb @@ -287,6 +287,7 @@ def ask_merge_shards shard_pool_name = ask("Enter shard pool name performing a merge operation (enter for default #{Jetpants.topology.default_shard_pool}):") shard_pool_name = Jetpants.topology.default_shard_pool if shard_pool_name.empty? shards_to_merge = Jetpants.shards(shard_pool_name).select{ |shard| !shard.combined_shard.nil? } + raise("No shards detected as merging!") if shards_to_merge.empty? shards_str = shards_to_merge.join(', ') answer = ask "Detected shards to merge as #{shards_str}, proceed (enter YES in all caps if so)?" raise "Aborting on user input" unless answer == "YES" From 4e0e19a4f21bd71e64b0faaae889a20f73cc780e Mon Sep 17 00:00:00 2001 From: Tom Christ Date: Thu, 4 Jun 2015 23:21:42 -0400 Subject: [PATCH 65/86] Fix sorting and shard pool selection --- lib/jetpants/topology.rb | 3 ++- plugins/jetpants_collins/topology.rb | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/jetpants/topology.rb b/lib/jetpants/topology.rb index ef20de3..5826aad 100644 --- a/lib/jetpants/topology.rb +++ b/lib/jetpants/topology.rb @@ -160,6 +160,7 @@ def shards(shard_pool_name = nil) if shard_pool_name.nil? shard_pool_name = default_shard_pool output "Using default shard pool #{default_shard_pool}" +puts caller end pools.select {|p| p.is_a? Shard}.select { |p| p.shard_pool && p.shard_pool.name.downcase == shard_pool_name.downcase } end @@ -195,7 +196,7 @@ def shard(min_id, max_id, shard_pool = nil) # Finds a ShardPool object by name def shard_pool(name) - shard_pools.select{|sp| sp.name == name}.first + shard_pools.select{|sp| sp.name.downcase == name.downcase}.first end # Returns the Jetpants::Shard that handles the given ID. diff --git a/plugins/jetpants_collins/topology.rb b/plugins/jetpants_collins/topology.rb index f37c962..ffd662b 100644 --- a/plugins/jetpants_collins/topology.rb +++ b/plugins/jetpants_collins/topology.rb @@ -371,17 +371,19 @@ def sort_assets_for_pool(pool, assets) def sort_pools_callback(pool) asset = pool.collins_asset role = asset.primary_role.upcase + shard_pool = '' case role when 'MYSQL_POOL' position = (asset.config_sort_order || 0).to_i when 'MYSQL_SHARD' position = asset.shard_min_id.to_i + shard_pool = pool.shard_pool.name else position = 0 end - [role, position] + [role, shard_pool, position] end end From 2ceb934ad5812b8239c5a9e8b566ec34582932ad Mon Sep 17 00:00:00 2001 From: Tom Christ Date: Fri, 5 Jun 2015 12:15:42 -0400 Subject: [PATCH 66/86] Touch up shardpool comments --- lib/jetpants/shardpool.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/jetpants/shardpool.rb b/lib/jetpants/shardpool.rb index 8e7a8c1..1812058 100644 --- a/lib/jetpants/shardpool.rb +++ b/lib/jetpants/shardpool.rb @@ -1,6 +1,6 @@ module Jetpants - # A ShardingPool is a sharding keyspace in Jetpants that contains - # man Shards. All shards within the pool partition a logically coherent + # A ShardPool is a sharding keyspace in Jetpants that contains + # many Shards. All shards within the pool partition a logically coherent # keyspace class ShardPool From 963105bb9f043c6a18547b395fa112e365ecc4f8 Mon Sep 17 00:00:00 2001 From: Bob Patterson Jr Date: Mon, 8 Jun 2015 16:00:54 -0400 Subject: [PATCH 67/86] fix the upgrade helper spare ask - was not selecting right version for clone because of the ordering of the check for empty - empty should default to spare ``` Please enter comma-separated list of targets (IPs or "spare") to clone to: => "" => 0 => "spare" => 1 ``` --- plugins/upgrade_helper/commandsuite.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/upgrade_helper/commandsuite.rb b/plugins/upgrade_helper/commandsuite.rb index f8e46ec..90e3694 100644 --- a/plugins/upgrade_helper/commandsuite.rb +++ b/plugins/upgrade_helper/commandsuite.rb @@ -17,8 +17,8 @@ def upgrade_clone_slave puts "You may clone to particular IP address(es), or can type \"spare\" to claim a node from the spare pool." target = options[:target] || ask('Please enter comma-separated list of targets (IPs or "spare") to clone to: ') - spares_needed = target.split(',').count {|t| t.strip.upcase == 'SPARE'} target = 'spare' if target.strip == '' || target.split(',').length == 0 + spares_needed = target.split(',').count {|t| t.strip.upcase == 'SPARE'} if spares_needed > 0 spares_available = Jetpants.topology.count_spares(role: :standby_slave, like: source, version: Plugin::UpgradeHelper.new_version) raise "Not enough upgraded spares with role of standby slave! Requested #{spares_needed} but only have #{spares_available} available." if spares_needed > spares_available From 27957bdf3d12ec4fe332dacbb422f96a4375c606 Mon Sep 17 00:00:00 2001 From: Tom Christ Date: Thu, 11 Jun 2015 15:19:22 -0400 Subject: [PATCH 68/86] Update comments --- plugins/jetpants_collins/shardpool.rb | 4 ++-- plugins/jetpants_collins/topology.rb | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/jetpants_collins/shardpool.rb b/plugins/jetpants_collins/shardpool.rb index a82eb29..bf239dc 100644 --- a/plugins/jetpants_collins/shardpool.rb +++ b/plugins/jetpants_collins/shardpool.rb @@ -1,6 +1,6 @@ module Jetpants - # A ShardingPool is a sharding keyspace in Jetpants that contains - # man Shards. All shards within the pool partition a logically coherent + # A ShardPool is a sharding keyspace in Jetpants that contains + # many Shards. All shards within the pool partition a logically coherent # keyspace class ShardPool diff --git a/plugins/jetpants_collins/topology.rb b/plugins/jetpants_collins/topology.rb index ffd662b..fea3a8d 100644 --- a/plugins/jetpants_collins/topology.rb +++ b/plugins/jetpants_collins/topology.rb @@ -40,7 +40,7 @@ def process_spare_selector_options(selector, options) def load_shard_pools @shard_pools = configuration_assets('MYSQL_SHARD_POOL').map(&:to_shard_pool) - @shard_pools.compact! # remove nils from pools that had no master + @shard_pools.compact! @shard_pools.sort_by! { |p| p.name } true From 3c958a854279149f79b58b2f5343ba1776396acd Mon Sep 17 00:00:00 2001 From: Tom Christ Date: Fri, 12 Jun 2015 11:19:49 -0400 Subject: [PATCH 69/86] Add default shard pool to prompt --- bin/jetpants | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/jetpants b/bin/jetpants index 6be377f..2518497 100755 --- a/bin/jetpants +++ b/bin/jetpants @@ -825,7 +825,7 @@ module Jetpants end def ask_shard_being_split - shard_pool = ask("Enter shard pool to take action on:") + shard_pool = ask("Enter shard pool to take action on (enter for default pool, #{Jetpants.topology.default_shard_pool}):") shards_being_split = Jetpants.shards(shard_pool).select {|s| s.children.count > 0} if shards_being_split.count == 0 raise 'No shards are currently being split. You can only use this task after running "jetpants shard_split".' From e0edf9d07e984f93539bc6bf535b6ff272302f96 Mon Sep 17 00:00:00 2001 From: Tom Christ Date: Mon, 15 Jun 2015 18:48:45 -0400 Subject: [PATCH 70/86] Accurately pass around shard pool --- bin/jetpants | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/jetpants b/bin/jetpants index 2518497..53be858 100755 --- a/bin/jetpants +++ b/bin/jetpants @@ -539,7 +539,7 @@ module Jetpants # has the same master/slaves but now has a non-infinity max ID. last_shard.state = :recycle last_shard.sync_configuration - last_shard_replace = Shard.new(last_shard.min_id, cutover_id - 1, last_shard_master, last_shard.shard_pool.name) + last_shard_replace = Shard.new(last_shard.min_id, cutover_id - 1, last_shard_master, :ready, shard_pool) last_shard_replace.sync_configuration Jetpants.topology.add_pool last_shard_replace @@ -568,12 +568,12 @@ module Jetpants end end - new_last_shard = Shard.new(cutover_id, 'INFINITY', new_last_shard_master) + new_last_shard = Shard.new(cutover_id, 'INFINITY', new_last_shard_master, :ready, shard_pool) new_last_shard.sync_configuration Jetpants.topology.add_pool new_last_shard # Create tables on the new shard's master, obtaining schema from previous last shard - tables = Table.from_config 'sharded_tables' + tables = Table.from_config('sharded_tables', shard_pool) last_shard_master.export_schemata tables last_shard_master.host.fast_copy_chain(Jetpants.export_location, new_last_shard_master.host, files: ["create_tables_#{last_shard_master.port}.sql"]) new_last_shard_master.import_schemata! From 45529d591fbb89981de7d84dd87a00cf28a0b19f Mon Sep 17 00:00:00 2001 From: Tom Christ Date: Tue, 16 Jun 2015 17:40:03 -0400 Subject: [PATCH 71/86] Code cleanup --- bin/jetpants | 11 ++++++----- lib/jetpants/db/import_export.rb | 2 +- lib/jetpants/table.rb | 8 ++++---- lib/jetpants/topology.rb | 9 ++++----- plugins/jetpants_collins/asset.rb | 2 +- plugins/jetpants_collins/shard.rb | 2 +- plugins/jetpants_collins/shardpool.rb | 2 +- plugins/jetpants_collins/topology.rb | 9 +++------ plugins/merge_helper/lib/commandsuite.rb | 9 +++++---- plugins/online_schema_change/lib/topology.rb | 2 +- 10 files changed, 27 insertions(+), 29 deletions(-) diff --git a/bin/jetpants b/bin/jetpants index 53be858..e7e63ce 100755 --- a/bin/jetpants +++ b/bin/jetpants @@ -592,7 +592,7 @@ module Jetpants method_option :min_id, :desc => 'Minimum ID of shard involved in master promotion' method_option :max_id, :desc => 'Maximum ID of shard involved in master promotion' method_option :new_master, :desc => 'New node to become master of the shard' - method_option :shard_pool, :desc => 'The sharding pool for which to perform the cutover' + method_option :shard_pool, :desc => 'The sharding pool for which to perform the promotion' def shard_promote_master shard_pool = options[:shard_pool] || ask("Please enter the sharding pool for which to perform the split (enter for default, #{Jetpants.topology.default_shard_pool}): ") shard_pool = Jetpants.topology.default_shard_pool if shard_pool.empty? @@ -624,7 +624,7 @@ module Jetpants desc 'shard_promote_master_reads', 'Lockless shard master promotion (step 2 of 4): move reads to new master' method_option :min_id, :desc => 'Minimum ID of shard involved in master promotion' method_option :max_id, :desc => 'Maximum ID of shard involved in master promotion' - method_option :shard_pool, :desc => 'The sharding pool for which to perform the cutover' + method_option :shard_pool, :desc => 'The sharding pool for which to perform the promotion' def shard_promote_master_reads shard_pool = options[:shard_pool] || ask("Please enter the sharding pool for which to perform the split (enter for default pool, #{Jetpants.topology.default_shard_pool}): ") shard_pool = Jetpants.topology.default_shard_pool if shard_pool.empty? @@ -654,7 +654,7 @@ module Jetpants end desc 'shard_promote_master_writes', 'Lockless shard master promotion (step 3 of 4): move writes to new master' - method_option :shard_pool, :desc => 'The sharding pool for which to perform the cutover' + method_option :shard_pool, :desc => 'The sharding pool for which to perform the promotion' def shard_promote_master_writes shard_pool = options[:shard_pool] || ask("Please enter the sharding pool for which to perform the split (enter for default pool, #{Jetpants.topology.default_shard_pool}): ") shard_pool = Jetpants.topology.default_shard_pool if shard_pool.empty? @@ -679,7 +679,7 @@ module Jetpants end desc 'shard_promote_master_cleanup', 'Lockless shard master promotion (step 4 of 4): clean up shard and eject old master' - method_option :shard_pool, :desc => 'The sharding pool for which to perform the cutover' + method_option :shard_pool, :desc => 'The sharding pool for which to perform the promotion' def shard_promote_master_cleanup shard_pool = options[:shard_pool] || ask("Please enter the sharding pool for which to perform the split (enter for default pool, #{Jetpants.topology.default_shard_pool}): ") shard_pool = Jetpants.topology.default_shard_pool if shard_pool.empty? @@ -826,6 +826,7 @@ module Jetpants def ask_shard_being_split shard_pool = ask("Enter shard pool to take action on (enter for default pool, #{Jetpants.topology.default_shard_pool}):") + shard_pool = Jetpants.topology.default_shard_pool if shard_pool.empty? shards_being_split = Jetpants.shards(shard_pool).select {|s| s.children.count > 0} if shards_being_split.count == 0 raise 'No shards are currently being split. You can only use this task after running "jetpants shard_split".' @@ -842,7 +843,7 @@ module Jetpants def ask_shard_being_promoted(stage = :prep, min_id = nil, max_id = nil, shard_pool) if stage == :writes || stage == :cleanup - shards_being_promoted = Jetpants.shards.select do |s| + shards_being_promoted = Jetpants.shards(shard_pool).select do |s| [:reads, :child, :needs_cleanup].include?(s.state) && !s.parent && s.master.master end diff --git a/lib/jetpants/db/import_export.rb b/lib/jetpants/db/import_export.rb index 087587c..cad4511 100644 --- a/lib/jetpants/db/import_export.rb +++ b/lib/jetpants/db/import_export.rb @@ -274,7 +274,7 @@ def rebuild!(tables=false, min_id=false, max_id=false) p = pool if p.is_a?(Shard) - tables ||= Table.from_config('sharded_tables', pool.shard_pool.name) + tables ||= Table.from_config('sharded_tables', p.shard_pool.name) min_id ||= p.min_id max_id ||= p.max_id if p.max_id != 'INFINITY' end diff --git a/lib/jetpants/table.rb b/lib/jetpants/table.rb index 7c0244e..33ac36e 100644 --- a/lib/jetpants/table.rb +++ b/lib/jetpants/table.rb @@ -161,10 +161,10 @@ def belongs_to?(pool) # of the given label. # TODO: integrate better with table schema detection code. Consider auto-detecting chunk # count based on file size and row count estimate. - def Table.from_config(label, shard_pool = nil) - shard_pool = Jetpants.topology.default_shard_pool if shard_pool.nil? - raise "Unable to find sharded tables for shard pool `#{shard_pool.downcase}`" if Jetpants.send(label)[shard_pool.downcase].nil? - Jetpants.send(label)[shard_pool.downcase].map {|name, attributes| Table.new name, attributes} + def Table.from_config(label, shard_pool_name = nil) + shard_pool_name = Jetpants.topology.default_shard_pool if shard_pool_name.nil? + raise "Unable to find sharded tables for shard pool `#{shard_pool_name.downcase}`" if Jetpants.send(label)[shard_pool_name.downcase].nil? + Jetpants.send(label)[shard_pool_name.downcase].map {|name, attributes| Table.new name, attributes} end def to_s diff --git a/lib/jetpants/topology.rb b/lib/jetpants/topology.rb index 5826aad..203831c 100644 --- a/lib/jetpants/topology.rb +++ b/lib/jetpants/topology.rb @@ -89,7 +89,7 @@ def load_pools synchronized # Plugin should override this to initialize @shard_pools def load_shard_pools - output "Notice: no plugin has overriddent Topology#load_shard_pools, so *no* shard pools are imported automaticaly" + output "Notice: no plugin has overridden Topology#load_shard_pools, so *no* shard pools are imported automaticaly" end synchronized @@ -160,7 +160,6 @@ def shards(shard_pool_name = nil) if shard_pool_name.nil? shard_pool_name = default_shard_pool output "Using default shard pool #{default_shard_pool}" -puts caller end pools.select {|p| p.is_a? Shard}.select { |p| p.shard_pool && p.shard_pool.name.downcase == shard_pool_name.downcase } end @@ -181,8 +180,8 @@ def pool(target) end # Finds and returns a single Jetpants::Shard - def shard(min_id, max_id, shard_pool = nil) - shard_pool = default_shard_pool if shard_pool.nil? + def shard(min_id, max_id, shard_pool_name = nil) + shard_pool_name = default_shard_pool if shard_pool_name.nil? if max_id.upcase == 'INFINITY' max_id.upcase! else @@ -191,7 +190,7 @@ def shard(min_id, max_id, shard_pool = nil) min_id = min_id.to_i - shards(shard_pool).select {|s| s.min_id == min_id.to_i && s.max_id == max_id}.first + shards(shard_pool_name).select {|s| s.min_id == min_id && s.max_id == max_id}.first end # Finds a ShardPool object by name diff --git a/plugins/jetpants_collins/asset.rb b/plugins/jetpants_collins/asset.rb index da636cc..d586a36 100644 --- a/plugins/jetpants_collins/asset.rb +++ b/plugins/jetpants_collins/asset.rb @@ -16,7 +16,7 @@ def to_host backend_ip_address.to_host end - # Convert a Collins:Asset to a Jetpants::Shard_pool + # Convert a Collins:Asset to a Jetpants::ShardPool def to_shard_pool raise "Can only call to_shard_pool on CONFIGURATION assets, but #{self} has type #{type}" unless type.upcase == 'CONFIGURATION' raise "Unknown primary role #{primary_role} for configuration asset #{self}" unless primary_role.upcase == 'MYSQL_SHARD_POOL' diff --git a/plugins/jetpants_collins/shard.rb b/plugins/jetpants_collins/shard.rb index 77863ee..ea3a8e2 100644 --- a/plugins/jetpants_collins/shard.rb +++ b/plugins/jetpants_collins/shard.rb @@ -15,7 +15,7 @@ def collins_asset(create_if_missing=false) operation: 'and', details: true, type: 'CONFIGURATION', - primary_role: 'MYSQL_SHARD', + primary_role: '^MYSQL_SHARD$', shard_min_id: "^#{@min_id}$", shard_max_id: "^#{@max_id}$", shard_pool: "^#{@shard_pool.name}$" diff --git a/plugins/jetpants_collins/shardpool.rb b/plugins/jetpants_collins/shardpool.rb index bf239dc..fa9fed1 100644 --- a/plugins/jetpants_collins/shardpool.rb +++ b/plugins/jetpants_collins/shardpool.rb @@ -16,7 +16,7 @@ def collins_asset(create_if_missing=false) operation: 'and', details: true, type: 'CONFIGURATION', - primary_role: 'MYSQL_SHARD_POOL', + primary_role: '^MYSQL_SHARD_POOL$', shard_pool: "^#{@name.upcase}$", status: 'Allocated', } diff --git a/plugins/jetpants_collins/topology.rb b/plugins/jetpants_collins/topology.rb index fea3a8d..72de3a2 100644 --- a/plugins/jetpants_collins/topology.rb +++ b/plugins/jetpants_collins/topology.rb @@ -323,10 +323,7 @@ def query_spare_assets(count, options={}) keep_assets = [] - # Probe concurrently for speed reasons nodes.map(&:to_db).concurrent_each {|db| db.probe rescue nil} - - # Now iterate in a single-threaded way for simplicity nodes.concurrent_each do |node| db = node.to_db if(db.usable_spare? && @@ -371,19 +368,19 @@ def sort_assets_for_pool(pool, assets) def sort_pools_callback(pool) asset = pool.collins_asset role = asset.primary_role.upcase - shard_pool = '' + shard_pool_name = '' case role when 'MYSQL_POOL' position = (asset.config_sort_order || 0).to_i when 'MYSQL_SHARD' position = asset.shard_min_id.to_i - shard_pool = pool.shard_pool.name + shard_pool_name = pool.shard_pool.name else position = 0 end - [role, shard_pool, position] + [role, shard_pool_name, position] end end diff --git a/plugins/merge_helper/lib/commandsuite.rb b/plugins/merge_helper/lib/commandsuite.rb index 9b42397..dff22bf 100644 --- a/plugins/merge_helper/lib/commandsuite.rb +++ b/plugins/merge_helper/lib/commandsuite.rb @@ -81,7 +81,8 @@ def merge_shards aggregate_shard_master_ip = ask("Enter the IP address of the new master or press enter to select a spare:") unless aggregate_shard_master_ip.empty? - aggregate_shard_master = aggregate_shard_master.to_db + error "Node (#{aggregate_shard_master_ip.blue}) does not appear to be an IP address." unless is_ip? aggregate_shard_master_ip + aggregate_shard_master = aggregate_shard_master_ip.to_db aggregate_shard_master.claim! if aggregate_shard_master.is_spare? else aggregate_shard_master = Jetpants.topology.claim_spare(role: :master, like: shards_to_merge.first.master) @@ -296,12 +297,12 @@ def ask_merge_shards end def ask_merge_shard_ranges - min_id = ask("Please provide the min ID of the shard range to merge:") - max_id = ask("Please provide the max ID of the shard range to merge:") - shard_pool = ask("Please enter the sharding pool which to perform the action on (enter for default pool #{Jetpants.topology.default_shard_pool}): ") shard_pool = Jetpants.topology.default_shard_pool if shard_pool.empty? + min_id = ask("Please provide the min ID of the shard range to merge:") + max_id = ask("Please provide the max ID of the shard range to merge:") + # for now we assume we'll never merge the shard at the head of the list shards_to_merge = Jetpants.shards(shard_pool).select do |shard| shard.min_id.to_i >= min_id.to_i && diff --git a/plugins/online_schema_change/lib/topology.rb b/plugins/online_schema_change/lib/topology.rb index 967486a..73f3f4c 100644 --- a/plugins/online_schema_change/lib/topology.rb +++ b/plugins/online_schema_change/lib/topology.rb @@ -35,7 +35,7 @@ def alter_table_shards(database, table, alter, dry_run=true, no_check_plan=false # also I will do the first shard and ask if you want to # continue, after that it will do each table serially def drop_old_alter_table_shards(database, table, shard_pool) - my_shards = shards.dup + my_shards = shards(shard_pool).dup first_shard = my_shards.shift output "Will run on first shard and prompt before going on to the rest\n\n" output "[#{Time.now.to_s.blue}] #{first_shard.pool.to_s}\n" From 42cf75602d512a2ce25600bac3015e893bf70610 Mon Sep 17 00:00:00 2001 From: Tom Christ Date: Tue, 16 Jun 2015 18:49:24 -0400 Subject: [PATCH 72/86] Remove explicit timestamps --- plugins/online_schema_change/lib/topology.rb | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/plugins/online_schema_change/lib/topology.rb b/plugins/online_schema_change/lib/topology.rb index 73f3f4c..808fde6 100644 --- a/plugins/online_schema_change/lib/topology.rb +++ b/plugins/online_schema_change/lib/topology.rb @@ -5,11 +5,12 @@ class Topology # if you specify dry run it will run a dry run on all the shards # otherwise it will run on the first shard and ask if you want to # continue on the rest of the shards, 10 shards at a time - def alter_table_shards(database, table, alter, dry_run=true, no_check_plan=false, shard_pool) + def alter_table_shards(database, table, alter, dry_run=true, no_check_plan=false, shard_pool = nil) + shard_pool = Jetpants.topology.default_shard_pool if shard_pool.nil? my_shards = shards(shard_pool).dup first_shard = my_shards.shift output "Will run on first shard and prompt for going past the dry run only on the first shard\n\n" - output "[#{Time.now.to_s.blue}] #{first_shard.pool.to_s}\n" + output "#{first_shard.pool.to_s}\n" unless first_shard.alter_table(database, table, alter, dry_run, false) output "First shard had an error, please check output\n" return @@ -20,7 +21,7 @@ def alter_table_shards(database, table, alter, dry_run=true, no_check_plan=false errors = [] my_shards.limited_concurrent_map(10) do |shard| - output "[#{Time.now.to_s.blue}] #{shard.pool.to_s}\n" + output "#{shard.pool.to_s}\n" errors << shard unless shard.alter_table(database, table, alter, dry_run, true, no_check_plan) end @@ -34,17 +35,18 @@ def alter_table_shards(database, table, alter, dry_run=true, no_check_plan=false # this is because we do not drop the old table in the osc # also I will do the first shard and ask if you want to # continue, after that it will do each table serially - def drop_old_alter_table_shards(database, table, shard_pool) + def drop_old_alter_table_shards(database, table, shard_pool = nil) + shard_pool = Jetpants.topology.default_shard_pool if shard_pool.nil? my_shards = shards(shard_pool).dup first_shard = my_shards.shift output "Will run on first shard and prompt before going on to the rest\n\n" - output "[#{Time.now.to_s.blue}] #{first_shard.pool.to_s}\n" + output "#{first_shard.pool.to_s}\n" first_shard.drop_old_alter_table(database, table) continue = ask('First shard complete would you like to continue with the rest of the shards?: (YES/no) - YES has to be in all caps and fully typed') if continue == 'YES' my_shards.each do |shard| - print "[#{Time.now.to_s.blue}] #{shard.pool.to_s}\n" + print "#{shard.pool.to_s}\n" shard.drop_old_alter_table(database, table) end end From 85ebee2efbd05470ac68569be3856a2fb8836804 Mon Sep 17 00:00:00 2001 From: Tom Christ Date: Fri, 19 Jun 2015 14:22:25 -0400 Subject: [PATCH 73/86] Select correct pool when promoting --- bin/jetpants | 2 +- lib/jetpants/topology.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/jetpants b/bin/jetpants index e7e63ce..5ab6887 100755 --- a/bin/jetpants +++ b/bin/jetpants @@ -863,7 +863,7 @@ module Jetpants max_id = ask("Enter max id of shard to perform master promotion: ") max_id = Integer(max_id) rescue max_id.upcase - s = Jetpants.topology.shard(min_id, max_id) + s = Jetpants.topology.shard(min_id, max_id, shard_pool) end raise "Invalid shard selected!" unless s.is_a? Shard diff --git a/lib/jetpants/topology.rb b/lib/jetpants/topology.rb index 203831c..c30ad2c 100644 --- a/lib/jetpants/topology.rb +++ b/lib/jetpants/topology.rb @@ -182,7 +182,7 @@ def pool(target) # Finds and returns a single Jetpants::Shard def shard(min_id, max_id, shard_pool_name = nil) shard_pool_name = default_shard_pool if shard_pool_name.nil? - if max_id.upcase == 'INFINITY' + if max_id.is_a?(String) && max_id.upcase == 'INFINITY' max_id.upcase! else max_id = max_id.to_i From 90760bd7cdcbc119aa350f9454aa19babbf4c69a Mon Sep 17 00:00:00 2001 From: Tom Christ Date: Fri, 17 Jul 2015 15:54:19 -0400 Subject: [PATCH 74/86] Implicitly load shard pools when loading shards --- plugins/jetpants_collins/topology.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/plugins/jetpants_collins/topology.rb b/plugins/jetpants_collins/topology.rb index 72de3a2..bc25758 100644 --- a/plugins/jetpants_collins/topology.rb +++ b/plugins/jetpants_collins/topology.rb @@ -48,6 +48,8 @@ def load_shard_pools # Initializes list of pools + shards from Collins def load_pools + load_shard_pools if @shard_pools.nil? + # We keep a cache of Collins::Asset objects, organized as pool_name => role => [asset, asset, ...] @pool_role_assets = {} From 4ccb9928429b54dd79e8ea3f022cac163825aeef Mon Sep 17 00:00:00 2001 From: Bob Patterson Jr Date: Wed, 29 Jul 2015 14:10:18 -0400 Subject: [PATCH 75/86] update upgrade_helper to be aware of backup slaves --- plugins/upgrade_helper/commandsuite.rb | 2 +- plugins/upgrade_helper/shard.rb | 20 ++++++++++++++++---- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/plugins/upgrade_helper/commandsuite.rb b/plugins/upgrade_helper/commandsuite.rb index 92acbe2..db4b9c2 100644 --- a/plugins/upgrade_helper/commandsuite.rb +++ b/plugins/upgrade_helper/commandsuite.rb @@ -125,7 +125,7 @@ def self.after_upgrade_promotion method_option :shard_pool, :desc => 'The sharding pool for which to perform the upgrade' def shard_upgrade shard_pool = options[:shard_pool] || ask('Please enter the sharding pool which to perform the action on (enter for default pool): ') - shard_pool = default_shard_pool if shard_pool.empty? + shard_pool = Jetpants.topology.default_shard_pool if shard_pool.empty? if options[:reads] raise 'The --reads, --writes, and --cleanup options are mutually exclusive' if options[:writes] || options[:cleanup] diff --git a/plugins/upgrade_helper/shard.rb b/plugins/upgrade_helper/shard.rb index 46c81f0..90d9bcd 100644 --- a/plugins/upgrade_helper/shard.rb +++ b/plugins/upgrade_helper/shard.rb @@ -6,10 +6,22 @@ def branched_upgrade_prep raise "Shard #{self} in wrong state to perform this action! expected :ready, found #{@state}" unless @state == :ready raise "Not enough standby slaves of this shard!" unless standby_slaves.size >= slaves_layout[:standby_slave] source = standby_slaves.last - spares_available = Jetpants.topology.count_spares(role: :standby_slave, like: source, version: Plugin::UpgradeHelper.new_version) - raise "Not enough spares available!" unless spares_available >= 1 + slaves_layout[:standby_slave] - - targets = Jetpants.topology.claim_spares(1 + slaves_layout[:standby_slave], role: :standby_slave, like: source, version: Plugin::UpgradeHelper.new_version) + + spares_needed = {'standby' => slaves_layout[:standby_slave] + 1, 'backup' => slaves_layout[:backup_slave]} + + # Array to hold all the target nodes + targets = [] + + spares_needed.map do |role, needed| + next if needed == 0 + available = Jetpants.topology.count_spares(role: "#{role}_slave".to_sym, like: source, version: Plugin::UpgradeHelper.new_version) + raise "Not enough spare machines with role of #{role} slave! Requested #{needed} but only have #{available} available." if needed > available + end + + spares_needed.map do |role, needed| + next if needed == 0 + targets.concat Jetpants.topology.claim_spares(needed, role: "#{role}_slave".to_sym, like: source, version: Plugin::UpgradeHelper.new_version) + end # Disable fast shutdown on the source source.mysql_root_cmd 'SET GLOBAL innodb_fast_shutdown = 0' From 45ae981c7478f68856961c4f3f5ce520b969b382 Mon Sep 17 00:00:00 2001 From: Bob Patterson Jr Date: Wed, 29 Jul 2015 14:38:47 -0400 Subject: [PATCH 76/86] update upgrade_helper to pause all child slaves together --- plugins/upgrade_helper/shard.rb | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/plugins/upgrade_helper/shard.rb b/plugins/upgrade_helper/shard.rb index 90d9bcd..6442641 100644 --- a/plugins/upgrade_helper/shard.rb +++ b/plugins/upgrade_helper/shard.rb @@ -40,11 +40,14 @@ def branched_upgrade_prep # Make the 1st new slave be the "future master" which the other new # slaves will replicate from future_master = targets.shift - targets.each do |t| - future_master.pause_replication_with t - t.change_master_to future_master - [future_master, t].each {|db| db.resume_replication; db.catch_up_to_master} + future_master.pause_replication_with *targets + targets.concurrent_each do |slave| + slave.change_master_to future_master + slave.resume_replication + slave.catch_up_to_master end + future_master.resume_replication + future_master.catch_up_to_master end # Hack the pool configuration to send reads to the new master, but still send From 84477276999774f98dd426c64b1c2d3e4c43a19d Mon Sep 17 00:00:00 2001 From: Bob Patterson Jr Date: Thu, 30 Jul 2015 14:36:09 -0400 Subject: [PATCH 77/86] each not map --- bin/jetpants | 4 ++-- plugins/upgrade_helper/shard.rb | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bin/jetpants b/bin/jetpants index 5ab6887..439b21b 100755 --- a/bin/jetpants +++ b/bin/jetpants @@ -268,13 +268,13 @@ module Jetpants end end - spares_needed.map do |role, needed| + spares_needed.each do |role, needed| next if needed == 0 available = Jetpants.topology.count_spares(role: "#{role}_slave".to_sym, like: compare) raise "Not enough spare machines with role of #{role} slave! Requested #{needed} but only have #{available} available." if needed > available end - spares_needed.map do |role, needed| + spares_needed.each do |role, needed| next if needed == 0 targets.concat Jetpants.topology.claim_spares(needed, role: "#{role}_slave".to_sym, like: compare) end diff --git a/plugins/upgrade_helper/shard.rb b/plugins/upgrade_helper/shard.rb index 6442641..956880e 100644 --- a/plugins/upgrade_helper/shard.rb +++ b/plugins/upgrade_helper/shard.rb @@ -12,13 +12,13 @@ def branched_upgrade_prep # Array to hold all the target nodes targets = [] - spares_needed.map do |role, needed| + spares_needed.each do |role, needed| next if needed == 0 available = Jetpants.topology.count_spares(role: "#{role}_slave".to_sym, like: source, version: Plugin::UpgradeHelper.new_version) raise "Not enough spare machines with role of #{role} slave! Requested #{needed} but only have #{available} available." if needed > available end - spares_needed.map do |role, needed| + spares_needed.each do |role, needed| next if needed == 0 targets.concat Jetpants.topology.claim_spares(needed, role: "#{role}_slave".to_sym, like: source, version: Plugin::UpgradeHelper.new_version) end From d78e79ddb0007774def12e037ff8c53c3897b86c Mon Sep 17 00:00:00 2001 From: Tom Christ Date: Mon, 3 Aug 2015 14:35:45 -0400 Subject: [PATCH 78/86] Only look at the sharded table config if the pool is a shard --- lib/jetpants/db/schema.rb | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/jetpants/db/schema.rb b/lib/jetpants/db/schema.rb index da9df24..43fe2f0 100644 --- a/lib/jetpants/db/schema.rb +++ b/lib/jetpants/db/schema.rb @@ -24,10 +24,12 @@ def detect_table_schema(table_name) 'columns' => connection.schema(table_name).map{|schema| schema[0]} } - config_params = Jetpants.send('sharded_tables')[pool.shard_pool.name.downcase] + if pool.is_a? Shard + config_params = Jetpants.send('sharded_tables')[pool.shard_pool.name.downcase] - unless(config_params[table_name].nil?) - params.merge!(config_params[table_name]) + unless(config_params[table_name].nil?) + params.merge!(config_params[table_name]) + end end Table.new(table_name, params) From 30222b062dc45a5b66ae3a4326f1919b8a238355 Mon Sep 17 00:00:00 2001 From: Bob Patterson Jr Date: Tue, 18 Aug 2015 13:04:06 -0400 Subject: [PATCH 79/86] upgrade helper --- clone from backup slave if available --- plugins/upgrade_helper/shard.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/upgrade_helper/shard.rb b/plugins/upgrade_helper/shard.rb index 956880e..479336b 100644 --- a/plugins/upgrade_helper/shard.rb +++ b/plugins/upgrade_helper/shard.rb @@ -5,7 +5,7 @@ class Shard def branched_upgrade_prep raise "Shard #{self} in wrong state to perform this action! expected :ready, found #{@state}" unless @state == :ready raise "Not enough standby slaves of this shard!" unless standby_slaves.size >= slaves_layout[:standby_slave] - source = standby_slaves.last + source = backup_slaves.empty? ? standby_slaves.last : backup_slaves.last spares_needed = {'standby' => slaves_layout[:standby_slave] + 1, 'backup' => slaves_layout[:backup_slave]} From 52a2b51dc31952ba57f811f85e0ad5345f322b97 Mon Sep 17 00:00:00 2001 From: Bob Patterson Jr Date: Fri, 21 Aug 2015 13:34:08 -0400 Subject: [PATCH 80/86] clone from backup slave if available via slave_for_clone funciton adding to upgrade helper plugin --- lib/jetpants/pool.rb | 4 ++++ plugins/upgrade_helper/shard.rb | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/jetpants/pool.rb b/lib/jetpants/pool.rb index 139990b..23a1c96 100644 --- a/lib/jetpants/pool.rb +++ b/lib/jetpants/pool.rb @@ -377,6 +377,10 @@ def method_missing(name, *args, &block) def respond_to?(name, include_private=false) super || @master.respond_to?(name) end + + def slave_for_clone + backup_slaves.empty? ? standby_slaves.last : backup_slaves.last + end end end diff --git a/plugins/upgrade_helper/shard.rb b/plugins/upgrade_helper/shard.rb index 479336b..9b9526a 100644 --- a/plugins/upgrade_helper/shard.rb +++ b/plugins/upgrade_helper/shard.rb @@ -5,7 +5,7 @@ class Shard def branched_upgrade_prep raise "Shard #{self} in wrong state to perform this action! expected :ready, found #{@state}" unless @state == :ready raise "Not enough standby slaves of this shard!" unless standby_slaves.size >= slaves_layout[:standby_slave] - source = backup_slaves.empty? ? standby_slaves.last : backup_slaves.last + source = slave_for_clone spares_needed = {'standby' => slaves_layout[:standby_slave] + 1, 'backup' => slaves_layout[:backup_slave]} From b15a1430c461f188a129eaabf7bd68002c5df63f Mon Sep 17 00:00:00 2001 From: Tom Christ Date: Mon, 24 Aug 2015 17:18:14 -0400 Subject: [PATCH 81/86] Add ssl replication support --- lib/jetpants.rb | 5 ++++- lib/jetpants/db/replication.rb | 28 +++++++++++++++++++++++++--- lib/jetpants/db/state.rb | 5 +++++ 3 files changed, 34 insertions(+), 4 deletions(-) diff --git a/lib/jetpants.rb b/lib/jetpants.rb index e774e45..7ef099e 100644 --- a/lib/jetpants.rb +++ b/lib/jetpants.rb @@ -49,7 +49,10 @@ module Jetpants 'local_private_interface' => nil, # local network interface corresponding to private IP of the machine jetpants is running on 'free_mem_min_mb' => 0, # Minimum amount of free memory in MB to be maintained on the node while performing the task (eg. network copy) 'default_shard_pool' => nil, # default pool for sharding operations - 'import_without_indices' => false, + 'import_without_indices' => false, + 'ssl_ca_path' => '/var/lib/mysql/ca.pem', + 'ssl_client_cert_path' => '/var/lib/mysql/client-cert.pem', + 'ssl_client_key_path' => '/var/lib/mysql/client-key.pem', } config_paths = ["/etc/jetpants.yaml", "~/.jetpants.yml", "~/.jetpants.yaml"] diff --git a/lib/jetpants/db/replication.rb b/lib/jetpants/db/replication.rb index ca5f2cd..c583ac0 100644 --- a/lib/jetpants/db/replication.rb +++ b/lib/jetpants/db/replication.rb @@ -28,17 +28,39 @@ def change_master_to(new_master, option_hash={}) repl_user = option_hash[:user] || replication_credentials[:user] repl_pass = option_hash[:password] || replication_credentials[:pass] + use_ssl = new_master.use_ssl_replication? && use_ssl_replication? pause_replication if @master && !@repl_paused - result = mysql_root_cmd "CHANGE MASTER TO " + + cmd_str = "CHANGE MASTER TO " + "MASTER_HOST='#{new_master.ip}', " + "MASTER_PORT=#{new_master.port}, " + "MASTER_LOG_FILE='#{logfile}', " + "MASTER_LOG_POS=#{pos}, " + "MASTER_USER='#{repl_user}', " + "MASTER_PASSWORD='#{repl_pass}'" - - output "Changing master to #{new_master} with coordinates (#{logfile}, #{pos}). #{result}" + + if use_ssl + ssl_ca_path = option_hash[:ssl_ca_path] || Jetpants.ssl_ca_path + ssl_client_cert_path = option_hash[:ssl_client_cert_path] || Jetpants.ssl_client_cert_path + ssl_client_key_path = option_hash[:ssl_client_key_path] || Jetpants.ssl_client_key_path + + cmd_str += ", MASTER_SSL=1" + cmd_str += ", MASTER_SSL_CA='#{ssl_ca_path}'" if ssl_ca_path + + if ssl_client_cert_path && ssl_client_key_path + cmd_str += + ", MASTER_SSL_CERT='#{ssl_client_cert_path}', " + + "MASTER_SSL_KEY='#{ssl_client_key_path}'" + end + end + + result = mysql_root_cmd cmd_str + + msg = "Changing master to #{new_master}" + msg += " using SSL" if use_ssl + msg += " with coordinates (#{logfile}, #{pos}). #{result}" + output msg + @master.slaves.delete(self) if @master rescue nil @master = new_master @repl_paused = true diff --git a/lib/jetpants/db/state.rb b/lib/jetpants/db/state.rb index 687f3b2..aa4869a 100644 --- a/lib/jetpants/db/state.rb +++ b/lib/jetpants/db/state.rb @@ -363,6 +363,11 @@ def avg_buffer_pool_hit_rate ((buffer_pool_hit_rate.split[4].to_f * 100) / buffer_pool_hit_rate.split[6].to_f).round(2) end + # Determine whether a server should use ssl as a replication source + def use_ssl_replication? + global_variables[:have_ssl] && global_variables[:have_ssl].downcase == "yes" + end + ###### Private methods ##################################################### private From caaf47fb01b88de0198b7b4bc1d0f627696f89bb Mon Sep 17 00:00:00 2001 From: Tom Christ Date: Wed, 26 Aug 2015 15:23:01 -0400 Subject: [PATCH 82/86] Add stream encryption for fast copy chain --- lib/jetpants/host.rb | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/lib/jetpants/host.rb b/lib/jetpants/host.rb index 1bd7784..9666059 100644 --- a/lib/jetpants/host.rb +++ b/lib/jetpants/host.rb @@ -214,6 +214,12 @@ def fast_copy_chain(base_dir, targets, options={}) else output "Compression disabled -- no compression method specified in Jetpants config file" end + + if Jetpants.encrypt_with && Jetpants.decrypt_with + enc_bin = Jetpants.encrypt_with.split(' ')[0] + confirm_installed enc_bin + output "Using #{enc_bin} for encryption" + end # On each destination host, do any initial setup (and optional validation/erasing), # and then listen for new files. If there are multiple destination hosts, all of them @@ -228,6 +234,12 @@ def fast_copy_chain(base_dir, targets, options={}) decomp_bin = Jetpants.decompress_with.split(' ')[0] t.confirm_installed decomp_bin end + + if Jetpants.encrypt_with && Jetpants.decrypt_with + decrypt_bin = Jetpants.decrypt_with.split(' ')[0] + t.confirm_installed decrypt_bin + end + t.ssh_cmd "mkdir -p #{dir}" # Check if contents already exist / non-empty. @@ -239,6 +251,7 @@ def fast_copy_chain(base_dir, targets, options={}) end decompression_pipe = Jetpants.decompress_with ? "| #{Jetpants.decompress_with}" : '' + decryption_pipe = Jetpants.decrypt_with ? "| #{Jetpants.decrypt_with}" : '' if i == 0 workers << Thread.new { t.ssh_cmd "cd #{dir} && nc -l #{port} #{decompression_pipe} | tar xv" } t.confirm_listening_on_port port @@ -249,7 +262,7 @@ def fast_copy_chain(base_dir, targets, options={}) workers << Thread.new { t.ssh_cmd "cd #{dir} && mkfifo #{fifo} && nc #{tt.ip} #{port} <#{fifo} && rm #{fifo}" } checker_th = Thread.new { t.ssh_cmd "while [ ! -p #{dir}/#{fifo} ] ; do sleep 1; done" } raise "FIFO not found on #{t} after 10 tries" unless checker_th.join(10) - workers << Thread.new { t.ssh_cmd "cd #{dir} && nc -l #{port} | tee #{fifo} #{decompression_pipe} | tar xv" } + workers << Thread.new { t.ssh_cmd "cd #{dir} && nc -l #{port} | tee #{fifo} #{decryption_pipe} #{decompression_pipe} | tar xv" } t.confirm_listening_on_port port t.output "Listening with netcat, and chaining to #{tt}." end @@ -259,7 +272,8 @@ def fast_copy_chain(base_dir, targets, options={}) # Start the copy chain. output "Sending files over to #{targets[0]}: #{file_list}" compression_pipe = Jetpants.compress_with ? "| #{Jetpants.compress_with}" : '' - ssh_cmd "cd #{base_dir} && tar vc #{file_list} #{compression_pipe} | nc #{targets[0].ip} #{port}" + encryption_pipe = Jetpants.encrypt_with ? "| #{Jetpants.encrypt_with}" : '' + ssh_cmd "cd #{base_dir} && tar vc #{file_list} #{compression_pipe} #{encryption_pipe} | nc #{targets[0].ip} #{port}" workers.each {|th| th.join} output "File copy complete." From df355162196e17f4dc90fcf80fb5999d6ec960cc Mon Sep 17 00:00:00 2001 From: Tom Christ Date: Thu, 27 Aug 2015 12:11:34 -0400 Subject: [PATCH 83/86] Add config defaults --- lib/jetpants.rb | 2 ++ lib/jetpants/host.rb | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/jetpants.rb b/lib/jetpants.rb index 7ef099e..02e2611 100644 --- a/lib/jetpants.rb +++ b/lib/jetpants.rb @@ -53,6 +53,8 @@ module Jetpants 'ssl_ca_path' => '/var/lib/mysql/ca.pem', 'ssl_client_cert_path' => '/var/lib/mysql/client-cert.pem', 'ssl_client_key_path' => '/var/lib/mysql/client-key.pem', + 'encrypt_with' => false, # command line stream encryption binary + 'decrypt_with' => false # command line stream decryption binary } config_paths = ["/etc/jetpants.yaml", "~/.jetpants.yml", "~/.jetpants.yaml"] diff --git a/lib/jetpants/host.rb b/lib/jetpants/host.rb index 9666059..b250f0a 100644 --- a/lib/jetpants/host.rb +++ b/lib/jetpants/host.rb @@ -253,7 +253,7 @@ def fast_copy_chain(base_dir, targets, options={}) decompression_pipe = Jetpants.decompress_with ? "| #{Jetpants.decompress_with}" : '' decryption_pipe = Jetpants.decrypt_with ? "| #{Jetpants.decrypt_with}" : '' if i == 0 - workers << Thread.new { t.ssh_cmd "cd #{dir} && nc -l #{port} #{decompression_pipe} | tar xv" } + workers << Thread.new { t.ssh_cmd "cd #{dir} && nc -l #{port} #{decryption_pipe} #{decompression_pipe} | tar xv" } t.confirm_listening_on_port port t.output "Listening with netcat." else From 9a4bff42638be7ba7f296a1337ecfdcfde254b02 Mon Sep 17 00:00:00 2001 From: Tom Christ Date: Mon, 31 Aug 2015 16:45:56 -0400 Subject: [PATCH 84/86] Add logic to conditionally encrypt based on target(s) --- lib/jetpants/host.rb | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/lib/jetpants/host.rb b/lib/jetpants/host.rb index b250f0a..3c14a4f 100644 --- a/lib/jetpants/host.rb +++ b/lib/jetpants/host.rb @@ -215,10 +215,17 @@ def fast_copy_chain(base_dir, targets, options={}) output "Compression disabled -- no compression method specified in Jetpants config file" end - if Jetpants.encrypt_with && Jetpants.decrypt_with + should_encrypt = false + targets.each do |t| + should_encrypt = should_encrypt || should_encrypt_with? t + end + + if Jetpants.encrypt_with && Jetpants.decrypt_with && should_encrypt enc_bin = Jetpants.encrypt_with.split(' ')[0] confirm_installed enc_bin output "Using #{enc_bin} for encryption" + else + output "Not encrypting data stream, either no encryption method specified or encryption unneeded with target" end # On each destination host, do any initial setup (and optional validation/erasing), @@ -235,7 +242,7 @@ def fast_copy_chain(base_dir, targets, options={}) t.confirm_installed decomp_bin end - if Jetpants.encrypt_with && Jetpants.decrypt_with + if Jetpants.encrypt_with && Jetpants.decrypt_with && should_encrypt decrypt_bin = Jetpants.decrypt_with.split(' ')[0] t.confirm_installed decrypt_bin end @@ -251,7 +258,7 @@ def fast_copy_chain(base_dir, targets, options={}) end decompression_pipe = Jetpants.decompress_with ? "| #{Jetpants.decompress_with}" : '' - decryption_pipe = Jetpants.decrypt_with ? "| #{Jetpants.decrypt_with}" : '' + decryption_pipe = (Jetpants.decrypt_with && should_encrypt) ? "| #{Jetpants.decrypt_with}" : '' if i == 0 workers << Thread.new { t.ssh_cmd "cd #{dir} && nc -l #{port} #{decryption_pipe} #{decompression_pipe} | tar xv" } t.confirm_listening_on_port port @@ -272,7 +279,7 @@ def fast_copy_chain(base_dir, targets, options={}) # Start the copy chain. output "Sending files over to #{targets[0]}: #{file_list}" compression_pipe = Jetpants.compress_with ? "| #{Jetpants.compress_with}" : '' - encryption_pipe = Jetpants.encrypt_with ? "| #{Jetpants.encrypt_with}" : '' + encryption_pipe = (Jetpants.encrypt_with && should_encrypt) ? "| #{Jetpants.encrypt_with}" : '' ssh_cmd "cd #{base_dir} && tar vc #{file_list} #{compression_pipe} #{encryption_pipe} | nc #{targets[0].ip} #{port}" workers.each {|th| th.join} output "File copy complete." @@ -284,6 +291,12 @@ def fast_copy_chain(base_dir, targets, options={}) compare_dir base_dir, destinations, options output "Verification successful." end + + # Add a hook point to determine whether a host should encrypt a data stream between two hosts + # This is useful to avoid encryption latency in a secure environment + def should_encrypt_with?(host) + true + end # Given the name of a directory or single file, returns a hash of filename => size of each file present. # Subdirectories will be returned with a size of '/', so you can process these differently as needed. From a1462a6eb8ff49a70f8650dc8bb2c89835e8238e Mon Sep 17 00:00:00 2001 From: Tom Christ Date: Wed, 2 Sep 2015 19:40:56 -0400 Subject: [PATCH 85/86] Use global config setting when testing for encryption --- lib/jetpants.rb | 3 ++- lib/jetpants/host.rb | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/jetpants.rb b/lib/jetpants.rb index 02e2611..d0353ad 100644 --- a/lib/jetpants.rb +++ b/lib/jetpants.rb @@ -54,7 +54,8 @@ module Jetpants 'ssl_client_cert_path' => '/var/lib/mysql/client-cert.pem', 'ssl_client_key_path' => '/var/lib/mysql/client-key.pem', 'encrypt_with' => false, # command line stream encryption binary - 'decrypt_with' => false # command line stream decryption binary + 'decrypt_with' => false, # command line stream decryption binary + 'encrypt_file_transfers' => false # flag to use stream encryption } config_paths = ["/etc/jetpants.yaml", "~/.jetpants.yml", "~/.jetpants.yaml"] diff --git a/lib/jetpants/host.rb b/lib/jetpants/host.rb index 3c14a4f..3c2f02e 100644 --- a/lib/jetpants/host.rb +++ b/lib/jetpants/host.rb @@ -217,7 +217,7 @@ def fast_copy_chain(base_dir, targets, options={}) should_encrypt = false targets.each do |t| - should_encrypt = should_encrypt || should_encrypt_with? t + should_encrypt = should_encrypt || should_encrypt_with?(t) end if Jetpants.encrypt_with && Jetpants.decrypt_with && should_encrypt @@ -295,7 +295,7 @@ def fast_copy_chain(base_dir, targets, options={}) # Add a hook point to determine whether a host should encrypt a data stream between two hosts # This is useful to avoid encryption latency in a secure environment def should_encrypt_with?(host) - true + Jetpants.encrypt_file_transfers end # Given the name of a directory or single file, returns a hash of filename => size of each file present. From 87c69d2631680da20ab7f7fc87376542adc88365 Mon Sep 17 00:00:00 2001 From: Amar Mudrankit Date: Tue, 8 Sep 2015 13:01:45 -0400 Subject: [PATCH 86/86] Better message during master promotion --- bin/jetpants | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bin/jetpants b/bin/jetpants index 439b21b..42f6a3a 100755 --- a/bin/jetpants +++ b/bin/jetpants @@ -594,7 +594,7 @@ module Jetpants method_option :new_master, :desc => 'New node to become master of the shard' method_option :shard_pool, :desc => 'The sharding pool for which to perform the promotion' def shard_promote_master - shard_pool = options[:shard_pool] || ask("Please enter the sharding pool for which to perform the split (enter for default, #{Jetpants.topology.default_shard_pool}): ") + shard_pool = options[:shard_pool] || ask("Please enter the sharding pool for which to perform the master promotion (enter for default, #{Jetpants.topology.default_shard_pool}): ") shard_pool = Jetpants.topology.default_shard_pool if shard_pool.empty? # find the shard we are going to do master promotion on s = ask_shard_being_promoted(:prep, options[:max_id], options[:max_id], shard_pool) @@ -626,7 +626,7 @@ module Jetpants method_option :max_id, :desc => 'Maximum ID of shard involved in master promotion' method_option :shard_pool, :desc => 'The sharding pool for which to perform the promotion' def shard_promote_master_reads - shard_pool = options[:shard_pool] || ask("Please enter the sharding pool for which to perform the split (enter for default pool, #{Jetpants.topology.default_shard_pool}): ") + shard_pool = options[:shard_pool] || ask("Please enter the sharding pool for which to perform the master promotion (enter for default pool, #{Jetpants.topology.default_shard_pool}): ") shard_pool = Jetpants.topology.default_shard_pool if shard_pool.empty? # find the shard we are going to do master promotion on s = ask_shard_being_promoted(:prep, options[:max_id], options[:max_id], shard_pool) @@ -656,7 +656,7 @@ module Jetpants desc 'shard_promote_master_writes', 'Lockless shard master promotion (step 3 of 4): move writes to new master' method_option :shard_pool, :desc => 'The sharding pool for which to perform the promotion' def shard_promote_master_writes - shard_pool = options[:shard_pool] || ask("Please enter the sharding pool for which to perform the split (enter for default pool, #{Jetpants.topology.default_shard_pool}): ") + shard_pool = options[:shard_pool] || ask("Please enter the sharding pool for which to perform the master promotion (enter for default pool, #{Jetpants.topology.default_shard_pool}): ") shard_pool = Jetpants.topology.default_shard_pool if shard_pool.empty? s = ask_shard_being_promoted(:writes, nil, nil, shard_pool) if s.state != :child @@ -681,7 +681,7 @@ module Jetpants desc 'shard_promote_master_cleanup', 'Lockless shard master promotion (step 4 of 4): clean up shard and eject old master' method_option :shard_pool, :desc => 'The sharding pool for which to perform the promotion' def shard_promote_master_cleanup - shard_pool = options[:shard_pool] || ask("Please enter the sharding pool for which to perform the split (enter for default pool, #{Jetpants.topology.default_shard_pool}): ") + shard_pool = options[:shard_pool] || ask("Please enter the sharding pool for which to perform the master promotion (enter for default pool, #{Jetpants.topology.default_shard_pool}): ") shard_pool = Jetpants.topology.default_shard_pool if shard_pool.empty? s = ask_shard_being_promoted(:cleanup, nil, nil, shard_pool) if s.state != :needs_cleanup