From 41cce837bc1f72e4278ec88ca3b86815d63151fe Mon Sep 17 00:00:00 2001 From: Glendon Solsberry Date: Mon, 4 Nov 2013 15:45:51 -0500 Subject: [PATCH 1/5] First pass at handling trigger conversion --- mysql2pgsql/lib/__init__.py | 5 +++++ mysql2pgsql/lib/converter.py | 7 +++++-- mysql2pgsql/lib/mysql_reader.py | 21 +++++++++++++++++++ mysql2pgsql/lib/postgres_db_writer.py | 13 ++++++++++++ mysql2pgsql/lib/postgres_file_writer.py | 11 ++++++++++ mysql2pgsql/lib/postgres_writer.py | 28 +++++++++++++++++++++++++ 6 files changed, 83 insertions(+), 2 deletions(-) diff --git a/mysql2pgsql/lib/__init__.py b/mysql2pgsql/lib/__init__.py index 27dbf36..bbf396b 100644 --- a/mysql2pgsql/lib/__init__.py +++ b/mysql2pgsql/lib/__init__.py @@ -51,6 +51,7 @@ def status_logger(f): constraints_template = 'ADDING CONSTRAINTS ON %s' write_contents_template = 'WRITING DATA TO %s' index_template = 'ADDING INDEXES TO %s' + trigger_template = 'ADDING TRIGGERS TO %s' statuses = { 'truncate': { 'start': start_template % truncate_template, @@ -72,6 +73,10 @@ def status_logger(f): 'start': start_template % index_template, 'finish': finish_template % index_template, }, + 'write_triggers': { + 'start': start_template % trigger_template, + 'finish': finish_template % trigger_template, + }, } @wraps(f) diff --git a/mysql2pgsql/lib/converter.py b/mysql2pgsql/lib/converter.py index 72251ad..02d5d41 100644 --- a/mysql2pgsql/lib/converter.py +++ b/mysql2pgsql/lib/converter.py @@ -55,7 +55,7 @@ def convert(self): if not self.supress_ddl: if self.verbose: - print_start_table('START CREATING INDEXES AND CONSTRAINTS') + print_start_table('START CREATING INDEXES, CONSTRAINTS, AND TRIGGERS') for table in tables: self.writer.write_indexes(table) @@ -63,8 +63,11 @@ def convert(self): for table in tables: self.writer.write_constraints(table) + for table in tables: + self.writer.write_triggers(table) + if self.verbose: - print_start_table('DONE CREATING INDEXES AND CONSTRAINTS') + print_start_table('DONE CREATING INDEXES, CONSTRAINTS, AND TRIGGERS') if self.verbose: print_start_table('\n\n>>>>>>>>>> FINISHED <<<<<<<<<<') diff --git a/mysql2pgsql/lib/mysql_reader.py b/mysql2pgsql/lib/mysql_reader.py index d74a27e..0a259bd 100644 --- a/mysql2pgsql/lib/mysql_reader.py +++ b/mysql2pgsql/lib/mysql_reader.py @@ -3,6 +3,7 @@ import re from contextlib import closing +from pprint import pprint import MySQLdb import MySQLdb.cursors @@ -83,8 +84,10 @@ def __init__(self, reader, name): self._name = name self._indexes = [] self._foreign_keys = [] + self._triggers = [] self._columns = self._load_columns() self._load_indexes() + self._load_triggers() def _convert_type(self, data_type): """Normalize MySQL `data_type`""" @@ -181,6 +184,20 @@ def _load_indexes(self): self._indexes.append(index) continue + def _load_triggers(self): + explain = self.reader.db.query('SHOW TRIGGERS WHERE `table` = \'%s\'' % self.name, one=True) + if type(explain) is tuple: + trigger = {} + trigger['name'] = explain[0] + trigger['event'] = explain[1] + trigger['statement'] = explain[3] + trigger['timing'] = explain[4] + + trigger['statement'] = re.sub('^BEGIN', '', trigger['statement']) + trigger['statement'] = re.sub('^END', '', trigger['statement'], flags=re.MULTILINE) + + self._triggers.append(trigger) + @property def name(self): return self._name @@ -197,6 +214,10 @@ def indexes(self): def foreign_keys(self): return self._foreign_keys + @property + def triggers(self): + return self._triggers + @property def query_for(self): return 'SELECT %(column_names)s FROM `%(table_name)s`' % { diff --git a/mysql2pgsql/lib/postgres_db_writer.py b/mysql2pgsql/lib/postgres_db_writer.py index 2e19170..fc37633 100644 --- a/mysql2pgsql/lib/postgres_db_writer.py +++ b/mysql2pgsql/lib/postgres_db_writer.py @@ -166,6 +166,19 @@ def write_indexes(self, table): for sql in index_sql: self.execute(sql) + @status_logger + def write_triggers(self, table): + """Send DDL to create the specified `table` triggers + + :Parameters: + - `table`: an instance of a :py:class:`mysql2pgsql.lib.mysql_reader.MysqlReader.Table` object that represents the table to read/write. + + Returns None + """ + index_sql = super(PostgresDbWriter, self).write_triggers(table) + for sql in index_sql: + self.execute(sql) + @status_logger def write_constraints(self, table): """Send DDL to create the specified `table` constraints diff --git a/mysql2pgsql/lib/postgres_file_writer.py b/mysql2pgsql/lib/postgres_file_writer.py index 7205f6f..ba5b209 100644 --- a/mysql2pgsql/lib/postgres_file_writer.py +++ b/mysql2pgsql/lib/postgres_file_writer.py @@ -100,6 +100,17 @@ def write_constraints(self, table): """ self.f.write('\n'.join(super(PostgresFileWriter, self).write_constraints(table))) + @status_logger + def write_triggers(self, table): + """Write TRIGGERs existing on `table` to the output file + + :Parameters: + - `table`: an instance of a :py:class:`mysql2pgsql.lib.mysql_reader.MysqlReader.Table` object that represents the table to read/write. + + Returns None + """ + self.f.write('\n'.join(super(PostgresFileWriter, self).write_triggers(table))) + @status_logger def write_contents(self, table, reader): """Write the data contents of `table` to the output file. diff --git a/mysql2pgsql/lib/postgres_writer.py b/mysql2pgsql/lib/postgres_writer.py index 3165520..d83c6f3 100644 --- a/mysql2pgsql/lib/postgres_writer.py +++ b/mysql2pgsql/lib/postgres_writer.py @@ -4,6 +4,7 @@ from cStringIO import StringIO from datetime import datetime, date, timedelta +from pprint import pprint from psycopg2.extensions import QuotedString, Binary, AsIs from pytz import timezone @@ -259,6 +260,33 @@ def write_constraints(self, table): 'ref_column_name': key['ref_column']}) return constraint_sql + def write_triggers(self, table): + trigger_sql = [] + for key in table.triggers: + trigger_sql.append("""CREATE OR REPLACE FUNCTION %(fn_trigger_name)s RETURNS TRIGGER AS $%(trigger_name)s$ + BEGIN + %(trigger_statement)s + RETURN NULL; + END; + $%(trigger_name)s$ LANGUAGE plpgsql;""" % { + 'table_name': table.name, + 'trigger_time': key['timing'], + 'trigger_event': key['event'], + 'trigger_name': key['name'], + 'fn_trigger_name': 'fn_' + key['name'] + '()', + 'trigger_statement': key['statement']}) + + trigger_sql.append("""CREATE TRIGGER %(trigger_name)s %(trigger_time)s %(trigger_event)s ON %(table_name)s + FOR EACH ROW + EXECUTE PROCEDURE fn_%(trigger_name)s();""" % { + 'table_name': table.name, + 'trigger_time': key['timing'], + 'trigger_event': key['event'], + 'trigger_name': key['name']}) + + print trigger_sql + return trigger_sql + def close(self): raise NotImplementedError From 90a8a0dd3db5d5c79c440b060de0bd095a9283c2 Mon Sep 17 00:00:00 2001 From: Glendon Solsberry Date: Tue, 5 Nov 2013 10:40:07 -0500 Subject: [PATCH 2/5] Handle all triggers on tables --- mysql2pgsql/lib/mysql_reader.py | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/mysql2pgsql/lib/mysql_reader.py b/mysql2pgsql/lib/mysql_reader.py index 0a259bd..e4e0fb7 100644 --- a/mysql2pgsql/lib/mysql_reader.py +++ b/mysql2pgsql/lib/mysql_reader.py @@ -185,18 +185,21 @@ def _load_indexes(self): continue def _load_triggers(self): - explain = self.reader.db.query('SHOW TRIGGERS WHERE `table` = \'%s\'' % self.name, one=True) - if type(explain) is tuple: - trigger = {} - trigger['name'] = explain[0] - trigger['event'] = explain[1] - trigger['statement'] = explain[3] - trigger['timing'] = explain[4] - - trigger['statement'] = re.sub('^BEGIN', '', trigger['statement']) - trigger['statement'] = re.sub('^END', '', trigger['statement'], flags=re.MULTILINE) - - self._triggers.append(trigger) + explain = self.reader.db.query('SHOW TRIGGERS WHERE `table` = \'%s\'' % self.name) + for row in explain: + pprint(row) + if type(row) is tuple: + trigger = {} + trigger['name'] = row[0] + trigger['event'] = row[1] + trigger['statement'] = row[3] + trigger['timing'] = row[4] + + trigger['statement'] = re.sub('^BEGIN', '', trigger['statement']) + trigger['statement'] = re.sub('^END', '', trigger['statement'], flags=re.MULTILINE) + trigger['statement'] = re.sub('`', '', trigger['statement']) + + self._triggers.append(trigger) @property def name(self): From 0ff16435dc84107444137de55f5ce982c0f16f29 Mon Sep 17 00:00:00 2001 From: Glendon Solsberry Date: Wed, 6 Nov 2013 08:33:09 -0500 Subject: [PATCH 3/5] Remove unnecessary debugging --- mysql2pgsql/lib/mysql_reader.py | 4 +--- mysql2pgsql/lib/postgres_writer.py | 2 -- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/mysql2pgsql/lib/mysql_reader.py b/mysql2pgsql/lib/mysql_reader.py index e4e0fb7..113ba75 100644 --- a/mysql2pgsql/lib/mysql_reader.py +++ b/mysql2pgsql/lib/mysql_reader.py @@ -3,7 +3,6 @@ import re from contextlib import closing -from pprint import pprint import MySQLdb import MySQLdb.cursors @@ -99,7 +98,7 @@ def _convert_type(self, data_type): return 'boolean' elif re.search(r'^smallint.* unsigned', data_type) or data_type.startswith('mediumint'): return 'integer' - elif data_type.startswith('smallint'): + elif data_type.startswith('smallint') or data_type.startswith('binary('): return 'tinyint' elif data_type.startswith('tinyint') or data_type.startswith('year('): return 'tinyint' @@ -187,7 +186,6 @@ def _load_indexes(self): def _load_triggers(self): explain = self.reader.db.query('SHOW TRIGGERS WHERE `table` = \'%s\'' % self.name) for row in explain: - pprint(row) if type(row) is tuple: trigger = {} trigger['name'] = row[0] diff --git a/mysql2pgsql/lib/postgres_writer.py b/mysql2pgsql/lib/postgres_writer.py index d83c6f3..fcf64e4 100644 --- a/mysql2pgsql/lib/postgres_writer.py +++ b/mysql2pgsql/lib/postgres_writer.py @@ -4,7 +4,6 @@ from cStringIO import StringIO from datetime import datetime, date, timedelta -from pprint import pprint from psycopg2.extensions import QuotedString, Binary, AsIs from pytz import timezone @@ -284,7 +283,6 @@ def write_triggers(self, table): 'trigger_event': key['event'], 'trigger_name': key['name']}) - print trigger_sql return trigger_sql def close(self): From b87af1b5f1e2fb400cda77ca5aa217f2b62673f1 Mon Sep 17 00:00:00 2001 From: Glendon Solsberry Date: Thu, 7 Nov 2013 08:11:32 -0500 Subject: [PATCH 4/5] Remove binary check, as it's not part of this PR --- mysql2pgsql/lib/mysql_reader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mysql2pgsql/lib/mysql_reader.py b/mysql2pgsql/lib/mysql_reader.py index 113ba75..d75b78d 100644 --- a/mysql2pgsql/lib/mysql_reader.py +++ b/mysql2pgsql/lib/mysql_reader.py @@ -98,7 +98,7 @@ def _convert_type(self, data_type): return 'boolean' elif re.search(r'^smallint.* unsigned', data_type) or data_type.startswith('mediumint'): return 'integer' - elif data_type.startswith('smallint') or data_type.startswith('binary('): + elif data_type.startswith('smallint'): return 'tinyint' elif data_type.startswith('tinyint') or data_type.startswith('year('): return 'tinyint' From 4c7e07f3633c58a10ca1f33ff5902f9e4def0268 Mon Sep 17 00:00:00 2001 From: Glendon Solsberry Date: Fri, 8 Nov 2013 12:02:58 -0500 Subject: [PATCH 5/5] Booleans need to be tested before the generic string test; otherwise, they'll always be strings --- mysql2pgsql/lib/postgres_writer.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/mysql2pgsql/lib/postgres_writer.py b/mysql2pgsql/lib/postgres_writer.py index fcf64e4..86c3adb 100644 --- a/mysql2pgsql/lib/postgres_writer.py +++ b/mysql2pgsql/lib/postgres_writer.py @@ -7,7 +7,6 @@ from psycopg2.extensions import QuotedString, Binary, AsIs from pytz import timezone - class PostgresWriter(object): """Base class for :py:class:`mysql2pgsql.lib.postgres_file_writer.PostgresFileWriter` and :py:class:`mysql2pgsql.lib.postgres_db_writer.PostgresDbWriter`. @@ -148,6 +147,9 @@ def process_row(self, table, row): row[index] = '1970-01-01 00:00:00' elif 'bit' in column_type: row[index] = bin(ord(row[index]))[2:] + elif column_type == 'boolean': + # We got here because you used a tinyint(1), if you didn't want a bool, don't use that type + row[index] = 't' if row[index] not in (None, 0) else 'f' if row[index] == 0 else row[index] elif isinstance(row[index], (str, unicode, basestring)): if column_type == 'bytea': row[index] = Binary(row[index]).getquoted()[1:-8] if row[index] else row[index] @@ -155,9 +157,6 @@ def process_row(self, table, row): row[index] = '{%s}' % ','.join('"%s"' % v.replace('"', r'\"') for v in row[index].split(',')) else: row[index] = row[index].replace('\\', r'\\').replace('\n', r'\n').replace('\t', r'\t').replace('\r', r'\r').replace('\0', '') - elif column_type == 'boolean': - # We got here because you used a tinyint(1), if you didn't want a bool, don't use that type - row[index] = 't' if row[index] not in (None, 0) else 'f' if row[index] == 0 else row[index] elif isinstance(row[index], (date, datetime)): if isinstance(row[index], datetime) and self.tz: try: