From 908300fb997dc32c69759ddc043668a57ca9bfc2 Mon Sep 17 00:00:00 2001 From: clevertension Date: Wed, 18 Jun 2014 11:25:32 +0800 Subject: [PATCH 1/4] add isColumnNameQuoted flag --- mysql2pgsql/lib/postgres_writer.py | 10 +++++++--- mysql2pgsql/mysql2pgsql.py | 6 ++++-- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/mysql2pgsql/lib/postgres_writer.py b/mysql2pgsql/lib/postgres_writer.py index 3910328..4fe2156 100644 --- a/mysql2pgsql/lib/postgres_writer.py +++ b/mysql2pgsql/lib/postgres_writer.py @@ -12,9 +12,10 @@ class PostgresWriter(object): """Base class for :py:class:`mysql2pgsql.lib.postgres_file_writer.PostgresFileWriter` and :py:class:`mysql2pgsql.lib.postgres_db_writer.PostgresDbWriter`. """ - def __init__(self, tz=False, index_prefix=''): + def __init__(self, tz=False, index_prefix='', isColumnNameQuoted=False): self.column_types = {} self.index_prefix = index_prefix + self.isColumnNameQuoted = isColumnNameQuoted if tz: self.tz = timezone('UTC') self.tz_offset = '+00:00' @@ -23,7 +24,10 @@ def __init__(self, tz=False, index_prefix=''): self.tz_offset = '' def column_description(self, column): - return '"%s" %s' % (column['name'], self.column_type_info(column)) + quote_tag = '"' + if not self.isColumnNameQuoted: + quote_tag='' + return '%s%s%s %s' % (quote_tag, column['name'] , quote_tag, self.column_type_info(column)) def column_type(self, column): hash_key = hash(frozenset(column.items())) @@ -141,7 +145,7 @@ def table_comments(self, table): def column_comment(self, tablename, column): if not column['comment']: - return (' COMMENT ON COLUMN %s.%s is %s;' % ( tablename, column['name'], QuotedString(column['comment']).getquoted())) + return (' COMMENT ON COLUMN %s.%s is %s;' % ( tablename, column['name'], QuotedString(column['comment'].encode('utf8')).getquoted())) else: return '' diff --git a/mysql2pgsql/mysql2pgsql.py b/mysql2pgsql/mysql2pgsql.py index c2136f0..badac6b 100644 --- a/mysql2pgsql/mysql2pgsql.py +++ b/mysql2pgsql/mysql2pgsql.py @@ -27,12 +27,14 @@ def convert(self): writer = PostgresFileWriter(self._get_file(self.file_options['destination']['file']), self.run_options.verbose, tz=self.file_options.get('timezone', False), - index_prefix=self.file_options.get("index_prefix", '')) + index_prefix=self.file_options.get("index_prefix", ''), + isColumnNameQuoted=self.file_options.get("column_name_quoted", False)) else: writer = PostgresDbWriter(self.file_options['destination']['postgres'], self.run_options.verbose, tz=self.file_options.get('timezone', False), - index_prefix=self.file_options.get("index_prefix", '')) + index_prefix=self.file_options.get("index_prefix", ''), + isColumnNameQuoted=self.file_options.get("column_name_quoted", False)) Converter(reader, writer, self.file_options, self.run_options.verbose).convert() From 49430dfbd3f272605effd4b213e27ccc59812df9 Mon Sep 17 00:00:00 2001 From: clevertension Date: Wed, 18 Jun 2014 11:29:46 +0800 Subject: [PATCH 2/4] add utf-8 encoding to table comment --- mysql2pgsql/lib/postgres_writer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mysql2pgsql/lib/postgres_writer.py b/mysql2pgsql/lib/postgres_writer.py index 4fe2156..822931f 100644 --- a/mysql2pgsql/lib/postgres_writer.py +++ b/mysql2pgsql/lib/postgres_writer.py @@ -150,7 +150,7 @@ def column_comment(self, tablename, column): return '' def table_comment(self, tablename, comment): - return (' COMMENT ON TABLE %s is %s;' % ( tablename, QuotedString(comment).getquoted())) + return (' COMMENT ON TABLE %s is %s;' % ( tablename, QuotedString(comment.encode('utf8')).getquoted())) def process_row(self, table, row): """Examines row data from MySQL and alters From 9f7bcb15198cb1a72970de85a58903056a57b20c Mon Sep 17 00:00:00 2001 From: clevertension Date: Wed, 18 Jun 2014 11:32:19 +0800 Subject: [PATCH 3/4] fix empty sql execution error --- mysql2pgsql/lib/postgres_db_writer.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mysql2pgsql/lib/postgres_db_writer.py b/mysql2pgsql/lib/postgres_db_writer.py index 2a67c15..9823b03 100644 --- a/mysql2pgsql/lib/postgres_db_writer.py +++ b/mysql2pgsql/lib/postgres_db_writer.py @@ -103,6 +103,9 @@ def query(self, sql, args=(), one=False): def execute(self, sql, args=(), many=False): with closing(self.conn.cursor()) as cur: + # avoid to execute empty sql + if not sql or sql.strip() == '': + return if many: cur.executemany(sql, args) else: From 048c94c0b86564c939c730999f6563d69736500f Mon Sep 17 00:00:00 2001 From: clevertension Date: Wed, 18 Jun 2014 12:09:29 +0800 Subject: [PATCH 4/4] add name_quoted flag --- README.rst | 3 ++ mysql2pgsql/lib/postgres_db_writer.py | 2 +- mysql2pgsql/lib/postgres_file_writer.py | 6 +-- mysql2pgsql/lib/postgres_writer.py | 49 +++++++++++++------------ mysql2pgsql/mysql2pgsql.py | 6 +-- tests/mysql2pgsql-test.yml | 3 ++ tests/mysql2pgsql-test.yml.example | 3 ++ 7 files changed, 42 insertions(+), 30 deletions(-) diff --git a/README.rst b/README.rst index 16cfbd7..96d1a81 100644 --- a/README.rst +++ b/README.rst @@ -148,6 +148,9 @@ to edit. For the impatient, here is what the file contains. # if index_prefix is given, indexes will be created whith a name prefixed with index_prefix index_prefix: + # if name_quoted is true, all table_name and column_name will be double quoted + name_quoted: false + Pretty self explainitory right? A couple things to note, first if `destination -> file` is populated all output will be dumped to the specified location regardless of what is contained in `destination -> diff --git a/mysql2pgsql/lib/postgres_db_writer.py b/mysql2pgsql/lib/postgres_db_writer.py index 9823b03..c6e4443 100644 --- a/mysql2pgsql/lib/postgres_db_writer.py +++ b/mysql2pgsql/lib/postgres_db_writer.py @@ -206,4 +206,4 @@ def write_contents(self, table, reader): Returns None """ f = self.FileObjFaker(table, reader.read(table), self.process_row, self.verbose) - self.copy_from(f, '"%s"' % table.name, ['"%s"' % c['name'] for c in table.columns]) + self.copy_from(f, '"%s"' % table.name, ['%s' % self.quoted_name(c['name']) for c in table.columns]) diff --git a/mysql2pgsql/lib/postgres_file_writer.py b/mysql2pgsql/lib/postgres_file_writer.py index ba5b209..cb33894 100644 --- a/mysql2pgsql/lib/postgres_file_writer.py +++ b/mysql2pgsql/lib/postgres_file_writer.py @@ -132,10 +132,10 @@ def write_contents(self, table, reader): -- Data for Name: %(table_name)s; Type: TABLE DATA; -- -COPY "%(table_name)s" (%(column_names)s) FROM stdin; +COPY %(table_name)s (%(column_names)s) FROM stdin; """ % { - 'table_name': table.name, - 'column_names': ', '.join(('"%s"' % col['name']) for col in table.columns)}) + 'table_name': self.quoted_name(table.name), + 'column_names': ', '.join(('%s' % self.quoted_name(col['name'])) for col in table.columns)}) if verbose: tt = time.time start_time = tt() diff --git a/mysql2pgsql/lib/postgres_writer.py b/mysql2pgsql/lib/postgres_writer.py index 822931f..5fc1f92 100644 --- a/mysql2pgsql/lib/postgres_writer.py +++ b/mysql2pgsql/lib/postgres_writer.py @@ -12,10 +12,10 @@ class PostgresWriter(object): """Base class for :py:class:`mysql2pgsql.lib.postgres_file_writer.PostgresFileWriter` and :py:class:`mysql2pgsql.lib.postgres_db_writer.PostgresDbWriter`. """ - def __init__(self, tz=False, index_prefix='', isColumnNameQuoted=False): + def __init__(self, tz=False, index_prefix='', isNameQuoted=False): self.column_types = {} self.index_prefix = index_prefix - self.isColumnNameQuoted = isColumnNameQuoted + self.isNameQuoted = isNameQuoted if tz: self.tz = timezone('UTC') self.tz_offset = '+00:00' @@ -24,11 +24,13 @@ def __init__(self, tz=False, index_prefix='', isColumnNameQuoted=False): self.tz_offset = '' def column_description(self, column): + return '%s %s' % (self.quoted_name(column['name']), self.column_type_info(column)) + + def quoted_name(self, value): quote_tag = '"' - if not self.isColumnNameQuoted: + if not self.isNameQuoted: quote_tag='' - return '%s%s%s %s' % (quote_tag, column['name'] , quote_tag, self.column_type_info(column)) - + return quote_tag + value + quote_tag def column_type(self, column): hash_key = hash(frozenset(column.items())) self.column_types[hash_key] = self.column_type_info(column).split(" ")[0] @@ -145,12 +147,12 @@ def table_comments(self, table): def column_comment(self, tablename, column): if not column['comment']: - return (' COMMENT ON COLUMN %s.%s is %s;' % ( tablename, column['name'], QuotedString(column['comment'].encode('utf8')).getquoted())) + return (' COMMENT ON COLUMN %s.%s is %s;' % ( self.quoted_name(tablename), self.quoted_name(column['name']), QuotedString(column['comment'].encode('utf8')).getquoted())) else: return '' def table_comment(self, tablename, comment): - return (' COMMENT ON TABLE %s is %s;' % ( tablename, QuotedString(comment.encode('utf8')).getquoted())) + return (' COMMENT ON TABLE %s is %s;' % ( self.quoted_name(tablename), QuotedString(comment.encode('utf8')).getquoted())) def process_row(self, table, row): """Examines row data from MySQL and alters @@ -219,12 +221,12 @@ def truncate(self, table): serial_key = column['name'] maxval = 1 if column['maxval'] < 1 else column['maxval'] + 1 - truncate_sql = 'TRUNCATE "%s" CASCADE;' % table.name + truncate_sql = 'TRUNCATE %s CASCADE;' % self.quoted_name(table.name) serial_key_sql = None if serial_key: serial_key_sql = "SELECT pg_catalog.setval(pg_get_serial_sequence(%(table_name)s, %(serial_key)s), %(maxval)s, true);" % { - 'table_name': QuotedString('"%s"' % table.name).getquoted(), + 'table_name': QuotedString('%s' % self.quoted_name(table.name)).getquoted(), 'serial_key': QuotedString(serial_key).getquoted(), 'maxval': maxval} @@ -241,8 +243,9 @@ def write_table(self, table): NO MAXVALUE NO MINVALUE CACHE 1;""" % serial_key_seq) serial_key_sql.append('SELECT pg_catalog.setval(\'"%s"\', %s, true);' % (serial_key_seq, maxval)) - table_sql.append('DROP TABLE IF EXISTS "%s" CASCADE;' % table.name) - table_sql.append('CREATE TABLE "%s" (\n%s\n)\nWITHOUT OIDS;' % (table.name.encode('utf8'), columns)) + + table_sql.append('DROP TABLE IF EXISTS %s CASCADE;' % self.quoted_name(table.name)) + table_sql.append('CREATE TABLE %s (\n%s\n)\nWITHOUT OIDS;' % (self.quoted_name(table.name.encode('utf8')), columns)) table_sql.append( self.table_comments(table)) return (table_sql, serial_key_sql) @@ -251,11 +254,11 @@ def write_indexes(self, table): primary_index = [idx for idx in table.indexes if idx.get('primary', None)] index_prefix = self.index_prefix if primary_index: - index_sql.append('ALTER TABLE "%(table_name)s" ADD CONSTRAINT "%(index_name)s_pkey" PRIMARY KEY(%(column_names)s);' % { - 'table_name': table.name, + index_sql.append('ALTER TABLE %(table_name)s ADD CONSTRAINT "%(index_name)s_pkey" PRIMARY KEY(%(column_names)s);' % { + 'table_name': self.quoted_name(table.name), 'index_name': '%s%s_%s' % (index_prefix, table.name, '_'.join(primary_index[0]['columns'])), - 'column_names': ', '.join('"%s"' % col for col in primary_index[0]['columns']), + 'column_names': ', '.join('%s' % self.quoted_name(col) for col in primary_index[0]['columns']), }) for index in table.indexes: if 'primary' in index: @@ -263,11 +266,11 @@ def write_indexes(self, table): unique = 'UNIQUE ' if index.get('unique', None) else '' index_name = '%s%s_%s' % (index_prefix, table.name, '_'.join(index['columns'])) index_sql.append('DROP INDEX IF EXISTS "%s" CASCADE;' % index_name) - index_sql.append('CREATE %(unique)sINDEX "%(index_name)s" ON "%(table_name)s" (%(column_names)s);' % { + index_sql.append('CREATE %(unique)sINDEX "%(index_name)s" ON %(table_name)s (%(column_names)s);' % { 'unique': unique, 'index_name': index_name, - 'table_name': table.name, - 'column_names': ', '.join('"%s"' % col for col in index['columns']), + 'table_name': self.quoted_name(table.name), + 'column_names': ', '.join('%s' % self.quoted_name(col) for col in index['columns']), }) return index_sql @@ -275,12 +278,12 @@ def write_indexes(self, table): def write_constraints(self, table): constraint_sql = [] for key in table.foreign_keys: - constraint_sql.append("""ALTER TABLE "%(table_name)s" ADD FOREIGN KEY ("%(column_name)s") - REFERENCES "%(ref_table_name)s"(%(ref_column_name)s);""" % { - 'table_name': table.name, - 'column_name': key['column'], - 'ref_table_name': key['ref_table'], - 'ref_column_name': key['ref_column']}) + constraint_sql.append("""ALTER TABLE %(table_name)s ADD FOREIGN KEY (%(column_name)s) + REFERENCES %(ref_table_name)s(%(ref_column_name)s);""" % { + 'table_name': self.quoted_name(table.name), + 'column_name': self.quoted_name(key['column']), + 'ref_table_name': self.quoted_name(key['ref_table']), + 'ref_column_name': self.quoted_name(key['ref_column'])}) return constraint_sql def write_triggers(self, table): diff --git a/mysql2pgsql/mysql2pgsql.py b/mysql2pgsql/mysql2pgsql.py index badac6b..56a0a6f 100644 --- a/mysql2pgsql/mysql2pgsql.py +++ b/mysql2pgsql/mysql2pgsql.py @@ -22,19 +22,19 @@ def __init__(self, options): def convert(self): reader = MysqlReader(self.file_options['mysql']) - + # isNameQuoted indicate that we need to add extra quotes on table and column name if self.file_options['destination']['file']: writer = PostgresFileWriter(self._get_file(self.file_options['destination']['file']), self.run_options.verbose, tz=self.file_options.get('timezone', False), index_prefix=self.file_options.get("index_prefix", ''), - isColumnNameQuoted=self.file_options.get("column_name_quoted", False)) + isNameQuoted=self.file_options.get("name_quoted", False)) else: writer = PostgresDbWriter(self.file_options['destination']['postgres'], self.run_options.verbose, tz=self.file_options.get('timezone', False), index_prefix=self.file_options.get("index_prefix", ''), - isColumnNameQuoted=self.file_options.get("column_name_quoted", False)) + isNameQuoted=self.file_options.get("name_quoted", False)) Converter(reader, writer, self.file_options, self.run_options.verbose).convert() diff --git a/tests/mysql2pgsql-test.yml b/tests/mysql2pgsql-test.yml index ab4a8bf..cc88f35 100644 --- a/tests/mysql2pgsql-test.yml +++ b/tests/mysql2pgsql-test.yml @@ -36,3 +36,6 @@ supress_ddl: false # if force_truncate is true, forces a table truncate before table loading force_truncate: false + +# if name_quoted is true, all table_name and column_name will be double quoted +name_quoted: false \ No newline at end of file diff --git a/tests/mysql2pgsql-test.yml.example b/tests/mysql2pgsql-test.yml.example index 61eb9a5..61e5e10 100644 --- a/tests/mysql2pgsql-test.yml.example +++ b/tests/mysql2pgsql-test.yml.example @@ -36,3 +36,6 @@ supress_ddl: false # if force_truncate is true, forces a table truncate before table loading force_truncate: false + +# if name_quoted is true, all table_name and column_name will be double quoted +name_quoted: false