From a06c428626de25efb5631b1cb1c2a91b88ba0660 Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Fri, 18 Aug 2017 18:05:58 +0100 Subject: [PATCH 1/2] Fix dbrestore of sqlite database to like newlines On Textfields that have line breaks, sqlite reports "unrecognized token" because it doesn't keep the previous line in memory to be ammended by the current line. This pull requests implements a very simple logic to only send SQL INSERT commands when the line finished with ");". Close #238 --- dbbackup/db/sqlite.py | 24 ++++++++++++++----- dbbackup/tests/test_connectors/test_sqlite.py | 2 +- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/dbbackup/db/sqlite.py b/dbbackup/db/sqlite.py index 04b89580..6d38e211 100644 --- a/dbbackup/db/sqlite.py +++ b/dbbackup/db/sqlite.py @@ -67,13 +67,25 @@ def restore_dump(self, dump): if not self.connection.is_usable(): self.connection.connect() cursor = self.connection.cursor() + sql_command = b"" + sql_is_complete = True for line in dump.readlines(): - try: - cursor.execute(line.decode('UTF-8')) - except OperationalError as err: - warnings.warn("Error in db restore: {}".format(err)) - except IntegrityError as err: - warnings.warn("Error in db restore: {}".format(err)) + sql_command = sql_command + line + line_str = line.decode('UTF-8') + if line_str.startswith("INSERT") and not line_str.endswith(");\n"): + sql_is_complete = False + continue + if not sql_is_complete and line_str.endswith(");\n"): + sql_is_complete = True + + if sql_is_complete: + try: + cursor.execute(sql_command.decode('UTF-8')) + except OperationalError as err: + warnings.warn("Error in db restore: {}".format(err)) + except IntegrityError as err: + warnings.warn("Error in db restore: {}".format(err)) + sql_command = b"" class SqliteCPConnector(BaseDBConnector): diff --git a/dbbackup/tests/test_connectors/test_sqlite.py b/dbbackup/tests/test_connectors/test_sqlite.py index 6f4dea95..92765924 100644 --- a/dbbackup/tests/test_connectors/test_sqlite.py +++ b/dbbackup/tests/test_connectors/test_sqlite.py @@ -6,7 +6,7 @@ from django.utils.six import BytesIO from dbbackup.db.sqlite import SqliteConnector, SqliteCPConnector -from dbbackup.tests.testapp.models import CharModel +from dbbackup.tests.testapp.models import CharModel, TextModel class SqliteConnectorTest(TestCase): From 757196993600a2f7f5256628ba0179f2bd67f8aa Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Mon, 16 Oct 2017 16:14:07 +0100 Subject: [PATCH 2/2] Add test to SQLite restore_dump with new line --- .../tests/test_connectors/test_postgresql.py | 1 + dbbackup/tests/test_connectors/test_sqlite.py | 6 +++++ .../tests/testapp/migrations/0001_initial.py | 23 ++++++++++++++----- dbbackup/tests/testapp/models.py | 4 ++++ 4 files changed, 28 insertions(+), 6 deletions(-) diff --git a/dbbackup/tests/test_connectors/test_postgresql.py b/dbbackup/tests/test_connectors/test_postgresql.py index 3df77d0f..79f97643 100644 --- a/dbbackup/tests/test_connectors/test_postgresql.py +++ b/dbbackup/tests/test_connectors/test_postgresql.py @@ -5,6 +5,7 @@ from django.test import TestCase from django.utils.six import BytesIO + from dbbackup.db.postgresql import (PgDumpConnector, PgDumpGisConnector, PgDumpBinaryConnector) diff --git a/dbbackup/tests/test_connectors/test_sqlite.py b/dbbackup/tests/test_connectors/test_sqlite.py index 92765924..eddba034 100644 --- a/dbbackup/tests/test_connectors/test_sqlite.py +++ b/dbbackup/tests/test_connectors/test_sqlite.py @@ -29,6 +29,12 @@ def test_create_dump_with_unicode(self): dump = connector.create_dump() self.assertTrue(dump.read()) + def test_create_dump_with_newline(self): + TextModel.objects.create(field='foo\nbar') + connector = SqliteConnector() + dump = connector.create_dump() + self.assertTrue(dump.read()) + def test_restore_dump(self): connector = SqliteConnector() dump = connector.create_dump() diff --git a/dbbackup/tests/testapp/migrations/0001_initial.py b/dbbackup/tests/testapp/migrations/0001_initial.py index 72c9f9ce..6f66346a 100644 --- a/dbbackup/tests/testapp/migrations/0001_initial.py +++ b/dbbackup/tests/testapp/migrations/0001_initial.py @@ -1,11 +1,15 @@ # -*- coding: utf-8 -*- +# Generated by Django 1.11.5 on 2017-10-17 07:15 from __future__ import unicode_literals -from django.db import models, migrations +from django.db import migrations, models +import django.db.models.deletion class Migration(migrations.Migration): + initial = True + dependencies = [ ] @@ -13,29 +17,36 @@ class Migration(migrations.Migration): migrations.CreateModel( name='CharModel', fields=[ - ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('field', models.CharField(max_length=10)), ], ), migrations.CreateModel( name='FileModel', fields=[ - ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('field', models.FileField(upload_to='.')), ], ), migrations.CreateModel( name='ForeignKeyModel', fields=[ - ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), - ('field', models.ForeignKey(to='testapp.CharModel')), + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('field', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='testapp.CharModel')), ], ), migrations.CreateModel( name='ManyToManyModel', fields=[ - ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('field', models.ManyToManyField(to='testapp.CharModel')), ], ), + migrations.CreateModel( + name='TextModel', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('field', models.TextField()), + ], + ), ] diff --git a/dbbackup/tests/testapp/models.py b/dbbackup/tests/testapp/models.py index 171599e8..34ec8888 100644 --- a/dbbackup/tests/testapp/models.py +++ b/dbbackup/tests/testapp/models.py @@ -10,7 +10,11 @@ class CharModel(models.Model): field = models.CharField(max_length=10) + +class TextModel(models.Model): + field = models.TextField() + class ForeignKeyModel(models.Model): field = models.ForeignKey(CharModel, on_delete=models.CASCADE)