From 8534c21b1cafc259b169802e4705d7c9cc573994 Mon Sep 17 00:00:00 2001 From: Lokkook Date: Sat, 28 Nov 2020 12:02:36 +0100 Subject: [PATCH 1/8] Update sqlite.py Adding @rgaiacs changes about restoring to sqlite db with newlines --- dbbackup/db/sqlite.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/dbbackup/db/sqlite.py b/dbbackup/db/sqlite.py index c65ddc99..fa5063a3 100644 --- a/dbbackup/db/sqlite.py +++ b/dbbackup/db/sqlite.py @@ -70,13 +70,25 @@ def restore_dump(self, dump): if not self.connection.is_usable(): self.connection.connect() cursor = self.connection.cursor() + sql_command = b"" + sql_is_complete = True for line in dump.readlines(): - try: - cursor.execute(line.decode('UTF-8')) - except OperationalError as err: - warnings.warn("Error in db restore: {}".format(err)) - except IntegrityError as err: - warnings.warn("Error in db restore: {}".format(err)) + sql_command = sql_command + line + line_str = line.decode('UTF-8') + if line_str.startswith("INSERT") and not line_str.endswith(");\n"): + sql_is_complete = False + continue + if not sql_is_complete and line_str.endswith(");\n"): + sql_is_complete = True + + if sql_is_complete: + try: + cursor.execute(sql_command.decode('UTF-8')) + except OperationalError as err: + warnings.warn("Error in db restore: {}".format(err)) + except IntegrityError as err: + warnings.warn("Error in db restore: {}".format(err)) + sql_command = b"" class SqliteCPConnector(BaseDBConnector): From a2d4203af339321e84ae86e2ee6d2d23fe425900 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Wed, 27 Apr 2022 22:42:54 -0700 Subject: [PATCH 2/8] add tests --- dbbackup/tests/settings.py | 1 + dbbackup/tests/test_connectors/test_sqlite.py | 8 +++++++- .../tests/testapp/migrations/0001_initial.py | 2 ++ .../testapp/migrations/0002_textmodel.py | 20 +++++++++++++++++++ dbbackup/tests/testapp/models.py | 8 ++++---- 5 files changed, 34 insertions(+), 5 deletions(-) create mode 100644 dbbackup/tests/testapp/migrations/0002_textmodel.py diff --git a/dbbackup/tests/settings.py b/dbbackup/tests/settings.py index 0aff0fd5..15d28f4f 100644 --- a/dbbackup/tests/settings.py +++ b/dbbackup/tests/settings.py @@ -29,6 +29,7 @@ 'dbbackup', 'dbbackup.tests.testapp', ) +DEFAULT_AUTO_FIELD = 'django.db.models.AutoField' DATABASES = { 'default': { diff --git a/dbbackup/tests/test_connectors/test_sqlite.py b/dbbackup/tests/test_connectors/test_sqlite.py index bfe7ed0c..97c4e53e 100644 --- a/dbbackup/tests/test_connectors/test_sqlite.py +++ b/dbbackup/tests/test_connectors/test_sqlite.py @@ -6,7 +6,7 @@ from mock import mock_open, patch from dbbackup.db.sqlite import SqliteConnector, SqliteCPConnector -from dbbackup.tests.testapp.models import CharModel +from dbbackup.tests.testapp.models import CharModel, TextModel class SqliteConnectorTest(TestCase): @@ -29,6 +29,12 @@ def test_create_dump_with_unicode(self): dump = connector.create_dump() self.assertTrue(dump.read()) + def test_create_dump_with_newline(self): + TextModel.objects.create(field='foo\nbar') + connector = SqliteConnector() + dump = connector.create_dump() + self.assertTrue(dump.read()) + def test_restore_dump(self): connector = SqliteConnector() dump = connector.create_dump() diff --git a/dbbackup/tests/testapp/migrations/0001_initial.py b/dbbackup/tests/testapp/migrations/0001_initial.py index d225561e..8e5ec2fe 100644 --- a/dbbackup/tests/testapp/migrations/0001_initial.py +++ b/dbbackup/tests/testapp/migrations/0001_initial.py @@ -6,6 +6,8 @@ class Migration(migrations.Migration): + initial = True + dependencies = [ ] diff --git a/dbbackup/tests/testapp/migrations/0002_textmodel.py b/dbbackup/tests/testapp/migrations/0002_textmodel.py new file mode 100644 index 00000000..32351354 --- /dev/null +++ b/dbbackup/tests/testapp/migrations/0002_textmodel.py @@ -0,0 +1,20 @@ +# Generated by Django 4.0.1 on 2022-04-27 22:36 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('testapp', '0001_initial'), + ] + + operations = [ + migrations.CreateModel( + name='TextModel', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('field', models.TextField()), + ], + ), + ] diff --git a/dbbackup/tests/testapp/models.py b/dbbackup/tests/testapp/models.py index 6012d250..d953b1b8 100644 --- a/dbbackup/tests/testapp/models.py +++ b/dbbackup/tests/testapp/models.py @@ -1,15 +1,15 @@ from __future__ import unicode_literals from django.db import models -___all__ = ('CharModel', 'IntegerModel', 'TextModel', 'BooleanModel' - 'DateModel', 'DateTimeModel', 'ForeignKeyModel', 'ManyToManyModel', - 'FileModel', 'TestModel',) - class CharModel(models.Model): field = models.CharField(max_length=10) +class TextModel(models.Model): + field = models.TextField() + + class ForeignKeyModel(models.Model): field = models.ForeignKey(CharModel, on_delete=models.CASCADE) From cac16383c3f9bc84d155c20eda55d2b97374d828 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Wed, 27 Apr 2022 22:56:16 -0700 Subject: [PATCH 3/8] attempt at fixing codecov hits --- dbbackup/tests/test_connectors/test_sqlite.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbbackup/tests/test_connectors/test_sqlite.py b/dbbackup/tests/test_connectors/test_sqlite.py index 97c4e53e..bb63bb64 100644 --- a/dbbackup/tests/test_connectors/test_sqlite.py +++ b/dbbackup/tests/test_connectors/test_sqlite.py @@ -30,7 +30,7 @@ def test_create_dump_with_unicode(self): self.assertTrue(dump.read()) def test_create_dump_with_newline(self): - TextModel.objects.create(field='foo\nbar') + TextModel.objects.create(field=f'INSERT({"foo" * 5000}\nbar WHERE baz);\n') connector = SqliteConnector() dump = connector.create_dump() self.assertTrue(dump.read()) From 20d72870f9c6e39d974852ecfd2540d6850b1404 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Wed, 27 Apr 2022 23:03:07 -0700 Subject: [PATCH 4/8] anotha one --- dbbackup/tests/test_connectors/test_sqlite.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbbackup/tests/test_connectors/test_sqlite.py b/dbbackup/tests/test_connectors/test_sqlite.py index bb63bb64..4d766323 100644 --- a/dbbackup/tests/test_connectors/test_sqlite.py +++ b/dbbackup/tests/test_connectors/test_sqlite.py @@ -30,7 +30,7 @@ def test_create_dump_with_unicode(self): self.assertTrue(dump.read()) def test_create_dump_with_newline(self): - TextModel.objects.create(field=f'INSERT({"foo" * 5000}\nbar WHERE baz);\n') + TextModel.objects.create(field=f'INSERT ({"foo" * 5000}\nbar\n WHERE \nbaz IS\n "great" );\n') connector = SqliteConnector() dump = connector.create_dump() self.assertTrue(dump.read()) From aaab2a56cf40ffaaca6226905800df43253b9c44 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 23 Aug 2024 20:56:59 +0000 Subject: [PATCH 5/8] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- dbbackup/db/sqlite.py | 4 ++-- dbbackup/tests/settings.py | 2 +- dbbackup/tests/test_connectors/test_sqlite.py | 4 +++- .../tests/testapp/migrations/0002_textmodel.py | 16 ++++++++++++---- dbbackup/tests/testapp/models.py | 2 +- 5 files changed, 19 insertions(+), 9 deletions(-) diff --git a/dbbackup/db/sqlite.py b/dbbackup/db/sqlite.py index d9582296..a372e2b3 100644 --- a/dbbackup/db/sqlite.py +++ b/dbbackup/db/sqlite.py @@ -75,7 +75,7 @@ def restore_dump(self, dump): sql_is_complete = True for line in dump.readlines(): sql_command = sql_command + line - line_str = line.decode('UTF-8') + line_str = line.decode("UTF-8") if line_str.startswith("INSERT") and not line_str.endswith(");\n"): sql_is_complete = False continue @@ -84,7 +84,7 @@ def restore_dump(self, dump): if sql_is_complete: try: - cursor.execute(sql_command.decode('UTF-8')) + cursor.execute(sql_command.decode("UTF-8")) except (OperationalError, IntegrityError) as err: warnings.warn(f"Error in db restore: {err}") sql_command = b"" diff --git a/dbbackup/tests/settings.py b/dbbackup/tests/settings.py index 51d326ea..c31a8ea3 100644 --- a/dbbackup/tests/settings.py +++ b/dbbackup/tests/settings.py @@ -29,7 +29,7 @@ "dbbackup", "dbbackup.tests.testapp", ) -DEFAULT_AUTO_FIELD = 'django.db.models.AutoField' +DEFAULT_AUTO_FIELD = "django.db.models.AutoField" DATABASES = { "default": { diff --git a/dbbackup/tests/test_connectors/test_sqlite.py b/dbbackup/tests/test_connectors/test_sqlite.py index f4e47cc5..86c9efe4 100644 --- a/dbbackup/tests/test_connectors/test_sqlite.py +++ b/dbbackup/tests/test_connectors/test_sqlite.py @@ -29,7 +29,9 @@ def test_create_dump_with_unicode(self): self.assertTrue(dump.read()) def test_create_dump_with_newline(self): - TextModel.objects.create(field=f'INSERT ({"foo" * 5000}\nbar\n WHERE \nbaz IS\n "great" );\n') + TextModel.objects.create( + field=f'INSERT ({"foo" * 5000}\nbar\n WHERE \nbaz IS\n "great" );\n' + ) connector = SqliteConnector() dump = connector.create_dump() self.assertTrue(dump.read()) diff --git a/dbbackup/tests/testapp/migrations/0002_textmodel.py b/dbbackup/tests/testapp/migrations/0002_textmodel.py index 32351354..ecc159c2 100644 --- a/dbbackup/tests/testapp/migrations/0002_textmodel.py +++ b/dbbackup/tests/testapp/migrations/0002_textmodel.py @@ -6,15 +6,23 @@ class Migration(migrations.Migration): dependencies = [ - ('testapp', '0001_initial'), + ("testapp", "0001_initial"), ] operations = [ migrations.CreateModel( - name='TextModel', + name="TextModel", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('field', models.TextField()), + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("field", models.TextField()), ], ), ] diff --git a/dbbackup/tests/testapp/models.py b/dbbackup/tests/testapp/models.py index 2daafe74..c68f101f 100644 --- a/dbbackup/tests/testapp/models.py +++ b/dbbackup/tests/testapp/models.py @@ -7,7 +7,7 @@ class CharModel(models.Model): class TextModel(models.Model): field = models.TextField() - + class ForeignKeyModel(models.Model): field = models.ForeignKey(CharModel, on_delete=models.CASCADE) From cedac65de751861d9b8aaa8e868b0081a8cf4e05 Mon Sep 17 00:00:00 2001 From: Lokkook Date: Fri, 23 Aug 2024 23:17:28 +0200 Subject: [PATCH 6/8] Update test_sqlite.py Adding test to restore a dump with a field containing newline --- dbbackup/tests/test_connectors/test_sqlite.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/dbbackup/tests/test_connectors/test_sqlite.py b/dbbackup/tests/test_connectors/test_sqlite.py index 86c9efe4..b802b28c 100644 --- a/dbbackup/tests/test_connectors/test_sqlite.py +++ b/dbbackup/tests/test_connectors/test_sqlite.py @@ -32,11 +32,15 @@ def test_create_dump_with_newline(self): TextModel.objects.create( field=f'INSERT ({"foo" * 5000}\nbar\n WHERE \nbaz IS\n "great" );\n' ) + connector = SqliteConnector() dump = connector.create_dump() self.assertTrue(dump.read()) def test_restore_dump(self): + TextModel.objects.create( + field='T\nf\nw\nnl' + ) connector = SqliteConnector() dump = connector.create_dump() connector.restore_dump(dump) From db4e66ed5103b433eaca4bec22851e03c3740f46 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 23 Aug 2024 21:17:37 +0000 Subject: [PATCH 7/8] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- dbbackup/tests/test_connectors/test_sqlite.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/dbbackup/tests/test_connectors/test_sqlite.py b/dbbackup/tests/test_connectors/test_sqlite.py index b802b28c..6ac737ea 100644 --- a/dbbackup/tests/test_connectors/test_sqlite.py +++ b/dbbackup/tests/test_connectors/test_sqlite.py @@ -32,15 +32,13 @@ def test_create_dump_with_newline(self): TextModel.objects.create( field=f'INSERT ({"foo" * 5000}\nbar\n WHERE \nbaz IS\n "great" );\n' ) - + connector = SqliteConnector() dump = connector.create_dump() self.assertTrue(dump.read()) def test_restore_dump(self): - TextModel.objects.create( - field='T\nf\nw\nnl' - ) + TextModel.objects.create(field="T\nf\nw\nnl") connector = SqliteConnector() dump = connector.create_dump() connector.restore_dump(dump) From 310632573fa49ce3740d712acecb6586262343c8 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Fri, 23 Aug 2024 14:49:58 -0700 Subject: [PATCH 8/8] Add changelog entry --- docs/changelog.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 34724520..2120040d 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,7 +4,8 @@ Changelog Unreleased ---------- -* Add `--no-drop` option to `dbrestore` command to prevent dropping tables before restoring data. +* Add --no-drop option to dbrestore command to prevent dropping tables before restoring data. +* Fix bug where sqlite dbrestore would fail if field data contains the line break character. 4.2.0 (2024-08-22) ------------------