From 5f045e338dbd37a0f0b986470a7d8e4aedd8018b Mon Sep 17 00:00:00 2001 From: Mingfei Shao Date: Thu, 27 Oct 2022 19:03:41 -0500 Subject: [PATCH 1/8] test --- .../6819874e85b9_remove_deprecated_metadata.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/migrations/versions/6819874e85b9_remove_deprecated_metadata.py b/migrations/versions/6819874e85b9_remove_deprecated_metadata.py index c0c4feaa..6d9aca44 100644 --- a/migrations/versions/6819874e85b9_remove_deprecated_metadata.py +++ b/migrations/versions/6819874e85b9_remove_deprecated_metadata.py @@ -40,13 +40,14 @@ def upgrade(): for r in results: guid, data = r[0], r[1] # scrub internal fields from metadata - for metadata_key in remove_metadata_keys: - if metadata_key in data.keys(): - data.pop(metadata_key) - sql_statement = f"""UPDATE metadata - SET data='{escape(json.dumps(data))}' - WHERE guid='{guid}'""" - connection.execute(sql_statement) + if data is not None: + for metadata_key in remove_metadata_keys: + if metadata_key in data.keys(): + data.pop(metadata_key) + sql_statement = f"""UPDATE metadata + SET data='{escape(json.dumps(data))}' + WHERE guid='{guid}'""" + connection.execute(sql_statement) # Grab another batch of rows offset += limit query = f"SELECT guid, data FROM metadata ORDER BY guid LIMIT {limit} OFFSET {offset} " From c3f8bb7a6e31d8dd3d23ff6ed866ec99cc49fdcf Mon Sep 17 00:00:00 2001 From: Mingfei Shao Date: Thu, 27 Oct 2022 21:20:47 -0500 Subject: [PATCH 2/8] fix --- .../versions/6819874e85b9_remove_deprecated_metadata.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/migrations/versions/6819874e85b9_remove_deprecated_metadata.py b/migrations/versions/6819874e85b9_remove_deprecated_metadata.py index 6d9aca44..aec5bf7f 100644 --- a/migrations/versions/6819874e85b9_remove_deprecated_metadata.py +++ b/migrations/versions/6819874e85b9_remove_deprecated_metadata.py @@ -19,8 +19,8 @@ def escape(str): - # escape single quotes for SQL statement - return str.replace("'", "''") + # escape % and single quotes for SQL statement + return str.replace("%", "%%").replace("'", "''") def upgrade(): From 5604b1240d356a231faf3af473c208f2a3991027 Mon Sep 17 00:00:00 2001 From: Mingfei Shao Date: Fri, 28 Oct 2022 09:43:02 -0500 Subject: [PATCH 3/8] fix tests --- .secrets.baseline | 4 ++-- tests/test_migrations.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.secrets.baseline b/.secrets.baseline index f21d3f0f..bf64bbd1 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -3,7 +3,7 @@ "files": null, "lines": null }, - "generated_at": "2022-10-12T21:15:02Z", + "generated_at": "2022-10-28T14:42:57Z", "plugins_used": [ { "name": "AWSKeyDetector" @@ -110,7 +110,7 @@ { "hashed_secret": "4dcba4ad1d671981e2d211ebe56da8a5b40f14ef", "is_verified": false, - "line_number": 225, + "line_number": 224, "type": "Hex High Entropy String" }, { diff --git a/tests/test_migrations.py b/tests/test_migrations.py index 9c8a2be3..5fb983c0 100644 --- a/tests/test_migrations.py +++ b/tests/test_migrations.py @@ -8,7 +8,6 @@ def escape(str): - # escape single quotes for SQL statement return str.replace("'", "''") @@ -253,12 +252,13 @@ async def test_6819874e85b9_upgrade(): old_metadata = { "foo": "bar", "bizz": "buzz", + "special": "50% for", "_uploader_id": "uploader", "_filename": "hello.txt", "_bucket": "mybucket", "_file_extension": ".txt", } - new_metadata = {"foo": "bar", "bizz": "buzz"} + new_metadata = {"foo": "bar", "bizz": "buzz", "special": "50% for"} authz_data = {"version": 0, "_resource_paths": ["/programs/DEV"]} async with db.with_bind(DB_DSN): From f35fe1219459cfc14044d15e9cdbc2e79808b1ce Mon Sep 17 00:00:00 2001 From: Mingfei Shao Date: Fri, 28 Oct 2022 09:44:05 -0500 Subject: [PATCH 4/8] fix tests --- .secrets.baseline | 6 +++--- tests/test_migrations.py | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.secrets.baseline b/.secrets.baseline index bf64bbd1..22a10a95 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -3,7 +3,7 @@ "files": null, "lines": null }, - "generated_at": "2022-10-28T14:42:57Z", + "generated_at": "2022-10-28T14:44:01Z", "plugins_used": [ { "name": "AWSKeyDetector" @@ -110,13 +110,13 @@ { "hashed_secret": "4dcba4ad1d671981e2d211ebe56da8a5b40f14ef", "is_verified": false, - "line_number": 224, + "line_number": 225, "type": "Hex High Entropy String" }, { "hashed_secret": "ecdb6b62dc6de954dbbef8185029415aecae5e5a", "is_verified": false, - "line_number": 283, + "line_number": 284, "type": "Hex High Entropy String" } ] diff --git a/tests/test_migrations.py b/tests/test_migrations.py index 5fb983c0..98275cf5 100644 --- a/tests/test_migrations.py +++ b/tests/test_migrations.py @@ -8,6 +8,7 @@ def escape(str): + # escape single quotes for SQL statement return str.replace("'", "''") From af2351bad261767473ec4dcf874d8981a0ca2997 Mon Sep 17 00:00:00 2001 From: Mingfei Shao Date: Fri, 28 Oct 2022 10:20:19 -0500 Subject: [PATCH 5/8] update version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index c8008d8a..428d0c5d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "mds" -version = "2.0.0" +version = "2.0.1" description = "Metadata Service" authors = ["CTDS UChicago "] license = "Apache-2.0" From fa545f9e83de52d08d5f55c4bf77172c3206f030 Mon Sep 17 00:00:00 2001 From: mfshao Date: Fri, 28 Oct 2022 15:22:44 +0000 Subject: [PATCH 6/8] Apply automatic documentation changes --- docs/openapi.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/openapi.yaml b/docs/openapi.yaml index 1c954c4c..e27f9a19 100644 --- a/docs/openapi.yaml +++ b/docs/openapi.yaml @@ -92,7 +92,7 @@ components: type: http info: title: Framework Services Object Management Service - version: 2.0.0 + version: 2.0.1 openapi: 3.0.2 paths: /_status: From dc2fa7df8df038b09011328d23175f6d6bd977c7 Mon Sep 17 00:00:00 2001 From: Mingfei Shao Date: Fri, 28 Oct 2022 11:27:28 -0500 Subject: [PATCH 7/8] use sa.text() --- .../versions/6819874e85b9_remove_deprecated_metadata.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/migrations/versions/6819874e85b9_remove_deprecated_metadata.py b/migrations/versions/6819874e85b9_remove_deprecated_metadata.py index aec5bf7f..0f7badf4 100644 --- a/migrations/versions/6819874e85b9_remove_deprecated_metadata.py +++ b/migrations/versions/6819874e85b9_remove_deprecated_metadata.py @@ -19,8 +19,8 @@ def escape(str): - # escape % and single quotes for SQL statement - return str.replace("%", "%%").replace("'", "''") + # escape colon for sa.text() and single quotes for SQL statement + return str.replace(":", "\:").replace("'", "''") def upgrade(): @@ -47,7 +47,7 @@ def upgrade(): sql_statement = f"""UPDATE metadata SET data='{escape(json.dumps(data))}' WHERE guid='{guid}'""" - connection.execute(sql_statement) + connection.execute(sa.text(sql_statement)) # Grab another batch of rows offset += limit query = f"SELECT guid, data FROM metadata ORDER BY guid LIMIT {limit} OFFSET {offset} " From 2da8ffd432a4b4f1c68f19864ff4b8f22b457684 Mon Sep 17 00:00:00 2001 From: Mingfei Shao Date: Fri, 28 Oct 2022 11:40:18 -0500 Subject: [PATCH 8/8] fix tests --- .secrets.baseline | 4 ++-- .../6819874e85b9_remove_deprecated_metadata.py | 2 +- tests/test_migrations.py | 13 ++++++++++--- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/.secrets.baseline b/.secrets.baseline index 22a10a95..d14a536a 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -3,7 +3,7 @@ "files": null, "lines": null }, - "generated_at": "2022-10-28T14:44:01Z", + "generated_at": "2022-10-28T16:39:53Z", "plugins_used": [ { "name": "AWSKeyDetector" @@ -116,7 +116,7 @@ { "hashed_secret": "ecdb6b62dc6de954dbbef8185029415aecae5e5a", "is_verified": false, - "line_number": 284, + "line_number": 291, "type": "Hex High Entropy String" } ] diff --git a/migrations/versions/6819874e85b9_remove_deprecated_metadata.py b/migrations/versions/6819874e85b9_remove_deprecated_metadata.py index 0f7badf4..1c551d5c 100644 --- a/migrations/versions/6819874e85b9_remove_deprecated_metadata.py +++ b/migrations/versions/6819874e85b9_remove_deprecated_metadata.py @@ -20,7 +20,7 @@ def escape(str): # escape colon for sa.text() and single quotes for SQL statement - return str.replace(":", "\:").replace("'", "''") + return str.replace(":", "\\:").replace("'", "''") def upgrade(): diff --git a/tests/test_migrations.py b/tests/test_migrations.py index 98275cf5..0348d994 100644 --- a/tests/test_migrations.py +++ b/tests/test_migrations.py @@ -9,7 +9,7 @@ def escape(str): # escape single quotes for SQL statement - return str.replace("'", "''") + return str.replace(":", "\\:").replace("'", "''") @pytest.mark.asyncio @@ -250,16 +250,23 @@ async def test_6819874e85b9_upgrade(): alembic_main(["--raiseerr", "downgrade", "3354f2c466ec"]) fake_guid = "7891011" + # "percent" and "colon" are some edge cases that was causing the migration to fail old_metadata = { "foo": "bar", "bizz": "buzz", - "special": "50% for", + "percent": "50% for", + "colon": "14(10):1534-47", "_uploader_id": "uploader", "_filename": "hello.txt", "_bucket": "mybucket", "_file_extension": ".txt", } - new_metadata = {"foo": "bar", "bizz": "buzz", "special": "50% for"} + new_metadata = { + "foo": "bar", + "bizz": "buzz", + "percent": "50% for", + "colon": "14(10):1534-47", + } authz_data = {"version": 0, "_resource_paths": ["/programs/DEV"]} async with db.with_bind(DB_DSN):