Merge pull request #79 from uc-cdis/fix/migration

Fix/migration
uc-cdis · Oct 28, 2022 · e34c7ab · e34c7ab
2 parents ffc7a5e + 2da8ffd
commit e34c7ab
Show file tree

Hide file tree

Showing 5 changed files with 24 additions and 15 deletions.
diff --git a/.secrets.baseline b/.secrets.baseline
@@ -3,7 +3,7 @@
     "files": null,
     "lines": null
   },
-  "generated_at": "2022-10-12T21:15:02Z",
+  "generated_at": "2022-10-28T16:39:53Z",
   "plugins_used": [
     {
       "name": "AWSKeyDetector"
@@ -116,7 +116,7 @@
       {
         "hashed_secret": "ecdb6b62dc6de954dbbef8185029415aecae5e5a",
         "is_verified": false,
-        "line_number": 283,
+        "line_number": 291,
         "type": "Hex High Entropy String"
       }
     ]

diff --git a/docs/openapi.yaml b/docs/openapi.yaml
@@ -92,7 +92,7 @@ components:
       type: http
 info:
   title: Framework Services Object Management Service
-  version: 2.0.0
+  version: 2.0.1
 openapi: 3.0.2
 paths:
   /_status:

diff --git a/migrations/versions/6819874e85b9_remove_deprecated_metadata.py b/migrations/versions/6819874e85b9_remove_deprecated_metadata.py
@@ -19,8 +19,8 @@
 
 
 def escape(str):
-    # escape single quotes for SQL statement
-    return str.replace("'", "''")
+    # escape colon for sa.text() and single quotes for SQL statement
+    return str.replace(":", "\\:").replace("'", "''")
 
 
 def upgrade():
@@ -40,13 +40,14 @@ def upgrade():
         for r in results:
             guid, data = r[0], r[1]
             # scrub internal fields from metadata
-            for metadata_key in remove_metadata_keys:
-                if metadata_key in data.keys():
-                    data.pop(metadata_key)
-            sql_statement = f"""UPDATE metadata
-                                SET data='{escape(json.dumps(data))}'
-                                WHERE guid='{guid}'"""
-            connection.execute(sql_statement)
+            if data is not None:
+                for metadata_key in remove_metadata_keys:
+                    if metadata_key in data.keys():
+                        data.pop(metadata_key)
+                sql_statement = f"""UPDATE metadata
+                                    SET data='{escape(json.dumps(data))}'
+                                    WHERE guid='{guid}'"""
+                connection.execute(sa.text(sql_statement))
         # Grab another batch of rows
         offset += limit
         query = f"SELECT guid, data FROM metadata ORDER BY guid LIMIT {limit} OFFSET {offset} "

diff --git a/pyproject.toml b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "mds"
-version = "2.0.0"
+version = "2.0.1"
 description = "Metadata Service"
 authors = ["CTDS UChicago <[email protected]>"]
 license = "Apache-2.0"

diff --git a/tests/test_migrations.py b/tests/test_migrations.py
@@ -9,7 +9,7 @@
 
 def escape(str):
     # escape single quotes for SQL statement
-    return str.replace("'", "''")
+    return str.replace(":", "\\:").replace("'", "''")
 
 
 @pytest.mark.asyncio
@@ -250,15 +250,23 @@ async def test_6819874e85b9_upgrade():
     alembic_main(["--raiseerr", "downgrade", "3354f2c466ec"])
 
     fake_guid = "7891011"
+    # "percent" and "colon" are some edge cases that was causing the migration to fail
     old_metadata = {
         "foo": "bar",
         "bizz": "buzz",
+        "percent": "50% for",
+        "colon": "14(10):1534-47",
         "_uploader_id": "uploader",
         "_filename": "hello.txt",
         "_bucket": "mybucket",
         "_file_extension": ".txt",
     }
-    new_metadata = {"foo": "bar", "bizz": "buzz"}
+    new_metadata = {
+        "foo": "bar",
+        "bizz": "buzz",
+        "percent": "50% for",
+        "colon": "14(10):1534-47",
+    }
     authz_data = {"version": 0, "_resource_paths": ["/programs/DEV"]}
 
     async with db.with_bind(DB_DSN):