libcove2/common.py: Handle new jsonschema

libcove2/common.py

@@ -11,34 +11,75 @@
     "^(.*_(((([A-Za-z]{2,3}(-([A-Za-z]{3}(-[A-Za-z]{3}){0,2}))?)|[A-Za-z]{4}|[A-Za-z]{5,8})(-([A-Za-z]{4}))?(-([A-Za-z]{2}|[0-9]{3}))?(-([A-Za-z0-9]{5,8}|[0-9][A-Za-z0-9]{3}))*(-([0-9A-WY-Za-wy-z](-[A-Za-z0-9]{2,8})+))*(-(x(-[A-Za-z0-9]{1,8})+))?)|(x(-[A-Za-z0-9]{1,8})+)))$"  # noqa
 )
 
+def process_items(schema_dict, registry=None, defs=None):
+    items_schema_dicts = []
+    if "oneOf" in schema_dict["items"] and isinstance(
+        schema_dict["items"]["oneOf"], list
+    ):
+        for oneOf in schema_dict["items"]["oneOf"]:
+            items_schema_dicts.append(oneOf)
+    elif '$ref' in schema_dict["items"]:
+        if schema_dict["items"]['$ref'].startswith('urn:'):
+            subschema = registry.contents(schema_dict["items"]['$ref'].split("#")[0])
+            if '#/$defs/' in schema_dict["items"]['$ref']:
+                name = schema_dict["items"]['$ref'].split('$defs/')[-1]
+                defs = subschema['$defs']
+                subschema = defs[name]
+            items_schema_dicts.append(subschema)
+        elif schema_dict["items"]['$ref'].startswith('#/$defs/'):
+            name = schema_dict["items"]['$ref'].split('$defs/')[-1]
+            items_schema_dicts.append(defs[name])
+    elif "properties" in schema_dict["items"] and isinstance(
+        schema_dict["items"]["properties"], dict
+    ):
+        items_schema_dicts.append(schema_dict["items"])
+    return items_schema_dicts
 
-def schema_dict_fields_generator(schema_dict):
+def schema_dict_fields_generator(schema_dict, registry=None, defs=None):
+    if '$defs' in schema_dict:
+        defs = schema_dict['$defs']
     if "properties" in schema_dict and isinstance(schema_dict["properties"], dict):
         for property_name, value in schema_dict["properties"].items():
-            if "oneOf" in value:
+            if '$ref' in value:
+                if value['$ref'].startswith('urn:'):
+                    property_schema_dicts = registry.contents(value['$ref'].split('#')[0])
+                    if '$defs/' in value['$ref']:
+                        name = value['$ref'].split('$defs/')[-1]
+                        property_schema_dicts = property_schema_dicts['$defs'][name]
+                else:
+                    name = value['$ref'].split('$defs/')[-1]
+                    property_schema_dicts = defs[name]
+                property_schema_dicts = [property_schema_dicts]
+            elif ("items" in value and isinstance(value["items"], dict) and
+                  (not "type" in value["items"] or value["items"]["type"] == "object")):
+                property_schema_dicts = process_items(value, registry=registry, defs=defs)
+            elif "oneOf" in value:
                 property_schema_dicts = value["oneOf"]
             else:
                 property_schema_dicts = [value]
             for property_schema_dict in property_schema_dicts:
                 if not isinstance(property_schema_dict, dict):
                     continue
                 if "properties" in property_schema_dict:
-                    for field in schema_dict_fields_generator(property_schema_dict):
+                    for field in schema_dict_fields_generator(property_schema_dict, registry=registry, defs=defs):
                         yield f"/{property_name}{field}"
                 elif "items" in property_schema_dict:
                     for field in schema_dict_fields_generator(
-                        property_schema_dict["items"]
+                        property_schema_dict["items"], registry=registry, defs=defs
                     ):
                         yield f"/{property_name}{field}"
                 yield f"/{property_name}"
-    if "items" in schema_dict and isinstance(schema_dict["items"], dict):
-        if "oneOf" in schema_dict["items"] and isinstance(
-            schema_dict["items"]["oneOf"], list
-        ):
-            for oneOf in schema_dict["items"]["oneOf"]:
-                for field in schema_dict_fields_generator(oneOf):
+    if "allOf" in schema_dict and isinstance(schema_dict["allOf"], list):
+        for clause in schema_dict["allOf"]:
+            if "then" in clause and isinstance(clause["then"], dict):
+                for field in schema_dict_fields_generator(clause["then"], registry=registry, defs=defs):
                     yield field
-
+    if ("items" in schema_dict and isinstance(schema_dict["items"], dict) and
+        (not "type" in schema_dict["items"] or schema_dict["items"]["type"] == "object")):
+        items_schema_dicts = process_items(schema_dict, registry=registry, defs=defs)
+        for items_schema_dict in items_schema_dicts:
+            for field in schema_dict_fields_generator(items_schema_dict, registry=registry, defs=defs):
+                yield field
 
 def get_additional_fields_info(json_data, schema_fields, fields_regex=False):
     fields_present = get_fields_present_with_examples(json_data)

tests/fixtures/bods-data-0-3-0.json

@@ -0,0 +1,102 @@
+[
+  {
+    "statementID": "1dc0e987-5c57-4a1c-b3ad-61353b66a9b7",
+    "statementType": "entityStatement",
+    "isComponent": false,
+    "statementDate": "2017-11-18",
+    "entityType": "registeredEntity",
+    "name": "CHRINON LTD",
+    "foundingDate": "2010-11-18",
+    "identifiers": [
+      {
+        "scheme": "GB-COH",
+        "id": "07444723"
+      }
+    ],
+    "publicListing": {
+      "hasPublicListing": true,
+      "companyFilingsURLs": ["http://example.com/"],
+      "securitiesListings": []
+    },
+    "publicationDetails": {
+      "publicationDate": "2018-02-13",
+      "bodsVersion": "0.3",
+      "publisher": {
+        "name": "CHRINON LTD"
+      }
+    }
+  },
+  {
+    "statementID": "019a93f1-e470-42e9-957b-03559861b2e2",
+    "statementType": "personStatement",
+    "isComponent": false,
+    "statementDate": "2017-11-18",
+    "personType": "knownPerson",
+    "nationalities": [
+      {
+        "code": "GB",
+        "name": "United Kingdom of Great Britain and Northern Ireland (the)"
+      }
+    ],
+    "names": [
+      {
+        "type": "individual",
+        "fullName": "Christopher Taggart",
+        "givenName": "Christopher",
+        "familyName": "Taggart"
+      },
+      {
+        "type": "alternative",
+        "fullName": "Chris Taggart"
+      }
+    ],
+    "birthDate": "1964-04",
+    "addresses": [
+      {
+        "type": "service",
+        "address": "Aston House, Cornwall Avenue, London",
+        "country": "GB",
+        "postCode": "N3 1LF"
+      }
+    ],
+    "publicationDetails": {
+      "publicationDate": "2018-02-13",
+      "bodsVersion": "0.3",
+      "publisher": {
+        "name": "CHRINON LTD"
+      }
+    }
+  },
+  {
+    "statementID": "fbfd0547-d0c6-4a00-b559-5c5e91c34f5c",
+    "statementType": "ownershipOrControlStatement",
+    "isComponent": false,
+    "statementDate": "2017-11-18",
+    "subject": {
+      "describedByEntityStatement": "1dc0e987-5c57-4a1c-b3ad-61353b66a9b7"
+    },
+    "interestedParty": {
+      "describedByPersonStatement": "019a93f1-e470-42e9-957b-03559861b2e2"
+    },
+    "interests": [
+      {
+        "type": "shareholding",
+        "directOrIndirect": "direct",
+        "beneficialOwnershipOrControl": true,
+        "startDate": "2016-04-06",
+        "share": {
+          "exact": 100,
+          "minimum": 100,
+          "maximum": 100
+        }
+      }
+    ],
+    "publicationDetails": {
+      "publicationDate": "2018-02-13",
+      "bodsVersion": "0.3",
+      "publisher": {
+        "name": "CHRINON LTD"
+      }
+    }
+  }
+]

tests/fixtures/bods-data-0-4-0.json

@@ -0,0 +1,111 @@
+[
+  {
+    "statementId": "1dc0e987-5c57-4a1c-b3ad-61353b66a9b7",
+    "declarationSubject": "c359f58d2977",
+    "statementDate": "2020-03-04",
+    "publicationDetails": {
+      "publicationDate": "2020-03-04",
+      "bodsVersion": "0.4",
+      "publisher": {
+        "name": "Profitech Ltd"
+      }
+    },
+    "recordId": "c359f58d2977",
+    "recordStatus": "new",
+    "recordType": "entity",
+    "recordDetails": {
+      "isComponent": false,
+      "entityType": {
+        "type": "registeredEntity"
+      },
+      "name": "Profitech Ltd",
+      "foundingDate": "2019-09-03",
+      "identifiers": [
+        {
+          "scheme": "GB-COH",
+          "id": "2063384560"
+        }
+      ]
+    }
+  },
+  {
+    "statementId": "019a93f1-e470-42e9-957b-03559861b2e2",
+    "declarationSubject": "c359f58d2977",
+    "statementDate": "2020-03-04",
+    "publicationDetails": {
+      "publicationDate": "2020-03-04",
+      "bodsVersion": "0.4",
+      "publisher": {
+        "name": "Profitech Ltd"
+      }
+    },
+    "recordId": "10478c6cf6de",
+    "recordStatus": "new",
+    "recordType": "person",
+    "recordDetails": {
+      "isComponent": false,
+      "personType": "knownPerson",
+      "nationalities": [
+        {
+          "code": "GB",
+          "name": "United Kingdom of Great Britain and Northern Ireland (the)"
+        }
+      ],
+      "names": [
+        {
+          "type": "legal",
+          "fullName": "Jennifer Hewitson-Smith",
+          "givenName": "Jennifer",
+          "familyName": "Hewitson-Smith"
+        },
+        {
+          "type": "alternative",
+          "fullName": "Jenny Hewitson-Smith"
+        }
+      ],
+      "birthDate": "1978-07",
+      "addresses": [
+        {
+          "type": "service",
+          "address": "76 York Road Bournemouth",
+          "postCode": "BH81 3LO",
+          "country": {
+            "name": "United Kingdom",
+            "code": "GB"
+          }
+        }
+      ]
+    }
+  },
+  {
+    "statementId": "fbfd0547-d0c6-4a00-b559-5c5e91c34f5c",
+    "declarationSubject": "c359f58d2977",
+    "statementDate": "2020-03-04",
+    "publicationDetails": {
+      "publicationDate": "2020-03-04",
+      "bodsVersion": "0.4",
+      "publisher": {
+        "name": "Profitech Ltd"
+      }
+    },
+    "recordId": "93b53022ae6a",
+    "recordStatus": "new",
+    "recordType": "relationship",
+    "recordDetails": {
+      "isComponent": false,
+      "subject": "c359f58d2977",
+      "interestedParty": "10478c6cf6de",
+      "interests": [
+        {
+          "type": "shareholding",
+          "beneficialOwnershipOrControl": true,
+          "directOrIndirect": "direct",
+          "startDate": "2016-04-06",
+          "share": {
+            "exact": 100
+          }
+        }
+      ]
+    }
+  }
+]