From ca43ea449c560bbb96f1ffa8ff693e26b1db81b7 Mon Sep 17 00:00:00 2001 From: Brett Date: Mon, 20 Mar 2023 15:27:52 -0400 Subject: [PATCH 1/4] WIP: jsonschema 4.17/4.18 check_schema not working --- asdf/schema.py | 53 +++++++++++++++++++++++++++++++++++++------------- 1 file changed, 39 insertions(+), 14 deletions(-) diff --git a/asdf/schema.py b/asdf/schema.py index 97dc8eeec..e3ba6cc75 100644 --- a/asdf/schema.py +++ b/asdf/schema.py @@ -11,8 +11,16 @@ import numpy as np import yaml +import importlib_metadata +if importlib_metadata.version('jsonschema') >= "4.18": + USE_REFERENCING = True + import referencing + from referencing.exceptions import Unretrievable as RefError +else: + from jsonschema.exceptions import RefResolutionError as RefError + USE_REFERENCING = False from jsonschema import validators as mvalidators -from jsonschema.exceptions import RefResolutionError, ValidationError +from jsonschema.exceptions import ValidationError from . import constants, extension, generic_io, reference, tagged, treeutil, util, versioning, yamlutil from .config import get_config @@ -215,7 +223,10 @@ def has_seen_pair(self, schema, instance): return (id(schema), id(instance)) in self._seen_id_pairs def create(self, schema, resolver): - return self._validator_class(schema, resolver=resolver) + if USE_REFERENCING: + return self._validator_class(schema, registry=resolver) + else: + return self._validator_class(schema, resolver=resolver) class _CycleCheckingValidatorProxy: @@ -384,13 +395,20 @@ def get_schema(url): # remote schemas here in `load_schema`, so we don't need # jsonschema to do it on our behalf. Setting it to `True` # counterintuitively makes things slower. - return mvalidators.RefResolver( - "", - {}, - cache_remote=False, - handlers=handlers, - urljoin_cache=urljoin_cache, - ) + if USE_REFERENCING: + def retrieve_schema(url): + schema = get_schema(url) + resource = referencing.Resource(schema, specification=referencing.jsonschema.DRAFT4) + return resource + return referencing.Registry({}, retrieve=retrieve_schema) + else: + return mvalidators.RefResolver( + "", + {}, + cache_remote=False, + handlers=handlers, + urljoin_cache=urljoin_cache, + ) @lru_cache @@ -569,11 +587,18 @@ def _iter_errors(self, instance, _schema=None): for schema_uri in schema_uris: try: - tag_schema = self._resolver.resolve(schema_uri)[1] - yield from self._json_schema_validator_factory.create(tag_schema, self._resolver).iter_errors( - node, - ) - except RefResolutionError: + if USE_REFERENCING: + tag_resource = self._resolver.get_or_retrieve(schema_uri) + resolver = tag_resource.value @ self._resolver + v = self._json_schema_validator_factory.create(tag_resource.value.contents, resolver) + v._resolver = resolver.resolver_with_root(tag_resource.value) + yield from v.iter_errors(node) + else: + tag_schema = self._resolver.resolve(schema_uri)[1] + yield from self._json_schema_validator_factory.create(tag_schema, self._resolver).iter_errors( + node, + ) + except RefError: warnings.warn(f"Unable to locate schema file for '{tag}': '{schema_uri}'", AsdfWarning) def validate(self, instance, _schema=None): From 7080d41b72b68a3b70e51d448eb11574b8b47fcc Mon Sep 17 00:00:00 2001 From: Brett Date: Mon, 20 Mar 2023 15:41:13 -0400 Subject: [PATCH 2/4] WIP: fixed jsonschema 4.18 schema path --- asdf/resource.py | 20 ++++++++++++++++---- asdf/schema.py | 7 ++++++- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/asdf/resource.py b/asdf/resource.py index adadfba75..d85423bba 100644 --- a/asdf/resource.py +++ b/asdf/resource.py @@ -4,6 +4,7 @@ """ import pkgutil from collections.abc import Mapping +import importlib_metadata from asdf_standard import DirectoryResourceMapping as _DirectoryResourceMapping @@ -168,9 +169,17 @@ def __repr__(self): return f"" -_JSONSCHEMA_URI_TO_FILENAME = { - "http://json-schema.org/draft-04/schema": "draft4.json", -} +if importlib_metadata.version('jsonschema') >= '4.18': + USE_JSONSCHEMA_SPECIFICATIONS = True + _JSONSCHEMA_URI_TO_FILENAME = { + "http://json-schema.org/draft-04/schema": "metaschema.json", + } +else: + USE_JSONSCHEMA_SPECIFICATIONS = False + _JSONSCHEMA_URI_TO_FILENAME = { + "http://json-schema.org/draft-04/schema": "draft4.json", + } + jsonschema_specifications/schemas/draft4 class JsonschemaResourceMapping(Mapping): @@ -181,7 +190,10 @@ class JsonschemaResourceMapping(Mapping): def __getitem__(self, uri): filename = _JSONSCHEMA_URI_TO_FILENAME[uri] - return pkgutil.get_data("jsonschema", f"schemas/{filename}") + if USE_JSONSCHEMA_SPECIFICATIONS: + return pkgutil.get_data("jsonschema_specifications", f"schemas/draft4/{filename}") + else: + return pkgutil.get_data("jsonschema", f"schemas/{filename}") def __len__(self): return len(_JSONSCHEMA_URI_TO_FILENAME) diff --git a/asdf/schema.py b/asdf/schema.py index e3ba6cc75..e93e69306 100644 --- a/asdf/schema.py +++ b/asdf/schema.py @@ -866,5 +866,10 @@ def applicable_validators(schema): id_of=mvalidators.Draft4Validator.ID_OF, applicable_validators=applicable_validators, ) - validator = cls(meta_schema, resolver=resolver) + if USE_REFERENCING: + validator = cls(meta_schema, registry=resolver) + resource = referencing.Resource(meta_schema, specification=referencing.jsonschema.DRAFT4) + validator._resolver = resolver.resolver_with_root(resource) + else: + validator = cls(meta_schema, resolver=resolver) validator.validate(schema) From 291c0086755956baf825c3f7769f03f96e8e7219 Mon Sep 17 00:00:00 2001 From: Brett Date: Mon, 20 Mar 2023 16:01:51 -0400 Subject: [PATCH 3/4] WIP: 4 failing tests with 4.18 --- asdf/resource.py | 1 - asdf/schema.py | 4 +++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/asdf/resource.py b/asdf/resource.py index d85423bba..f7f0daf56 100644 --- a/asdf/resource.py +++ b/asdf/resource.py @@ -179,7 +179,6 @@ def __repr__(self): _JSONSCHEMA_URI_TO_FILENAME = { "http://json-schema.org/draft-04/schema": "draft4.json", } - jsonschema_specifications/schemas/draft4 class JsonschemaResourceMapping(Mapping): diff --git a/asdf/schema.py b/asdf/schema.py index e93e69306..b3ac52203 100644 --- a/asdf/schema.py +++ b/asdf/schema.py @@ -838,6 +838,8 @@ def _validate_default(validator, default, instance, schema): return if "default" in instance: + get_validator(instance).validate(instance['default']) + return instance_validator.resolver.push_scope(instance_scope) try: yield from instance_validator.descend(instance["default"], instance) @@ -869,7 +871,7 @@ def applicable_validators(schema): if USE_REFERENCING: validator = cls(meta_schema, registry=resolver) resource = referencing.Resource(meta_schema, specification=referencing.jsonschema.DRAFT4) - validator._resolver = resolver.resolver_with_root(resource) + #validator._resolver = resolver.resolver_with_root(resource) else: validator = cls(meta_schema, resolver=resolver) validator.validate(schema) From 4f13930cd50408ff4e1c3ddb8b7f72dc3a6aee6d Mon Sep 17 00:00:00 2001 From: Brett Date: Mon, 20 Mar 2023 16:09:38 -0400 Subject: [PATCH 4/4] some code cleanup --- asdf/schema.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/asdf/schema.py b/asdf/schema.py index b3ac52203..463e830b7 100644 --- a/asdf/schema.py +++ b/asdf/schema.py @@ -828,11 +828,6 @@ def check_schema(schema, validate_default=True): validators = util.HashableDict(mvalidators.Draft4Validator.VALIDATORS.copy()) if validate_default: - # The jsonschema library doesn't validate defaults - # on its own. - instance_validator = _create_json_schema_validator_factory().create(schema, _make_resolver(None)) - instance_scope = schema.get("id", "") - def _validate_default(validator, default, instance, schema): if not validator.is_type(instance, "object"): return @@ -840,11 +835,6 @@ def _validate_default(validator, default, instance, schema): if "default" in instance: get_validator(instance).validate(instance['default']) return - instance_validator.resolver.push_scope(instance_scope) - try: - yield from instance_validator.descend(instance["default"], instance) - finally: - instance_validator.resolver.pop_scope() validators.update({"default": _validate_default}) @@ -870,8 +860,6 @@ def applicable_validators(schema): ) if USE_REFERENCING: validator = cls(meta_schema, registry=resolver) - resource = referencing.Resource(meta_schema, specification=referencing.jsonschema.DRAFT4) - #validator._resolver = resolver.resolver_with_root(resource) else: validator = cls(meta_schema, resolver=resolver) validator.validate(schema)