Skip to content

oci_discovery/fetch_json: Return the final URI along with the JSON #25

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
xiekeyang merged 1 commit into from
Sep 21, 2017
Merged

oci_discovery/fetch_json: Return the final URI along with the JSON #25

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 18 additions & 15 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,22 +44,25 @@ $ python3 -m oci_discovery.ref_engine_discovery -l debug example.com/app#1.0 2>/
"example.com/app#1.0": {
"roots": [
{
"annotations": {
"org.opencontainers.image.ref.name": "1.0"
"root": {
"annotations": {
"org.opencontainers.image.ref.name": "1.0"
},
"casEngines": [
{
"protocol": "oci-cas-template-v1",
"uri": "https://a.example.com/cas/{algorithm}/{encoded:2}/{encoded}"
}
],
"digest": "sha256:e9770a03fbdccdd4632895151a93f9af58bbe2c91fdfaaf73160648d250e6ec3",
"mediaType": "application/vnd.oci.image.manifest.v1+json",
"platform": {
"architecture": "ppc64le",
"os": "linux"
},
"size": 799
},
"casEngines": [
{
"protocol": "oci-cas-template-v1",
"uri": "https://a.example.com/cas/{algorithm}/{encoded:2}/{encoded}"
}
],
"digest": "sha256:e9770a03fbdccdd4632895151a93f9af58bbe2c91fdfaaf73160648d250e6ec3",
"mediaType": "application/vnd.oci.image.manifest.v1+json",
"platform": {
"architecture": "ppc64le",
"os": "linux"
},
"size": 799
"uri": "http://example.com/oci-index/app"
Copy link
Copy Markdown
Owner

@xiekeyang xiekeyang Sep 20, 2017

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Here needn't protocol entry? Maybe both uri and protocol are all necessary.

Copy link
Copy Markdown
Contributor Author

@wking wking Sep 20, 2017

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Here needn't protocol entry?

This isn't an engine config, it's the location from which this root was extracted. It would be passed into a subsequent CAS-engine initialization as base, although in this case

.["example.com/app#1.0"].roots[0].root.casEngines[0].uri is a full URI (not a reference), so the base URI won't be needed. It helps with things like:

{
  "example.com/app#1.0": {
    "roots": [
      {
        "root": {
          ...,
          "casEngines": [
            {
              "protocol": "oci-cas-template-v1",
              "uri": "../cas/{algorithm}/{encoded:2}/{encoded}"
            }
          ]
        },
        "uri": "http://example.com/oci-index/app"
      }
    ]
  }
}

So I don't think we need protocol here; just the base URI(s) for resolving any references in the returned root(s).

}
]
}
Expand Down
27 changes: 19 additions & 8 deletions oci_discovery/fetch_json/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,18 +13,26 @@
# limitations under the License.

import json as _json
import logging as _logging
import urllib.request as _urllib_request


_LOGGER = _logging.getLogger(__name__)


def fetch(uri, media_type='application/json'):
"""Fetch a JSON resource."""
response = _urllib_request.urlopen(uri)
content_type = response.headers.get_content_type()
if content_type != media_type:
raise ValueError(
'{} returned {}, not {}'.format(uri, content_type, media_type))
body_bytes = response.read()
charset = response.headers.get_content_charset()
with _urllib_request.urlopen(uri) as response:
content_type = response.headers.get_content_type()
if content_type != media_type:
raise ValueError(
'{} returned {}, not {}'.format(uri, content_type, media_type))
body_bytes = response.read()
charset = response.headers.get_content_charset()
finalURI = response.geturl()
if finalURI != uri:
_LOGGER.debug('redirects lead from {} to {}'.format(uri, finalURI))
uri = finalURI
if charset is None:
raise ValueError('{} does not declare a charset'.format(uri))
try:
Expand All @@ -34,6 +42,9 @@ def fetch(uri, media_type='application/json'):
'{} returned content which did not match the declared {} charset'
.format(uri, charset)) from error
try:
return _json.loads(body)
return {
'uri': uri,
'json': _json.loads(body),
}
except ValueError as error:
raise ValueError('{} returned invalid JSON'.format(uri)) from error
44 changes: 36 additions & 8 deletions oci_discovery/fetch_json/test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,24 +19,47 @@
from . import fetch


class ContextManager(object):
def __init__(self, target, return_value):
self._target = target
self._return_value = return_value

def __enter__(self):
context = unittest.mock.MagicMock()
context.__enter__ = lambda a: self._return_value
context.__exit__ = lambda a, b, c, d: None
self._patch = unittest.mock.patch(
target=self._target, return_value=context)
return self._patch.__enter__()

def __exit__(self, *args, **kwargs):
self._patch.__exit__(*args, **kwargs)


class HTTPResponse(object):
def __init__(self, code=200, body=None, headers=None):
def __init__(self, url, code=200, body=None, headers=None):
self._url = url
self.code = code
self._body = body
self.headers = email.message.Message()
for key, value in headers.items():
self.headers[key] = value

def geturl(self):
return self._url

def read(self):
return self._body or ''


class TestFetchJSON(unittest.TestCase):
def test_good(self):
uri = 'https://example.com'
for name, response, expected in [
(
'empty object',
HTTPResponse(
url=uri,
body=b'{}',
headers={
'Content-Type': 'application/json; charset=UTF-8',
Expand All @@ -47,6 +70,7 @@ def test_good(self):
(
'basic object',
HTTPResponse(
url=uri,
body=b'{"a": "b", "c": 1}',
headers={
'Content-Type': 'application/json; charset=UTF-8',
Expand All @@ -56,18 +80,19 @@ def test_good(self):
),
]:
with self.subTest(name=name):
with unittest.mock.patch(
target='oci_discovery.fetch_json._urllib_request.urlopen',
return_value=response
) as patch_context:
json = fetch(uri='https://example.com')
self.assertEqual(json, expected)
with ContextManager(
target='oci_discovery.fetch_json._urllib_request.urlopen',
return_value=response):
fetched = fetch(uri=uri)
self.assertEqual(fetched, {'uri': uri, 'json': expected})

def test_bad(self):
uri = 'https://example.com'
for name, response, error, regex in [
(
'no charset',
HTTPResponse(
url=uri,
body=b'{}',
headers={
'Content-Type': 'application/json',
Expand All @@ -79,6 +104,7 @@ def test_bad(self):
(
'declared charset does not match body',
HTTPResponse(
url=uri,
body=b'\xff',
headers={
'Content-Type': 'application/json; charset=UTF-8',
Expand All @@ -90,6 +116,7 @@ def test_bad(self):
(
'invalid JSON',
HTTPResponse(
url=uri,
body=b'{',
headers={
'Content-Type': 'application/json; charset=UTF-8',
Expand All @@ -101,6 +128,7 @@ def test_bad(self):
(
'unexpected media type',
HTTPResponse(
url=uri,
body=b'{}',
headers={
'Content-Type': 'text/plain; charset=UTF-8',
Expand All @@ -111,7 +139,7 @@ def test_bad(self):
),
]:
with self.subTest(name=name):
with unittest.mock.patch(
with ContextManager(
target='oci_discovery.fetch_json._urllib_request.urlopen',
return_value=response):
self.assertRaisesRegex(
Expand Down
3 changes: 2 additions & 1 deletion oci_discovery/ref_engine/dummy.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,9 @@ def __str__(self):
self.__class__.__name__,
self._response)

def __init__(self, response):
def __init__(self, response, base=None):
self._response = response
self.base = base

def resolve(self, name):
return _copy.deepcopy(self._response)
8 changes: 6 additions & 2 deletions oci_discovery/ref_engine/oci_index_template.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,9 +45,10 @@ def resolve(self, name):
if self.base:
uri = _urllib_parse.urljoin(base=self.base, url=uri)
_LOGGER.debug('fetching an OCI index for {} from {}'.format(name, uri))
index = _fetch_json.fetch(
fetched = _fetch_json.fetch(
uri=uri,
media_type='application/vnd.oci.image.index.v1+json')
index = fetched['json']
_LOGGER.debug('received OCI index object:\n{}'.format(
_pprint.pformat(index)))
if not isinstance(index, dict):
Expand All @@ -72,4 +73,7 @@ def resolve(self, name):
'org.opencontainers.image.ref.name', None)
if (name_parts['fragment'] == '' or
name_parts['fragment'] == entry_name):
yield entry
yield {
'uri': fetched['uri'],
'root': entry,
}
30 changes: 24 additions & 6 deletions oci_discovery/ref_engine/test_oci_index_template.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,14 +94,24 @@ def test_good(self):
),
]:
engine = oci_index_template.Engine(uri='https://example.com/index')
responseURI = 'https://x.example.com/y'
with self.subTest(label=label):
with unittest.mock.patch(
target='oci_discovery.ref_engine.oci_index_template._fetch_json.fetch',
return_value=response):
return_value={
'uri': responseURI,
'json': response,
}):
resolved = list(engine.resolve(name=name))
self.assertEqual(resolved, expected)
self.assertEqual(
resolved,
[
{'uri': responseURI, 'root': root}
for root in expected
])

def test_bad(self):
uri = 'https://example.com/index'
for label, response, error, regex in [
(
'index is not a JSON object',
Expand All @@ -128,11 +138,14 @@ def test_bad(self):
"https://example.com/index claimed to return application/vnd.oci.image.index.v1\+json, but actually returned \{'manifests': \[\{'annotations': None}]}",
),
]:
engine = oci_index_template.Engine(uri='https://example.com/index')
engine = oci_index_template.Engine(uri=uri)
with self.subTest(label=label):
with unittest.mock.patch(
target='oci_discovery.ref_engine.oci_index_template._fetch_json.fetch',
return_value=response):
return_value={
'uri': uri,
'json': response,
}):
generator = engine.resolve(name='example.com/a')
self.assertRaisesRegex(error, regex, list, generator)

Expand Down Expand Up @@ -193,9 +206,14 @@ def test_reference_expansion(self):
engine = oci_index_template.Engine(uri=uri, base=base)
with unittest.mock.patch(
target='oci_discovery.ref_engine.oci_index_template._fetch_json.fetch',
return_value=response) as mock:
return_value={
'uri': expected,
'json': response
}) as mock:
resolved = list(engine.resolve(name='example.com/a#1.0'))
mock.assert_called_with(
uri=expected,
media_type='application/vnd.oci.image.index.v1+json')
self.assertEqual(resolved, response['manifests'])
self.assertEqual(
resolved,
[{'uri': expected, 'root': root} for root in response['manifests']])
5 changes: 3 additions & 2 deletions oci_discovery/ref_engine_discovery/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,14 +40,15 @@ def resolve(name, protocols=('https', 'http'), port=None):
protocol, host)
_LOGGER.debug('discovering ref engines via {}'.format(uri))
try:
ref_engines_object = _fetch_json.fetch(
fetched = _fetch_json.fetch(
uri=uri,
media_type='application/vnd.oci.ref-engines.v1+json')
except (_ssl.CertificateError,
_ssl.SSLError,
_urllib_error.URLError) as error:
_LOGGER.warning('failed to fetch {} ({})'.format(uri, error))
continue
ref_engines_object = fetched['json']
_LOGGER.debug('received ref-engine discovery object:\n{}'.format(
_pprint.pformat(ref_engines_object)))
if not isinstance(ref_engines_object, dict):
Expand All @@ -58,7 +59,7 @@ def resolve(name, protocols=('https', 'http'), port=None):
continue
for ref_engine_object in ref_engines_object.get('refEngines', []):
try:
ref_engine = _ref_engine.new(**ref_engine_object)
ref_engine = _ref_engine.new(base=fetched['uri'], **ref_engine_object)
except KeyError as error:
_LOGGER.warning(error)
continue
Expand Down
34 changes: 28 additions & 6 deletions oci_discovery/ref_engine_discovery/test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,26 +36,48 @@ def test(self):
{
'protocol': '_dummy',
'response': [
{'name': 'dummy Merkle root 1'},
{'name': 'dummy Merkle root 2'},
{
'uri': 'https://x.example.com/y',
'root': {'name': 'dummy Merkle root 1'},
},
{
'uri': 'https://x.example.com/z',
'root': {'name': 'dummy Merkle root 2'},
},
],
}
]
},
{
'roots': [
{'name': 'dummy Merkle root 1'},
{'name': 'dummy Merkle root 2'},
{
'uri': 'https://x.example.com/y',
'root': {'name': 'dummy Merkle root 1'},
},
{
'uri': 'https://x.example.com/z',
'root': {'name': 'dummy Merkle root 2'},
},
],
}
),
]:
responseURI = 'https://x.example.com/y'
with self.subTest(label=label):
with unittest.mock.patch(
target='oci_discovery.ref_engine_discovery._fetch_json.fetch',
return_value=response):
return_value={
'uri': responseURI,
'json': response,
}):
resolved = resolve(name=name)
self.assertEqual(resolved, expected)
self.assertEqual(
resolved,
[
{'uri': responseURI, 'root': root}
for root in expected
])


def test_bad(self):
for label, name, response, error, regex in [
Expand Down