diff --git a/EXTERNAL_LICENSES.md b/EXTERNAL_LICENSES.md index e826f6230..9d3a1c3a5 100644 --- a/EXTERNAL_LICENSES.md +++ b/EXTERNAL_LICENSES.md @@ -33,7 +33,7 @@ gitdb (0.6.4) imagesize (0.7.1) License: MIT -jsonschema (2.4.0) +jsonschema (2.5.1) License: MIT pager (3.3) diff --git a/cli/binary/binary.spec b/cli/binary/binary.spec index 5439a5449..0022e3dc7 100644 --- a/cli/binary/binary.spec +++ b/cli/binary/binary.spec @@ -11,6 +11,7 @@ a = Analysis(['../dcoscli/main.py'], ], binaries=None, datas=[('../dcoscli/data/help/*', 'dcoscli/data/help'), + ('../dcoscli/data/schemas/*', 'dcoscli/data/schemas'), ('../../dcos/data/config-schema/*', 'dcos/data/config-schema'), ('../../dcos/data/marathon/*', 'dcos/data/marathon') ], diff --git a/cli/dcoscli/data/help/package.txt b/cli/dcoscli/data/help/package.txt index f10bfae6d..de3ab991c 100644 --- a/cli/dcoscli/data/help/package.txt +++ b/cli/dcoscli/data/help/package.txt @@ -5,6 +5,7 @@ Usage: dcos package --config-schema dcos package --help dcos package --info + dcos package build [--output-directory=] dcos package describe [--app --cli --config] [--render] [--package-versions] @@ -26,6 +27,8 @@ Usage: dcos package update Commands: + build + Build a package to install to DC/OS or share with Universe. describe Get specific details for packages. install @@ -68,6 +71,9 @@ Options: Print a short description of this subcommand. --options= Path to a JSON file that contains customized package installation options. + --output-directory= + Path to the directory where the data should be stored. + Defaults to the current working directory. --package-version= The package version to install. --package-versions @@ -81,6 +87,8 @@ Options: Turn off interactive mode and assume "yes" is the answer to all prompts. Positional Arguments: + + Path to a DC/OS Package Build Definition. Name of the DC/OS package in the package repository. diff --git a/cli/dcoscli/data/schemas/build-definition-schema.json b/cli/dcoscli/data/schemas/build-definition-schema.json new file mode 100644 index 000000000..037f08d5c --- /dev/null +++ b/cli/dcoscli/data/schemas/build-definition-schema.json @@ -0,0 +1,492 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + + "definitions": { + + "localReference": { + "type": "string", + "pattern": "^@" + }, + + "dcosReleaseVersion": { + "type": "string", + "pattern": "^(?:0|[1-9][0-9]*)(?:\\.(?:0|[1-9][0-9]*))*$", + "description": "A string representation of a DC/OS Release Version" + }, + + "url": { + "type": "string", + "allOf": [ + { "format": "uri" }, + { "pattern": "^https?://" } + ] + }, + + "base64String": { + "type": "string", + "pattern": "^([A-Za-z0-9+/]{4})*([A-Za-z0-9+/]{4}|[A-Za-z0-9+/]{3}=|[A-Za-z0-9+/]{2}==)$" + }, + + "cliInfo": { + "additionalProperties": false, + "properties": { + "contentHash": { + "items": { + "$ref": "#/definitions/hash" + }, + "minItems": 1, + "type": "array" + }, + "kind": { + "enum": [ + "executable", + "zip" + ], + "type": "string" + }, + "url": { + "$ref": "#/definitions/url" + } + }, + "required": [ + "url", + "kind", + "contentHash" + ], + "type": "object" + }, + + "hash": { + "additionalProperties": false, + "properties": { + "algo": { + "enum": [ + "sha256" + ], + "type": "string" + }, + "value": { + "type": "string" + } + }, + "required": [ + "algo", + "value" + ], + "type": "object" + }, + + + "marathon": { + "type": "object", + "properties": { + "v2AppMustacheTemplate": { + "oneOf": [ + {"$ref": "#/definitions/base64String"}, + {"$ref": "#/definitions/localReference"} + ] + } + }, + "required": [ "v2AppMustacheTemplate" ], + "additionalProperties": false + }, + + + "v20resource": { + "additionalProperties": false, + "type": "object", + "properties": { + "assets": { + "type": "object", + "properties": { + "uris": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "container": { + "type": "object", + "properties": { + "docker": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false + }, + "images": { + "type": "object", + "properties": { + "icon-small": { + "type": "string", + "description": "PNG icon URL, preferably 48 by 48 pixels." + }, + "icon-medium": { + "type": "string", + "description": "PNG icon URL, preferably 128 by 128 pixels." + }, + "icon-large": { + "type": "string", + "description": "PNG icon URL, preferably 256 by 256 pixels." + }, + "screenshots": { + "type": "array", + "items": { + "type": "string", + "description": "PNG screen URL, preferably 1024 by 1024 pixels." + } + } + }, + "additionalProperties": false + } + } + }, + + + "v30resource": { + "additionalProperties": false, + "type": "object", + "properties": { + "assets": { + "type": "object", + "properties": { + "uris": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "container": { + "type": "object", + "properties": { + "docker": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false + }, + "cli": { + "additionalProperties": false, + "properties": { + "binaries": { + "additionalProperties": false, + "minProperties": 1, + "properties": { + "darwin": { + "additionalProperties": false, + "properties": { + "x86-64": { + "$ref": "#/definitions/cliInfo" + } + }, + "required": [ + "x86-64" + ], + "type": "object" + }, + "linux": { + "additionalProperties": false, + "properties": { + "x86-64": { + "$ref": "#/definitions/cliInfo" + } + }, + "required": [ + "x86-64" + ], + "type": "object" + }, + "windows": { + "additionalProperties": false, + "properties": { + "x86-64": { + "$ref": "#/definitions/cliInfo" + } + }, + "required": [ + "x86-64" + ], + "type": "object" + } + }, + "type": "object" + } + }, + "required": [ + "binaries" + ], + "type": "object" + }, + "images": { + "type": "object", + "properties": { + "icon-small": { + "type": "string", + "description": "PNG icon URL, preferably 48 by 48 pixels." + }, + "icon-medium": { + "type": "string", + "description": "PNG icon URL, preferably 128 by 128 pixels." + }, + "icon-large": { + "type": "string", + "description": "PNG icon URL, preferably 256 by 256 pixels." + }, + "screenshots": { + "type": "array", + "items": { + "type": "string", + "description": "PNG screen URL, preferably 1024 by 1024 pixels." + } + } + }, + "additionalProperties": false + } + } + }, + + + "config": { + "$ref": "http://json-schema.org/draft-04/schema#" + }, + + + "command": { + "additionalProperties": false, + "required": ["pip"], + "properties": { + "pip": { + "type": "array", + "items": { + "type": "string" + }, + "title": "Embedded Requirements File", + "description": "[Deprecated v3.x] An array of strings representing of the requirements file to use for installing the subcommand for Pip. Each item is interpreted as a line in the requirements file." + } + } + }, + + + + "v20Package": { + "properties": { + "packagingVersion": { + "type": "string", + "enum": ["2.0"] + }, + "name": { + "type": "string" + }, + "version": { + "type": "string" + }, + "scm": { + "type": "string" + }, + "maintainer": { + "type": "string" + }, + "website": { + "type": "string" + }, + "description": { + "type": "string" + }, + "framework": { + "type": "boolean", + "default": false, + "description": "True if this package installs a new Mesos framework." + }, + "preInstallNotes": { + "type": "string", + "description": "Pre installation notes that would be useful to the user of this package." + }, + "postInstallNotes": { + "type": "string", + "description": "Post installation notes that would be useful to the user of this package." + }, + "postUninstallNotes": { + "type": "string", + "description": "Post uninstallation notes that would be useful to the user of this package." + }, + "tags": { + "type": "array", + "items": { + "type": "string", + "pattern": "^[^\\s]+$" + } + }, + "licenses": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "The name of the license. For example one of [Apache License Version 2.0 | MIT License | BSD License | Proprietary]" + }, + "url": { + "$ref": "#/definitions/url" + } + }, + "additionalProperties": false, + "required": [ + "name", + "url" + ] + } + }, + "marathon": { + "$ref": "#/definitions/marathon" + }, + "resource": { + "oneOf": [ + {"$ref": "#/definitions/v20resource"}, + {"$ref": "#/definitions/localReference"} + ] + }, + "config": { + "oneOf": [ + {"$ref": "#/definitions/config"}, + {"$ref": "#/definitions/localReference"} + ] + }, + "command": { + "$ref": "#/definitions/command" + } + }, + "required": [ + "packagingVersion", + "name", + "version", + "maintainer", + "marathon", + "description", + "tags" + ], + "additionalProperties": false + }, + + "v30Package": { + "properties": { + "packagingVersion": { + "type": "string", + "enum": ["3.0"] + }, + "name": { + "type": "string" + }, + "version": { + "type": "string" + }, + "scm": { + "type": "string" + }, + "maintainer": { + "type": "string" + }, + "website": { + "type": "string" + }, + "description": { + "type": "string" + }, + "framework": { + "type": "boolean", + "default": false, + "description": "True if this package installs a new Mesos framework." + }, + "preInstallNotes": { + "type": "string", + "description": "Pre installation notes that would be useful to the user of this package." + }, + "postInstallNotes": { + "type": "string", + "description": "Post installation notes that would be useful to the user of this package." + }, + "postUninstallNotes": { + "type": "string", + "description": "Post uninstallation notes that would be useful to the user of this package." + }, + "tags": { + "type": "array", + "items": { + "type": "string", + "pattern": "^[^\\s]+$" + } + }, + "licenses": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "The name of the license. For example one of [Apache License Version 2.0 | MIT License | BSD License | Proprietary]" + }, + "url": { + "$ref": "#/definitions/url", + "description": "The URL where the license can be accessed" + } + }, + "additionalProperties": false, + "required": [ + "name", + "url" + ] + } + }, + "minDcosReleaseVersion": { + "$ref": "#/definitions/dcosReleaseVersion", + "description": "The minimum DC/OS Release Version the package can run on." + }, + "marathon": { + "$ref": "#/definitions/marathon" + }, + "resource": { + "oneOf": [ + {"$ref": "#/definitions/v30resource"}, + {"$ref": "#/definitions/localReference"} + ] + }, + "config": { + "oneOf": [ + {"$ref": "#/definitions/config"}, + {"$ref": "#/definitions/localReference"} + ] + }, + "command": { + "$ref": "#/definitions/command" + } + }, + "required": [ + "packagingVersion", + "name", + "version", + "maintainer", + "description", + "tags" + ], + "additionalProperties": false + } + + }, + + "type": "object", + "oneOf": [ + { "$ref": "#/definitions/v20Package" }, + { "$ref": "#/definitions/v30Package" } + ] +} diff --git a/cli/dcoscli/data/schemas/manifest-schema.json b/cli/dcoscli/data/schemas/manifest-schema.json new file mode 100644 index 000000000..c9083238c --- /dev/null +++ b/cli/dcoscli/data/schemas/manifest-schema.json @@ -0,0 +1,12 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "properties": { + "built-by": { + "type": "string" + }, + "created-by": { + "type": "string" + } + } +} diff --git a/cli/dcoscli/data/schemas/metadata-schema.json b/cli/dcoscli/data/schemas/metadata-schema.json new file mode 100644 index 000000000..4efc73436 --- /dev/null +++ b/cli/dcoscli/data/schemas/metadata-schema.json @@ -0,0 +1,472 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + + "definitions": { + + "dcosReleaseVersion": { + "type": "string", + "pattern": "^(?:0|[1-9][0-9]*)(?:\\.(?:0|[1-9][0-9]*))*$", + "description": "A string representation of a DC/OS Release Version" + }, + + "url": { + "type": "string", + "allOf": [ + { "format": "uri" }, + { "pattern": "^https?://" } + ] + }, + + "base64String": { + "type": "string", + "pattern": "^([A-Za-z0-9+/]{4})*([A-Za-z0-9+/]{4}|[A-Za-z0-9+/]{3}=|[A-Za-z0-9+/]{2}==)$" + }, + + "cliInfo": { + "additionalProperties": false, + "properties": { + "contentHash": { + "items": { + "$ref": "#/definitions/hash" + }, + "minItems": 1, + "type": "array" + }, + "kind": { + "enum": [ + "executable", + "zip" + ], + "type": "string" + }, + "url": { + "$ref": "#/definitions/url" + } + }, + "required": [ + "url", + "kind", + "contentHash" + ], + "type": "object" + }, + + "hash": { + "additionalProperties": false, + "properties": { + "algo": { + "enum": [ + "sha256" + ], + "type": "string" + }, + "value": { + "type": "string" + } + }, + "required": [ + "algo", + "value" + ], + "type": "object" + }, + + + "marathon": { + "type": "object", + "properties": { + "v2AppMustacheTemplate": { + "$ref": "#/definitions/base64String" + } + }, + "required": [ "v2AppMustacheTemplate" ], + "additionalProperties": false + }, + + + "v20resource": { + "additionalProperties": false, + "type": "object", + "properties": { + "assets": { + "type": "object", + "properties": { + "uris": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "container": { + "type": "object", + "properties": { + "docker": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false + }, + "images": { + "type": "object", + "properties": { + "icon-small": { + "type": "string", + "description": "PNG icon URL, preferably 48 by 48 pixels." + }, + "icon-medium": { + "type": "string", + "description": "PNG icon URL, preferably 128 by 128 pixels." + }, + "icon-large": { + "type": "string", + "description": "PNG icon URL, preferably 256 by 256 pixels." + }, + "screenshots": { + "type": "array", + "items": { + "type": "string", + "description": "PNG screen URL, preferably 1024 by 1024 pixels." + } + } + }, + "additionalProperties": false + } + } + }, + + + "v30resource": { + "additionalProperties": false, + "type": "object", + "properties": { + "assets": { + "type": "object", + "properties": { + "uris": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "container": { + "type": "object", + "properties": { + "docker": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false + }, + "cli": { + "additionalProperties": false, + "properties": { + "binaries": { + "additionalProperties": false, + "minProperties": 1, + "properties": { + "darwin": { + "additionalProperties": false, + "properties": { + "x86-64": { + "$ref": "#/definitions/cliInfo" + } + }, + "required": [ + "x86-64" + ], + "type": "object" + }, + "linux": { + "additionalProperties": false, + "properties": { + "x86-64": { + "$ref": "#/definitions/cliInfo" + } + }, + "required": [ + "x86-64" + ], + "type": "object" + }, + "windows": { + "additionalProperties": false, + "properties": { + "x86-64": { + "$ref": "#/definitions/cliInfo" + } + }, + "required": [ + "x86-64" + ], + "type": "object" + } + }, + "type": "object" + } + }, + "required": [ + "binaries" + ], + "type": "object" + }, + "images": { + "type": "object", + "properties": { + "icon-small": { + "type": "string", + "description": "PNG icon URL, preferably 48 by 48 pixels." + }, + "icon-medium": { + "type": "string", + "description": "PNG icon URL, preferably 128 by 128 pixels." + }, + "icon-large": { + "type": "string", + "description": "PNG icon URL, preferably 256 by 256 pixels." + }, + "screenshots": { + "type": "array", + "items": { + "type": "string", + "description": "PNG screen URL, preferably 1024 by 1024 pixels." + } + } + }, + "additionalProperties": false + } + } + }, + + + "config": { + "$ref": "http://json-schema.org/draft-04/schema#" + }, + + + "command": { + "additionalProperties": false, + "required": ["pip"], + "properties": { + "pip": { + "type": "array", + "items": { + "type": "string" + }, + "title": "Embedded Requirements File", + "description": "[Deprecated v3.x] An array of strings representing of the requirements file to use for installing the subcommand for Pip. Each item is interpreted as a line in the requirements file." + } + } + }, + + + + "v20Package": { + "properties": { + "packagingVersion": { + "type": "string", + "enum": ["2.0"] + }, + "name": { + "type": "string" + }, + "version": { + "type": "string" + }, + "scm": { + "type": "string" + }, + "maintainer": { + "type": "string" + }, + "website": { + "type": "string" + }, + "description": { + "type": "string" + }, + "framework": { + "type": "boolean", + "default": false, + "description": "True if this package installs a new Mesos framework." + }, + "preInstallNotes": { + "type": "string", + "description": "Pre installation notes that would be useful to the user of this package." + }, + "postInstallNotes": { + "type": "string", + "description": "Post installation notes that would be useful to the user of this package." + }, + "postUninstallNotes": { + "type": "string", + "description": "Post uninstallation notes that would be useful to the user of this package." + }, + "tags": { + "type": "array", + "items": { + "type": "string", + "pattern": "^[^\\s]+$" + } + }, + "licenses": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "The name of the license. For example one of [Apache License Version 2.0 | MIT License | BSD License | Proprietary]" + }, + "url": { + "$ref": "#/definitions/url" + } + }, + "additionalProperties": false, + "required": [ + "name", + "url" + ] + } + }, + "marathon": { + "$ref": "#/definitions/marathon" + }, + "resource": { + "$ref": "#/definitions/v20resource" + }, + "config": { + "$ref": "#/definitions/config" + }, + "command": { + "$ref": "#/definitions/command" + } + }, + "required": [ + "packagingVersion", + "name", + "version", + "maintainer", + "marathon", + "description", + "tags" + ], + "additionalProperties": false + }, + + "v30Package": { + "properties": { + "packagingVersion": { + "type": "string", + "enum": ["3.0"] + }, + "name": { + "type": "string" + }, + "version": { + "type": "string" + }, + "scm": { + "type": "string" + }, + "maintainer": { + "type": "string" + }, + "website": { + "type": "string" + }, + "description": { + "type": "string" + }, + "framework": { + "type": "boolean", + "default": false, + "description": "True if this package installs a new Mesos framework." + }, + "preInstallNotes": { + "type": "string", + "description": "Pre installation notes that would be useful to the user of this package." + }, + "postInstallNotes": { + "type": "string", + "description": "Post installation notes that would be useful to the user of this package." + }, + "postUninstallNotes": { + "type": "string", + "description": "Post uninstallation notes that would be useful to the user of this package." + }, + "tags": { + "type": "array", + "items": { + "type": "string", + "pattern": "^[^\\s]+$" + } + }, + "licenses": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "The name of the license. For example one of [Apache License Version 2.0 | MIT License | BSD License | Proprietary]" + }, + "url": { + "$ref": "#/definitions/url", + "description": "The URL where the license can be accessed" + } + }, + "additionalProperties": false, + "required": [ + "name", + "url" + ] + } + }, + "minDcosReleaseVersion": { + "$ref": "#/definitions/dcosReleaseVersion", + "description": "The minimum DC/OS Release Version the package can run on." + }, + "marathon": { + "$ref": "#/definitions/marathon" + }, + "resource": { + "$ref": "#/definitions/v30resource" + }, + "config": { + "$ref": "#/definitions/config" + }, + "command": { + "$ref": "#/definitions/command" + } + }, + "required": [ + "packagingVersion", + "name", + "version", + "maintainer", + "description", + "tags" + ], + "additionalProperties": false + } + + }, + + "type": "object", + "oneOf": [ + { "$ref": "#/definitions/v20Package" }, + { "$ref": "#/definitions/v30Package" } + ] +} diff --git a/cli/dcoscli/main.py b/cli/dcoscli/main.py index 272fe33d2..80f360532 100644 --- a/cli/dcoscli/main.py +++ b/cli/dcoscli/main.py @@ -5,10 +5,10 @@ import docopt from six.moves import urllib -import dcoscli from dcos import config, constants, emitting, errors, http, subcommand, util from dcos.errors import DCOSException from dcoscli.subcommand import default_doc, SubcommandMain +from dcoscli.util import formatted_cli_version logger = util.get_logger(__name__) @@ -44,7 +44,7 @@ def _get_versions(dcos_url): pass emitter.publish( - "dcoscli.version={}\n".format(dcoscli.version) + + formatted_cli_version() + "\n" + "dcos.version={}\n".format(dcos_info.get("version", "N/A")) + "dcos.commit={}\n".format(dcos_info.get( "dcos-image-commit", "N/A")) + diff --git a/cli/dcoscli/package/main.py b/cli/dcoscli/package/main.py index 8a222d1a5..7b76c0281 100644 --- a/cli/dcoscli/package/main.py +++ b/cli/dcoscli/package/main.py @@ -1,17 +1,23 @@ +import base64 import json import os +import shutil import sys +import tempfile +import zipfile import docopt import pkg_resources +import six import dcoscli from dcos import (cmds, config, cosmospackage, emitting, http, options, package, subcommand, util) from dcos.errors import DCOSException +from dcos.util import md5_hash_file from dcoscli import tables from dcoscli.subcommand import default_command_info, default_doc -from dcoscli.util import decorate_docopt_usage +from dcoscli.util import (decorate_docopt_usage, formatted_cli_version) logger = util.get_logger(__name__) emitter = emitting.FlatEmitter() @@ -63,6 +69,11 @@ def _cmds(): arg_keys=[''], function=_remove_repo), + cmds.Command( + hierarchy=['package', 'build'], + arg_keys=['', '--output-directory'], + function=_build, + ), cmds.Command( hierarchy=['package', 'describe'], arg_keys=['', '--app', '--cli', '--options', @@ -150,8 +161,8 @@ def _update(): def _list_repos(is_json): """List configured package repositories. - :param json_: output json if True - :type json_: bool + :param is_json: output json if True + :type is_json: bool :returns: Process status :rtype: int """ @@ -206,6 +217,256 @@ def _remove_repo(repo_name): return 0 +def _build(build_definition, + output_directory): + """ Creates a DC/OS Package from a DC/OS Package Build Definition + + :param build_definition: The path to a DC/OS Package Build Definition + :type build_definition: str + :param output_directory: The directory where the DC/OS Package + will be stored + :type output_directory: str + :returns: The process status + :rtype: int + """ + # get the path of the build definition + cwd = os.getcwd() + build_definition_path = build_definition + if not os.path.isabs(build_definition_path): + build_definition_path = os.path.join(cwd, build_definition_path) + + build_definition_directory = os.path.dirname(build_definition_path) + + if not os.path.exists(build_definition_path): + raise DCOSException( + "The file [{}] does not exist".format(build_definition_path)) + + # get the path to the output directory + if output_directory is None: + output_directory = cwd + + if not os.path.exists(output_directory): + raise DCOSException( + "The output directory [{}]" + " does not exist".format(output_directory)) + + logger.debug("Using [%s] as output directory", output_directory) + + # load raw build definition + with util.open_file(build_definition_path) as bd: + build_definition_raw = util.load_json(bd, keep_order=True) + + # validate DC/OS Package Build Definition with local references + build_definition_schema_path = "data/schemas/build-definition-schema.json" + build_definition_schema = util.load_jsons( + pkg_resources.resource_string( + "dcoscli", build_definition_schema_path).decode()) + + errs = util.validate_json(build_definition_raw, build_definition_schema) + + if errs: + logger.debug("Failed before resolution: \n" + "\tbuild definition: {}" + "".format(build_definition_raw)) + raise DCOSException(_validation_error(build_definition_path)) + + # resolve local references in build definition + _resolve_local_references( + build_definition_raw, + build_definition_schema, + build_definition_directory + ) + + # at this point all the local references have been resolved + build_definition_resolved = build_definition_raw + + # validate resolved build definition + metadata_schema_path = "data/schemas/metadata-schema.json" + metadata_schema = util.load_jsons( + pkg_resources.resource_string( + "dcoscli", metadata_schema_path).decode()) + + errs = util.validate_json(build_definition_resolved, metadata_schema) + + if errs: + logger.debug("Failed after resolution: \n" + "\tbuild definition: {}" + "".format(build_definition_resolved)) + raise DCOSException('Error validating package: ' + 'there was a problem resolving ' + 'the local references in ' + '[{}]'.format(build_definition_path)) + + # create the manifest + manifest_json = {'built-by': formatted_cli_version()} + + # create the metadata + metadata_json = build_definition_resolved + + # create zip file + with tempfile.NamedTemporaryFile() as temp_file: + with zipfile.ZipFile( + temp_file.file, + mode='w', + compression=zipfile.ZIP_DEFLATED, + allowZip64=True) as zip_file: + + metadata = json.dumps(metadata_json, indent=2).encode() + zip_file.writestr("metadata.json", metadata) + + manifest = json.dumps(manifest_json, indent=2).encode() + zip_file.writestr("manifest.json", manifest) + + # name the package appropriately + temp_file.file.seek(0) + dcos_package_name = '{}-{}-{}.dcos'.format( + metadata_json['name'], + metadata_json['version'], + md5_hash_file(temp_file.file)) + + # get the dcos package path + dcos_package_path = os.path.join(output_directory, dcos_package_name) + + if os.path.exists(dcos_package_path): + raise DCOSException( + 'Output file [{}] already exists'.format( + dcos_package_path)) + + # create a new file to contain the package + temp_file.file.seek(0) + with util.open_file(dcos_package_path, 'w+b') as dcos_package: + shutil.copyfileobj(temp_file.file, dcos_package) + + emitter.publish( + 'Created DCOS Universe package [{}].'.format(dcos_package_path)) + + return 0 + + +def _resolve_local_references(build_definition, + build_schema, + build_definition_directory): + """ Resolves all local references in a DC/OS Package Build Definition + + :param build_definition: The DC/OS Package Build Definition that may + contain local references + :type build_definition: dict + :param build_definition_directory: The directory of the Build Definition + :type build_definition_directory: str + :param build_schema: The schema for the Build Definition + :type build_schema: dict + """ + _replace_marathon(build_definition, + build_schema, + build_definition_directory) + + _replace_directly(build_definition, + build_schema, + build_definition_directory, + "config") + + _replace_directly(build_definition, + build_schema, + build_definition_directory, + "resource") + + +def _replace_directly(build_definition, + build_schema, + build_definition_directory, + ref): + """ Replaces the local reference ref with the contents of + the file pointed to by ref + + :param build_definition: The DC/OS Package Build Definition that + may contain local references + :type build_definition: dict + :param build_definition_directory: The directory of the Build Definition + :type build_definition_directory: str + :param build_schema: The schema for the Build Definition + :type build_schema: dict + :param ref: The key in build_definition that will be replaced + :type ref: str + """ + if ref in build_definition and _is_local_reference(build_definition[ref]): + location = build_definition[ref][1:] + if not os.path.isabs(location): + location = os.path.join(build_definition_directory, location) + + with util.open_file(location) as f: + contents = util.load_json(f, True) + + build_definition[ref] = contents + + errs = util.validate_json(build_definition, build_schema) + if errs: + logger.debug("Failed during resolution of {}: \n" + "\tbuild definition: {}" + "".format(ref, build_definition)) + raise DCOSException(_validation_error(location)) + + +def _replace_marathon(build_definition, + build_schema, + build_definition_directory): + """ Replaces the marathon v2AppMustacheTemplate ref with + the base64 encoding of the file pointed to by the reference + + :param build_definition: The DC/OS Package Build Definition that + may contain local references + :type build_definition: dict + :param build_definition_directory: The directory of the Build Definition + :type build_definition_directory: str + :param build_schema: The schema for the Build Definition + :type build_schema: dict + """ + ref = "marathon" + template = "v2AppMustacheTemplate" + if ref in build_definition and \ + _is_local_reference(build_definition[ref][template]): + location = (build_definition[ref])[template][1:] + if not os.path.isabs(location): + location = os.path.join(build_definition_directory, location) + + # convert the contents of the marathon file into base64 + with util.open_file(location) as f: + contents = base64.b64encode( + f.read().encode()).decode() + + build_definition[ref][template] = contents + + errs = util.validate_json(build_definition, build_schema) + if errs: + logger.debug("Failed during resolution of marathon: \n" + "\tbuild definition: {}" + "".format(build_definition)) + raise DCOSException(_validation_error(location)) + + +def _validation_error(filename): + """Renders a human readable validation error + + :param filename: the file that failed to validate + :type filename: str + :returns: validation error message + :rtype: str + """ + return 'Error validating package: ' \ + '[{}] does not conform to the' \ + ' specified schema'.format(filename) + + +def _is_local_reference(item): + """Checks if an object is a local reference + + :param item: the object that may be a reference + :type item: object + :returns: true if item is a local reference else false + :rtype: bool + """ + return isinstance(item, six.string_types) and item.startswith("@") + + def _describe(package_name, app, cli, diff --git a/cli/dcoscli/util.py b/cli/dcoscli/util.py index 9b3ede470..cc7f03deb 100644 --- a/cli/dcoscli/util.py +++ b/cli/dcoscli/util.py @@ -1,6 +1,7 @@ from functools import wraps import docopt +import dcoscli from dcos import emitting emitter = emitting.FlatEmitter() @@ -25,3 +26,12 @@ def wrapper(*args, **kwargs): return 1 return result return wrapper + + +def formatted_cli_version(): + """Formats the CLI version + + :return: formatted cli version + :rtype: str + """ + return "dcoscli.version={}".format(dcoscli.version) diff --git a/cli/setup.py b/cli/setup.py index b8cb99c04..efc1060fa 100644 --- a/cli/setup.py +++ b/cli/setup.py @@ -78,6 +78,7 @@ 'dcoscli': [ 'data/*.json', 'data/help/*.txt', + 'data/schemas/*.json' ], }, diff --git a/cli/tests/data/build/config.json b/cli/tests/data/build/config.json new file mode 100644 index 000000000..d3050cc9c --- /dev/null +++ b/cli/tests/data/build/config.json @@ -0,0 +1,17 @@ +{ + "$schema": "http://json-schema.org/schema#", + "properties": { + "bitbucket": { + "properties": { + "name": { + "default": "bitbucket", + "description": "name", + "type": "string" + } + }, + "required": ["name"], + "type": "object" + } + }, + "type": "object" +} diff --git a/cli/tests/data/build/marathon.json.mustache b/cli/tests/data/build/marathon.json.mustache new file mode 100644 index 000000000..3b712cee1 --- /dev/null +++ b/cli/tests/data/build/marathon.json.mustache @@ -0,0 +1,47 @@ +{ + "id": "/{{bitbucket.name}}", + "instances": {{bitbucket.instances}}, + "cpus": {{bitbucket.cpus}}, + "mem": {{bitbucket.mem}}, + "maintainer": "support@mesosphere.io", + "container": { + "type": "DOCKER", + "docker": { + "image": "{{resource.assets.container.docker.bitbucket-docker}}", + "network": "BRIDGE", + "portMappings": [ + { "containerPort": 7990, "hostPort": 0 }, + { "containerPort": 7999, "hostPort": 0 } + ] + }, + "volumes": [ + { + "containerPath": "/var/atlassian/application-data/bitbucket", + "hostPath": "{{bitbucket.host-volume}}", + "mode": "RW" + } + ] + }, + "healthChecks": [ + { + "protocol": "COMMAND", + "command": { "value": "curl --fail ${HOST}:${PORT0}" }, + "gracePeriodSeconds": 300, + "intervalSeconds": 60, + "timeoutSeconds": 20, + "maxConsecutiveFailures": 3 + } + ], + "acceptedResourceRoles": [ + "{{bitbucket.role}}" + ], + "labels": { + {{#bitbucket.virtual-host}} + "HAPROXY_GROUP":"external", + "HAPROXY_0_VHOST":"{{bitbucket.virtual-host}}", + {{/bitbucket.virtual-host}} + "DCOS_SERVICE_NAME": "{{bitbucket.name}}", + "DCOS_SERVICE_SCHEME": "http", + "DCOS_SERVICE_PORT_INDEX": "0" + } +} diff --git a/cli/tests/data/build/package_all_references.json b/cli/tests/data/build/package_all_references.json new file mode 100644 index 000000000..ecfcb2f36 --- /dev/null +++ b/cli/tests/data/build/package_all_references.json @@ -0,0 +1,13 @@ +{ + "packagingVersion": "3.0", + "name": "bitbucket", + "version": "4.5", + "description": "Bitbucket Server gives you secure, fast, enterprise-grade controls, like fine-grained permissions and powerful management features.", + "maintainer": "support@mesosphere.io", + "tags": [ "git", "bitbucket", "bit", "bucket", "vcs", "scm" ], + "resource": "@resource.json", + "config": "@config.json", + "marathon": { + "v2AppMustacheTemplate": "@marathon.json.mustache" + } +} diff --git a/cli/tests/data/build/package_badly_formed_reference.json b/cli/tests/data/build/package_badly_formed_reference.json new file mode 100644 index 000000000..1f17ea027 --- /dev/null +++ b/cli/tests/data/build/package_badly_formed_reference.json @@ -0,0 +1,29 @@ +{ + "packagingVersion": "3.0", + "name": "bitbucket", + "version": "4.5", + "description": "Bitbucket Server gives you secure, fast, enterprise-grade controls, like fine-grained permissions and powerful management features.", + "maintainer": "support@mesosphere.io", + "tags": [ "git", "bitbucket", "bit", "bucket", "vcs", "scm" ], + "resource": "resource.json", + "config": { + "$schema": "http://json-schema.org/schema#", + "properties": { + "bitbucket": { + "properties": { + "name": { + "default": "bitbucket", + "description": "name", + "type": "string" + } + }, + "required": ["name"], + "type": "object" + } + }, + "type": "object" + }, + "marathon": { + "v2AppMustacheTemplate": "@marathon.json" + } +} diff --git a/cli/tests/data/build/package_config_reference.json b/cli/tests/data/build/package_config_reference.json new file mode 100644 index 000000000..84c490fd5 --- /dev/null +++ b/cli/tests/data/build/package_config_reference.json @@ -0,0 +1,9 @@ +{ + "packagingVersion": "3.0", + "name": "bitbucket", + "version": "4.5", + "description": "Bitbucket", + "maintainer": "support@mesosphere.io", + "tags": [ "bitbucket" ], + "config": "@config.json" +} diff --git a/cli/tests/data/build/package_config_reference_expected.json b/cli/tests/data/build/package_config_reference_expected.json new file mode 100644 index 000000000..4cbab6a44 --- /dev/null +++ b/cli/tests/data/build/package_config_reference_expected.json @@ -0,0 +1,25 @@ +{ + "packagingVersion": "3.0", + "name": "bitbucket", + "version": "4.5", + "description": "Bitbucket", + "maintainer": "support@mesosphere.io", + "tags": [ "bitbucket" ], + "config": { + "$schema": "http://json-schema.org/schema#", + "properties": { + "bitbucket": { + "properties": { + "name": { + "default": "bitbucket", + "description": "name", + "type": "string" + } + }, + "required": ["name"], + "type": "object" + } + }, + "type": "object" + } +} diff --git a/cli/tests/data/build/package_marathon_reference.json b/cli/tests/data/build/package_marathon_reference.json new file mode 100644 index 000000000..5e9586f63 --- /dev/null +++ b/cli/tests/data/build/package_marathon_reference.json @@ -0,0 +1,25 @@ +{ + "packagingVersion": "3.0", + "name": "bitbucket", + "version": "4.5", + "description": "Bitbucket Server gives you secure, fast, enterprise-grade controls, like fine-grained permissions and powerful management features.", + "maintainer": "support@mesosphere.io", + "tags": [ "git", "bitbucket", "bit", "bucket", "vcs", "scm" ], + "resource": { + "images": { + "icon-small": "http://i.imgur.com/QGc420u.png", + "icon-medium": "http://i.imgur.com/QGc420u.png", + "icon-large": "http://i.imgur.com/QGc420u.png" + }, + "assets": { + "container": { + "docker": { + "bitbucket-docker": "atlassian/bitbucket-server:4.5" + } + } + } + }, + "marathon": { + "v2AppMustacheTemplate": "@marathon.json.mustache" + } +} diff --git a/cli/tests/data/build/package_marathon_reference_expected.json b/cli/tests/data/build/package_marathon_reference_expected.json new file mode 100644 index 000000000..fc5c2fc53 --- /dev/null +++ b/cli/tests/data/build/package_marathon_reference_expected.json @@ -0,0 +1,25 @@ +{ + "packagingVersion": "3.0", + "name": "bitbucket", + "version": "4.5", + "description": "Bitbucket Server gives you secure, fast, enterprise-grade controls, like fine-grained permissions and powerful management features.", + "maintainer": "support@mesosphere.io", + "tags": [ "git", "bitbucket", "bit", "bucket", "vcs", "scm" ], + "resource": { + "images": { + "icon-small": "http://i.imgur.com/QGc420u.png", + "icon-medium": "http://i.imgur.com/QGc420u.png", + "icon-large": "http://i.imgur.com/QGc420u.png" + }, + "assets": { + "container": { + "docker": { + "bitbucket-docker": "atlassian/bitbucket-server:4.5" + } + } + } + }, + "marathon": { + "v2AppMustacheTemplate": "ewogICJpZCI6ICIve3tiaXRidWNrZXQubmFtZX19IiwKICAiaW5zdGFuY2VzIjoge3tiaXRidWNrZXQuaW5zdGFuY2VzfX0sCiAgImNwdXMiOiB7e2JpdGJ1Y2tldC5jcHVzfX0sCiAgIm1lbSI6IHt7Yml0YnVja2V0Lm1lbX19LAogICJtYWludGFpbmVyIjogInN1cHBvcnRAbWVzb3NwaGVyZS5pbyIsCiAgImNvbnRhaW5lciI6IHsKICAgICJ0eXBlIjogIkRPQ0tFUiIsCiAgICAiZG9ja2VyIjogewogICAgICAiaW1hZ2UiOiAie3tyZXNvdXJjZS5hc3NldHMuY29udGFpbmVyLmRvY2tlci5iaXRidWNrZXQtZG9ja2VyfX0iLAogICAgICAibmV0d29yayI6ICJCUklER0UiLAogICAgICAicG9ydE1hcHBpbmdzIjogWwogICAgICAgIHsgImNvbnRhaW5lclBvcnQiOiA3OTkwLCAiaG9zdFBvcnQiOiAwIH0sCiAgICAgICAgeyAiY29udGFpbmVyUG9ydCI6IDc5OTksICJob3N0UG9ydCI6IDAgfQogICAgICBdCiAgICB9LAogICAgInZvbHVtZXMiOiBbCiAgICB7CiAgICAgICAgImNvbnRhaW5lclBhdGgiOiAiL3Zhci9hdGxhc3NpYW4vYXBwbGljYXRpb24tZGF0YS9iaXRidWNrZXQiLAogICAgICAgICJob3N0UGF0aCI6ICJ7e2JpdGJ1Y2tldC5ob3N0LXZvbHVtZX19IiwKICAgICAgICAibW9kZSI6ICJSVyIKICAgIH0KICAgIF0KICB9LAogICJoZWFsdGhDaGVja3MiOiBbCiAgICB7CiAgICAgICJwcm90b2NvbCI6ICJDT01NQU5EIiwKICAgICAgImNvbW1hbmQiOiB7ICJ2YWx1ZSI6ICJjdXJsIC0tZmFpbCAke0hPU1R9OiR7UE9SVDB9IiB9LAogICAgICAiZ3JhY2VQZXJpb2RTZWNvbmRzIjogMzAwLAogICAgICAiaW50ZXJ2YWxTZWNvbmRzIjogNjAsCiAgICAgICJ0aW1lb3V0U2Vjb25kcyI6IDIwLAogICAgICAibWF4Q29uc2VjdXRpdmVGYWlsdXJlcyI6IDMKICAgIH0KICBdLAogICJhY2NlcHRlZFJlc291cmNlUm9sZXMiOiBbCiAgICAie3tiaXRidWNrZXQucm9sZX19IgogIF0sCiAgImxhYmVscyI6IHsKICAgIHt7I2JpdGJ1Y2tldC52aXJ0dWFsLWhvc3R9fQogICAgIkhBUFJPWFlfR1JPVVAiOiJleHRlcm5hbCIsCiAgICAiSEFQUk9YWV8wX1ZIT1NUIjoie3tiaXRidWNrZXQudmlydHVhbC1ob3N0fX0iLAogICAge3svYml0YnVja2V0LnZpcnR1YWwtaG9zdH19CiAgICAiRENPU19TRVJWSUNFX05BTUUiOiAie3tiaXRidWNrZXQubmFtZX19IiwKICAgICJEQ09TX1NFUlZJQ0VfU0NIRU1FIjogImh0dHAiLAogICAgIkRDT1NfU0VSVklDRV9QT1JUX0lOREVYIjogIjAiCiAgfQp9Cg==" + } +} diff --git a/cli/tests/data/build/package_missing_references.json b/cli/tests/data/build/package_missing_references.json new file mode 100644 index 000000000..ee8e8d67a --- /dev/null +++ b/cli/tests/data/build/package_missing_references.json @@ -0,0 +1,29 @@ +{ + "packagingVersion": "3.0", + "name": "bitbucket", + "version": "4.5", + "description": "Bitbucket Server gives you secure, fast, enterprise-grade controls, like fine-grained permissions and powerful management features.", + "maintainer": "support@mesosphere.io", + "tags": [ "git", "bitbucket", "bit", "bucket", "vcs", "scm" ], + "resource": "@resource.json", + "config": { + "$schema": "http://json-schema.org/schema#", + "properties": { + "bitbucket": { + "properties": { + "name": { + "default": "bitbucket", + "description": "name", + "type": "string" + } + }, + "required": ["name"], + "type": "object" + } + }, + "type": "object" + }, + "marathon": { + "v2AppMustacheTemplate": "@marathon.json" + } +} diff --git a/cli/tests/data/build/package_no_match_schema.json b/cli/tests/data/build/package_no_match_schema.json new file mode 100644 index 000000000..2421381e8 --- /dev/null +++ b/cli/tests/data/build/package_no_match_schema.json @@ -0,0 +1,27 @@ +{ + "packagingVersion": "3.0", + "name": "bitbucket", + "version": "4.5", + "description": "Bitbucket Server gives you secure, fast, enterprise-grade controls, like fine-grained permissions and powerful management features.", + "maintainer": "support@mesosphere.io", + "tags": [ "git", "bitbucket", "bit", "bucket", "vcs", "scm" ], + "resource": "@resource.json", + "config": { + "$schema": "http://json-schema.org/schema#", + "properties": { + "bitbucket": { + "properties": { + "name": { + "default": "bitbucket", + "description": "name", + "type": "string" + } + }, + "required": ["name"], + "type": "object" + } + }, + "type": "object" + }, + "marathon": "ewogICJpZCI6ICIve3tiaXRidWNrZXQubmFtZX19IiwKICAiaW5zdGFuY2VzIjoge3tiaXRidWNrZXQuaW5zdGFuY2VzfX0sCiAgImNwdXMiOiB7e2JpdGJ1Y2tldC5jcHVzfX0sCiAgIm1lbSI6IHt7Yml0YnVja2V0Lm1lbX19LAogICJtYWludGFpbmVyIjogInN1cHBvcnRAbWVzb3NwaGVyZS5pbyIsCiAgImNvbnRhaW5lciI6IHsKICAgICJ0eXBlIjogIkRPQ0tFUiIsCiAgICAiZG9ja2VyIjogewogICAgICAiaW1hZ2UiOiAie3tyZXNvdXJjZS5hc3NldHMuY29udGFpbmVyLmRvY2tlci5iaXRidWNrZXQtZG9ja2VyfX0iLAogICAgICAibmV0d29yayI6ICJCUklER0UiLAogICAgICAicG9ydE1hcHBpbmdzIjogWwogICAgICAgIHsgImNvbnRhaW5lclBvcnQiOiA3OTkwLCAiaG9zdFBvcnQiOiAwIH0sCiAgICAgICAgeyAiY29udGFpbmVyUG9ydCI6IDc5OTksICJob3N0UG9ydCI6IDAgfQogICAgICBdCiAgICB9LAogICAgInZvbHVtZXMiOiBbCiAgICB7CiAgICAgICAgImNvbnRhaW5lclBhdGgiOiAiL3Zhci9hdGxhc3NpYW4vYXBwbGljYXRpb24tZGF0YS9iaXRidWNrZXQiLAogICAgICAgICJob3N0UGF0aCI6ICJ7e2JpdGJ1Y2tldC5ob3N0LXZvbHVtZX19IiwKICAgICAgICAibW9kZSI6ICJSVyIKICAgIH0KICAgIF0KICB9LAogICJoZWFsdGhDaGVja3MiOiBbCiAgICB7CiAgICAgICJwcm90b2NvbCI6ICJDT01NQU5EIiwKICAgICAgImNvbW1hbmQiOiB7ICJ2YWx1ZSI6ICJjdXJsIC0tZmFpbCAke0hPU1R9OiR7UE9SVDB9IiB9LAogICAgICAiZ3JhY2VQZXJpb2RTZWNvbmRzIjogMzAwLAogICAgICAiaW50ZXJ2YWxTZWNvbmRzIjogNjAsCiAgICAgICJ0aW1lb3V0U2Vjb25kcyI6IDIwLAogICAgICAibWF4Q29uc2VjdXRpdmVGYWlsdXJlcyI6IDMKICAgIH0KICBdLAogICJhY2NlcHRlZFJlc291cmNlUm9sZXMiOiBbCiAgICAie3tiaXRidWNrZXQucm9sZX19IgogIF0sCiAgImxhYmVscyI6IHsKICAgIHt7I2JpdGJ1Y2tldC52aXJ0dWFsLWhvc3R9fQogICAgIkhBUFJPWFlfR1JPVVAiOiJleHRlcm5hbCIsCiAgICAiSEFQUk9YWV8wX1ZIT1NUIjoie3tiaXRidWNrZXQudmlydHVhbC1ob3N0fX0iLAogICAge3svYml0YnVja2V0LnZpcnR1YWwtaG9zdH19CiAgICAiRENPU19TRVJWSUNFX05BTUUiOiAie3tiaXRidWNrZXQubmFtZX19IiwKICAgICJEQ09TX1NFUlZJQ0VfU0NIRU1FIjogImh0dHAiLAogICAgIkRDT1NfU0VSVklDRV9QT1JUX0lOREVYIjogIjAiCiAgfQp9Cg==" +} diff --git a/cli/tests/data/build/package_no_references.json b/cli/tests/data/build/package_no_references.json new file mode 100644 index 000000000..c16c73265 --- /dev/null +++ b/cli/tests/data/build/package_no_references.json @@ -0,0 +1,42 @@ +{ + "packagingVersion": "3.0", + "name": "bitbucket", + "version": "4.5", + "description": "Bitbucket Server gives you secure, fast, enterprise-grade controls, like fine-grained permissions and powerful management features.", + "maintainer": "support@mesosphere.io", + "tags": [ "git", "bitbucket", "bit", "bucket", "vcs", "scm" ], + "resource": { + "images": { + "icon-small": "http://i.imgur.com/QGc420u.png", + "icon-medium": "http://i.imgur.com/QGc420u.png", + "icon-large": "http://i.imgur.com/QGc420u.png" + }, + "assets": { + "container": { + "docker": { + "bitbucket-docker": "atlassian/bitbucket-server:4.5" + } + } + } + }, + "config": { + "$schema": "http://json-schema.org/schema#", + "properties": { + "bitbucket": { + "properties": { + "name": { + "default": "bitbucket", + "description": "name", + "type": "string" + } + }, + "required": ["name"], + "type": "object" + } + }, + "type": "object" + }, + "marathon": { + "v2AppMustacheTemplate": "ewogICJpZCI6ICIve3tiaXRidWNrZXQubmFtZX19IiwKICAiaW5zdGFuY2VzIjoge3tiaXRidWNrZXQuaW5zdGFuY2VzfX0sCiAgImNwdXMiOiB7e2JpdGJ1Y2tldC5jcHVzfX0sCiAgIm1lbSI6IHt7Yml0YnVja2V0Lm1lbX19LAogICJtYWludGFpbmVyIjogInN1cHBvcnRAbWVzb3NwaGVyZS5pbyIsCiAgImNvbnRhaW5lciI6IHsKICAgICJ0eXBlIjogIkRPQ0tFUiIsCiAgICAiZG9ja2VyIjogewogICAgICAiaW1hZ2UiOiAie3tyZXNvdXJjZS5hc3NldHMuY29udGFpbmVyLmRvY2tlci5iaXRidWNrZXQtZG9ja2VyfX0iLAogICAgICAibmV0d29yayI6ICJCUklER0UiLAogICAgICAicG9ydE1hcHBpbmdzIjogWwogICAgICAgIHsgImNvbnRhaW5lclBvcnQiOiA3OTkwLCAiaG9zdFBvcnQiOiAwIH0sCiAgICAgICAgeyAiY29udGFpbmVyUG9ydCI6IDc5OTksICJob3N0UG9ydCI6IDAgfQogICAgICBdCiAgICB9LAogICAgInZvbHVtZXMiOiBbCiAgICB7CiAgICAgICAgImNvbnRhaW5lclBhdGgiOiAiL3Zhci9hdGxhc3NpYW4vYXBwbGljYXRpb24tZGF0YS9iaXRidWNrZXQiLAogICAgICAgICJob3N0UGF0aCI6ICJ7e2JpdGJ1Y2tldC5ob3N0LXZvbHVtZX19IiwKICAgICAgICAibW9kZSI6ICJSVyIKICAgIH0KICAgIF0KICB9LAogICJoZWFsdGhDaGVja3MiOiBbCiAgICB7CiAgICAgICJwcm90b2NvbCI6ICJDT01NQU5EIiwKICAgICAgImNvbW1hbmQiOiB7ICJ2YWx1ZSI6ICJjdXJsIC0tZmFpbCAke0hPU1R9OiR7UE9SVDB9IiB9LAogICAgICAiZ3JhY2VQZXJpb2RTZWNvbmRzIjogMzAwLAogICAgICAiaW50ZXJ2YWxTZWNvbmRzIjogNjAsCiAgICAgICJ0aW1lb3V0U2Vjb25kcyI6IDIwLAogICAgICAibWF4Q29uc2VjdXRpdmVGYWlsdXJlcyI6IDMKICAgIH0KICBdLAogICJhY2NlcHRlZFJlc291cmNlUm9sZXMiOiBbCiAgICAie3tiaXRidWNrZXQucm9sZX19IgogIF0sCiAgImxhYmVscyI6IHsKICAgIHt7I2JpdGJ1Y2tldC52aXJ0dWFsLWhvc3R9fQogICAgIkhBUFJPWFlfR1JPVVAiOiJleHRlcm5hbCIsCiAgICAiSEFQUk9YWV8wX1ZIT1NUIjoie3tiaXRidWNrZXQudmlydHVhbC1ob3N0fX0iLAogICAge3svYml0YnVja2V0LnZpcnR1YWwtaG9zdH19CiAgICAiRENPU19TRVJWSUNFX05BTUUiOiAie3tiaXRidWNrZXQubmFtZX19IiwKICAgICJEQ09TX1NFUlZJQ0VfU0NIRU1FIjogImh0dHAiLAogICAgIkRDT1NfU0VSVklDRV9QT1JUX0lOREVYIjogIjAiCiAgfQp9Cg==" + } +} diff --git a/cli/tests/data/build/package_reference_does_not_match_schema.json b/cli/tests/data/build/package_reference_does_not_match_schema.json new file mode 100644 index 000000000..231521c1e --- /dev/null +++ b/cli/tests/data/build/package_reference_does_not_match_schema.json @@ -0,0 +1,13 @@ +{ + "packagingVersion": "3.0", + "name": "bitbucket", + "version": "4.5", + "description": "Bitbucket Server gives you secure, fast, enterprise-grade controls, like fine-grained permissions and powerful management features.", + "maintainer": "support@mesosphere.io", + "tags": [ "git", "bitbucket", "bit", "bucket", "vcs", "scm" ], + "resource": "@resource-bad.json", + "config": "@config.json", + "marathon": { + "v2AppMustacheTemplate": "@marathon.json.mustache" + } +} diff --git a/cli/tests/data/build/package_resource_only_reference.json b/cli/tests/data/build/package_resource_only_reference.json new file mode 100644 index 000000000..39a6096a0 --- /dev/null +++ b/cli/tests/data/build/package_resource_only_reference.json @@ -0,0 +1,9 @@ +{ + "packagingVersion": "3.0", + "name": "bitbucket", + "version": "4.5", + "description": "Bitbucket Server gives you secure, fast, enterprise-grade controls, like fine-grained permissions and powerful management features.", + "maintainer": "support@mesosphere.io", + "tags": [ "git", "bitbucket", "bit", "bucket", "vcs", "scm" ], + "resource": "@resource.json" +} diff --git a/cli/tests/data/build/package_resource_only_reference_expected.json b/cli/tests/data/build/package_resource_only_reference_expected.json new file mode 100644 index 000000000..712c48034 --- /dev/null +++ b/cli/tests/data/build/package_resource_only_reference_expected.json @@ -0,0 +1,22 @@ +{ + "packagingVersion": "3.0", + "name": "bitbucket", + "version": "4.5", + "description": "Bitbucket Server gives you secure, fast, enterprise-grade controls, like fine-grained permissions and powerful management features.", + "maintainer": "support@mesosphere.io", + "tags": [ "git", "bitbucket", "bit", "bucket", "vcs", "scm" ], + "resource": { + "images": { + "icon-small": "http://i.imgur.com/QGc420u.png", + "icon-medium": "http://i.imgur.com/QGc420u.png", + "icon-large": "http://i.imgur.com/QGc420u.png" + }, + "assets": { + "container": { + "docker": { + "bitbucket-docker": "atlassian/bitbucket-server:4.5" + } + } + } + } +} diff --git a/cli/tests/data/build/package_resource_reference.json b/cli/tests/data/build/package_resource_reference.json new file mode 100644 index 000000000..c4eb0603a --- /dev/null +++ b/cli/tests/data/build/package_resource_reference.json @@ -0,0 +1,29 @@ +{ + "packagingVersion": "3.0", + "name": "bitbucket", + "version": "4.5", + "description": "Bitbucket Server gives you secure, fast, enterprise-grade controls, like fine-grained permissions and powerful management features.", + "maintainer": "support@mesosphere.io", + "tags": [ "git", "bitbucket", "bit", "bucket", "vcs", "scm" ], + "resource": "@resource.json", + "config": { + "$schema": "http://json-schema.org/schema#", + "properties": { + "bitbucket": { + "properties": { + "name": { + "default": "bitbucket", + "description": "name", + "type": "string" + } + }, + "required": ["name"], + "type": "object" + } + }, + "type": "object" + }, + "marathon": { + "v2AppMustacheTemplate": "ewogICJpZCI6ICIve3tiaXRidWNrZXQubmFtZX19IiwKICAiaW5zdGFuY2VzIjoge3tiaXRidWNrZXQuaW5zdGFuY2VzfX0sCiAgImNwdXMiOiB7e2JpdGJ1Y2tldC5jcHVzfX0sCiAgIm1lbSI6IHt7Yml0YnVja2V0Lm1lbX19LAogICJtYWludGFpbmVyIjogInN1cHBvcnRAbWVzb3NwaGVyZS5pbyIsCiAgImNvbnRhaW5lciI6IHsKICAgICJ0eXBlIjogIkRPQ0tFUiIsCiAgICAiZG9ja2VyIjogewogICAgICAiaW1hZ2UiOiAie3tyZXNvdXJjZS5hc3NldHMuY29udGFpbmVyLmRvY2tlci5iaXRidWNrZXQtZG9ja2VyfX0iLAogICAgICAibmV0d29yayI6ICJCUklER0UiLAogICAgICAicG9ydE1hcHBpbmdzIjogWwogICAgICAgIHsgImNvbnRhaW5lclBvcnQiOiA3OTkwLCAiaG9zdFBvcnQiOiAwIH0sCiAgICAgICAgeyAiY29udGFpbmVyUG9ydCI6IDc5OTksICJob3N0UG9ydCI6IDAgfQogICAgICBdCiAgICB9LAogICAgInZvbHVtZXMiOiBbCiAgICB7CiAgICAgICAgImNvbnRhaW5lclBhdGgiOiAiL3Zhci9hdGxhc3NpYW4vYXBwbGljYXRpb24tZGF0YS9iaXRidWNrZXQiLAogICAgICAgICJob3N0UGF0aCI6ICJ7e2JpdGJ1Y2tldC5ob3N0LXZvbHVtZX19IiwKICAgICAgICAibW9kZSI6ICJSVyIKICAgIH0KICAgIF0KICB9LAogICJoZWFsdGhDaGVja3MiOiBbCiAgICB7CiAgICAgICJwcm90b2NvbCI6ICJDT01NQU5EIiwKICAgICAgImNvbW1hbmQiOiB7ICJ2YWx1ZSI6ICJjdXJsIC0tZmFpbCAke0hPU1R9OiR7UE9SVDB9IiB9LAogICAgICAiZ3JhY2VQZXJpb2RTZWNvbmRzIjogMzAwLAogICAgICAiaW50ZXJ2YWxTZWNvbmRzIjogNjAsCiAgICAgICJ0aW1lb3V0U2Vjb25kcyI6IDIwLAogICAgICAibWF4Q29uc2VjdXRpdmVGYWlsdXJlcyI6IDMKICAgIH0KICBdLAogICJhY2NlcHRlZFJlc291cmNlUm9sZXMiOiBbCiAgICAie3tiaXRidWNrZXQucm9sZX19IgogIF0sCiAgImxhYmVscyI6IHsKICAgIHt7I2JpdGJ1Y2tldC52aXJ0dWFsLWhvc3R9fQogICAgIkhBUFJPWFlfR1JPVVAiOiJleHRlcm5hbCIsCiAgICAiSEFQUk9YWV8wX1ZIT1NUIjoie3tiaXRidWNrZXQudmlydHVhbC1ob3N0fX0iLAogICAge3svYml0YnVja2V0LnZpcnR1YWwtaG9zdH19CiAgICAiRENPU19TRVJWSUNFX05BTUUiOiAie3tiaXRidWNrZXQubmFtZX19IiwKICAgICJEQ09TX1NFUlZJQ0VfU0NIRU1FIjogImh0dHAiLAogICAgIkRDT1NfU0VSVklDRV9QT1JUX0lOREVYIjogIjAiCiAgfQp9Cg==" + } +} diff --git a/cli/tests/data/build/resource-bad.json b/cli/tests/data/build/resource-bad.json new file mode 100644 index 000000000..f8c372060 --- /dev/null +++ b/cli/tests/data/build/resource-bad.json @@ -0,0 +1,15 @@ +{ + "images": { + "icon-small": "http://i.imgur.com/QGc420u.png", + "icon-medium": "http://i.imgur.com/QGc420u.png", + "icon-large": "http://i.imgur.com/QGc420u.png", + "icon-bigilionbigaggg": "tuehasoentuh" + }, + "assets": { + "container": { + "docker": { + "bitbucket-docker": "atlassian/bitbucket-server:4.5" + } + } + } +} diff --git a/cli/tests/data/build/resource.json b/cli/tests/data/build/resource.json new file mode 100644 index 000000000..9a0761e87 --- /dev/null +++ b/cli/tests/data/build/resource.json @@ -0,0 +1,14 @@ +{ + "images": { + "icon-small": "http://i.imgur.com/QGc420u.png", + "icon-medium": "http://i.imgur.com/QGc420u.png", + "icon-large": "http://i.imgur.com/QGc420u.png" + }, + "assets": { + "container": { + "docker": { + "bitbucket-docker": "atlassian/bitbucket-server:4.5" + } + } + } +} diff --git a/cli/tests/data/help/package.txt b/cli/tests/data/help/package.txt index f10bfae6d..de3ab991c 100644 --- a/cli/tests/data/help/package.txt +++ b/cli/tests/data/help/package.txt @@ -5,6 +5,7 @@ Usage: dcos package --config-schema dcos package --help dcos package --info + dcos package build [--output-directory=] dcos package describe [--app --cli --config] [--render] [--package-versions] @@ -26,6 +27,8 @@ Usage: dcos package update Commands: + build + Build a package to install to DC/OS or share with Universe. describe Get specific details for packages. install @@ -68,6 +71,9 @@ Options: Print a short description of this subcommand. --options= Path to a JSON file that contains customized package installation options. + --output-directory= + Path to the directory where the data should be stored. + Defaults to the current working directory. --package-version= The package version to install. --package-versions @@ -81,6 +87,8 @@ Options: Turn off interactive mode and assume "yes" is the answer to all prompts. Positional Arguments: + + Path to a DC/OS Package Build Definition. Name of the DC/OS package in the package repository. diff --git a/cli/tests/integrations/test_package_build.py b/cli/tests/integrations/test_package_build.py new file mode 100644 index 000000000..4193f98e3 --- /dev/null +++ b/cli/tests/integrations/test_package_build.py @@ -0,0 +1,169 @@ +import json +import os +import re +import tempfile +import zipfile + +from shutil import rmtree + +from dcos import util +from dcos.util import md5_hash_file +from dcoscli.util import formatted_cli_version +from .common import exec_command + + +def _success_test(build_definition, + expected_package_path="tests/data/build/" + "package_no_references.json"): + output_folder = tempfile.mkdtemp() + + # perform the operation + return_code, stdout, stderr = exec_command( + ['dcos', 'package', 'build', '--output-directory', + output_folder, build_definition] + ) + + # check that the output is correct + pattern = re.compile("^Created DCOS Universe package") + assert stderr == b"" + assert return_code == 0 + assert pattern.match(stdout.decode()), stdout.decode() + + # check that the files created are correct + zip_file_name = re.search('^Created DCOS Universe package \[(.+?)\].', + stdout.decode()).group(1) + + results = re.search('^(.+)-(.+)-(.+)\.dcos', zip_file_name) + + name_result = results.group(1) + name_expected = os.path.join(output_folder, "bitbucket") + assert name_result == name_expected + + version_result = results.group(2) + version_expected = "4.5" + assert version_result == version_expected + + hash_result = results.group(3) + with util.open_file(zip_file_name, 'rb') as zp: + hash_expected = md5_hash_file(zp) + assert hash_result == hash_expected + + # check that the contents of the zip file created are correct + with zipfile.ZipFile(zip_file_name) as zip_file: + # package.json + with util.open_file(expected_package_path) as pj: + expected_metadata = util.load_json(pj) + metadata = json.loads(zip_file.read("metadata.json").decode()) + assert metadata == expected_metadata + + # manifest.json + expected_manifest = {'built-by': formatted_cli_version()} + manifest = json.loads(zip_file.read("manifest.json").decode()) + assert manifest == expected_manifest + + # delete the files created + rmtree(output_folder) + + +def _failure_test(build_definition, error_pattern): + output_folder = tempfile.mkdtemp() + + # perform the operation + return_code, stdout, stderr = exec_command( + ['dcos', 'package', 'build', '--output-directory', + output_folder, build_definition] + ) + + # check that the output is correct + assert return_code == 1 + + p = re.compile(error_pattern) + m = p.match(stderr.decode()) + + assert m, '[[' + stderr.decode() + ']]' \ + + ' did not match ' \ + + '[[' + error_pattern + ']]' + assert stderr.decode() == m.string + + # check that no files were created in the temp folder + assert len(os.listdir(output_folder)) == 0 + + # delete the files created + rmtree(output_folder) + + +def test_package_resource_only_reference(): + _success_test( + "tests/data/build/" + "package_resource_only_reference.json", + expected_package_path="tests/data/build/" + "package_resource_only" + "_reference_expected.json") + + +def test_package_config_no_reference(): + _success_test( + "tests/data/build/package_config_reference_expected.json", + expected_package_path="tests/data/build/" + "package_config_reference_expected.json") + + +def test_package_config_reference(): + _success_test( + "tests/data/build/package_config_reference.json", + expected_package_path="tests/data/build/" + "package_config_reference_expected.json") + + +def test_package_marathon_reference(): + _success_test( + "tests/data/build/package_marathon_reference.json", + expected_package_path="tests/data/build/" + "package_marathon_reference_expected.json") + + +def test_package_resource_reference(): + _success_test("tests/data/build/package_resource_reference.json") + + +def test_package_no_references(): + _success_test("tests/data/build/package_no_references.json") + + +def test_package_all_references(): + _success_test("tests/data/build/package_all_references.json") + + +def test_package_does_not_exist(): + _failure_test("tests/data/build/does_not_exist.json", + "^The file \[(.+)\] does not exist.*") + + +def test_package_missing_references(): + _failure_test("tests/data/build/package_missing_references.json", + "^Error opening file " + "\[(.+)marathon\.json\]: " + "No such file or directory.*") + + +def test_package_reference_does_not_match_schema(): + _failure_test("tests/data/build/" + "package_reference_does_not_match_schema.json", + "^Error validating package: " + "\[(.+)resource-bad\.json\] " + "does not conform to the specified schema.*") + + +def test_package_no_match_schema(): + _failure_test("tests/data/build/package_no_match_schema.json", + "^Error validating package: " + "\[(.+)package_no_match_schema\.json\]" + " does not conform to the specified schema.*") + + +def test_package_badly_formed_reference(): + _failure_test("tests/data/build/package_badly_formed_reference.json", + "^Error validating package: " + "\[(.+)" + "package_badly_formed_reference\.json\]" + " does not conform to the specified schema.*") diff --git a/dcos/util.py b/dcos/util.py index 153f61ea4..9e2b0eabd 100644 --- a/dcos/util.py +++ b/dcos/util.py @@ -2,6 +2,7 @@ import concurrent.futures import contextlib import functools +import hashlib import json import logging import os @@ -253,17 +254,22 @@ def configure_logger(log_level): msg.format(log_level, constants.VALID_LOG_LEVEL_VALUES)) -def load_json(reader): +def load_json(reader, keep_order=False): """Deserialize a reader into a python object :param reader: the json reader :type reader: a :code:`.read()`-supporting object + :param keep_order: whether the return should be an ordered dictionary + :type keep_order: bool :returns: the deserialized JSON object :rtype: dict | list | str | int | float | bool """ try: - return json.load(reader) + if keep_order: + return json.load(reader, object_pairs_hook=collections.OrderedDict) + else: + return json.load(reader) except Exception as error: logger.error( 'Unhandled exception while loading JSON: %r', @@ -610,3 +616,17 @@ def normalize_marathon_id_path(id_path): logger = get_logger(__name__) + + +def md5_hash_file(file): + """Calculates the md5 of a file + + :param file: file to hash + :type file: file + :returns: digest in hexadecimal + :rtype: str + """ + hasher = hashlib.md5() + for chunk in iter(lambda: file.read(4096), b''): + hasher.update(chunk) + return hasher.hexdigest() diff --git a/setup.py b/setup.py index e363a3570..3fae6a629 100755 --- a/setup.py +++ b/setup.py @@ -59,7 +59,7 @@ packages=find_packages(exclude=['pydoc', 'tests', 'cli', 'bin']), install_requires=[ - 'jsonschema==2.4', # pin the exact version, jsonschema 2.5 broke py3 + 'jsonschema>=2.5', 'pager>=3.3, <4.0', 'prettytable>=0.7, <1.0', 'pygments>=2.0, <3.0', @@ -75,7 +75,8 @@ package_data={ 'dcos': [ 'data/config-schema/*.json', - 'data/marathon/*.json' + 'data/marathon/*.json', + 'data/schemas/*.json' ], }, )