diff --git a/docs/yaml-config.rst b/docs/yaml-config.rst index db2c1b0199b..0a8b6eb69f3 100644 --- a/docs/yaml-config.rst +++ b/docs/yaml-config.rst @@ -69,6 +69,40 @@ The file option specified the Conda `environment file`_ to use. .. note:: Conda is only supported via the YAML file. + +build +~~~~~ + +The ``build`` block configures specific aspects of the documentation build. + +.. _yaml_build_image: + +build.image +``````````` + + +* Default: :djangosetting:`DOCKER_IMAGE` +* Options: ``1.0``, ``2.0``, ``latest`` + +The build image to use for specific builds. +This lets users specify a more experimental build image, +if they want to be on the cutting edge. + +Certain Python versions require a certain build image, +as defined here:: + +* `'1.0': 2, 2.7, 3, 3.4` +* `'2.0': 2, 2.7, 3, 3.5` +* `'latest': 2, 2.7, 3, 3.3, 3.4, 3.5, 3.6` + +.. code-block:: yaml + + build: + image: latest + + python: + version: 3.6 + python ~~~~~~ @@ -85,15 +119,12 @@ This is the version of Python to use when building your documentation. If you specify only the major version of Python, the highest supported minor version will be selected. -The supported Python versions depends on the version of the build image your -project is using. The default build image that is used to build documentation -contains support for Python ``2.7`` and ``3.5``. - -There is also an image in testing that supports Python versions ``2.7``, -``3.3``, ``3.4``, ``3.5``, and ``3.6``. If you would like access to this build -image, you can sign up for beta access here: +.. warning:: -https://goo.gl/forms/AKEoeWHixlzVfqKT2 + The supported Python versions depends on the version of the build image your + project is using. The default build image that is used to build documentation + contains support for Python ``2.7`` and ``3.5``. + See the :ref:`yaml_build_image` for more information on supported Python versions. .. code-block:: yaml diff --git a/readthedocs/doc_builder/config.py b/readthedocs/doc_builder/config.py index 2ce1db0ad9c..a8d10efd7ed 100644 --- a/readthedocs/doc_builder/config.py +++ b/readthedocs/doc_builder/config.py @@ -8,7 +8,7 @@ from readthedocs_build.config import load as load_config from readthedocs_build.config import BuildConfig, ConfigError, InvalidConfig -from .constants import DOCKER_BUILD_IMAGES, DOCKER_IMAGE +from .constants import DOCKER_IMAGE_SETTINGS, DOCKER_IMAGE class ConfigWrapper(object): @@ -108,6 +108,13 @@ def formats(self): formats += ['pdf'] return formats + @property + def build_image(self): + if self._project.container_image: + # Allow us to override per-project still + return self._project.container_image + return self._yaml_config['build']['image'] + # Not implemented until we figure out how to keep in sync with the webs. # Probably needs to be version-specific as well, not project. # @property @@ -126,19 +133,21 @@ def load_yaml_config(version): parsing consistent between projects. """ checkout_path = version.project.checkout_path(version.slug) - env_config = {} # Get build image to set up the python version validation. Pass in the # build image python limitations to the loaded config so that the versions # can be rejected at validation - build_image = DOCKER_BUILD_IMAGES.get( - version.project.container_image, - DOCKER_BUILD_IMAGES.get(DOCKER_IMAGE, None), - ) - if build_image: - env_config = { - 'python': build_image['python'], + + img_name = version.project.container_image or DOCKER_IMAGE + env_config = { + 'build': { + 'image': img_name, } + } + img_settings = DOCKER_IMAGE_SETTINGS.get(img_name, None) + if img_settings: + env_config.update(img_settings) + env_config['DOCKER_IMAGE_SETTINGS'] = img_settings try: sphinx_env_config = env_config.copy() diff --git a/readthedocs/doc_builder/constants.py b/readthedocs/doc_builder/constants.py index 18caa705028..14598ea00bf 100644 --- a/readthedocs/doc_builder/constants.py +++ b/readthedocs/doc_builder/constants.py @@ -4,11 +4,14 @@ from __future__ import ( absolute_import, division, print_function, unicode_literals) +import logging import os import re from django.conf import settings +log = logging.getLogger(__name__) + SPHINX_TEMPLATE_DIR = os.path.join( settings.SITE_ROOT, 'readthedocs', @@ -33,6 +36,13 @@ ) DOCKER_VERSION = getattr(settings, 'DOCKER_VERSION', 'auto') DOCKER_IMAGE = getattr(settings, 'DOCKER_IMAGE', 'readthedocs/build:2.0') +DOCKER_IMAGE_SETTINGS = getattr(settings, 'DOCKER_IMAGE_SETTINGS', {}) + +old_config = getattr(settings, 'DOCKER_BUILD_IMAGES', None) +if old_config: + log.warning('Old config detected, DOCKER_BUILD_IMAGES->DOCKER_IMAGE_SETTINGS') + DOCKER_IMAGE_SETTINGS.update(old_config) + DOCKER_LIMITS = {'memory': '200m', 'time': 600} DOCKER_LIMITS.update(getattr(settings, 'DOCKER_LIMITS', {})) @@ -40,17 +50,3 @@ DOCKER_OOM_EXIT_CODE = 137 DOCKER_HOSTNAME_MAX_LEN = 64 - -# Build images -DOCKER_BUILD_IMAGES = { - 'readthedocs/build:1.0': { - 'python': {'supported_versions': [2, 2.7, 3, 3.4]}, - }, - 'readthedocs/build:2.0': { - 'python': {'supported_versions': [2, 2.7, 3, 3.5]}, - }, - 'readthedocs/build:latest': { - 'python': {'supported_versions': [2, 2.7, 3, 3.3, 3.4, 3.5, 3.6]}, - }, -} -DOCKER_BUILD_IMAGES.update(getattr(settings, 'DOCKER_BUILD_IMAGES', {})) diff --git a/readthedocs/doc_builder/environments.py b/readthedocs/doc_builder/environments.py index 2ccc70107c3..1fb1a9431b8 100644 --- a/readthedocs/doc_builder/environments.py +++ b/readthedocs/doc_builder/environments.py @@ -294,11 +294,12 @@ class BuildEnvironment(object): successful """ - def __init__(self, project=None, version=None, build=None, record=True, - environment=None, update_on_success=True): + def __init__(self, project=None, version=None, build=None, config=None, + record=True, environment=None, update_on_success=True): self.project = project self.version = version self.build = build + self.config = config self.record = record self.environment = environment or {} self.update_on_success = update_on_success @@ -526,6 +527,8 @@ def __init__(self, *args, **kwargs): project_name=self.project.slug, )[:DOCKER_HOSTNAME_MAX_LEN] ) + if self.config and self.config.build_image: + self.container_image = self.config.build_image if self.project.container_mem_limit: self.container_mem_limit = self.project.container_mem_limit if self.project.container_time_limit: diff --git a/readthedocs/projects/tasks.py b/readthedocs/projects/tasks.py index 4bd6e959ba8..9e8cf1d0fe0 100644 --- a/readthedocs/projects/tasks.py +++ b/readthedocs/projects/tasks.py @@ -245,7 +245,7 @@ def run_build(self, docker=False, record=True): env_cls = DockerEnvironment else: env_cls = LocalEnvironment - self.build_env = env_cls(project=self.project, version=self.version, + self.build_env = env_cls(project=self.project, version=self.version, config=self.config, build=self.build, record=record, environment=env_vars) # Environment used for building code, usually with Docker diff --git a/readthedocs/rtd_tests/tests/test_config_wrapper.py b/readthedocs/rtd_tests/tests/test_config_wrapper.py index aaa374ae7ce..cae7bec62f5 100644 --- a/readthedocs/rtd_tests/tests/test_config_wrapper.py +++ b/readthedocs/rtd_tests/tests/test_config_wrapper.py @@ -54,7 +54,7 @@ def test_python_supported_versions_default_image_1_0(self, load_config): self.assertEqual(load_config.call_count, 1) load_config.assert_has_calls([ mock.call(path=mock.ANY, env_config={ - 'python': {'supported_versions': [2, 2.7, 3, 3.4]}, + 'build': {'image': 'readthedocs/build:1.0'}, 'type': 'sphinx', 'output_base': '', 'name': mock.ANY @@ -62,37 +62,29 @@ def test_python_supported_versions_default_image_1_0(self, load_config): ]) self.assertEqual(config.python_version, 2) + def test_python_supported_versions_image_1_0(self, load_config): + load_config.side_effect = create_load() + self.project.container_image = 'readthedocs/build:1.0' + self.project.save() + config = load_yaml_config(self.version) + self.assertEqual(config._yaml_config.get_valid_python_versions(), + [2, 2.7, 3, 3.4]) + def test_python_supported_versions_image_2_0(self, load_config): load_config.side_effect = create_load() self.project.container_image = 'readthedocs/build:2.0' self.project.save() config = load_yaml_config(self.version) - self.assertEqual(load_config.call_count, 1) - load_config.assert_has_calls([ - mock.call(path=mock.ANY, env_config={ - 'python': {'supported_versions': [2, 2.7, 3, 3.5]}, - 'type': 'sphinx', - 'output_base': '', - 'name': mock.ANY - }), - ]) - self.assertEqual(config.python_version, 2) + self.assertEqual(config._yaml_config.get_valid_python_versions(), + [2, 2.7, 3, 3.5]) def test_python_supported_versions_image_latest(self, load_config): load_config.side_effect = create_load() self.project.container_image = 'readthedocs/build:latest' self.project.save() config = load_yaml_config(self.version) - self.assertEqual(load_config.call_count, 1) - load_config.assert_has_calls([ - mock.call(path=mock.ANY, env_config={ - 'python': {'supported_versions': [2, 2.7, 3, 3.3, 3.4, 3.5, 3.6]}, - 'type': 'sphinx', - 'output_base': '', - 'name': mock.ANY - }), - ]) - self.assertEqual(config.python_version, 2) + self.assertEqual(config._yaml_config.get_valid_python_versions(), + [2, 2.7, 3, 3.3, 3.4, 3.5, 3.6]) def test_python_default_version(self, load_config): load_config.side_effect = create_load() diff --git a/requirements/pip.txt b/requirements/pip.txt index 92f4e5328b0..63dd94f3063 100644 --- a/requirements/pip.txt +++ b/requirements/pip.txt @@ -10,7 +10,9 @@ mkdocs==0.14.0 django==1.9.12 six==1.10.0 future==0.16.0 -readthedocs-build==2.0.7 +#readthedocs-build==2.0.8 +# For testing +git+https://github.com/rtfd/readthedocs-build.git@79f78d23d367f71#egg=readthedocs_build django-tastypie==0.13.0 django-haystack==2.6.0