diff --git a/readthedocs/config/config.py b/readthedocs/config/config.py index 0607fd3eea1..f17fb88f13d 100644 --- a/readthedocs/config/config.py +++ b/readthedocs/config/config.py @@ -526,6 +526,17 @@ def build(self): def doctype(self): return self.defaults['doctype'] + @property + def sphinx(self): + config_file = self.defaults['sphinx_configuration'] + if config_file is not None: + config_file = os.path.join(self.base_path, config_file) + return Sphinx( + builder=self.doctype, + configuration=config_file, + fail_on_warning=False, + ) + class BuildConfigV2(BuildConfigBase): @@ -959,7 +970,8 @@ def get_configuration_class(version): version = int(version) return configurations_class[version] except (KeyError, ValueError): - raise ConfigError( - 'Invalid version of the configuration file', + raise InvalidConfig( + 'version', code=VERSION_INVALID, + error_message='Invalid version of the configuration file', ) diff --git a/readthedocs/doc_builder/backends/sphinx.py b/readthedocs/doc_builder/backends/sphinx.py index ba719cf6226..571d53be57e 100644 --- a/readthedocs/doc_builder/backends/sphinx.py +++ b/readthedocs/doc_builder/backends/sphinx.py @@ -41,9 +41,14 @@ class BaseSphinx(BaseBuilder): def __init__(self, *args, **kwargs): super(BaseSphinx, self).__init__(*args, **kwargs) + self.config_file = self.config.sphinx.configuration try: + if not self.config_file: + self.config_file = self.project.conf_file(self.version.slug) self.old_artifact_path = os.path.join( - self.project.conf_dir(self.version.slug), self.sphinx_build_dir) + os.path.dirname(self.config_file), + self.sphinx_build_dir + ) except ProjectConfigurationError: docs_dir = self.docs_dir() self.old_artifact_path = os.path.join( @@ -69,7 +74,7 @@ def get_config_params(self): # TODO this should be handled better in the theme conf_py_path = os.path.join( os.path.sep, - self.version.get_conf_py_path(), + self.config_file, '', ) remote_version = self.version.commit_name @@ -151,15 +156,16 @@ def get_config_params(self): def append_conf(self, **__): """Find or create a ``conf.py`` with a rendered ``doc_builder/conf.py.tmpl`` appended""" - try: - self.version.get_conf_py_path() - except ProjectConfigurationError: + if self.config_file is None: master_doc = self.create_index(extension='rst') self._write_config(master_doc=master_doc) try: - outfile_path = self.project.conf_file(self.version.slug) - outfile = codecs.open(outfile_path, encoding='utf-8', mode='a') + self.config_file = ( + self.config_file or + self.project.conf_file(self.version.slug) + ) + outfile = codecs.open(self.config_file, encoding='utf-8', mode='a') except (ProjectConfigurationError, IOError): trace = sys.exc_info()[2] six.reraise( @@ -183,7 +189,7 @@ def append_conf(self, **__): self.run( 'cat', os.path.relpath( - outfile_path, + self.config_file, self.project.checkout_path(self.version.slug), ), cwd=self.project.checkout_path(self.version.slug), @@ -199,6 +205,8 @@ def build(self): ] if self._force: build_command.append('-E') + if self.config.sphinx.fail_on_warning: + build_command.append('-W') build_command.extend([ '-b', self.sphinx_builder, @@ -210,8 +218,10 @@ def build(self): self.sphinx_build_dir, ]) cmd_ret = self.run( - *build_command, cwd=project.conf_dir(self.version.slug), - bin_path=self.python_env.venv_bin()) + *build_command, + cwd=os.path.dirname(self.config_file), + bin_path=self.python_env.venv_bin() + ) return cmd_ret.successful @@ -347,7 +357,7 @@ class PdfBuilder(BaseSphinx): def build(self): self.clean() - cwd = self.project.conf_dir(self.version.slug) + cwd = os.path.dirname(self.config_file) # Default to this so we can return it always. self.run( diff --git a/readthedocs/doc_builder/base.py b/readthedocs/doc_builder/base.py index fcb70964709..1ee7df4ccb6 100644 --- a/readthedocs/doc_builder/base.py +++ b/readthedocs/doc_builder/base.py @@ -46,6 +46,7 @@ def __init__(self, build_env, python_env, force=False): self.python_env = python_env self.version = build_env.version self.project = build_env.project + self.config = python_env.config self._force = force self.target = self.project.artifact_path( version=self.version.slug, type_=self.type) diff --git a/readthedocs/doc_builder/config.py b/readthedocs/doc_builder/config.py index 289d0c85eb5..9d714af604f 100644 --- a/readthedocs/doc_builder/config.py +++ b/readthedocs/doc_builder/config.py @@ -31,7 +31,10 @@ def load_yaml_config(version): python_version = 3 if project.python_interpreter == 'python3' else 2 allow_v2 = project.has_feature(Feature.ALLOW_V2_CONFIG_FILE) try: - sphinx_configuration = version.get_conf_py_path() + sphinx_configuration = path.join( + version.get_conf_py_path(), + 'conf.py' + ) except ProjectConfigurationError: sphinx_configuration = None diff --git a/readthedocs/projects/tasks.py b/readthedocs/projects/tasks.py index 0ca28d358ef..d5db9297c32 100644 --- a/readthedocs/projects/tasks.py +++ b/readthedocs/projects/tasks.py @@ -701,7 +701,7 @@ def build_docs(self): def build_docs_html(self): """Build HTML docs.""" - html_builder = get_builder_class(self.project.documentation_type)( + html_builder = get_builder_class(self.config.doctype)( build_env=self.build_env, python_env=self.python_env, ) diff --git a/readthedocs/rtd_tests/tests/test_config_integration.py b/readthedocs/rtd_tests/tests/test_config_integration.py index 1673cacf2b6..3387c24dae9 100644 --- a/readthedocs/rtd_tests/tests/test_config_integration.py +++ b/readthedocs/rtd_tests/tests/test_config_integration.py @@ -232,7 +232,6 @@ def test_requirements_file(self, load_config): self.assertEqual(config.python.requirements, '__init__.py') -@pytest.mark.skip @pytest.mark.django_db @mock.patch('readthedocs.projects.models.Project.checkout_path') class TestLoadConfigV2(object): @@ -290,12 +289,10 @@ def test_report_using_invalid_version(self, checkout_path, tmpdir): @pytest.mark.parametrize('config', [{}, {'formats': []}]) @patch('readthedocs.projects.models.Project.repo_nonblockinglock', new=MagicMock()) - @patch('readthedocs.doc_builder.backends.sphinx.SearchBuilder.build') @patch('readthedocs.doc_builder.backends.sphinx.HtmlBuilder.build') @patch('readthedocs.doc_builder.backends.sphinx.HtmlBuilder.append_conf') def test_build_formats_default_empty( - self, append_conf, html_build, search_build, - checkout_path, config, tmpdir): + self, append_conf, html_build, checkout_path, config, tmpdir): """ The default value for formats is [], which means no extra formats are build. @@ -304,22 +301,26 @@ def test_build_formats_default_empty( self.create_config_file(tmpdir, config) update_docs = self.get_update_docs_task() + python_env = Virtualenv( + version=self.version, + build_env=update_docs.build_env, + config=update_docs.config + ) + update_docs.python_env = python_env outcomes = update_docs.build_docs() # No extra formats were triggered assert outcomes['html'] - assert outcomes['search'] assert not outcomes['localmedia'] assert not outcomes['pdf'] assert not outcomes['epub'] @patch('readthedocs.projects.models.Project.repo_nonblockinglock', new=MagicMock()) @patch('readthedocs.projects.tasks.UpdateDocsTaskStep.build_docs_class') - @patch('readthedocs.doc_builder.backends.sphinx.SearchBuilder.build') @patch('readthedocs.doc_builder.backends.sphinx.HtmlBuilder.build') @patch('readthedocs.doc_builder.backends.sphinx.HtmlBuilder.append_conf') def test_build_formats_only_pdf( - self, append_conf, html_build, search_build, build_docs_class, + self, append_conf, html_build, build_docs_class, checkout_path, tmpdir): """ Only the pdf format is build. @@ -328,11 +329,17 @@ def test_build_formats_only_pdf( self.create_config_file(tmpdir, {'formats': ['pdf']}) update_docs = self.get_update_docs_task() + python_env = Virtualenv( + version=self.version, + build_env=update_docs.build_env, + config=update_docs.config + ) + update_docs.python_env = python_env + outcomes = update_docs.build_docs() # Only pdf extra format was triggered assert outcomes['html'] - assert outcomes['search'] build_docs_class.assert_called_with('sphinx_pdf') assert outcomes['pdf'] assert not outcomes['localmedia'] @@ -642,6 +649,35 @@ def test_sphinx_configuration_default( append_conf.assert_called_once() move.assert_called_once() + @patch('readthedocs.doc_builder.backends.sphinx.BaseSphinx.move') + @patch('readthedocs.doc_builder.backends.sphinx.BaseSphinx.append_conf') + @patch('readthedocs.doc_builder.backends.sphinx.BaseSphinx.run') + def test_sphinx_configuration_default( + self, run, append_conf, move, checkout_path, tmpdir): + """Should be default to find a conf.py file.""" + checkout_path.return_value = str(tmpdir) + + apply_fs(tmpdir, {'conf.py': ''}) + self.create_config_file(tmpdir, {}) + self.project.conf_py_file = '' + self.project.save() + + update_docs = self.get_update_docs_task() + config = update_docs.config + python_env = Virtualenv( + version=self.version, + build_env=update_docs.build_env, + config=config + ) + update_docs.python_env = python_env + + update_docs.build_docs_html() + + args, kwargs = run.call_args + assert kwargs['cwd'] == str(tmpdir) + append_conf.assert_called_once() + move.assert_called_once() + @patch('readthedocs.doc_builder.backends.sphinx.BaseSphinx.move') @patch('readthedocs.doc_builder.backends.sphinx.BaseSphinx.append_conf') @patch('readthedocs.doc_builder.backends.sphinx.BaseSphinx.run') @@ -716,6 +752,7 @@ def test_sphinx_fail_on_warning( append_conf.assert_called_once() move.assert_called_once() + @pytest.mark.skip @patch('readthedocs.doc_builder.backends.mkdocs.BaseMkdocs.move') @patch('readthedocs.doc_builder.backends.mkdocs.BaseMkdocs.append_conf') @patch('readthedocs.doc_builder.backends.mkdocs.BaseMkdocs.run') @@ -754,6 +791,7 @@ def test_mkdocs_configuration( append_conf.assert_called_once() move.assert_called_once() + @pytest.mark.skip @patch('readthedocs.doc_builder.backends.mkdocs.BaseMkdocs.move') @patch('readthedocs.doc_builder.backends.mkdocs.BaseMkdocs.append_conf') @patch('readthedocs.doc_builder.backends.mkdocs.BaseMkdocs.run') diff --git a/readthedocs/rtd_tests/tests/test_doc_builder.py b/readthedocs/rtd_tests/tests/test_doc_builder.py index 0c6e877ce4b..4c2f7396991 100644 --- a/readthedocs/rtd_tests/tests/test_doc_builder.py +++ b/readthedocs/rtd_tests/tests/test_doc_builder.py @@ -18,6 +18,7 @@ from readthedocs.builds.models import Version from readthedocs.doc_builder.backends.mkdocs import BaseMkdocs, MkdocsHTML from readthedocs.doc_builder.backends.sphinx import BaseSphinx +from readthedocs.doc_builder.python_environments import Virtualenv from readthedocs.projects.exceptions import ProjectConfigurationError from readthedocs.projects.models import Project @@ -30,13 +31,12 @@ def setUp(self): self.project = Project.objects.get(slug='pip') self.version = self.project.versions.first() - build_env = namedtuple('project', 'version') - build_env.project = self.project - build_env.version = self.version + self.build_env = namedtuple('project', 'version') + self.build_env.project = self.project + self.build_env.version = self.version BaseSphinx.type = 'base' BaseSphinx.sphinx_build_dir = tempfile.mkdtemp() - self.base_sphinx = BaseSphinx(build_env=build_env, python_env=None) @patch( 'readthedocs.doc_builder.backends.sphinx.SPHINX_TEMPLATE_DIR', @@ -47,8 +47,10 @@ def setUp(self): @patch('readthedocs.doc_builder.backends.sphinx.BaseSphinx.get_config_params') @patch('readthedocs.doc_builder.backends.sphinx.BaseSphinx.run') @patch('readthedocs.builds.models.Version.get_conf_py_path') - @patch('readthedocs.builds.models.Project.conf_file') - def test_create_conf_py(self, conf_file, get_conf_py_path, _, get_config_params, create_index, docs_dir): + @patch('readthedocs.projects.models.Project.checkout_path') + def test_create_conf_py( + self, checkout_path, get_conf_py_path, _, + get_config_params, create_index, docs_dir): """ Test for a project without ``conf.py`` file. @@ -60,22 +62,41 @@ def test_create_conf_py(self, conf_file, get_conf_py_path, _, get_config_params, any kind of exception raised by ``append_conf`` (we were originally having a ``TypeError`` because of an encoding problem in Python3) """ - docs_dir.return_value = tempfile.mkdtemp() + tmp_dir = tempfile.mkdtemp() + checkout_path.return_value = tmp_dir + docs_dir.return_value = tmp_dir create_index.return_value = 'README.rst' get_config_params.return_value = {} get_conf_py_path.side_effect = ProjectConfigurationError - conf_file.return_value = tempfile.mktemp() + python_env = Virtualenv( + version=self.version, + build_env=self.build_env, + config=None, + ) + base_sphinx = BaseSphinx( + build_env=self.build_env, + python_env=python_env, + ) try: - self.base_sphinx.append_conf() + base_sphinx.append_conf() except Exception: pytest.fail('Exception was generated when append_conf called.') # Check the content generated by our method is the same than what we # expects from a pre-generated file - generated_conf_py = os.path.join(self.base_sphinx.docs_dir(), 'conf.py') - expected_conf_py = os.path.join(os.path.dirname(__file__), '..', 'files', 'conf.py') + generated_conf_py = os.path.join(base_sphinx.docs_dir(), 'conf.py') + expected_conf_py = os.path.join( + os.path.dirname(__file__), + '..', + 'files', + 'conf.py' + ) with open(generated_conf_py) as gf, open(expected_conf_py) as ef: - self.assertEqual(gf.read(), ef.read()) + autogenerated_confpy_lines = 28 + self.assertEqual( + gf.readlines()[:autogenerated_confpy_lines], + ef.readlines()[:autogenerated_confpy_lines] + ) @patch( 'readthedocs.doc_builder.backends.sphinx.SPHINX_TEMPLATE_DIR', @@ -86,8 +107,9 @@ def test_create_conf_py(self, conf_file, get_conf_py_path, _, get_config_params, @patch('readthedocs.doc_builder.backends.sphinx.BaseSphinx.get_config_params') @patch('readthedocs.doc_builder.backends.sphinx.BaseSphinx.run') @patch('readthedocs.builds.models.Version.get_conf_py_path') - def test_create_conf_py( - self, get_conf_py_path, _, get_config_params, + @patch('readthedocs.projects.models.Project.checkout_path') + def test_multiple_conf_py( + self, checkout_path, get_conf_py_path, _, get_config_params, create_index, docs_dir): """ Test for a project with multiple ``conf.py`` files. @@ -97,14 +119,24 @@ def test_create_conf_py( """ tmp_docs_dir = py.path.local(tempfile.mkdtemp()) - tmp_docs_dir.join('conf.py').new() - tmp_docs_dir.join('test').mkdir().join('conf.py').new() + tmp_docs_dir.join('conf.py').write('') + tmp_docs_dir.join('test').mkdir().join('conf.py').write('') docs_dir.return_value = str(tmp_docs_dir) + checkout_path.return_value = str(tmp_docs_dir) create_index.return_value = 'README.rst' get_config_params.return_value = {} get_conf_py_path.side_effect = ProjectConfigurationError + python_env = Virtualenv( + version=self.version, + build_env=self.build_env, + config=None, + ) + base_sphinx = BaseSphinx( + build_env=self.build_env, + python_env=python_env, + ) with pytest.raises(ProjectConfigurationError): - self.base_sphinx.append_conf() + base_sphinx.append_conf() @override_settings(PRODUCTION_DOMAIN='readthedocs.org') @@ -125,9 +157,14 @@ def test_append_conf_create_yaml(self, checkout_path, run): os.mkdir(os.path.join(tmpdir, 'docs')) checkout_path.return_value = tmpdir + python_env = Virtualenv( + version=self.version, + build_env=self.build_env, + config=None, + ) self.searchbuilder = MkdocsHTML( build_env=self.build_env, - python_env=None + python_env=python_env, ) self.searchbuilder.append_conf() @@ -180,9 +217,14 @@ def test_append_conf_existing_yaml_on_root(self, checkout_path, run): ) checkout_path.return_value = tmpdir + python_env = Virtualenv( + version=self.version, + build_env=self.build_env, + config=None, + ) self.searchbuilder = MkdocsHTML( build_env=self.build_env, - python_env=None + python_env=python_env, ) self.searchbuilder.append_conf() @@ -234,9 +276,14 @@ def test_override_theme_new_style(self, checkout_path, run): ) checkout_path.return_value = tmpdir + python_env = Virtualenv( + version=self.version, + build_env=self.build_env, + config=None, + ) self.searchbuilder = MkdocsHTML( build_env=self.build_env, - python_env=None + python_env=python_env, ) self.searchbuilder.append_conf() @@ -267,9 +314,14 @@ def test_override_theme_old_style(self, checkout_path, run): ) checkout_path.return_value = tmpdir + python_env = Virtualenv( + version=self.version, + build_env=self.build_env, + config=None, + ) self.searchbuilder = MkdocsHTML( build_env=self.build_env, - python_env=None + python_env=python_env, ) self.searchbuilder.append_conf() @@ -298,9 +350,14 @@ def test_dont_override_theme(self, checkout_path, run): ) checkout_path.return_value = tmpdir + python_env = Virtualenv( + version=self.version, + build_env=self.build_env, + config=None, + ) self.searchbuilder = MkdocsHTML( build_env=self.build_env, - python_env=None + python_env=python_env, ) self.searchbuilder.append_conf()