Append core requirements to Conda environment file

docs/guides/feature-flags.rst

@@ -20,6 +20,11 @@ The version of ``conda`` used in the build process could not be the latest one.
 This is because we use Miniconda, which its release process is a little more slow than ``conda`` itself.
 In case you prefer to use the latest ``conda`` version available, this is the flag you need.
 
+``CONDA_APPEND_CORE_REQUIREMENTS``: :featureflags:`CONDA_APPEND_CORE_REQUIREMENTS`
+
+Makes Read the Docs to install all the requirements at once on ``conda create`` step.
+This helps users to pin dependencies on conda and to improve build time.
+
 ``DONT_OVERWRITE_SPHINX_CONTEXT``: :featureflags:`DONT_OVERWRITE_SPHINX_CONTEXT`
 
 ``DONT_SHALLOW_CLONE``: :featureflags:`DONT_SHALLOW_CLONE`

readthedocs/doc_builder/python_environments.py

@@ -1,14 +1,16 @@
 """An abstraction over virtualenv and Conda environments."""
 
 import copy
+import codecs
 import hashlib
 import itertools
 import json
 import logging
 import os
 import shutil
+import yaml
 
-from readthedocs.config import PIP, SETUPTOOLS
+from readthedocs.config import PIP, SETUPTOOLS, ParseError, parse as parse_yaml
 from readthedocs.config.models import PythonInstall, PythonInstallRequirements
 from readthedocs.doc_builder.config import load_yaml_config
 from readthedocs.doc_builder.constants import DOCKER_IMAGE
@@ -442,6 +444,10 @@ def setup_base(self):
         if self.project.has_feature(Feature.UPDATE_CONDA_STARTUP):
             self._update_conda_startup()
 
+        if self.project.has_feature(Feature.CONDA_APPEND_CORE_REQUIREMENTS):
+            self._append_core_requirements()
+            self._show_environment_yaml()
+
         self.build_env.run(
             'conda',
             'env',
@@ -455,10 +461,75 @@ def setup_base(self):
             cwd=self.checkout_path,
         )
 
-    def install_core_requirements(self):
-        """Install basic Read the Docs requirements into the Conda env."""
+    def _show_environment_yaml(self):
+        """Show ``environment.yml`` file in the Build output."""
+        self.build_env.run(
+            'cat',
+            self.config.conda.environment,
+            cwd=self.checkout_path,
+        )
+
+    def _append_core_requirements(self):
+        """
+        Append Read the Docs dependencies to Conda environment file.
+
+        This help users to pin their dependencies properly without us upgrading
+        them in the second ``conda install`` run.
+
+        See https://github.com/readthedocs/readthedocs.org/pull/5631
+        """
+        try:
+            inputfile = codecs.open(
+                os.path.join(
+                    self.checkout_path,
+                    self.config.conda.environment,
+                ),
+                encoding='utf-8',
+                mode='r',
+            )
+            environment = parse_yaml(inputfile)
+        except IOError:
+            log.warning(
+                'There was an error while reading Conda environment file.',
+            )
+        except ParseError:
+            log.warning(
+                'There was an error while parsing Conda environment file.',
+            )
+        else:
+            # Append conda dependencies directly to ``dependencies`` and pip
+            # dependencies to ``dependencies.pip``
+            pip_requirements, conda_requirements = self._get_core_requirements()
+            dependencies = environment.get('dependencies', [])
+            pip_dependencies = {'pip': pip_requirements}
+
+            for item in dependencies:
+                if isinstance(item, dict) and 'pip' in item:
+                    pip_requirements.extend(item.get('pip', []))
+                    dependencies.remove(item)
+                    break
+
+            dependencies.append(pip_dependencies)
+            environment.update({'dependencies': dependencies})
+            try:
+                outputfile = codecs.open(
+                    os.path.join(
+                        self.checkout_path,
+                        self.config.conda.environment,
+                    ),
+                    encoding='utf-8',
+                    mode='w',
+                )
+                yaml.safe_dump(environment, outputfile)
+            except IOError:
+                log.warning(
+                    'There was an error while writing the new Conda '
+                    'environment file.',
+                )
+
+    def _get_core_requirements(self):
         # Use conda for requirements it packages
-        requirements = [
+        conda_requirements = [
             'mock',
             'pillow',
         ]
@@ -472,8 +543,22 @@ def install_core_requirements(self):
             pip_requirements.append('mkdocs')
         else:
             pip_requirements.append('readthedocs-sphinx-ext')
-            requirements.extend(['sphinx', 'sphinx_rtd_theme'])
+            conda_requirements.extend(['sphinx', 'sphinx_rtd_theme'])
+
+        return pip_requirements, conda_requirements
+
+    def install_core_requirements(self):
+        """Install basic Read the Docs requirements into the Conda env."""
 
+        if self.project.has_feature(Feature.CONDA_APPEND_CORE_REQUIREMENTS):
+            # Skip install core requirements since they were already appended to
+            # the user's ``environment.yml`` and installed at ``conda env
+            # create`` step.
+            return
+
+        pip_requirements, conda_requirements = self._get_core_requirements()
+        # Install requirements via ``conda install`` command if they were
+        # not appended to the ``environment.yml`` file.
         cmd = [
             'conda',
             'install',
@@ -482,12 +567,13 @@ def install_core_requirements(self):
             '--name',
             self.version.slug,
         ]
-        cmd.extend(requirements)
+        cmd.extend(conda_requirements)
         self.build_env.run(
             *cmd,
             cwd=self.checkout_path,
         )
 
+        # Install requirements via ``pip install``
         pip_cmd = [
             self.venv_bin(filename='python'),
             '-m',

readthedocs/projects/models.py

@@ -1374,6 +1374,7 @@ def add_features(sender, **kwargs):
     DEFAULT_TO_MKDOCS_0_17_3 = 'default_to_mkdocs_0_17_3'
     CLEAN_AFTER_BUILD = 'clean_after_build'
     UPDATE_CONDA_STARTUP = 'update_conda_startup'
+    CONDA_APPEND_CORE_REQUIREMENTS = 'conda_append_core_requirements'
 
     FEATURES = (
         (USE_SPHINX_LATEST, _('Use latest version of Sphinx')),
@@ -1418,6 +1419,10 @@ def add_features(sender, **kwargs):
             UPDATE_CONDA_STARTUP,
             _('Upgrade conda before creating the environment'),
         ),
+        (
+            CONDA_APPEND_CORE_REQUIREMENTS,
+            _('Append Read the Docs core requirements to environment.yml file'),
+        ),
     )
 
     projects = models.ManyToManyField(