diff --git a/.buildkite/doc.rayci.yml b/.buildkite/doc.rayci.yml index dccaa28b383a..38705bfad289 100644 --- a/.buildkite/doc.rayci.yml +++ b/.buildkite/doc.rayci.yml @@ -76,8 +76,7 @@ steps: key: doc_api_policy_check instance_type: medium depends_on: docbuild - # TODO(aslonnie): migrate to Python 3.12 - job_env: docbuild-py3.9 + job_env: docbuild-py3.12 commands: - bash ci/lint/lint.sh api_policy_check diff --git a/ci/ray_ci/doc/BUILD.bazel b/ci/ray_ci/doc/BUILD.bazel index 4aed0f52537c..b683d28ce587 100644 --- a/ci/ray_ci/doc/BUILD.bazel +++ b/ci/ray_ci/doc/BUILD.bazel @@ -11,7 +11,10 @@ py_binary( name = "cmd_build", srcs = ["cmd_build.py"], exec_compatible_with = ["//:hermetic_python"], - deps = [":doc"], + deps = [ + ci_require("click"), + ":doc", + ], ) py_library( @@ -23,9 +26,8 @@ py_library( "cmd_*.py", ], ), - visibility = ["//ci/ray_ci/doc:__subpackages__"], deps = [ - "//ci/ray_ci:ray_ci_lib", + ci_require("boto3"), ci_require("sphinx"), ci_require("myst_parser"), ci_require("myst-nb"), diff --git a/ci/ray_ci/doc/build_cache.py b/ci/ray_ci/doc/build_cache.py index 2db4abaadaa4..bdf32938dcfb 100644 --- a/ci/ray_ci/doc/build_cache.py +++ b/ci/ray_ci/doc/build_cache.py @@ -1,18 +1,17 @@ import os import pickle import subprocess +import sys import tempfile from typing import Set import boto3 -from ci.ray_ci.utils import logger - -from ray_release.util import get_write_state_machine_aws_bucket - -AWS_CACHE_KEY = "doc_build" ENVIRONMENT_PICKLE = "_build/doctrees/environment.pickle" +_BUILD_CACHE_S3_BUCKET = "ray-ci-results" +_BUILD_CACHE_PATH_PREFIX = "doc_build/" + class BuildCache: """ @@ -28,27 +27,17 @@ def __init__(self, cache_dir: str): self._cache_dir = cache_dir def upload(self, dry_run: bool) -> None: - """ - Upload the build artifacts to S3 - """ - logger.info("Massage the build artifacts to be used as a cache.") + """Upload the build artifacts to S3.""" self._massage_cache(ENVIRONMENT_PICKLE) - - logger.info("Obtaining the list of cache files.") cache_files = self._get_cache() - - logger.info("Creating a tarball of the cache files.") doc_tarball = self._zip_cache(cache_files) if dry_run: - logger.info(f"Skipping upload of {doc_tarball} to S3.") + print(f"Skipping upload of {doc_tarball} to S3.", file=sys.stderr) return - logger.info("Upload the tarball to S3.") self._upload_cache(doc_tarball) - logger.info(f"Successfully uploaded {doc_tarball} to S3.") - def _massage_cache(self, environment_cache_file: str) -> None: """ Massage the build artifacts, remove the unnecessary files so that they can @@ -102,6 +91,6 @@ def _zip_cache(self, cache_files: Set[str]) -> str: def _upload_cache(self, doc_tarball: str) -> None: boto3.client("s3").upload_file( os.path.join(self._cache_dir, doc_tarball), - get_write_state_machine_aws_bucket(), - f"{AWS_CACHE_KEY}/{doc_tarball}", + _BUILD_CACHE_S3_BUCKET, + f"{_BUILD_CACHE_PATH_PREFIX}{doc_tarball}", ) diff --git a/ci/ray_ci/doc/cmd_build.py b/ci/ray_ci/doc/cmd_build.py index 2973f03589d1..ad304a2321ad 100644 --- a/ci/ray_ci/doc/cmd_build.py +++ b/ci/ray_ci/doc/cmd_build.py @@ -1,12 +1,10 @@ import os import subprocess +import sys import click from ci.ray_ci.doc.build_cache import BuildCache -from ci.ray_ci.utils import ci_init, logger - -from ray_release.configs.global_config import get_global_config @click.command() @@ -18,37 +16,32 @@ def main(ray_checkout_dir: str) -> None: """ This script builds ray doc and upload build artifacts to S3. """ - ci_init() # Add the safe.directory config to the global git config so that the doc build subprocess.run( ["git", "config", "--global", "--add", "safe.directory", ray_checkout_dir], check=True, ) - logger.info("Building ray doc.") + print("--- Building ray doc.", file=sys.stderr) _build(ray_checkout_dir) dry_run = False - if ( - os.environ.get("BUILDKITE_PIPELINE_ID") - not in get_global_config()["ci_pipeline_postmerge"] - ): + if os.environ.get("RAYCI_STAGE", "") != "postmerge": dry_run = True - logger.info( - "Not uploading build artifacts because this is not a postmerge pipeline." + print( + "Not uploading build artifacts because this is not a postmerge pipeline.", + file=sys.stderr, ) - - if os.environ.get("BUILDKITE_BRANCH") != "master": + elif os.environ.get("BUILDKITE_BRANCH") != "master": dry_run = True - logger.info( - "Not uploading build artifacts because this is not the master branch." + print( + "Not uploading build artifacts because this is not the master branch.", + file=sys.stderr, ) - logger.info("Uploading build artifacts to S3.") + print("--- Uploading build artifacts to S3.", file=sys.stderr) BuildCache(os.path.join(ray_checkout_dir, "doc")).upload(dry_run=dry_run) - return - def _build(ray_checkout_dir): env = os.environ.copy() diff --git a/ci/ray_ci/doc/cmd_check_api_discrepancy.py b/ci/ray_ci/doc/cmd_check_api_discrepancy.py index 2fd8bfdb3f0d..c159f4da79ab 100644 --- a/ci/ray_ci/doc/cmd_check_api_discrepancy.py +++ b/ci/ray_ci/doc/cmd_check_api_discrepancy.py @@ -1,9 +1,10 @@ +import sys + import click from ci.ray_ci.doc.api import API from ci.ray_ci.doc.autodoc import Autodoc from ci.ray_ci.doc.module import Module -from ci.ray_ci.utils import logger TEAM_API_CONFIGS = { "data": { @@ -102,24 +103,28 @@ def _check_team(ray_checkout_dir: str, team: str) -> bool: white_list_apis = TEAM_API_CONFIGS[team]["white_list_apis"] # Policy 01: all public APIs should be documented - logger.info(f"Validating that public {team} APIs should be documented...") + print( + f"--- Validating that public {team} APIs should be documented...", + file=sys.stderr, + ) good_apis, bad_apis = API.split_good_and_bad_apis( api_in_codes, api_in_docs, white_list_apis ) if good_apis: - logger.info("Public APIs that are documented:") + print("Public APIs that are documented:", file=sys.stderr) for api in good_apis: - logger.info(f"\t{api}") + print(f"\t{api}", file=sys.stderr) if bad_apis: - logger.info("Public APIs that are NOT documented:") + print("Public APIs that are NOT documented:", file=sys.stderr) for api in bad_apis: - logger.info(f"\t{api}") + print(f"\t{api}", file=sys.stderr) if bad_apis: - logger.info( - f"Some public {team} APIs are not documented. Please document them." + print( + f"Some public {team} APIs are not documented. Please document them.", + file=sys.stderr, ) return False return True