[doc build] use rayci.anyscale.dev to fetch doc build cache

stop fetching from s3 directly

Signed-off-by: Lonnie Liu <[email protected]>

Author: aslonnie

doc/load_doc_cache.py

@@ -1,20 +1,17 @@
-import boto3
-import botocore
 import subprocess
 import tarfile
 import os
-import click
-from botocore import UNSIGNED
-from botocore.client import Config
 import time
+
+import click
 import requests
 
 S3_BUCKET = "ray-ci-results"
 DOC_BUILD_DIR_S3 = "doc_build"
 LAST_BUILD_CUTOFF = 3  # how many days ago to consider a build outdated
 PENDING_FILES_PATH = "pending_files.txt"
 ENVIRONMENT_PICKLE = "_build/doctrees/environment.pickle"
-DOC_BUILD_S3_URL = "https://ray-ci-results.s3.us-west-2.amazonaws.com/doc_build"
+DOC_BUILD_S3_URL = "https://rayci.anyscale.dev/doc-build-cache"
 
 
 def find_latest_master_commit():
@@ -34,16 +31,16 @@ def find_latest_master_commit():
         .split("\n")
     )
     for commit in latest_commits:
-        result = requests.head(f"{DOC_BUILD_S3_URL}/{commit}.tgz")
-        if result.status_code == 200:
-            return commit
+        with requests.head(f"{DOC_BUILD_S3_URL}/{commit}.tgz", allow_redirects=True) as response:
+            if response.status_code == 200:
+                return commit
     raise Exception(
-        "No cache found for latest master commit."
+        f"No cache found for latest master commit {commit}. "
         "Please merge with upstream master or use 'make develop'."
     )
 
 
-def fetch_cache_from_s3(commit, target_file_path):
+def fetch_cache(commit, target_file_path):
     """
     Fetch doc cache archive from ray-ci-results S3 bucket
 
@@ -52,15 +49,12 @@ def fetch_cache_from_s3(commit, target_file_path):
         target_file_path: The file path to save the doc cache archive
     """
     # Create an S3 client
-    s3 = boto3.client("s3", config=Config(signature_version=UNSIGNED))
-    s3_file_path = f"{DOC_BUILD_DIR_S3}/{commit}.tgz"
-    try:
-        print(f"Fetching doc cache from commit {commit}...")
-        s3.download_file(S3_BUCKET, s3_file_path, target_file_path)
-        print(f"Successfully downloaded {s3_file_path} to {target_file_path}")
-    except botocore.exceptions.ClientError as e:
-        print(f"Failed to download {s3_file_path} from S3: {str(e)}")
-        raise e
+    with requests.get(f"{DOC_BUILD_S3_URL}/{commit}.tgz", allow_redirects=True) as response:
+        response.raise_for_status()
+        with open(target_file_path, "wb") as f:
+            for chunk in response.iter_content(chunk_size=8192):
+                f.write(chunk)
+        print(f"Successfully downloaded {target_file_path}")
 
 
 def extract_cache(cache_path: str, doc_dir: str):
@@ -150,7 +144,7 @@ def main(ray_dir: str) -> None:
 
     cache_path = f"{ray_dir}/doc.tgz"
     # Fetch cache of that commit from S3 to cache_path
-    fetch_cache_from_s3(latest_master_commit, cache_path)
+    fetch_cache(latest_master_commit, cache_path)
     # Extract cache to override ray/doc directory
     extract_cache(cache_path, f"{ray_dir}/doc")
     os.remove(cache_path)