[core] Add .rayignore (#58500)

## Description

### Status Quo
Previously, `.gitignore` files handled both uploading to cluster _and_
uploading to github. This PR essentially allows the ability to break
those 2 functionalities apart by creating a `.rayignore` file which will
handle uploading to cluster.

### Purpose
Any path or file specified in `.rayignore` will be ignored when
uploading to the cluster. This is useful for local development when you
don't want random files being uploaded and taking up space.

### How it works
By default, directories containing both `.gitignore` and `.rayignore`
will both be considered (so existing behavior is preserved). To make
`.gitignore` only ignore files uploaded to github, and `.rayignore` only
ignore files uploaded to cluster (essentially making them independent of
each other), you can use the existing `RAY_RUNTIME_ENV_IGNORE_GITIGNORE`
and set that to `1`

## Related issues
#53648

## Additional information
Since `.rayignore` is part of the ray ecosystem, I did not create an env
var to disable ignoring all-together. If users do not want to ignore
files, they can leave `.rayignore` empty, or not create the file at all.

---------

Signed-off-by: iamjustinhsu <[email protected]>

Author: iamjustinhsu

doc/source/ray-core/handling-dependencies.rst

@@ -505,7 +505,7 @@ The ``runtime_env`` is a Python dictionary or a Python class :class:`ray.runtime
 
   Note: Setting a local directory per-task or per-actor is currently unsupported; it can only be set per-job (i.e., in ``ray.init()``).
 
-  Note: If the local directory contains a ``.gitignore`` file, the files and paths specified there are not uploaded to the cluster.  You can disable this by setting the environment variable `RAY_RUNTIME_ENV_IGNORE_GITIGNORE=1` on the machine doing the uploading.
+  Note: By default, if the local directory contains a ``.gitignore`` and/or ``.rayignore`` file, the specified files are not uploaded to the cluster. To disable the ``.gitignore`` from being considered, set ``RAY_RUNTIME_ENV_IGNORE_GITIGNORE=1`` on the machine doing the uploading.
 
   Note: If the local directory contains symbolic links, Ray follows the links and the files they point to are uploaded to the cluster.
 
@@ -532,7 +532,8 @@ The ``runtime_env`` is a Python dictionary or a Python class :class:`ray.runtime
 
   Note: Setting options (1), (3) and (4) per-task or per-actor is currently unsupported, it can only be set per-job (i.e., in ``ray.init()``).
 
-  Note: For option (1), if the local directory contains a ``.gitignore`` file, the files and paths specified there are not uploaded to the cluster.  You can disable this by setting the environment variable `RAY_RUNTIME_ENV_IGNORE_GITIGNORE=1` on the machine doing the uploading.
+  Note: For option (1), by default, if the local directory contains a ``.gitignore`` and/or ``.rayignore`` file, the specified files are not uploaded to the cluster. To disable the ``.gitignore`` from being considered, set ``RAY_RUNTIME_ENV_IGNORE_GITIGNORE=1`` on the machine doing the uploading.
+
 
 - ``py_executable`` (str): Specifies the executable used for running the Ray workers. It can include arguments as well. The executable can be
   located in the `working_dir`. This runtime environment is useful to run workers in a custom debugger or profiler as well as to run workers

python/ray/_private/runtime_env/packaging.py

@@ -15,7 +15,6 @@
 from ray._private.path_utils import is_path
 from ray._private.ray_constants import (
     GRPC_CPP_MAX_MESSAGE_SIZE,
-    RAY_RUNTIME_ENV_IGNORE_GITIGNORE,
     RAY_RUNTIME_ENV_URI_PIN_EXPIRATION_S_DEFAULT,
     RAY_RUNTIME_ENV_URI_PIN_EXPIRATION_S_ENV_VAR,
 )
@@ -86,16 +85,17 @@ def _dir_travel(
     path: Path,
     excludes: List[Callable],
     handler: Callable,
+    include_gitignore: bool,
     logger: Optional[logging.Logger] = default_logger,
 ):
     """Travels the path recursively, calling the handler on each subpath.
 
     Respects excludes, which will be called to check if this path is skipped.
     """
-    e = _get_gitignore(path)
-
-    if e is not None:
-        excludes.append(e)
+    new_excludes = get_excludes_from_ignore_files(
+        path, include_gitignore=include_gitignore, logger=logger
+    )
+    excludes.extend(new_excludes)
 
     skip = any(e(path) for e in excludes)
     if not skip:
@@ -106,9 +106,15 @@ def _dir_travel(
             raise e
         if path.is_dir():
             for sub_path in path.iterdir():
-                _dir_travel(sub_path, excludes, handler, logger=logger)
+                _dir_travel(
+                    sub_path,
+                    excludes,
+                    handler,
+                    include_gitignore=include_gitignore,
+                    logger=logger,
+                )
 
-    if e is not None:
+    for _ in range(len(new_excludes)):
         excludes.pop()
 
 
@@ -166,6 +172,7 @@ def _hash_directory(
     root: Path,
     relative_path: Path,
     excludes: Optional[Callable],
+    include_gitignore: bool,
     logger: Optional[logging.Logger] = default_logger,
 ) -> bytes:
     """Helper function to create hash of a directory.
@@ -183,7 +190,9 @@ def handler(path: Path):
         hash_val = _xor_bytes(hash_val, file_hash)
 
     excludes = [] if excludes is None else [excludes]
-    _dir_travel(root, excludes, handler, logger=logger)
+    _dir_travel(
+        root, excludes, handler, include_gitignore=include_gitignore, logger=logger
+    )
     return hash_val
 
 
@@ -280,24 +289,21 @@ def match(p: Path):
     return match
 
 
-def _get_gitignore(path: Path) -> Optional[Callable]:
+def _get_ignore_file(path: Path, ignore_file: str) -> Optional[Callable]:
     """Returns a function that returns True if the path should be excluded.
 
-    Returns None if there is no .gitignore file in the path, or if the
-    RAY_RUNTIME_ENV_IGNORE_GITIGNORE environment variable is set to 1.
+    Returns None if there is no ignore_file in the path.
 
     Args:
-        path: The path to the directory to check for a .gitignore file.
+        path: The path to the directory to check for an ignore file.
+        ignore_file: The name of the ignore file.
 
     Returns:
         A function that returns True if the path should be excluded.
     """
-    ignore_gitignore = os.environ.get(RAY_RUNTIME_ENV_IGNORE_GITIGNORE, "0") == "1"
-    if ignore_gitignore:
-        return None
 
     path = path.absolute()
-    ignore_file = path / ".gitignore"
+    ignore_file = path / ignore_file
     if ignore_file.is_file():
         with ignore_file.open("r") as f:
             pathspec = PathSpec.from_lines("gitwildmatch", f.readlines())
@@ -311,6 +317,43 @@ def match(p: Path):
         return None
 
 
+def get_excludes_from_ignore_files(
+    path: Path,
+    include_gitignore: bool,
+    logger: Optional[logging.Logger] = default_logger,
+) -> List[Callable]:
+    """Get exclusion functions from .gitignore and .rayignore files in the current path.
+
+    Args:
+        path: The path to check for ignore files.
+        include_gitignore: Whether to respect .gitignore files.
+        logger: Logger to use.
+
+    Returns:
+        List[Callable]: List of exclusion functions. Each function takes a Path
+            and returns True if the path should be excluded based on the ignore
+            patterns in the respective ignore file.
+    """
+    ignore_files = []
+
+    to_ignore: List[Optional[Callable]] = []
+    if include_gitignore:
+        g = _get_ignore_file(path, ignore_file=".gitignore")
+        if g is not None:
+            to_ignore.append(g)
+            ignore_files.append(path / ".gitignore")
+
+    r = _get_ignore_file(path, ignore_file=".rayignore")
+    if r is not None:
+        to_ignore.append(r)
+        ignore_files.append(path / ".rayignore")
+
+    if ignore_files:
+        logger.info(f"Ignoring upload to cluster for these files: {ignore_files}")
+
+    return to_ignore
+
+
 def pin_runtime_env_uri(uri: str, *, expiration_s: Optional[int] = None) -> None:
     """Pin a reference to a runtime_env URI in the GCS on a timeout.
 
@@ -403,6 +446,7 @@ def _zip_files(
     path_str: str,
     excludes: List[str],
     output_path: str,
+    include_gitignore: bool,
     include_parent_dir: bool = False,
     logger: Optional[logging.Logger] = default_logger,
 ) -> None:
@@ -440,7 +484,13 @@ def handler(path: Path):
                 zip_handler.write(path, to_path)
 
         excludes = [_get_excludes(file_path, excludes)]
-        _dir_travel(file_path, excludes, handler, logger=logger)
+        _dir_travel(
+            file_path,
+            excludes,
+            handler,
+            include_gitignore=include_gitignore,
+            logger=logger,
+        )
 
 
 def package_exists(pkg_uri: str) -> bool:
@@ -508,7 +558,11 @@ def get_uri_for_file(file: str) -> str:
     )
 
 
-def get_uri_for_directory(directory: str, excludes: Optional[List[str]] = None) -> str:
+def get_uri_for_directory(
+    directory: str,
+    include_gitignore: bool,
+    excludes: Optional[List[str]] = None,
+) -> str:
     """Get a content-addressable URI from a directory's contents.
 
     This function generates the name of the package by the directory.
@@ -524,6 +578,7 @@ def get_uri_for_directory(directory: str, excludes: Optional[List[str]] = None)
 
     Args:
         directory: The directory.
+        include_gitignore: Whether to respect .gitignore files.
         excludes (list[str]): The dir or files that should be excluded.
 
     Returns:
@@ -539,7 +594,12 @@ def get_uri_for_directory(directory: str, excludes: Optional[List[str]] = None)
     if not directory.exists() or not directory.is_dir():
         raise ValueError(f"directory {directory} must be an existing directory")
 
-    hash_val = _hash_directory(directory, directory, _get_excludes(directory, excludes))
+    hash_val = _hash_directory(
+        directory,
+        directory,
+        _get_excludes(directory, excludes),
+        include_gitignore=include_gitignore,
+    )
 
     return "{protocol}://{pkg_name}.zip".format(
         protocol=Protocol.GCS.value, pkg_name=RAY_PKG_PREFIX + hash_val.hex()
@@ -574,6 +634,7 @@ def upload_package_to_gcs(pkg_uri: str, pkg_bytes: bytes) -> None:
 def create_package(
     module_path: str,
     target_path: Path,
+    include_gitignore: bool,
     include_parent_dir: bool = False,
     excludes: Optional[List[str]] = None,
     logger: Optional[logging.Logger] = default_logger,
@@ -590,6 +651,7 @@ def create_package(
             module_path,
             excludes,
             str(target_path),
+            include_gitignore=include_gitignore,
             include_parent_dir=include_parent_dir,
             logger=logger,
         )
@@ -599,6 +661,7 @@ def upload_package_if_needed(
     pkg_uri: str,
     base_directory: str,
     module_path: str,
+    include_gitignore: bool,
     include_parent_dir: bool = False,
     excludes: Optional[List[str]] = None,
     logger: Optional[logging.Logger] = default_logger,
@@ -617,6 +680,7 @@ def upload_package_if_needed(
         include_parent_dir: If true, includes the top-level directory as a
             directory inside the zip file.
         excludes: List specifying files to exclude.
+        include_gitignore: Whether to respect .gitignore files. Default is True.
 
     Raises:
         RuntimeError: If the upload fails.
@@ -646,6 +710,7 @@ def upload_package_if_needed(
     create_package(
         module_path,
         package_file,
+        include_gitignore=include_gitignore,
         include_parent_dir=include_parent_dir,
         excludes=excludes,
     )

python/ray/_private/runtime_env/py_modules.py

@@ -48,6 +48,7 @@ def _check_is_uri(s: str) -> bool:
 
 def upload_py_modules_if_needed(
     runtime_env: Dict[str, Any],
+    include_gitignore: bool,
     scratch_dir: Optional[str] = os.getcwd(),
     logger: Optional[logging.Logger] = default_logger,
     upload_fn=None,
@@ -102,7 +103,11 @@ def upload_py_modules_if_needed(
                 is_dir = Path(module_path).is_dir()
                 excludes = runtime_env.get("excludes", None)
                 if is_dir:
-                    module_uri = get_uri_for_directory(module_path, excludes=excludes)
+                    module_uri = get_uri_for_directory(
+                        module_path,
+                        include_gitignore=include_gitignore,
+                        excludes=excludes,
+                    )
                 else:
                     module_uri = get_uri_for_file(module_path)
                 if upload_fn is None:
@@ -111,8 +116,9 @@ def upload_py_modules_if_needed(
                             module_uri,
                             scratch_dir,
                             module_path,
-                            excludes=excludes,
+                            include_gitignore=include_gitignore,
                             include_parent_dir=is_dir,
+                            excludes=excludes,
                             logger=logger,
                         )
                     except Exception as e:

python/ray/_private/runtime_env/working_dir.py

@@ -30,6 +30,7 @@
 
 def upload_working_dir_if_needed(
     runtime_env: Dict[str, Any],
+    include_gitignore: bool,
     scratch_dir: Optional[str] = os.getcwd(),
     logger: Optional[logging.Logger] = default_logger,
     upload_fn: Optional[Callable[[str, Optional[List[str]]], None]] = None,
@@ -64,7 +65,11 @@ def upload_working_dir_if_needed(
 
     excludes = runtime_env.get("excludes", None)
     try:
-        working_dir_uri = get_uri_for_directory(working_dir, excludes=excludes)
+        working_dir_uri = get_uri_for_directory(
+            working_dir,
+            include_gitignore=include_gitignore,
+            excludes=excludes,
+        )
     except ValueError:  # working_dir is not a directory
         package_path = Path(working_dir)
         if not package_path.exists() or package_path.suffix != ".zip":
@@ -90,6 +95,7 @@ def upload_working_dir_if_needed(
                 working_dir,
                 include_parent_dir=False,
                 excludes=excludes,
+                include_gitignore=include_gitignore,
                 logger=logger,
             )
         except Exception as e:

python/ray/_private/worker.py

@@ -2610,13 +2610,24 @@ def connect(
     # environment here. If it's ray client, the environment will be prepared
     # at the server side.
     if mode == SCRIPT_MODE and not job_config._client_job and job_config.runtime_env:
+        from ray._private.ray_constants import RAY_RUNTIME_ENV_IGNORE_GITIGNORE
+
         scratch_dir: str = worker.node.get_runtime_env_dir_path()
         runtime_env = job_config.runtime_env or {}
+        # Determine whether to respect .gitignore files based on environment variable
+        # Default is True (respect .gitignore). Set to False if env var is "1".
+        include_gitignore = os.environ.get(RAY_RUNTIME_ENV_IGNORE_GITIGNORE, "0") != "1"
         runtime_env = upload_py_modules_if_needed(
-            runtime_env, scratch_dir, logger=logger
+            runtime_env,
+            include_gitignore=include_gitignore,
+            scratch_dir=scratch_dir,
+            logger=logger,
         )
         runtime_env = upload_working_dir_if_needed(
-            runtime_env, scratch_dir, logger=logger
+            runtime_env,
+            include_gitignore=include_gitignore,
+            scratch_dir=scratch_dir,
+            logger=logger,
         )
         runtime_env = upload_worker_process_setup_hook_if_needed(
             runtime_env,