[DO NOT MERGE] preview of pep8 formatting

.style.yapf

@@ -14,9 +14,19 @@ allow_multiline_dictionary_keys=False
 # Allow lambdas to be formatted on more than one line.
 allow_multiline_lambdas=False
 
+# Allow splits before the dictionary value.
+allow_split_before_dict_value=True
+
+# Number of blank lines surrounding top-level function and class
+# definitions.
+blank_lines_around_top_level_definition=2
+
 # Insert a blank line before a class-level docstring.
 blank_line_before_class_docstring=False
 
+# Insert a blank line before a module docstring.
+blank_line_before_module_docstring=False
+
 # Insert a blank line before a 'def' or 'class' immediately nested
 # within another 'def' or 'class'. For example:
 #
@@ -45,7 +55,23 @@ blank_line_before_nested_class_or_def=False
 coalesce_brackets=True
 
 # The column limit.
-column_limit=79
+column_limit=80
+
+# The style for continuation alignment. Possible values are:
+#
+# - SPACE: Use spaces for continuation alignment. This is default behavior.
+# - FIXED: Use fixed number (CONTINUATION_INDENT_WIDTH) of columns
+#   (ie: CONTINUATION_INDENT_WIDTH/INDENT_WIDTH tabs) for continuation
+#   alignment.
+# - LESS: Slightly left if cannot vertically align continuation lines with
+#   indent characters.
+# - VALIGN-RIGHT: Vertically align continuation lines with indent
+#   characters. Slightly right (one more indent character) if cannot
+#   vertically align continuation lines with indent characters.
+#
+# For options FIXED, and VALIGN-RIGHT are only available when USE_TABS is
+# enabled.
+continuation_align_style=SPACE
 
 # Indent width used for line continuations.
 continuation_indent_width=4
@@ -66,7 +92,7 @@ continuation_indent_width=4
 #       start_ts=now()-timedelta(days=3),
 #       end_ts=now(),
 #   )        # <--- this bracket is dedented and on a separate line
-dedent_closing_brackets=False
+dedent_closing_brackets=True
 
 # Place each dictionary entry onto its own line.
 each_dict_entry_on_separate_line=True
@@ -79,7 +105,7 @@ i18n_comment=
 # The i18n function call names. The presence of this function stops
 # reformattting on that line, because the string it has cannot be moved
 # away from the i18n comment.
-i18n_function_call=
+i18n_function_call=['']
 
 # Indent the dictionary value if it cannot fit on the same line as the
 # dictionary key. For example:
@@ -90,13 +116,13 @@ i18n_function_call=
 #       'key2': value1 +
 #               value2,
 #   }
-indent_dictionary_value=True
+indent_dictionary_value=False
 
 # The number of columns to use for indentation.
 indent_width=4
 
 # Join short lines into one line. E.g., single line 'if' statements.
-join_multiple_lines=True
+join_multiple_lines=False
 
 # Do not include spaces around selected binary operators. For example:
 #
@@ -106,7 +132,7 @@ join_multiple_lines=True
 #
 #   1 + 2*3 - 4/5
 #
-no_spaces_around_selected_binary_operators=set([])
+no_spaces_around_selected_binary_operators={'set()'}
 
 # Use spaces around default or named assigns.
 spaces_around_default_or_named_assign=False
@@ -123,12 +149,16 @@ space_between_ending_comma_and_closing_bracket=True
 
 # Split before arguments if the argument list is terminated by a
 # comma.
-split_arguments_when_comma_terminated=False
+split_arguments_when_comma_terminated=True
 
 # Set to True to prefer splitting before '&', '|' or '^' rather than
 # after.
 split_before_bitwise_operator=True
 
+# Split before the closing bracket if a list or dict literal doesn't fit on
+# a single line.
+split_before_closing_bracket=True
+
 # Split before a dictionary or set generator (comp_for). For example, note
 # the split before the 'for':
 #
@@ -138,6 +168,10 @@ split_before_bitwise_operator=True
 #   }
 split_before_dict_set_generator=True
 
+# Split after the opening paren which surrounds an expression if it doesn't
+# fit on a single line.
+split_before_expression_after_opening_paren=False
+
 # If an argument / parameter list is going to be split, then split before
 # the first argument.
 split_before_first_argument=False
@@ -149,6 +183,22 @@ split_before_logical_operator=True
 # Split named assignments onto individual lines.
 split_before_named_assigns=True
 
+# Set to True to split list comprehensions and generators that have
+# non-trivial expressions and multiple clauses before each of these
+# clauses. For example:
+#
+#   result = [
+#       a_long_var + 100 for a_long_var in xrange(1000)
+#       if a_long_var % 10]
+#
+# would reformat to something like:
+#
+#   result = [
+#       a_long_var + 100
+#       for a_long_var in xrange(1000)
+#       if a_long_var % 10]
+split_complex_comprehension=True
+
 # The penalty for splitting right after the opening bracket.
 split_penalty_after_opening_bracket=30
 
@@ -162,8 +212,12 @@ split_penalty_before_if_expr=0
 # operators.
 split_penalty_bitwise_operator=300
 
+# The penalty for splitting a list comprehension or generator
+# expression.
+split_penalty_comprehension=80
+
 # The penalty for characters over the column limit.
-split_penalty_excess_character=4500
+split_penalty_excess_character=1000
 
 # The penalty incurred by adding a line split to the unwrapped line. The
 # more line splits added the higher the penalty.
@@ -187,3 +241,4 @@ split_penalty_logical_operator=300
 
 # Use the Tab character for indentation.
 use_tabs=False
+

.travis/yapf.sh

@@ -1,27 +1,30 @@
 #!/usr/bin/env bash
 
 # Cause the script to exit if a single command fails
-set -e
-
-ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
-
-pushd $ROOT_DIR/../test
-  find . -name '*.py' -type f -exec yapf --style=pep8 -i -r {} \;
-popd
-
-pushd $ROOT_DIR/../python
-  find . -name '*.py' -type f -not -path './ray/dataframe/*' -not -path './ray/rllib/*' -not -path './ray/cloudpickle/*' -exec yapf --style=pep8 -i -r {} \;
-popd
-
-CHANGED_FILES=(`git diff --name-only`)
-if [ "$CHANGED_FILES" ]; then
-  echo 'Reformatted staged files. Please review and stage the changes.'
-  echo
-  echo 'Files updated:'
-  for file in ${CHANGED_FILES[@]}; do
-    echo "  $file"
-  done
-  exit 1
-else
-  exit 0
+set -eo pipefail
+
+# this stops git rev-parse from failing if we run this from the .git directory
+builtin cd "$(dirname "${BASH_SOURCE:-$0}")"
+
+ROOT="$(git rev-parse --show-toplevel)"
+builtin cd "$ROOT"
+
+yapf \
+    --style "$ROOT/.style.yapf" \
+    --in-place --recursive --parallel \
+    --exclude 'python/ray/cloudpickle/' \
+    -- \
+    'test/' 'python/'
+
+CHANGED_FILES=($(git diff --name-only))
+
+if [[ "${#CHANGED_FILES[@]}" -gt 0 ]]; then
+    echo 'Reformatted staged files. Please review and stage the changes.'
+    echo 'Files updated:'
+
+    for file in "${CHANGED_FILES[@]}"; do
+        echo "$file"
+    done
+
+    exit 1
 fi

python/ray/__init__.py

@@ -6,14 +6,17 @@
 import sys
 
 if "pyarrow" in sys.modules:
-    raise ImportError("Ray must be imported before pyarrow because Ray "
-                      "requires a specific version of pyarrow (which is "
-                      "packaged along with Ray).")
+    raise ImportError(
+        "Ray must be imported before pyarrow because Ray "
+        "requires a specific version of pyarrow (which is "
+        "packaged along with Ray)."
+    )
 
 # Add the directory containing pyarrow to the Python path so that we find the
 # pyarrow version packaged with ray and not a pre-existing pyarrow.
 pyarrow_path = os.path.join(
-    os.path.abspath(os.path.dirname(__file__)), "pyarrow_files")
+    os.path.abspath(os.path.dirname(__file__)), "pyarrow_files"
+)
 sys.path.insert(0, pyarrow_path)
 
 # See https://github.com/ray-project/ray/issues/131.
@@ -27,15 +30,21 @@
 try:
     import pyarrow  # noqa: F401
 except ImportError as e:
-    if ((hasattr(e, "msg") and isinstance(e.msg, str)
-         and ("libstdc++" in e.msg or "CXX" in e.msg))):
+    if ((
+        hasattr(e, "msg") and isinstance(e.msg, str)
+        and ("libstdc++" in e.msg or "CXX" in e.msg)
+    )):
         # This code path should be taken with Python 3.
         e.msg += helpful_message
-    elif (hasattr(e, "message") and isinstance(e.message, str)
-          and ("libstdc++" in e.message or "CXX" in e.message)):
+    elif (
+        hasattr(e, "message") and isinstance(e.message, str)
+        and ("libstdc++" in e.message or "CXX" in e.message)
+    ):
         # This code path should be taken with Python 2.
-        condition = (hasattr(e, "args") and isinstance(e.args, tuple)
-                     and len(e.args) == 1 and isinstance(e.args[0], str))
+        condition = (
+            hasattr(e, "args") and isinstance(e.args, tuple)
+            and len(e.args) == 1 and isinstance(e.args[0], str)
+        )
         if condition:
             e.args = (e.args[0] + helpful_message, )
         else:
@@ -47,12 +56,13 @@
     raise
 
 from ray.local_scheduler import _config  # noqa: E402
-from ray.worker import (error_info, init, connect, disconnect, get, put, wait,
-                        remote, log_event, log_span, flush_log, get_gpu_ids,
-                        get_webui_url,
-                        register_custom_serializer)  # noqa: E402
-from ray.worker import (SCRIPT_MODE, WORKER_MODE, PYTHON_MODE,
-                        SILENT_MODE)  # noqa: E402
+from ray.worker import (
+    error_info, init, connect, disconnect, get, put, wait, remote, log_event,
+    log_span, flush_log, get_gpu_ids, get_webui_url, register_custom_serializer
+)  # noqa: E402
+from ray.worker import (
+    SCRIPT_MODE, WORKER_MODE, PYTHON_MODE, SILENT_MODE
+)  # noqa: E402
 from ray.worker import global_state  # noqa: E402
 # We import ray.actor because some code is run in actor.py which initializes
 # some functions in the worker.
@@ -66,9 +76,9 @@
 __all__ = [
     "error_info", "init", "connect", "disconnect", "get", "put", "wait",
     "remote", "log_event", "log_span", "flush_log", "actor", "method",
-    "get_gpu_ids", "get_webui_url", "register_custom_serializer",
-    "SCRIPT_MODE", "WORKER_MODE", "PYTHON_MODE", "SILENT_MODE", "global_state",
-    "_config", "__version__"
+    "get_gpu_ids", "get_webui_url", "register_custom_serializer", "SCRIPT_MODE",
+    "WORKER_MODE", "PYTHON_MODE", "SILENT_MODE", "global_state", "_config",
+    "__version__"
 ]
 
 import ctypes  # noqa: E402