chore: add ruff linter

.github/image/Dockerfile

@@ -56,7 +56,7 @@ RUN apt install -y \
     rm -rf /var/lib/apt/lists/*
 
 # Upgrade pip and install some common Python packages
-RUN pip3 install --break-system-packages pytest tox flake8
+RUN pip3 install --break-system-packages pytest tox flake8 ruff
 
 RUN apt update && apt install -y build-essential git
 RUN mkdir -p /usr/lib/jvm && ln -s /usr/lib/jvm/java-17-openjdk-amd64/ /usr/lib/jvm/default-jvm

.github/workflows/test_python.yaml

@@ -44,14 +44,22 @@ jobs:
           python3 -m venv chronon_py_env
           source chronon_py_env/bin/activate
 
-      - name: Run Chronon Python lint
+      - name: Run Chronon Python lint (flake8)
         shell: bash
         run: |
           source chronon_py_env/bin/activate
           cd api/py/ai/chronon
           pip install importlib-metadata==4.11.4
           pip install flake8
           flake8 --extend-ignore=W605,Q000,F631 --exclude=cli
+
+      - name: Run Chronon Python lint (ruff)
+        shell: bash
+        run: |
+          source chronon_py_env/bin/activate
+          cd api/py
+          pip install ruff
+          ruff check .
 
       - name: Run Chronon Python tests
         shell: bash

api/py/ai/chronon/cli/compile/compile_context.py

@@ -1,18 +1,17 @@
-from dataclasses import dataclass
 import os
+from dataclasses import dataclass
 from typing import Any, Dict, List, Type
 
+import ai.chronon.cli.compile.parse_teams as teams
 from ai.chronon.api.common.ttypes import ConfigType
 from ai.chronon.api.ttypes import GroupBy, Join, StagingQuery, Team
-from ai.chronon.cli.compile.display.compiled_obj import CompiledObj
+from ai.chronon.cli.compile.conf_validator import ConfValidator
 from ai.chronon.cli.compile.display.compile_status import CompileStatus
+from ai.chronon.cli.compile.display.compiled_obj import CompiledObj
 from ai.chronon.cli.compile.serializer import file2thrift
-from ai.chronon.cli.compile.conf_validator import ConfValidator
-import ai.chronon.cli.compile.parse_teams as teams
-from ai.chronon.cli.logger import require, get_logger
+from ai.chronon.cli.logger import get_logger, require
 from ai.chronon.model import Model
 
-
 logger = get_logger()
 
 
@@ -129,7 +128,7 @@ def _parse_existing_confs(self, obj_class: type) -> Dict[str, object]:
         if not os.path.exists(output_dir):
             return result
 
-        for sub_root, sub_dirs, sub_files in os.walk(output_dir):
+        for sub_root, _sub_dirs, sub_files in os.walk(output_dir):
 
             for f in sub_files:
 

api/py/ai/chronon/cli/compile/compiler.py

@@ -1,18 +1,15 @@
-from dataclasses import dataclass
 import os
 import shutil
 import traceback
-from typing import Dict, Any, List, Optional, Tuple
-
-from ai.chronon.cli.compile.compile_context import CompileContext, ConfigInfo
+from dataclasses import dataclass
+from typing import Any, Dict, List, Optional, Tuple
 
-from ai.chronon.cli.compile.display.class_tracker import ClassTracker
 import ai.chronon.cli.compile.display.compiled_obj
-from ai.chronon.cli.compile.display.compiled_obj import CompiledObj
 import ai.chronon.cli.compile.parse_configs as parser
 import ai.chronon.cli.logger as logger
-
 from ai.chronon.api.common.ttypes import ConfigType
+from ai.chronon.cli.compile.compile_context import CompileContext, ConfigInfo
+from ai.chronon.cli.compile.display.compiled_obj import CompiledObj
 
 logger = logger.get_logger()
 

api/py/ai/chronon/cli/compile/conf_validator.py

@@ -17,26 +17,23 @@
 
 import json
 import logging
-import os
 import re
-import ai.chronon.api.common.ttypes as common
-import glom
-
+from collections import defaultdict
+from typing import Dict, List, Set
 
+import ai.chronon.api.common.ttypes as common
 from ai.chronon.api.ttypes import (
     Accuracy,
     Aggregation,
+    Derivation,
+    ExternalPart,
     GroupBy,
     Join,
     Source,
-    Derivation,
-    ExternalPart,
 )
 from ai.chronon.group_by import get_output_col_names
 from ai.chronon.logger import get_logger
 from ai.chronon.repo.serializer import thrift_simple_json
-from collections import defaultdict
-from typing import List, Dict, Set
 
 # Fields that indicate stutus of the entities.
 SKIPPED_FIELDS = frozenset(["metaData"])

api/py/ai/chronon/cli/compile/display/class_tracker.py

@@ -1,5 +1,6 @@
 import difflib
-from typing import Dict, List, Any
+from typing import Any, Dict, List
+
 from rich.text import Text
 
 from ai.chronon.cli.compile.display.compiled_obj import CompiledObj
@@ -79,7 +80,7 @@ def to_status(self) -> Text:
         if self.files_to_obj:
             text.append("  Compiled ")
             text.append(f"{len(self.new_objs)} ", style="bold green")
-            text.append(f"objects from ")
+            text.append("objects from ")
             text.append(f"{len(self.files_to_obj)} ", style="bold green")
             text.append("files.\n")
 

api/py/ai/chronon/cli/compile/display/compile_status.py

@@ -1,11 +1,11 @@
-from typing import Dict, List
+from collections import OrderedDict
+from typing import Dict
+
 from rich.live import Live
 from rich.text import Text
-from rich.console import Console
-from collections import OrderedDict
 
-from ai.chronon.cli.compile.display.compiled_obj import CompiledObj
 from ai.chronon.cli.compile.display.class_tracker import ClassTracker
+from ai.chronon.cli.compile.display.compiled_obj import CompiledObj
 
 
 class CompileStatus:

api/py/ai/chronon/cli/compile/display/diff_result.py

@@ -1,8 +1,6 @@
-from dataclasses import dataclass
-from typing import List, Dict
+from typing import List
+
 from rich.text import Text
-import difflib
-from ai.chronon.cli.compile.display.compiled_obj import CompiledObj
 
 
 class DiffResult:

api/py/ai/chronon/cli/compile/parse_configs.py

@@ -4,8 +4,8 @@
 from typing import List
 
 from ai.chronon.cli.compile import parse_teams, serializer
-from ai.chronon.cli.compile.display.compiled_obj import CompiledObj
 from ai.chronon.cli.compile.compile_context import CompileContext
+from ai.chronon.cli.compile.display.compiled_obj import CompiledObj
 from ai.chronon.cli.logger import get_logger
 
 logger = get_logger()

api/py/ai/chronon/cli/compile/parse_teams.py

@@ -1,14 +1,15 @@
-from copy import deepcopy
 import importlib
 import os
+from copy import deepcopy
+from typing import Any, Dict, List, Union
+
 from ai.chronon.api.common.ttypes import (
-    ExecutionInfo,
     ConfigProperties,
     EnvironmentVariables,
+    ExecutionInfo,
 )
 from ai.chronon.api.ttypes import Team
 from ai.chronon.cli.logger import get_logger, require
-from typing import Any, List, Dict, Union
 
 logger = get_logger()
 

api/py/ai/chronon/cli/compile/serializer.py

@@ -13,12 +13,12 @@
 #     limitations under the License.
 
 import json
-from thrift.Thrift import TType
-from thrift.protocol.TJSONProtocol import TSimpleJSONProtocolFactory
 
-from thrift.transport.TTransport import TMemoryBuffer
-from thrift.protocol.TBinaryProtocol import TBinaryProtocolAccelerated
 from thrift import TSerialization
+from thrift.protocol.TBinaryProtocol import TBinaryProtocolAccelerated
+from thrift.protocol.TJSONProtocol import TSimpleJSONProtocolFactory
+from thrift.Thrift import TType
+from thrift.transport.TTransport import TMemoryBuffer
 
 
 class ThriftJSONDecoder(json.JSONDecoder):

api/py/ai/chronon/cli/entrypoint.py

@@ -1,16 +1,17 @@
+from datetime import datetime, timedelta
 from typing import Dict
+
 import click
-from datetime import datetime, timedelta
 
 from ai.chronon.api.common.ttypes import ConfigType
+from ai.chronon.cli.compile.compile_context import CompileContext
+from ai.chronon.cli.compile.compiler import Compiler, CompileResult
+from ai.chronon.cli.git_utils import get_current_branch
 from ai.chronon.cli.plan.physical_index import (
     PhysicalIndex,
     get_backfill_physical_graph,
     submit_physical_graph,
 )
-from ai.chronon.cli.compile.compiler import CompileResult, Compiler
-from ai.chronon.cli.compile.compile_context import CompileContext
-from ai.chronon.cli.git_utils import get_current_branch
 
 
 @click.group()

api/py/ai/chronon/cli/git_utils.py

@@ -1,7 +1,7 @@
 import subprocess
 import sys
 from pathlib import Path
-from typing import List, Optional, Set
+from typing import List, Optional
 
 from ai.chronon.cli.logger import get_logger
 
@@ -30,7 +30,7 @@ def get_current_branch() -> str:
                 if content.startswith("ref: refs/heads/"):
                     return content.split("/")[-1]
 
-        except:
+        except Exception:
             pass
 
         print(
@@ -70,7 +70,7 @@ def get_file_content_at_commit(file_path: str, commit: str) -> Optional[str]:
 def get_current_file_content(file_path: str) -> Optional[str]:
     try:
         return Path(file_path).read_text()
-    except:
+    except Exception:
         return None
 
 

api/py/ai/chronon/cli/plan/controller_iface.py

@@ -2,7 +2,6 @@
 from typing import Dict, List, Optional
 
 from ai.chronon.cli.plan.physical_graph import PhysicalGraph
-from ai.chronon.cli.plan.physical_index import PhysicalNode
 
 
 class ControllerIface(ABC):

api/py/ai/chronon/cli/plan/physical_index.py

@@ -5,8 +5,8 @@
 from ai.chronon.cli.compile.compiler import CompileResult
 from ai.chronon.cli.plan.controller_iface import ControllerIface
 from ai.chronon.cli.plan.physical_graph import PhysicalGraph
-from ai.chronon.lineage.ttypes import Column, ColumnLineage
 from ai.chronon.cli.plan.physical_node import PhysicalNode
+from ai.chronon.lineage.ttypes import Column, ColumnLineage
 
 
 @dataclass

api/py/ai/chronon/cli/plan/physical_node.py

@@ -1,9 +1,6 @@
-from dataclasses import dataclass
 from typing import List
 
-from ai.chronon.api.common.ttypes import TableDependency
 from ai.chronon.api.ttypes import GroupBy, Join, Model, StagingQuery
-from ai.chronon.cli.compile.compile_context import CompiledObj
 from ai.chronon.orchestration.ttypes import PhysicalNode
 
 

api/py/ai/chronon/eval/__init__.py

@@ -1,4 +1,8 @@
 from typing import Any, List
+
+from pyspark.sql import DataFrame, SparkSession
+
+import ai.chronon.api.ttypes as chronon
 from ai.chronon.eval.query_parsing import get_tables_from_query
 from ai.chronon.eval.sample_tables import sample_tables, sample_with_query
 from ai.chronon.eval.table_scan import (
@@ -8,8 +12,6 @@
     table_scans_in_join,
     table_scans_in_source,
 )
-import ai.chronon.api.ttypes as chronon
-from pyspark.sql import DataFrame, SparkSession
 
 
 def eval(obj: Any) -> List[DataFrame]:

api/py/ai/chronon/eval/sample_tables.py

@@ -1,5 +1,6 @@
 import os
 from typing import List
+
 from ai.chronon.eval.table_scan import local_warehouse
 
 
@@ -55,13 +56,13 @@ def _sample_bigquery(query, output_path):
 
 
 def _sample_bigquery_fast(query, destination_path):
-    from google.cloud.bigquery_storage import BigQueryReadClient
-    from google.cloud.bigquery_storage_v1.types import ReadSession
-    from google.cloud.bigquery_storage_v1.types import DataFormat
-    from google.cloud import bigquery
-    import pyarrow.parquet as pq
     import os
 
+    import pyarrow.parquet as pq
+    from google.cloud import bigquery
+    from google.cloud.bigquery_storage import BigQueryReadClient
+    from google.cloud.bigquery_storage_v1.types import DataFormat, ReadSession
+
     project_id = os.getenv("GCP_PROJECT_ID")
     assert project_id, "Please set the GCP_PROJECT_ID environment variable"
 

api/py/ai/chronon/eval/table_scan.py

@@ -1,10 +1,11 @@
-from dataclasses import dataclass
-from datetime import datetime, timedelta
 import hashlib
 import os
-from pathlib import Path
 import re
+from dataclasses import dataclass
+from datetime import datetime, timedelta
+from pathlib import Path
 from typing import List, Tuple
+
 import ai.chronon.api.ttypes as chronon
 
 

api/py/ai/chronon/group_by.py

@@ -12,17 +12,16 @@
 #     See the License for the specific language governing permissions and
 #     limitations under the License.
 
-import ai.chronon.api.ttypes as ttypes
-import ai.chronon.api.common.ttypes as common
-import ai.chronon.utils as utils
-import ai.chronon.windows as window_utils
-
-from copy import deepcopy
-import logging
 import inspect
 import json
-from typing import List, Optional, Union, Dict, Callable, Tuple
+import logging
+from copy import deepcopy
+from typing import Callable, Dict, List, Optional, Tuple, Union
 
+import ai.chronon.api.common.ttypes as common
+import ai.chronon.api.ttypes as ttypes
+import ai.chronon.utils as utils
+import ai.chronon.windows as window_utils
 
 OperationType = int  # type(zthrift.Operation.FIRST)
 
@@ -361,7 +360,7 @@ def validate_group_by(group_by: ttypes.GroupBy):
                         raise ValueError(
                             "[Percentiles] Unable to decode percentiles value, expected json array with values between"
                             f" 0 and 1 inclusive (ex: [0.6, 0.1]), received: {agg.argMap['percentiles']}"
-                        )
+                        ) from e
                 else:
                     raise ValueError(
                         f"[Percentiles] Unsupported arguments for {op_to_str(agg.operation)}, "