compiler cutover

api/py/ai/chronon/cli/compile/compile_context.py

@@ -3,14 +3,13 @@
 from typing import Any, Dict, List, Type
 
 from ai.chronon.api.common.ttypes import ConfigType
-from ai.chronon.api.ttypes import GroupBy, Join, StagingQuery, Team
+from ai.chronon.api.ttypes import GroupBy, Join, StagingQuery, Team, Model
 from ai.chronon.cli.compile.display.compiled_obj import CompiledObj
 from ai.chronon.cli.compile.display.compile_status import CompileStatus
 from ai.chronon.cli.compile.serializer import file2thrift
 from ai.chronon.cli.compile.conf_validator import ConfValidator
 import ai.chronon.cli.compile.parse_teams as teams
 from ai.chronon.cli.logger import require, get_logger
-from ai.chronon.model import Model
 
 
 logger = get_logger()

api/py/ai/chronon/cli/compile/compiler.py

@@ -102,10 +102,6 @@ def _write_objects_in_folder(
 
                     for error in errors:
                         print(f"\nError processing conf {co.name}: {error}")
-                        traceback.print_exception(
-                            type(error), error, error.__traceback__
-                        )
-                        print("\n")
 
                 else:
                     object_dict[co.name] = co.obj

api/py/ai/chronon/cli/compile/parse_configs.py

@@ -71,7 +71,9 @@ def from_file(file_path: str, cls: type, input_dir: str):
     rel_path_without_extension = os.path.splitext(rel_path)[0]
 
     module_name = rel_path_without_extension.replace("/", ".")
-    conf_type, team_name, rest = module_name.split(".", 2)
+
+    conf_type, team_name_with_path = module_name.split(".", 1)
+    mod_path  = team_name_with_path.replace("/", ".")
 
     module = importlib.import_module(module_name)
 
@@ -81,9 +83,9 @@ def from_file(file_path: str, cls: type, input_dir: str):
 
         if isinstance(obj, cls):
 
-            name = f"{team_name}.{rest}.{var_name}"
+            name = f"{mod_path}.{var_name}"
             obj.metaData.name = name
-            obj.metaData.team = team_name
+            obj.metaData.team = mod_path.split(".")[0]
 
             result[name] = obj
 

api/py/ai/chronon/cli/compile/parse_teams.py

@@ -1,5 +1,7 @@
 from copy import deepcopy
 import importlib
+import importlib.util
+import sys
 import os
 from ai.chronon.api.common.ttypes import (
     ExecutionInfo,
@@ -12,9 +14,27 @@
 
 logger = get_logger()
 
-_DEFAULT_CONF_TEAM = "common"
+_DEFAULT_CONF_TEAM = "default"
 
 
+def import_module_from_file(file_path):
+    # Get the module name from the file path (without .py extension)
+    module_name = file_path.split("/")[-1].replace(".py", "")
+
+    # Create the module spec
+    spec = importlib.util.spec_from_file_location(module_name, file_path)
+
+    # Create the module based on the spec
+    module = importlib.util.module_from_spec(spec)
+
+    # Add the module to sys.modules
+    sys.modules[module_name] = module
+
+    # Execute the module
+    spec.loader.exec_module(module)
+
+    return module
+
 def load_teams(conf_root: str) -> Dict[str, Team]:
 
     teams_file = os.path.join(conf_root, "teams.py")
@@ -24,15 +44,15 @@ def load_teams(conf_root: str) -> Dict[str, Team]:
         f"Team config file: {teams_file} not found. You might be running this from the wrong directory.",
     )
 
-    team_module = importlib.import_module("teams")
+    team_module = import_module_from_file(teams_file)
 
     require(
         team_module is not None,
         f"Team config file {teams_file} is not on the PYTHONPATH. You might need to add the your config directory to the PYTHONPATH.",
     )
 
     team_dict = {}
-
+    logger.info(f"Processing teams from path {teams_file}")
     for name, obj in team_module.__dict__.items():
         if isinstance(obj, Team):
             obj.name = name
@@ -64,7 +84,7 @@ def update_metadata(obj: Any, team_dict: Dict[str, Team]):
 
     require(
         _DEFAULT_CONF_TEAM in team_dict,
-        f"'{_DEFAULT_CONF_TEAM}' team not found in teams.py, please add an entry 🙏",
+        f"'{_DEFAULT_CONF_TEAM}' team not found in teams.py, please add an entry 🙏.",
     )
 
     metadata.outputNamespace = team_dict[team].outputNamespace

api/py/ai/chronon/cli/logger.py

@@ -53,9 +53,9 @@ def green(text):
 
 def require(cond, message):
     if not cond:
-        print(f"{red("❌")} {message}")
+        print(f"X: {message}")
         sys.exit(1)
 
 
 def done(cond, message):
-    print(f"{green("✅")} {message}")
+    print(f"DONE: {message}")

api/py/ai/chronon/repo/compile.py

@@ -309,7 +309,3 @@ def _print_error(left, right):
 def _print_warning(string):
     # print in yellow - \u001b[33m
     print(f"\u001b[33m{string}\u001b[0m")
-
-
-if __name__ == "__main__":
-    extract_and_convert()

api/py/ai/chronon/repo/zipline.py

@@ -1,6 +1,8 @@
+import os
 import click
 
-from ai.chronon.repo.compile import extract_and_convert
+from ai.chronon.cli.compile.compile_context import CompileContext
+from ai.chronon.cli.compile.compiler import Compiler
 from ai.chronon.repo.run import main as run_main
 from ai.chronon.repo.init import main as init_main
 
@@ -10,6 +12,22 @@ def zipline():
     pass
 
 
-zipline.add_command(extract_and_convert)
+@click.command(name="compile")
+def compile():
+    # check that a "teams.py" file exists in the current directory
+    if not (os.path.exists("teams.py") or os.path.exists("teams.json")):
+        raise click.ClickException(
+            (
+                "teams.py or teams.json file not found in current directory."
+                " Please run from the top level of conf directory."
+            )
+        )
+
+    compile_context = CompileContext()
+    compiler = Compiler(compile_context)
+    compiler.compile(compile_context)
+
+
+zipline.add_command(compile)
 zipline.add_command(run_main)
 zipline.add_command(init_main)

api/py/requirements/base.in

@@ -7,4 +7,5 @@ sqlglot
 crcmod==1.7
 glom
 boto3
-importlib-resources==6.5.2
+importlib-resources==6.5.2
+rich

api/py/requirements/base.txt

@@ -46,6 +46,8 @@ jmespath==1.0.1
     # via
     #   boto3
     #   botocore
+markdown-it-py==3.0.0
+    # via rich
 proto-plus==1.25.0
     # via google-api-core
 protobuf==5.29.3
@@ -59,12 +61,16 @@ pyasn1==0.6.1
     #   rsa
 pyasn1-modules==0.4.1
     # via google-auth
+pygments==2.19.1
+    # via rich
 python-dateutil==2.9.0.post0
     # via botocore
 requests==2.32.3
     # via
     #   google-api-core
     #   google-cloud-storage
+rich==13.9.4
+    # via -r requirements/base.in
 rsa==4.9
     # via google-auth
 s3transfer==0.11.4

api/py/test/sample/group_bys/sample_team/sample_group_by_missing_input_column.py

@@ -25,6 +25,7 @@
     sources=test_sources.staging_entities,
     keys=["s2CellId", "place_id"],
     aggregations=[
+        # Intentionally left out `input_column` to test error handling
         Aggregation(operation=Operation.COUNT),
         Aggregation(operation=Operation.COUNT),
     ],

api/py/test/sample/teams.py

@@ -42,7 +42,7 @@
 
 
 test = Team(
-    namespace="test",
+    outputNamespace="test",
     env=EnvironmentVariables(
         common={
             "GCP_BIGTABLE_INSTANCE_ID": "test-instance"  # example, custom bigtable instance
@@ -63,7 +63,7 @@
 
 
 sample_team = Team(
-    namespace="test",
+    outputNamespace="test",
     env=EnvironmentVariables(
         common={
             "GCP_BIGTABLE_INSTANCE_ID": "test-instance"  # example, custom bigtable instance
@@ -82,10 +82,10 @@
     ),
 )
 
-etsy_search = Team(namespace="etsy-search")
+etsy_search = Team(outputNamespace="etsy-search")
 
-kaggle = Team(namespace="kaggle")
+kaggle = Team(outputNamespace="kaggle")
 
-quickstart = Team(namespace="quickstart")
+quickstart = Team(outputNamespace="quickstart")
 
-risk = Team(namespace="risk")
+risk = Team(outputNamespace="risk")