Trackio when available

docs/source/saving-and-reading-results.mdx

@@ -36,7 +36,7 @@ This will create a Tensorboard dashboard in a HF org set with the `--results-org
 option.
 
 
-## Pushing results to WandB
+## Pushing results to Trackio or WandB
 
 You can push the results to WandB by setting `--wandb`. This will init a WandB
 run and log the results.
@@ -49,6 +49,15 @@ export WANDB_PROJECT="lighteval"
 
 You can find a list of variable in the [wandb documentation](https://docs.wandb.ai/guides/track/environment-variables/).
 
+If trackio is available in your environment `pip install
+lighteval[trackio]`, it will be used to log and push the results on a
+huggingface dataset. Choose the dataset name and org with:
+
+```
+export WANDB_SPACE_ID="org/name"
+```
+
+
 
 ## How to load and investigate details
 

pyproject.toml

@@ -114,6 +114,7 @@ multilingual = [
 ]
 math = ["latex2sympy2_extended==1.0.6"]
 wandb = ["wandb"]
+trackio = ["trackio"]
 
 [project.urls]
 Homepage = "https://github.com/huggingface/lighteval"

src/lighteval/logging/evaluation_tracker.py

@@ -130,7 +130,7 @@ def __init__(
         tensorboard_metric_prefix: str = "eval",
         public: bool = False,
         nanotron_run_info: "GeneralArgs" = None,
-        wandb: bool = False,
+        use_wandb: bool = False,
     ) -> None:
         """Creates all the necessary loggers for evaluation tracking."""
         self.details_logger = DetailsLogger()
@@ -150,7 +150,7 @@ def __init__(
 
         self.should_push_to_hub = push_to_hub
         self.should_save_details = save_details
-        self.wandb = wandb
+        self.use_wandb = use_wandb
 
         self.should_push_results_to_tensorboard = push_to_tensorboard
         self.tensorboard_repo = f"{hub_results_org}/tensorboard_logs"
@@ -160,18 +160,32 @@ def __init__(
 
         self.public = public
 
-        if wandb is True:
-            import wandb
+        if use_wandb is True:
+            try:
+                import trackio as wandb
+
+                logger.warning("Trackio was found available in your environment, using it instead of wandb")
+                self.wandb_project = os.environ.get("WANDB_PROJECT", None)
+                self.space_id = os.environ.get("WANDB_SPACE_ID", None)
+
+                wandb_kwargs = {
+                    "space_id": self.space_id,
+                }
+
+            except ImportError:
+                import wandb
 
-            self.wandb_project = os.environ.get("WANDB_PROJECT", None)
+                self.wandb_project = os.environ.get("WANDB_PROJECT", None)
+                wandb.login()
+                wandb_kwargs = {}
 
             if self.wandb_project is None:
-                raise ValueError("You need to specify the project name in wandb_args")
+                raise ValueError("You need to specify the project name using the WANDB_PROJECT environment variable")
 
-            wandb.login()
             self.wandb_run = wandb.init(
                 project=self.wandb_project,
                 resume="allow",
+                **wandb_kwargs,
             )
 
     @property
@@ -245,7 +259,7 @@ def save(self) -> None:
                 results_dict=results_dict,
             )
 
-        if self.wandb is True:
+        if self.use_wandb is True:
             self.push_to_wandb(
                 results_dict=results_dict,
                 details_datasets=details_datasets,

src/lighteval/main_accelerate.py

@@ -99,7 +99,7 @@ def accelerate(  # noqa C901
     wandb: Annotated[
         bool,
         Option(
-            help="Push results to wandb. This will only work if you have wandb installed and logged in. We use env variable to configure wandb. see here: https://docs.wandb.ai/guides/track/environment-variables/",
+            help="Push results to wandb or trackio if available. We use env variable to configure trackio or wandb. see here: https://docs.wandb.ai/guides/track/environment-variables/, https://github.com/gradio-app/trackio",
             rich_help_panel=HELP_PANEL_NAME_2,
         ),
     ] = False,
@@ -132,7 +132,7 @@ def accelerate(  # noqa C901
         push_to_tensorboard=push_to_tensorboard,
         public=public_run,
         hub_results_org=results_org,
-        wandb=wandb,
+        use_wandb=wandb,
     )
     pipeline_params = PipelineParameters(
         launcher_type=ParallelismManager.ACCELERATE,

src/lighteval/main_endpoint.py

@@ -101,7 +101,7 @@ def inference_endpoint(
     wandb: Annotated[
         bool,
         Option(
-            help="Push results to wandb. This will only work if you have wandb installed and logged in. We use env variable to configure wandb. see here: https://docs.wandb.ai/guides/track/environment-variables/",
+            help="Push results to wandb or trackio if available. We use env variable to configure trackio or wandb. see here: https://docs.wandb.ai/guides/track/environment-variables/, https://github.com/gradio-app/trackio",
             rich_help_panel=HELP_PANEL_NAME_2,
         ),
     ] = False,
@@ -128,7 +128,7 @@ def inference_endpoint(
         push_to_tensorboard=push_to_tensorboard,
         public=public_run,
         hub_results_org=results_org,
-        wandb=wandb,
+        use_wandb=wandb,
     )
 
     parallelism_manager = ParallelismManager.NONE  # since we're using inference endpoints in remote
@@ -226,7 +226,7 @@ def tgi(
     wandb: Annotated[
         bool,
         Option(
-            help="Push results to wandb. This will only work if you have wandb installed and logged in. We use env variable to configure wandb. see here: https://docs.wandb.ai/guides/track/environment-variables/",
+            help="Push results to wandb or trackio if available. We use env variable to configure trackio or wandb. see here: https://docs.wandb.ai/guides/track/environment-variables/, https://github.com/gradio-app/trackio",
             rich_help_panel=HELP_PANEL_NAME_2,
         ),
     ] = False,
@@ -256,7 +256,7 @@ def tgi(
         push_to_tensorboard=push_to_tensorboard,
         public=public_run,
         hub_results_org=results_org,
-        wandb=wandb,
+        use_wandb=wandb,
     )
 
     parallelism_manager = ParallelismManager.TGI
@@ -358,7 +358,7 @@ def litellm(
     wandb: Annotated[
         bool,
         Option(
-            help="Push results to wandb. This will only work if you have wandb installed and logged in. We use env variable to configure wandb. see here: https://docs.wandb.ai/guides/track/environment-variables/",
+            help="Push results to wandb or trackio if available. We use env variable to configure trackio or wandb. see here: https://docs.wandb.ai/guides/track/environment-variables/, https://github.com/gradio-app/trackio",
             rich_help_panel=HELP_PANEL_NAME_2,
         ),
     ] = False,
@@ -388,7 +388,7 @@ def litellm(
         push_to_tensorboard=push_to_tensorboard,
         public=public_run,
         hub_results_org=results_org,
-        wandb=wandb,
+        use_wandb=wandb,
     )
 
     parallelism_manager = ParallelismManager.NONE
@@ -481,7 +481,7 @@ def inference_providers(
     wandb: Annotated[
         bool,
         Option(
-            help="Push results to wandb. This will only work if you have wandb installed and logged in. We use env variable to configure wandb. see here: https://docs.wandb.ai/guides/track/environment-variables/",
+            help="Push results to wandb or trackio if available. We use env variable to configure trackio or wandb. see here: https://docs.wandb.ai/guides/track/environment-variables/, https://github.com/gradio-app/trackio",
             rich_help_panel=HELP_PANEL_NAME_2,
         ),
     ] = False,
@@ -521,7 +521,7 @@ def inference_providers(
         push_to_tensorboard=push_to_tensorboard,
         public=public_run,
         hub_results_org=results_org,
-        wandb=wandb,
+        use_wandb=wandb,
     )
 
     parallelism_manager = ParallelismManager.NONE

src/lighteval/main_sglang.py

@@ -92,7 +92,7 @@ def sglang(
     wandb: Annotated[
         bool,
         Option(
-            help="Push results to wandb. This will only work if you have wandb installed and logged in. We use env variable to configure wandb. see here: https://docs.wandb.ai/guides/track/environment-variables/",
+            help="Push results to wandb or trackio if available. We use env variable to configure trackio or wandb. see here: https://docs.wandb.ai/guides/track/environment-variables/, https://github.com/gradio-app/trackio",
             rich_help_panel=HELP_PANEL_NAME_2,
         ),
     ] = False,
@@ -121,7 +121,7 @@ def sglang(
         push_to_tensorboard=push_to_tensorboard,
         public=public_run,
         hub_results_org=results_org,
-        wandb=wandb,
+        use_wandb=wandb,
     )
 
     pipeline_params = PipelineParameters(

src/lighteval/main_vllm.py

@@ -95,7 +95,7 @@ def vllm(
     wandb: Annotated[
         bool,
         Option(
-            help="Push results to wandb. This will only work if you have wandb installed and logged in. We use env variable to configure wandb. see here: https://docs.wandb.ai/guides/track/environment-variables/",
+            help="Push results to wandb or trackio if available. We use env variable to configure trackio or wandb. see here: https://docs.wandb.ai/guides/track/environment-variables/, https://github.com/gradio-app/trackio",
             rich_help_panel=HELP_PANEL_NAME_2,
         ),
     ] = False,
@@ -124,7 +124,7 @@ def vllm(
         push_to_tensorboard=push_to_tensorboard,
         public=public_run,
         hub_results_org=results_org,
-        wandb=wandb,
+        use_wandb=wandb,
     )
 
     pipeline_params = PipelineParameters(