From 08de178698c0fc551a8dfecabdae0aa709055555 Mon Sep 17 00:00:00 2001 From: Nathan Habib Date: Wed, 20 Aug 2025 11:49:09 +0000 Subject: [PATCH 1/6] use trackio when available --- src/lighteval/logging/evaluation_tracker.py | 28 +++++++++++++++------ src/lighteval/main_accelerate.py | 2 +- src/lighteval/main_endpoint.py | 8 +++--- src/lighteval/main_sglang.py | 2 +- src/lighteval/main_vllm.py | 2 +- 5 files changed, 28 insertions(+), 14 deletions(-) diff --git a/src/lighteval/logging/evaluation_tracker.py b/src/lighteval/logging/evaluation_tracker.py index 32a638d1e..d23f3e7dd 100644 --- a/src/lighteval/logging/evaluation_tracker.py +++ b/src/lighteval/logging/evaluation_tracker.py @@ -130,7 +130,7 @@ def __init__( tensorboard_metric_prefix: str = "eval", public: bool = False, nanotron_run_info: "GeneralArgs" = None, - wandb: bool = False, + use_wandb: bool = False, ) -> None: """Creates all the necessary loggers for evaluation tracking.""" self.details_logger = DetailsLogger() @@ -150,7 +150,7 @@ def __init__( self.should_push_to_hub = push_to_hub self.should_save_details = save_details - self.wandb = wandb + self.wandb = use_wandb self.should_push_results_to_tensorboard = push_to_tensorboard self.tensorboard_repo = f"{hub_results_org}/tensorboard_logs" @@ -160,18 +160,32 @@ def __init__( self.public = public - if wandb is True: - import wandb + if use_wandb is True: + try: + import trackio as wandb + + logger.warning("Trackio was found avaialable in your environment, using it instead of wandb") + self.wandb_project = os.environ.get("WANDB_PROJECT", None) + self.space_id = os.environ.get("WANDB_SPACE_ID", None) + + wandb_kwargs = { + "space_id": self.space_id, + } + + except ImportError: + import wandb - self.wandb_project = os.environ.get("WANDB_PROJECT", None) + self.wandb_project = os.environ.get("WANDB_PROJECT", None) + wandb.login() + wandb_kwargs = {} if self.wandb_project is None: - raise ValueError("You need to specify the project name in wandb_args") + raise ValueError("You need to specify the project name using the WANDB_PROJECT environment variable") - wandb.login() self.wandb_run = wandb.init( project=self.wandb_project, resume="allow", + **wandb_kwargs, ) @property diff --git a/src/lighteval/main_accelerate.py b/src/lighteval/main_accelerate.py index ce275092c..7c7df8a6d 100644 --- a/src/lighteval/main_accelerate.py +++ b/src/lighteval/main_accelerate.py @@ -132,7 +132,7 @@ def accelerate( # noqa C901 push_to_tensorboard=push_to_tensorboard, public=public_run, hub_results_org=results_org, - wandb=wandb, + use_wandb=wandb, ) pipeline_params = PipelineParameters( launcher_type=ParallelismManager.ACCELERATE, diff --git a/src/lighteval/main_endpoint.py b/src/lighteval/main_endpoint.py index 793a578a5..b71b83dda 100644 --- a/src/lighteval/main_endpoint.py +++ b/src/lighteval/main_endpoint.py @@ -128,7 +128,7 @@ def inference_endpoint( push_to_tensorboard=push_to_tensorboard, public=public_run, hub_results_org=results_org, - wandb=wandb, + use_wandb=wandb, ) parallelism_manager = ParallelismManager.NONE # since we're using inference endpoints in remote @@ -256,7 +256,7 @@ def tgi( push_to_tensorboard=push_to_tensorboard, public=public_run, hub_results_org=results_org, - wandb=wandb, + use_wandb=wandb, ) parallelism_manager = ParallelismManager.TGI @@ -388,7 +388,7 @@ def litellm( push_to_tensorboard=push_to_tensorboard, public=public_run, hub_results_org=results_org, - wandb=wandb, + use_wandb=wandb, ) parallelism_manager = ParallelismManager.NONE @@ -521,7 +521,7 @@ def inference_providers( push_to_tensorboard=push_to_tensorboard, public=public_run, hub_results_org=results_org, - wandb=wandb, + use_wandb=wandb, ) parallelism_manager = ParallelismManager.NONE diff --git a/src/lighteval/main_sglang.py b/src/lighteval/main_sglang.py index a10964ed5..baaa2b8b3 100644 --- a/src/lighteval/main_sglang.py +++ b/src/lighteval/main_sglang.py @@ -121,7 +121,7 @@ def sglang( push_to_tensorboard=push_to_tensorboard, public=public_run, hub_results_org=results_org, - wandb=wandb, + use_wandb=wandb, ) pipeline_params = PipelineParameters( diff --git a/src/lighteval/main_vllm.py b/src/lighteval/main_vllm.py index ba30777d2..aee1cbdf2 100644 --- a/src/lighteval/main_vllm.py +++ b/src/lighteval/main_vllm.py @@ -124,7 +124,7 @@ def vllm( push_to_tensorboard=push_to_tensorboard, public=public_run, hub_results_org=results_org, - wandb=wandb, + use_wandb=wandb, ) pipeline_params = PipelineParameters( From e504804cfbd72207eb5866a08e6d007ad8f5f9c4 Mon Sep 17 00:00:00 2001 From: Nathan Habib <30601243+NathanHB@users.noreply.github.com> Date: Wed, 20 Aug 2025 14:06:05 +0200 Subject: [PATCH 2/6] Update src/lighteval/logging/evaluation_tracker.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/lighteval/logging/evaluation_tracker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lighteval/logging/evaluation_tracker.py b/src/lighteval/logging/evaluation_tracker.py index d23f3e7dd..8d82c9fd0 100644 --- a/src/lighteval/logging/evaluation_tracker.py +++ b/src/lighteval/logging/evaluation_tracker.py @@ -164,7 +164,7 @@ def __init__( try: import trackio as wandb - logger.warning("Trackio was found avaialable in your environment, using it instead of wandb") + logger.warning("Trackio was found available in your environment, using it instead of wandb") self.wandb_project = os.environ.get("WANDB_PROJECT", None) self.space_id = os.environ.get("WANDB_SPACE_ID", None) From b0d4bdad2ce56168e23a08efd321a694e82c9ca4 Mon Sep 17 00:00:00 2001 From: Nathan Habib Date: Wed, 20 Aug 2025 12:25:08 +0000 Subject: [PATCH 3/6] update doc --- docs/source/saving-and-reading-results.mdx | 11 ++++++++++- pyproject.toml | 1 + src/lighteval/main_accelerate.py | 2 +- src/lighteval/main_endpoint.py | 8 ++++---- src/lighteval/main_sglang.py | 2 +- src/lighteval/main_vllm.py | 2 +- 6 files changed, 18 insertions(+), 8 deletions(-) diff --git a/docs/source/saving-and-reading-results.mdx b/docs/source/saving-and-reading-results.mdx index 2a54aeaf4..f8e32d676 100644 --- a/docs/source/saving-and-reading-results.mdx +++ b/docs/source/saving-and-reading-results.mdx @@ -36,7 +36,7 @@ This will create a Tensorboard dashboard in a HF org set with the `--results-org option. -## Pushing results to WandB +## Pushing results to Trackio or WandB You can push the results to WandB by setting `--wandb`. This will init a WandB run and log the results. @@ -49,6 +49,15 @@ export WANDB_PROJECT="lighteval" You can find a list of variable in the [wandb documentation](https://docs.wandb.ai/guides/track/environment-variables/). +If trackio is available in your environment `pip install +lighteval[trackio]`, it will be used to log and push the results on a +huggingface dataset. Choose the daatset name and org with: + +``` +export WANDB_SPACE_ID="org/name" +``` + + ## How to load and investigate details diff --git a/pyproject.toml b/pyproject.toml index 8a5a9cf90..08cc31a5c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -114,6 +114,7 @@ multilingual = [ ] math = ["latex2sympy2_extended==1.0.6"] wandb = ["wandb"] +trakcio = ["trakcio"] [project.urls] Homepage = "https://github.com/huggingface/lighteval" diff --git a/src/lighteval/main_accelerate.py b/src/lighteval/main_accelerate.py index 7c7df8a6d..0c0049040 100644 --- a/src/lighteval/main_accelerate.py +++ b/src/lighteval/main_accelerate.py @@ -99,7 +99,7 @@ def accelerate( # noqa C901 wandb: Annotated[ bool, Option( - help="Push results to wandb. This will only work if you have wandb installed and logged in. We use env variable to configure wandb. see here: https://docs.wandb.ai/guides/track/environment-variables/", + help="Push results to wandb or trackio if available. We use env variable to configure trackio or wandb. see here: https://docs.wandb.ai/guides/track/environment-variables/, https://github.com/gradio-app/trackio", rich_help_panel=HELP_PANEL_NAME_2, ), ] = False, diff --git a/src/lighteval/main_endpoint.py b/src/lighteval/main_endpoint.py index b71b83dda..a6a29f2b5 100644 --- a/src/lighteval/main_endpoint.py +++ b/src/lighteval/main_endpoint.py @@ -101,7 +101,7 @@ def inference_endpoint( wandb: Annotated[ bool, Option( - help="Push results to wandb. This will only work if you have wandb installed and logged in. We use env variable to configure wandb. see here: https://docs.wandb.ai/guides/track/environment-variables/", + help="Push results to wandb or trackio if available. We use env variable to configure trackio or wandb. see here: https://docs.wandb.ai/guides/track/environment-variables/, https://github.com/gradio-app/trackio", rich_help_panel=HELP_PANEL_NAME_2, ), ] = False, @@ -226,7 +226,7 @@ def tgi( wandb: Annotated[ bool, Option( - help="Push results to wandb. This will only work if you have wandb installed and logged in. We use env variable to configure wandb. see here: https://docs.wandb.ai/guides/track/environment-variables/", + help="Push results to wandb or trackio if available. We use env variable to configure trackio or wandb. see here: https://docs.wandb.ai/guides/track/environment-variables/, https://github.com/gradio-app/trackio", rich_help_panel=HELP_PANEL_NAME_2, ), ] = False, @@ -358,7 +358,7 @@ def litellm( wandb: Annotated[ bool, Option( - help="Push results to wandb. This will only work if you have wandb installed and logged in. We use env variable to configure wandb. see here: https://docs.wandb.ai/guides/track/environment-variables/", + help="Push results to wandb or trackio if available. We use env variable to configure trackio or wandb. see here: https://docs.wandb.ai/guides/track/environment-variables/, https://github.com/gradio-app/trackio", rich_help_panel=HELP_PANEL_NAME_2, ), ] = False, @@ -481,7 +481,7 @@ def inference_providers( wandb: Annotated[ bool, Option( - help="Push results to wandb. This will only work if you have wandb installed and logged in. We use env variable to configure wandb. see here: https://docs.wandb.ai/guides/track/environment-variables/", + help="Push results to wandb or trackio if available. We use env variable to configure trackio or wandb. see here: https://docs.wandb.ai/guides/track/environment-variables/, https://github.com/gradio-app/trackio", rich_help_panel=HELP_PANEL_NAME_2, ), ] = False, diff --git a/src/lighteval/main_sglang.py b/src/lighteval/main_sglang.py index baaa2b8b3..2129a7281 100644 --- a/src/lighteval/main_sglang.py +++ b/src/lighteval/main_sglang.py @@ -92,7 +92,7 @@ def sglang( wandb: Annotated[ bool, Option( - help="Push results to wandb. This will only work if you have wandb installed and logged in. We use env variable to configure wandb. see here: https://docs.wandb.ai/guides/track/environment-variables/", + help="Push results to wandb or trackio if available. We use env variable to configure trackio or wandb. see here: https://docs.wandb.ai/guides/track/environment-variables/, https://github.com/gradio-app/trackio", rich_help_panel=HELP_PANEL_NAME_2, ), ] = False, diff --git a/src/lighteval/main_vllm.py b/src/lighteval/main_vllm.py index aee1cbdf2..f5ba384ed 100644 --- a/src/lighteval/main_vllm.py +++ b/src/lighteval/main_vllm.py @@ -95,7 +95,7 @@ def vllm( wandb: Annotated[ bool, Option( - help="Push results to wandb. This will only work if you have wandb installed and logged in. We use env variable to configure wandb. see here: https://docs.wandb.ai/guides/track/environment-variables/", + help="Push results to wandb or trackio if available. We use env variable to configure trackio or wandb. see here: https://docs.wandb.ai/guides/track/environment-variables/, https://github.com/gradio-app/trackio", rich_help_panel=HELP_PANEL_NAME_2, ), ] = False, From 00dd5ec1efda70ff4878f97d6a071c608f58c62c Mon Sep 17 00:00:00 2001 From: Nathan Habib Date: Wed, 20 Aug 2025 12:29:20 +0000 Subject: [PATCH 4/6] change attr name --- src/lighteval/logging/evaluation_tracker.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lighteval/logging/evaluation_tracker.py b/src/lighteval/logging/evaluation_tracker.py index 8d82c9fd0..c9a336fc1 100644 --- a/src/lighteval/logging/evaluation_tracker.py +++ b/src/lighteval/logging/evaluation_tracker.py @@ -150,7 +150,7 @@ def __init__( self.should_push_to_hub = push_to_hub self.should_save_details = save_details - self.wandb = use_wandb + self.use_wandb = use_wandb self.should_push_results_to_tensorboard = push_to_tensorboard self.tensorboard_repo = f"{hub_results_org}/tensorboard_logs" @@ -259,7 +259,7 @@ def save(self) -> None: results_dict=results_dict, ) - if self.wandb is True: + if self.use_wandb is True: self.push_to_wandb( results_dict=results_dict, details_datasets=details_datasets, From 4e9d97aa14984df03f58db76c4a9df58194afe46 Mon Sep 17 00:00:00 2001 From: Nathan Habib Date: Wed, 20 Aug 2025 12:31:25 +0000 Subject: [PATCH 5/6] fix trackio typo --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 08cc31a5c..4100add41 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -114,7 +114,7 @@ multilingual = [ ] math = ["latex2sympy2_extended==1.0.6"] wandb = ["wandb"] -trakcio = ["trakcio"] +trackio = ["trackio"] [project.urls] Homepage = "https://github.com/huggingface/lighteval" From c815df3af3b53ed5f3c9b9fc266dc3b0fc9fbac8 Mon Sep 17 00:00:00 2001 From: Nathan Habib <30601243+NathanHB@users.noreply.github.com> Date: Wed, 20 Aug 2025 14:36:45 +0200 Subject: [PATCH 6/6] Update docs/source/saving-and-reading-results.mdx Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- docs/source/saving-and-reading-results.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/saving-and-reading-results.mdx b/docs/source/saving-and-reading-results.mdx index f8e32d676..e37cfb88e 100644 --- a/docs/source/saving-and-reading-results.mdx +++ b/docs/source/saving-and-reading-results.mdx @@ -51,7 +51,7 @@ You can find a list of variable in the [wandb documentation](https://docs.wandb. If trackio is available in your environment `pip install lighteval[trackio]`, it will be used to log and push the results on a -huggingface dataset. Choose the daatset name and org with: +huggingface dataset. Choose the dataset name and org with: ``` export WANDB_SPACE_ID="org/name"