From eb94e9c8f69ebbea75dc8b3b8c8caa96aded08af Mon Sep 17 00:00:00 2001 From: Albert Villanova del Moral <8515462+albertvillanova@users.noreply.github.com> Date: Thu, 19 Dec 2024 09:37:52 +0100 Subject: [PATCH 01/14] Fix definition of public in docstring --- src/lighteval/logging/evaluation_tracker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lighteval/logging/evaluation_tracker.py b/src/lighteval/logging/evaluation_tracker.py index 017055348..a2bc828c0 100644 --- a/src/lighteval/logging/evaluation_tracker.py +++ b/src/lighteval/logging/evaluation_tracker.py @@ -119,7 +119,7 @@ def __init__( more details about the datasets organisation in [`EvaluationTracker.save`] tensorboard_metric_prefix (str): Prefix for the metrics in the tensorboard logs - public (bool): If True, results and details are pushed in private orgs + public (`bool`, defaults to False): If True, results and details are pushed to public orgs. nanotron_run_info (GeneralArgs): Reference to informations about Nanotron models runs """ self.details_logger = DetailsLogger() From 0a27162af58aeee88ea1dd95ff081f28f206b5f6 Mon Sep 17 00:00:00 2001 From: Albert Villanova del Moral <8515462+albertvillanova@users.noreply.github.com> Date: Thu, 19 Dec 2024 09:39:09 +0100 Subject: [PATCH 02/14] Fix push_to_tensorboard param name in docstring --- src/lighteval/logging/evaluation_tracker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lighteval/logging/evaluation_tracker.py b/src/lighteval/logging/evaluation_tracker.py index a2bc828c0..78bf6f02b 100644 --- a/src/lighteval/logging/evaluation_tracker.py +++ b/src/lighteval/logging/evaluation_tracker.py @@ -114,7 +114,7 @@ def __init__( push_to_hub (bool): If True, details are pushed to the hub. Results are pushed to `{hub_results_org}/details__{sanitized model_name}` for the model `model_name`, a public dataset, if `public` is True else `{hub_results_org}/details__{sanitized model_name}_private`, a private dataset. - push_results_to_tensorboard (bool): If True, will create and push the results for a tensorboard folder on the hub + push_to_tensorboard (bool): If True, will create and push the results for a tensorboard folder on the hub hub_results_org (str): The organisation to push the results to. See more details about the datasets organisation in [`EvaluationTracker.save`] From 9a9ba1b04cdd0edf7a28188dbe9e184d97ab8cc0 Mon Sep 17 00:00:00 2001 From: Albert Villanova del Moral <8515462+albertvillanova@users.noreply.github.com> Date: Thu, 19 Dec 2024 09:47:41 +0100 Subject: [PATCH 03/14] Fix docstring style --- src/lighteval/logging/evaluation_tracker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lighteval/logging/evaluation_tracker.py b/src/lighteval/logging/evaluation_tracker.py index 78bf6f02b..6100a44c1 100644 --- a/src/lighteval/logging/evaluation_tracker.py +++ b/src/lighteval/logging/evaluation_tracker.py @@ -114,7 +114,7 @@ def __init__( push_to_hub (bool): If True, details are pushed to the hub. Results are pushed to `{hub_results_org}/details__{sanitized model_name}` for the model `model_name`, a public dataset, if `public` is True else `{hub_results_org}/details__{sanitized model_name}_private`, a private dataset. - push_to_tensorboard (bool): If True, will create and push the results for a tensorboard folder on the hub + push_to_tensorboard (`bool`, defaults to False): If True, will create and push the results for a tensorboard folder on the hub. hub_results_org (str): The organisation to push the results to. See more details about the datasets organisation in [`EvaluationTracker.save`] From 01879aaa070340110f7c3ffc2db7a4d036435b22 Mon Sep 17 00:00:00 2001 From: Albert Villanova del Moral <8515462+albertvillanova@users.noreply.github.com> Date: Thu, 19 Dec 2024 09:48:01 +0100 Subject: [PATCH 04/14] Add EvaluationTracker to docs --- docs/source/package_reference/logging.mdx | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/source/package_reference/logging.mdx b/docs/source/package_reference/logging.mdx index 9fd01154e..9102755c1 100644 --- a/docs/source/package_reference/logging.mdx +++ b/docs/source/package_reference/logging.mdx @@ -1,4 +1,7 @@ -# Loggers +# Logging + +## EvaluationTracker +[[autodoc]] logging.evaluation_tracker.EvaluationTracker ## GeneralConfigLogger [[autodoc]] logging.info_loggers.GeneralConfigLogger From 2920dfd7b89ef6c25d35769871e723e441a92553 Mon Sep 17 00:00:00 2001 From: Albert Villanova del Moral <8515462+albertvillanova@users.noreply.github.com> Date: Thu, 19 Dec 2024 09:56:51 +0100 Subject: [PATCH 05/14] Fix docstring style --- src/lighteval/logging/evaluation_tracker.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/lighteval/logging/evaluation_tracker.py b/src/lighteval/logging/evaluation_tracker.py index 6100a44c1..d83bd0bc5 100644 --- a/src/lighteval/logging/evaluation_tracker.py +++ b/src/lighteval/logging/evaluation_tracker.py @@ -83,7 +83,7 @@ def default(self, o): class EvaluationTracker: """ - Keeps track of the overall evaluation process and relevant informations. + Keeps track of the overall evaluation process and relevant information. The [`EvaluationTracker`] contains specific loggers for experiments details ([`DetailsLogger`]), metrics ([`MetricsLogger`]), task versions @@ -109,18 +109,17 @@ def __init__( Creates all the necessary loggers for evaluation tracking. Args: - output_dir (str): Local folder path where you want results to be saved - save_details (bool): If True, details are saved to the output_dir - push_to_hub (bool): If True, details are pushed to the hub. + output_dir (`str`): Local folder path where you want results to be saved. + save_details (`bool`, defaults to True): If True, details are saved to the `output_dir`. + push_to_hub (`bool`, defaults to False): If True, details are pushed to the hub. Results are pushed to `{hub_results_org}/details__{sanitized model_name}` for the model `model_name`, a public dataset, if `public` is True else `{hub_results_org}/details__{sanitized model_name}_private`, a private dataset. push_to_tensorboard (`bool`, defaults to False): If True, will create and push the results for a tensorboard folder on the hub. - hub_results_org (str): The organisation to push the results to. See - more details about the datasets organisation in - [`EvaluationTracker.save`] - tensorboard_metric_prefix (str): Prefix for the metrics in the tensorboard logs + hub_results_org (`str`, *optional*): The organisation to push the results to. + See more details about the datasets organisation in [`EvaluationTracker.save`]. + tensorboard_metric_prefix (`str`, defaults to "eval"): Prefix for the metrics in the tensorboard logs. public (`bool`, defaults to False): If True, results and details are pushed to public orgs. - nanotron_run_info (GeneralArgs): Reference to informations about Nanotron models runs + nanotron_run_info ([`GeneralArgs`], *optional*): Reference to information about Nanotron models runs. """ self.details_logger = DetailsLogger() self.metrics_logger = MetricsLogger() From e872679aea62182303accb849df6f376bcc94cf6 Mon Sep 17 00:00:00 2001 From: Albert Villanova del Moral <8515462+albertvillanova@users.noreply.github.com> Date: Thu, 19 Dec 2024 10:00:57 +0100 Subject: [PATCH 06/14] Move docstring to class header --- src/lighteval/logging/evaluation_tracker.py | 33 ++++++++++----------- 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/src/lighteval/logging/evaluation_tracker.py b/src/lighteval/logging/evaluation_tracker.py index d83bd0bc5..e738229a8 100644 --- a/src/lighteval/logging/evaluation_tracker.py +++ b/src/lighteval/logging/evaluation_tracker.py @@ -82,8 +82,7 @@ def default(self, o): class EvaluationTracker: - """ - Keeps track of the overall evaluation process and relevant information. + """Keeps track of the overall evaluation process and relevant information. The [`EvaluationTracker`] contains specific loggers for experiments details ([`DetailsLogger`]), metrics ([`MetricsLogger`]), task versions @@ -92,6 +91,19 @@ class EvaluationTracker: ([`GeneralConfigLogger`]). It compiles the data from these loggers and writes it to files, which can be published to the Hugging Face hub if requested. + + Args: + output_dir (`str`): Local folder path where you want results to be saved. + save_details (`bool`, defaults to True): If True, details are saved to the `output_dir`. + push_to_hub (`bool`, defaults to False): If True, details are pushed to the hub. + Results are pushed to `{hub_results_org}/details__{sanitized model_name}` for the model `model_name`, a public dataset, + if `public` is True else `{hub_results_org}/details__{sanitized model_name}_private`, a private dataset. + push_to_tensorboard (`bool`, defaults to False): If True, will create and push the results for a tensorboard folder on the hub. + hub_results_org (`str`, *optional*): The organisation to push the results to. + See more details about the datasets organisation in [`EvaluationTracker.save`]. + tensorboard_metric_prefix (`str`, defaults to "eval"): Prefix for the metrics in the tensorboard logs. + public (`bool`, defaults to False): If True, results and details are pushed to public orgs. + nanotron_run_info ([`GeneralArgs`], *optional*): Reference to information about Nanotron models runs. """ def __init__( @@ -105,22 +117,7 @@ def __init__( public: bool = False, nanotron_run_info: "GeneralArgs" = None, ) -> None: - """ - Creates all the necessary loggers for evaluation tracking. - - Args: - output_dir (`str`): Local folder path where you want results to be saved. - save_details (`bool`, defaults to True): If True, details are saved to the `output_dir`. - push_to_hub (`bool`, defaults to False): If True, details are pushed to the hub. - Results are pushed to `{hub_results_org}/details__{sanitized model_name}` for the model `model_name`, a public dataset, - if `public` is True else `{hub_results_org}/details__{sanitized model_name}_private`, a private dataset. - push_to_tensorboard (`bool`, defaults to False): If True, will create and push the results for a tensorboard folder on the hub. - hub_results_org (`str`, *optional*): The organisation to push the results to. - See more details about the datasets organisation in [`EvaluationTracker.save`]. - tensorboard_metric_prefix (`str`, defaults to "eval"): Prefix for the metrics in the tensorboard logs. - public (`bool`, defaults to False): If True, results and details are pushed to public orgs. - nanotron_run_info ([`GeneralArgs`], *optional*): Reference to information about Nanotron models runs. - """ + """Creates all the necessary loggers for evaluation tracking.""" self.details_logger = DetailsLogger() self.metrics_logger = MetricsLogger() self.versions_logger = VersionsLogger() From 39420c69cf18fed69d3fa32aecc8b60428be36de Mon Sep 17 00:00:00 2001 From: Albert Villanova del Moral <8515462+albertvillanova@users.noreply.github.com> Date: Thu, 19 Dec 2024 10:08:57 +0100 Subject: [PATCH 07/14] Add attributes to docstring --- src/lighteval/logging/evaluation_tracker.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/lighteval/logging/evaluation_tracker.py b/src/lighteval/logging/evaluation_tracker.py index e738229a8..7d9f19b10 100644 --- a/src/lighteval/logging/evaluation_tracker.py +++ b/src/lighteval/logging/evaluation_tracker.py @@ -104,6 +104,13 @@ class EvaluationTracker: tensorboard_metric_prefix (`str`, defaults to "eval"): Prefix for the metrics in the tensorboard logs. public (`bool`, defaults to False): If True, results and details are pushed to public orgs. nanotron_run_info ([`GeneralArgs`], *optional*): Reference to information about Nanotron models runs. + + Attributes: + details_logger ([`DetailsLogger`]): Logger for experiment details. + metrics_logger ([`MetricsLogger`]): Logger for experiment metrics. + versions_logger ([`VersionsLogger`]): Logger for task versions. + general_config_logger ([`GeneralConfigLogger`]): Logger for general configuration. + task_config_logger ([`TaskConfigLogger`]): Logger for task configuration. """ def __init__( From 1dae9e487b13421edf501b800bcfb8d6acd59bfc Mon Sep 17 00:00:00 2001 From: Albert Villanova del Moral <8515462+albertvillanova@users.noreply.github.com> Date: Thu, 19 Dec 2024 10:12:06 +0100 Subject: [PATCH 08/14] Fix style --- src/lighteval/logging/evaluation_tracker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lighteval/logging/evaluation_tracker.py b/src/lighteval/logging/evaluation_tracker.py index 7d9f19b10..5287d5923 100644 --- a/src/lighteval/logging/evaluation_tracker.py +++ b/src/lighteval/logging/evaluation_tracker.py @@ -105,7 +105,7 @@ class EvaluationTracker: public (`bool`, defaults to False): If True, results and details are pushed to public orgs. nanotron_run_info ([`GeneralArgs`], *optional*): Reference to information about Nanotron models runs. - Attributes: + **Attributes**: details_logger ([`DetailsLogger`]): Logger for experiment details. metrics_logger ([`MetricsLogger`]): Logger for experiment metrics. versions_logger ([`VersionsLogger`]): Logger for task versions. From 0af6ea277e1a83eef102e5ec041044b5f7c95272 Mon Sep 17 00:00:00 2001 From: Albert Villanova del Moral <8515462+albertvillanova@users.noreply.github.com> Date: Thu, 19 Dec 2024 10:25:02 +0100 Subject: [PATCH 09/14] Fix style --- src/lighteval/logging/evaluation_tracker.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/lighteval/logging/evaluation_tracker.py b/src/lighteval/logging/evaluation_tracker.py index 5287d5923..a03058be7 100644 --- a/src/lighteval/logging/evaluation_tracker.py +++ b/src/lighteval/logging/evaluation_tracker.py @@ -106,11 +106,11 @@ class EvaluationTracker: nanotron_run_info ([`GeneralArgs`], *optional*): Reference to information about Nanotron models runs. **Attributes**: - details_logger ([`DetailsLogger`]): Logger for experiment details. - metrics_logger ([`MetricsLogger`]): Logger for experiment metrics. - versions_logger ([`VersionsLogger`]): Logger for task versions. - general_config_logger ([`GeneralConfigLogger`]): Logger for general configuration. - task_config_logger ([`TaskConfigLogger`]): Logger for task configuration. + - details_logger ([`DetailsLogger`]): Logger for experiment details. + - metrics_logger ([`MetricsLogger`]): Logger for experiment metrics. + - versions_logger ([`VersionsLogger`]): Logger for task versions. + - general_config_logger ([`GeneralConfigLogger`]): Logger for general configuration. + - task_config_logger ([`TaskConfigLogger`]): Logger for task configuration. """ def __init__( From cec1da683200c3adfdc667ec2a4f7a2fdc9cd3f7 Mon Sep 17 00:00:00 2001 From: Albert Villanova del Moral <8515462+albertvillanova@users.noreply.github.com> Date: Thu, 19 Dec 2024 10:31:51 +0100 Subject: [PATCH 10/14] Fix style --- src/lighteval/logging/evaluation_tracker.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/lighteval/logging/evaluation_tracker.py b/src/lighteval/logging/evaluation_tracker.py index a03058be7..2686a5032 100644 --- a/src/lighteval/logging/evaluation_tracker.py +++ b/src/lighteval/logging/evaluation_tracker.py @@ -106,11 +106,11 @@ class EvaluationTracker: nanotron_run_info ([`GeneralArgs`], *optional*): Reference to information about Nanotron models runs. **Attributes**: - - details_logger ([`DetailsLogger`]): Logger for experiment details. - - metrics_logger ([`MetricsLogger`]): Logger for experiment metrics. - - versions_logger ([`VersionsLogger`]): Logger for task versions. - - general_config_logger ([`GeneralConfigLogger`]): Logger for general configuration. - - task_config_logger ([`TaskConfigLogger`]): Logger for task configuration. + - **details_logger** ([`DetailsLogger`]): Logger for experiment details. + - **metrics_logger** ([`MetricsLogger`]): Logger for experiment metrics. + - **versions_logger** ([`VersionsLogger`]): Logger for task versions. + - **general_config_logger** ([`GeneralConfigLogger`]): Logger for general configuration. + - **task_config_logger** ([`TaskConfigLogger`]): Logger for task configuration. """ def __init__( From 4b191aba48ba679445a69ea7e48bb9d33684842a Mon Sep 17 00:00:00 2001 From: Albert Villanova del Moral <8515462+albertvillanova@users.noreply.github.com> Date: Thu, 19 Dec 2024 10:32:05 +0100 Subject: [PATCH 11/14] Fix style --- src/lighteval/logging/evaluation_tracker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lighteval/logging/evaluation_tracker.py b/src/lighteval/logging/evaluation_tracker.py index 2686a5032..4ae2385f3 100644 --- a/src/lighteval/logging/evaluation_tracker.py +++ b/src/lighteval/logging/evaluation_tracker.py @@ -106,7 +106,7 @@ class EvaluationTracker: nanotron_run_info ([`GeneralArgs`], *optional*): Reference to information about Nanotron models runs. **Attributes**: - - **details_logger** ([`DetailsLogger`]): Logger for experiment details. + - **details_logger** ([`DetailsLogger`]) -- Logger for experiment details. - **metrics_logger** ([`MetricsLogger`]): Logger for experiment metrics. - **versions_logger** ([`VersionsLogger`]): Logger for task versions. - **general_config_logger** ([`GeneralConfigLogger`]): Logger for general configuration. From 5d713fa441005e38586f64713945cb0ec66b6996 Mon Sep 17 00:00:00 2001 From: Albert Villanova del Moral <8515462+albertvillanova@users.noreply.github.com> Date: Thu, 19 Dec 2024 10:36:59 +0100 Subject: [PATCH 12/14] Fix style --- src/lighteval/logging/evaluation_tracker.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lighteval/logging/evaluation_tracker.py b/src/lighteval/logging/evaluation_tracker.py index 4ae2385f3..827054cd9 100644 --- a/src/lighteval/logging/evaluation_tracker.py +++ b/src/lighteval/logging/evaluation_tracker.py @@ -106,8 +106,8 @@ class EvaluationTracker: nanotron_run_info ([`GeneralArgs`], *optional*): Reference to information about Nanotron models runs. **Attributes**: - - **details_logger** ([`DetailsLogger`]) -- Logger for experiment details. - - **metrics_logger** ([`MetricsLogger`]): Logger for experiment metrics. + - **details_logger** ([`DetailsLogger`]) -- Logger for experiment details. + - **metrics_logger** ([`MetricsLogger`]) -- Logger for experiment metrics. - **versions_logger** ([`VersionsLogger`]): Logger for task versions. - **general_config_logger** ([`GeneralConfigLogger`]): Logger for general configuration. - **task_config_logger** ([`TaskConfigLogger`]): Logger for task configuration. From 4fb23edfa65519ebb39becac1fccfa23c3038596 Mon Sep 17 00:00:00 2001 From: Albert Villanova del Moral <8515462+albertvillanova@users.noreply.github.com> Date: Thu, 19 Dec 2024 10:40:15 +0100 Subject: [PATCH 13/14] Fix style --- src/lighteval/logging/evaluation_tracker.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/lighteval/logging/evaluation_tracker.py b/src/lighteval/logging/evaluation_tracker.py index 827054cd9..f7feb364d 100644 --- a/src/lighteval/logging/evaluation_tracker.py +++ b/src/lighteval/logging/evaluation_tracker.py @@ -106,11 +106,11 @@ class EvaluationTracker: nanotron_run_info ([`GeneralArgs`], *optional*): Reference to information about Nanotron models runs. **Attributes**: - - **details_logger** ([`DetailsLogger`]) -- Logger for experiment details. - - **metrics_logger** ([`MetricsLogger`]) -- Logger for experiment metrics. - - **versions_logger** ([`VersionsLogger`]): Logger for task versions. - - **general_config_logger** ([`GeneralConfigLogger`]): Logger for general configuration. - - **task_config_logger** ([`TaskConfigLogger`]): Logger for task configuration. + - **details_logger** ([`DetailsLogger`]) -- Logger for experiment details. + - **metrics_logger** ([`MetricsLogger`]) -- Logger for experiment metrics. + - **versions_logger** ([`VersionsLogger`]) -- Logger for task versions. + - **general_config_logger** ([`GeneralConfigLogger`]) -- Logger for general configuration. + - **task_config_logger** ([`TaskConfigLogger`]) -- Logger for task configuration. """ def __init__( From e25616dd6309c56be33f57441222685bd06ba2ac Mon Sep 17 00:00:00 2001 From: Albert Villanova del Moral <8515462+albertvillanova@users.noreply.github.com> Date: Thu, 19 Dec 2024 10:50:48 +0100 Subject: [PATCH 14/14] Fix internal links in docstring --- src/lighteval/logging/evaluation_tracker.py | 22 ++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/lighteval/logging/evaluation_tracker.py b/src/lighteval/logging/evaluation_tracker.py index f7feb364d..8cc8c09e9 100644 --- a/src/lighteval/logging/evaluation_tracker.py +++ b/src/lighteval/logging/evaluation_tracker.py @@ -84,11 +84,11 @@ def default(self, o): class EvaluationTracker: """Keeps track of the overall evaluation process and relevant information. - The [`EvaluationTracker`] contains specific loggers for experiments details - ([`DetailsLogger`]), metrics ([`MetricsLogger`]), task versions - ([`VersionsLogger`]) as well as for the general configurations of both the - specific task ([`TaskConfigLogger`]) and overall evaluation run - ([`GeneralConfigLogger`]). It compiles the data from these loggers and + The [`~logging.evaluation_tracker.EvaluationTracker`] contains specific loggers for experiments details + ([`~logging.evaluation_tracker.DetailsLogger`]), metrics ([`~logging.evaluation_tracker.MetricsLogger`]), task versions + ([`~logging.evaluation_tracker.VersionsLogger`]) as well as for the general configurations of both the + specific task ([`~logging.evaluation_tracker.TaskConfigLogger`]) and overall evaluation run + ([`~logging.evaluation_tracker.GeneralConfigLogger`]). It compiles the data from these loggers and writes it to files, which can be published to the Hugging Face hub if requested. @@ -103,14 +103,14 @@ class EvaluationTracker: See more details about the datasets organisation in [`EvaluationTracker.save`]. tensorboard_metric_prefix (`str`, defaults to "eval"): Prefix for the metrics in the tensorboard logs. public (`bool`, defaults to False): If True, results and details are pushed to public orgs. - nanotron_run_info ([`GeneralArgs`], *optional*): Reference to information about Nanotron models runs. + nanotron_run_info ([`~nanotron.config.GeneralArgs`], *optional*): Reference to information about Nanotron models runs. **Attributes**: - - **details_logger** ([`DetailsLogger`]) -- Logger for experiment details. - - **metrics_logger** ([`MetricsLogger`]) -- Logger for experiment metrics. - - **versions_logger** ([`VersionsLogger`]) -- Logger for task versions. - - **general_config_logger** ([`GeneralConfigLogger`]) -- Logger for general configuration. - - **task_config_logger** ([`TaskConfigLogger`]) -- Logger for task configuration. + - **details_logger** ([`~logging.info_loggers.DetailsLogger`]) -- Logger for experiment details. + - **metrics_logger** ([`~logging.info_loggers.MetricsLogger`]) -- Logger for experiment metrics. + - **versions_logger** ([`~logging.info_loggers.VersionsLogger`]) -- Logger for task versions. + - **general_config_logger** ([`~logging.info_loggers.GeneralConfigLogger`]) -- Logger for general configuration. + - **task_config_logger** ([`~logging.info_loggers.TaskConfigLogger`]) -- Logger for task configuration. """ def __init__(