From 1ff89a7e6f849274bf013f71439983ade9547022 Mon Sep 17 00:00:00 2001 From: Michael Wyatt Date: Wed, 25 Jan 2023 15:43:40 -0800 Subject: [PATCH 1/3] use pydantic model for profiling config --- deepspeed/profiling/config.py | 52 ++++++++------------------------ deepspeed/profiling/constants.py | 43 -------------------------- 2 files changed, 13 insertions(+), 82 deletions(-) delete mode 100644 deepspeed/profiling/constants.py diff --git a/deepspeed/profiling/config.py b/deepspeed/profiling/config.py index 0671211132c6..e2286d35380e 100644 --- a/deepspeed/profiling/config.py +++ b/deepspeed/profiling/config.py @@ -3,47 +3,21 @@ Licensed under the MIT license. """ -from deepspeed.runtime.config_utils import get_scalar_param, DeepSpeedConfigObject -from deepspeed.profiling.constants import * +from pydantic import Field +from deepspeed.runtime.config_utils import DeepSpeedConfigModel +FLOPS_PROFILER = "flops_profiler" -class DeepSpeedFlopsProfilerConfig(DeepSpeedConfigObject): - def __init__(self, param_dict): - super(DeepSpeedFlopsProfilerConfig, self).__init__() - self.enabled = None - self.profile_step = None - self.module_depth = None - self.top_modules = None +def get_flops_profiler_config(param_dict): + flops_profiler_config_dict = param_dict.get(FLOPS_PROFILER, {}) + return DeepSpeedFlopsProfilerConfig(**flops_profiler_config_dict) - if FLOPS_PROFILER in param_dict.keys(): - flops_profiler_dict = param_dict[FLOPS_PROFILER] - else: - flops_profiler_dict = {} - self._initialize(flops_profiler_dict) - - def _initialize(self, flops_profiler_dict): - self.enabled = get_scalar_param(flops_profiler_dict, - FLOPS_PROFILER_ENABLED, - FLOPS_PROFILER_ENABLED_DEFAULT) - - self.profile_step = get_scalar_param(flops_profiler_dict, - FLOPS_PROFILER_PROFILE_STEP, - FLOPS_PROFILER_PROFILE_STEP_DEFAULT) - - self.module_depth = get_scalar_param(flops_profiler_dict, - FLOPS_PROFILER_MODULE_DEPTH, - FLOPS_PROFILER_MODULE_DEPTH_DEFAULT) - - self.top_modules = get_scalar_param(flops_profiler_dict, - FLOPS_PROFILER_TOP_MODULES, - FLOPS_PROFILER_TOP_MODULES_DEFAULT) - - self.detailed = get_scalar_param(flops_profiler_dict, - FLOPS_PROFILER_DETAILED, - FLOPS_PROFILER_DETAILED_DEFAULT) - - self.output_file = get_scalar_param(flops_profiler_dict, - FLOPS_PROFILER_OUTPUT_FILE, - FLOPS_PROFILER_OUTPUT_FILE_DEFAULT) +class DeepSpeedFlopsProfilerConfig(DeepSpeedConfigModel): + enabled: bool = False + profile_step: int = Field(1, ge=1) + module_depth: int = -1 + top_modules: int = 1 + detailed: bool = True + output_file: str = None diff --git a/deepspeed/profiling/constants.py b/deepspeed/profiling/constants.py deleted file mode 100644 index d999dc61bd9f..000000000000 --- a/deepspeed/profiling/constants.py +++ /dev/null @@ -1,43 +0,0 @@ -""" -Copyright (c) Microsoft Corporation -Licensed under the MIT license. -""" - -######################################### -# flops profiler -######################################### -# Flops profiler. By default, this feature is not enabled. -# Users can configure in ds_config.json as below example: -FLOPS_PROFILER_FORMAT = ''' -flops profiler should be enabled as: -"session_params": { - "flops_profiler": { - "enabled": true, - "profile_step": 1, - "module_depth": -1, - "top_modules": 3, - "detailed": true, - "output_file": null - } -} -''' - -FLOPS_PROFILER = "flops_profiler" - -FLOPS_PROFILER_ENABLED = "enabled" -FLOPS_PROFILER_ENABLED_DEFAULT = False - -FLOPS_PROFILER_PROFILE_STEP = "profile_step" -FLOPS_PROFILER_PROFILE_STEP_DEFAULT = 1 - -FLOPS_PROFILER_MODULE_DEPTH = "module_depth" -FLOPS_PROFILER_MODULE_DEPTH_DEFAULT = -1 - -FLOPS_PROFILER_TOP_MODULES = "top_modules" -FLOPS_PROFILER_TOP_MODULES_DEFAULT = 1 - -FLOPS_PROFILER_DETAILED = "detailed" -FLOPS_PROFILER_DETAILED_DEFAULT = True - -FLOPS_PROFILER_OUTPUT_FILE = "output_file" -FLOPS_PROFILER_OUTPUT_FILE_DEFAULT = None From 8050c56a419dde7da0cea5560ae1b3851eb967fc Mon Sep 17 00:00:00 2001 From: Michael Wyatt Date: Wed, 25 Jan 2023 15:46:21 -0800 Subject: [PATCH 2/3] modify runtime config to use new pydantic config --- deepspeed/runtime/config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deepspeed/runtime/config.py b/deepspeed/runtime/config.py index 98548c45873d..7252363113fa 100755 --- a/deepspeed/runtime/config.py +++ b/deepspeed/runtime/config.py @@ -48,7 +48,7 @@ NUM_GPUS_PER_NODE_DEFAULT, ) -from ..profiling.config import DeepSpeedFlopsProfilerConfig +from ..profiling.config import get_flops_profiler_config from ..autotuning.config import DeepSpeedAutotuningConfig from ..nebula.config import DeepSpeedNebulaConfig @@ -860,7 +860,7 @@ def _initialize_params(self, param_dict): self.scheduler_name = get_scheduler_name(param_dict) self.scheduler_params = get_scheduler_params(param_dict) - self.flops_profiler_config = DeepSpeedFlopsProfilerConfig(param_dict) + self.flops_profiler_config = get_flops_profiler_config(param_dict) self.wall_clock_breakdown = (get_wall_clock_breakdown(param_dict) | self.flops_profiler_config.enabled) self.memory_breakdown = get_memory_breakdown(param_dict) From 739bf0f0db9b8627dca5794f84229ff2b75a7b52 Mon Sep 17 00:00:00 2001 From: Michael Wyatt Date: Mon, 30 Jan 2023 16:31:53 -0800 Subject: [PATCH 3/3] Added docs for config fields --- deepspeed/profiling/config.py | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/deepspeed/profiling/config.py b/deepspeed/profiling/config.py index e2286d35380e..7076fa25c993 100644 --- a/deepspeed/profiling/config.py +++ b/deepspeed/profiling/config.py @@ -6,18 +6,36 @@ from pydantic import Field from deepspeed.runtime.config_utils import DeepSpeedConfigModel -FLOPS_PROFILER = "flops_profiler" - def get_flops_profiler_config(param_dict): - flops_profiler_config_dict = param_dict.get(FLOPS_PROFILER, {}) + flops_profiler_config_dict = param_dict.get("flops_profiler", {}) return DeepSpeedFlopsProfilerConfig(**flops_profiler_config_dict) class DeepSpeedFlopsProfilerConfig(DeepSpeedConfigModel): + """ Sets parameters for the flops profiler. """ + enabled: bool = False + """ Enables the flops profiler. This also enables wall_clock_breakdown. """ + profile_step: int = Field(1, ge=1) + """ + The global training step at which to profile. Note that warm up steps are + needed for accurate time measurement. + """ + module_depth: int = -1 + """ + The depth of the model at which to print the aggregated module information. + When set to `-1`, it prints information from the top module to the + innermost modules (the maximum depth). + """ + top_modules: int = 1 + """ Limits the aggregated profile output to the number of top modules specified. """ + detailed: bool = True + """ Whether to print the detailed model profile. """ + output_file: str = None + """ Path to the output file. If None, the profiler prints to stdout. """