From 5d104f23537eff0f59d4e1bae5372b8e9c92af38 Mon Sep 17 00:00:00 2001 From: Jeff Rasley Date: Tue, 19 Jul 2022 10:46:12 -0700 Subject: [PATCH 1/5] reduce ds-inference log verbosity --- deepspeed/ops/transformer/inference/transformer_inference.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/deepspeed/ops/transformer/inference/transformer_inference.py b/deepspeed/ops/transformer/inference/transformer_inference.py index 342f4e1b595f..7b29ac340543 100755 --- a/deepspeed/ops/transformer/inference/transformer_inference.py +++ b/deepspeed/ops/transformer/inference/transformer_inference.py @@ -11,6 +11,8 @@ from ... import op_builder import torch.nn as nn from deepspeed import comm as dist +from deepspeed.utils.logging import log_dist + # Cuda modules will be imported if needed inference_cuda_module = None minus_inf = -10000.0 @@ -768,7 +770,7 @@ def __init__(self, builder = op_builder.InferenceBuilder() inference_cuda_module = builder.load() - print("DeepSpeed Transformer Inference config is ", self.config.__dict__) + log_dist(f"DeepSpeed-Inference config: {self.config.__dict__}", [0]) self.attention = DeepSpeedSelfAttention(self.config, mp_group, From c3ad1f8ea9dd70be393ff98bcf2b934894cbeccd Mon Sep 17 00:00:00 2001 From: Jeff Rasley Date: Tue, 19 Jul 2022 11:09:09 -0700 Subject: [PATCH 2/5] reduce quantize logs --- deepspeed/inference/engine.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/deepspeed/inference/engine.py b/deepspeed/inference/engine.py index ea29282da744..6761e06caec7 100755 --- a/deepspeed/inference/engine.py +++ b/deepspeed/inference/engine.py @@ -208,9 +208,9 @@ def _init_quantization_setting(self, quantization_setting): self.quantize_groups = quantization_setting elif quantization_setting is not None: self.quantize_groups = quantization_setting - logger.info(f"quantize_bits = {self.quantize_bits} " - f"mlp_extra_grouping = {self.mlp_extra_grouping}, " - f"quantize_groups = {self.quantize_groups}") + log_dist(f"quantize_bits = {self.quantize_bits} " + f"mlp_extra_grouping = {self.mlp_extra_grouping}, " + f"quantize_groups = {self.quantize_groups}", [0]) def _validate_args(self, mpu): if not isinstance(self.module, Module): From fcbb319df6e9d796016a5cbf9322bd49f7730c49 Mon Sep 17 00:00:00 2001 From: Jeff Rasley Date: Tue, 19 Jul 2022 10:22:26 -0700 Subject: [PATCH 3/5] print config for only one layer and formatting fixes --- deepspeed/inference/engine.py | 8 +++++--- .../ops/transformer/inference/transformer_inference.py | 3 ++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/deepspeed/inference/engine.py b/deepspeed/inference/engine.py index 6761e06caec7..159967248619 100755 --- a/deepspeed/inference/engine.py +++ b/deepspeed/inference/engine.py @@ -208,9 +208,11 @@ def _init_quantization_setting(self, quantization_setting): self.quantize_groups = quantization_setting elif quantization_setting is not None: self.quantize_groups = quantization_setting - log_dist(f"quantize_bits = {self.quantize_bits} " - f"mlp_extra_grouping = {self.mlp_extra_grouping}, " - f"quantize_groups = {self.quantize_groups}", [0]) + log_dist( + f"quantize_bits = {self.quantize_bits} " + f"mlp_extra_grouping = {self.mlp_extra_grouping}, " + f"quantize_groups = {self.quantize_groups}", + [0]) def _validate_args(self, mpu): if not isinstance(self.module, Module): diff --git a/deepspeed/ops/transformer/inference/transformer_inference.py b/deepspeed/ops/transformer/inference/transformer_inference.py index 7b29ac340543..85857e0e8a4d 100755 --- a/deepspeed/ops/transformer/inference/transformer_inference.py +++ b/deepspeed/ops/transformer/inference/transformer_inference.py @@ -770,7 +770,8 @@ def __init__(self, builder = op_builder.InferenceBuilder() inference_cuda_module = builder.load() - log_dist(f"DeepSpeed-Inference config: {self.config.__dict__}", [0]) + if DeepSpeedTransformerInference.layer_id == 1: + log_dist(f"DeepSpeed-Inference config: {self.config.__dict__}", [0]) self.attention = DeepSpeedSelfAttention(self.config, mp_group, From 622bcb81dea405baf6e922772ac9262bd8b90864 Mon Sep 17 00:00:00 2001 From: Jeff Rasley Date: Tue, 19 Jul 2022 11:10:20 -0700 Subject: [PATCH 4/5] add missing import --- deepspeed/inference/engine.py | 1 + 1 file changed, 1 insertion(+) diff --git a/deepspeed/inference/engine.py b/deepspeed/inference/engine.py index 159967248619..9bfe954aa528 100755 --- a/deepspeed/inference/engine.py +++ b/deepspeed/inference/engine.py @@ -6,6 +6,7 @@ from deepspeed import comm as dist from deepspeed.utils import groups +from deepspeed.utils.logging import log_dist from torch.nn.modules import Module from packaging import version as pkg_version From 92ff91e6f96f0f211f2d5d239d282c20bf460142 Mon Sep 17 00:00:00 2001 From: Jeff Rasley Date: Tue, 19 Jul 2022 12:20:13 -0700 Subject: [PATCH 5/5] remove groups log --- deepspeed/comm/torch.py | 1 - 1 file changed, 1 deletion(-) diff --git a/deepspeed/comm/torch.py b/deepspeed/comm/torch.py index dcfae9161213..17802c2e03ef 100644 --- a/deepspeed/comm/torch.py +++ b/deepspeed/comm/torch.py @@ -163,7 +163,6 @@ def get_backend(self, group=None): return torch.distributed.get_backend(group=group) def new_group(self, ranks): - utils.logger.info(f"new group called with {ranks}") return torch.distributed.new_group(ranks) def get_global_rank(self, group, group_rank):