From c020cebd1a7adc55be0b4c6420f0e35eb777fccb Mon Sep 17 00:00:00 2001 From: Sherlock Huang Date: Wed, 23 Jun 2021 05:03:12 +0000 Subject: [PATCH 1/2] Ortmodule override torch.manual_seed() --- .../python/training/ortmodule/__init__.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/orttraining/orttraining/python/training/ortmodule/__init__.py b/orttraining/orttraining/python/training/ortmodule/__init__.py index 76e26b346e87c..20392358e414e 100644 --- a/orttraining/orttraining/python/training/ortmodule/__init__.py +++ b/orttraining/orttraining/python/training/ortmodule/__init__.py @@ -58,5 +58,20 @@ from ._custom_autograd_function import enable_custom_autograd_support enable_custom_autograd_support() +# Override torch.manual_seed and torch.cuda.manual_seed +def override_torch_manual_seed(seed): + from onnxruntime import set_seed + set_seed(seed) + return torch_manual_seed(seed) +torch_manual_seed = torch.manual_seed +torch.manual_seed = override_torch_manual_seed + +def override_torch_cuda_manual_seed(seed): + from onnxruntime import set_seed + set_seed(seed) + return torch_cuda_manual_seed(seed) +torch_cuda_manual_seed = torch.cuda.manual_seed +torch.cuda.manual_seed = override_torch_cuda_manual_seed + # ORTModule must be loaded only after all validation passes from .ortmodule import ORTModule From 3c2988c5cb57b785e98945cd5db7f665b5360442 Mon Sep 17 00:00:00 2001 From: Sherlock Huang Date: Thu, 24 Jun 2021 04:03:02 +0000 Subject: [PATCH 2/2] address comments --- .../python/training/ortmodule/__init__.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/orttraining/orttraining/python/training/ortmodule/__init__.py b/orttraining/orttraining/python/training/ortmodule/__init__.py index 20392358e414e..27aa287622c2f 100644 --- a/orttraining/orttraining/python/training/ortmodule/__init__.py +++ b/orttraining/orttraining/python/training/ortmodule/__init__.py @@ -58,17 +58,22 @@ from ._custom_autograd_function import enable_custom_autograd_support enable_custom_autograd_support() +# Initalized ORT's random seed with pytorch's initial seed +# Initalized ORT's random seed with pytorch's current seed, +# in case user has set pytorch seed before importing ORTModule +import sys +from onnxruntime import set_seed +set_seed((torch.initial_seed() % sys.maxsize)) + # Override torch.manual_seed and torch.cuda.manual_seed def override_torch_manual_seed(seed): - from onnxruntime import set_seed - set_seed(seed) + set_seed(seed % sys.maxsize) return torch_manual_seed(seed) torch_manual_seed = torch.manual_seed torch.manual_seed = override_torch_manual_seed def override_torch_cuda_manual_seed(seed): - from onnxruntime import set_seed - set_seed(seed) + set_seed(seed % sys.maxsize) return torch_cuda_manual_seed(seed) torch_cuda_manual_seed = torch.cuda.manual_seed torch.cuda.manual_seed = override_torch_cuda_manual_seed