Pull request pytorch#63: [NO UPSTREAM] Unify unit test seeding for input data

Merge in AITEC/executorch from feature/nxg11066/EIEX-155-Unify-unit-test-seeding-for-input-data to main-nxp

* commit '97a27f2d47369a71597c622f487a06638deee3bc':
  [NO UPSTREAM] Unify unit test seeding for input data

Author: roman-janik-nxp

backends/nxp/tests/ir/converter/node_converter/test_avg_pool2d_converter.py

@@ -11,7 +11,7 @@
 
 @pytest.fixture(autouse=True)
 def reseed_model_per_test_run():
-    torch.seed()
+    torch.manual_seed(23)
     np.random.seed(23)
 
 

backends/nxp/tests/ir/converter/node_converter/test_constant_pad_nd_converter.py

@@ -8,9 +8,12 @@
 from executorch.backends.nxp.tests.models import ConstantPadNDModule, ConstantPadNDConvModule, Conv2dConstantPadNDModule
 from torch.export import ExportedProgram
 
+
 @pytest.fixture(autouse=True)
 def reseed_model_per_test_run():
-    torch.seed()
+    torch.manual_seed(23)
+    np.random.seed(23)
+
 
 @pytest.mark.parametrize("constant", [0.0, 42., -13.37])
 def test_constant_pad_nd_conversion__specific_constant(constant):
@@ -19,8 +22,7 @@ def test_constant_pad_nd_conversion__specific_constant(constant):
 
     edge_program = to_edge_program(ConstantPadNDModule(paddings, constant), input_shape).exported_program()
 
-    torch.manual_seed(23)
-    input_data = torch.randn(input_shape, dtype=torch.float32).detach().numpy()
+    input_data = np.random.random(input_shape).astype(np.float32)
 
     convert_run_compare(edge_program, input_data)
 
@@ -41,7 +43,6 @@ def test_constant_pad_nd_quant_conversion__specific_constant(mocker, constant):
     # Capture converted program
     edge_program: ExportedProgram = converter_spy.call_args.args[1]
 
-    np.random.seed(23)
     input_data = (np.random.random(input_shape).astype(np.float32) * 50).astype(np.int8)
 
     convert_run_compare(edge_program, input_data, tfl_model=tflite_flatbuffers_model, atol=1.,
@@ -54,8 +55,7 @@ def test_constant_pad_nd_conversion__default_constant():
 
     edge_program = to_edge_program(ConstantPadNDModule(paddings), input_shape).exported_program()
 
-    torch.manual_seed(23)
-    input_data = torch.randn(input_shape, dtype=torch.float32).detach().numpy()
+    input_data = np.random.random(input_shape).astype(np.float32)
 
     convert_run_compare(edge_program, input_data)
 
@@ -84,8 +84,7 @@ def test_constant_pad_nd_conversion__default_constant():
 def test_constant_pad_nd_conversion__format_less(input_shape, paddings):
     edge_program = to_edge_program(ConstantPadNDModule(paddings), input_shape).exported_program()
 
-    torch.manual_seed(23)
-    input_data = torch.randn(input_shape, dtype=torch.float32).detach().numpy()
+    input_data = np.random.random(input_shape).astype(np.float32)
 
     convert_run_compare(edge_program, input_data)
 
@@ -99,8 +98,7 @@ def test_constant_pad_nd_conversion__format_less(input_shape, paddings):
 def test_constant_pad_nd_conversion__channels_first(input_shape, paddings):
     edge_program = to_edge_program(ConstantPadNDConvModule(paddings), input_shape).exported_program()  # Extra `Conv` after the padding.
 
-    torch.manual_seed(23)
-    input_data = torch.randn(input_shape, dtype=torch.float32).detach().numpy()
+    input_data = np.random.random(input_shape).astype(np.float32)
 
     convert_run_compare(edge_program, input_data, tflite_input_preprocess=ToNHWCPreprocess(),
                         tflite_output_preprocess=ToNCHWPreprocess())

backends/nxp/tests/ir/converter/node_converter/test_conv_converter.py

@@ -11,7 +11,7 @@
 
 @pytest.fixture(autouse=True)
 def reseed_model_per_test_run():
-    torch.seed()
+    torch.manual_seed(23)
     np.random.seed(23)
 
 

backends/nxp/tests/ir/converter/node_converter/test_linear_converter.py

@@ -6,24 +6,25 @@
 from executorch.backends.nxp.tests.executors import convert_run_compare
 from executorch.backends.nxp.tests.models import LinearModule
 
+
 @pytest.fixture(autouse=True)
 def reseed_model_per_test_run():
-    torch.seed()
+    torch.manual_seed(23)
+    np.random.seed(23)
+
 
 def test_linear_conversion__with_bias():
     input_shape = (10, 32)
     edge_program = to_edge_program(LinearModule(bias=True), input_shape).exported_program()
 
-    torch.manual_seed(23)
-    input_data = torch.randn(input_shape, dtype=torch.float32).detach().numpy()
+    input_data = np.random.random(input_shape).astype(np.float32)
 
     convert_run_compare(edge_program, input_data=input_data, atol=1.e-6)
 
 def test_linear_conversion__without_bias():
     input_shape = (10, 32)
     edge_program = to_edge_program(LinearModule(bias=True), input_shape).exported_program()
 
-    torch.manual_seed(23)
-    input_data = torch.randn(input_shape, dtype=torch.float32).detach().numpy()
+    input_data = np.random.random(input_shape).astype(np.float32)
 
     convert_run_compare(edge_program, input_data=input_data, atol=1.e-6)

backends/nxp/tests/ir/converter/node_converter/test_max_pool_2d_converter.py

@@ -13,7 +13,7 @@
 
 @pytest.fixture(autouse=True)
 def reseed_model_per_test_run():
-    torch.seed()
+    torch.manual_seed(23)
     np.random.seed(23)
 
 

backends/nxp/tests/ir/converter/node_converter/test_permute_copy_converter.py

@@ -8,9 +8,12 @@
 from executorch.backends.nxp.tests.models import Conv2dModule
 from torch.export import ExportedProgram
 
+
 @pytest.fixture(autouse=True)
 def reseed_model_per_test_run():
-    torch.seed()
+    torch.manual_seed(23)
+    np.random.seed(23)
+
 
 class Conv2dPermuteCopyModule(torch.nn.Module):
     def __init__(self, new_dims: tuple[int, ...]):
@@ -38,7 +41,6 @@ def test_permute_copy_quant_conversion__with_bias(mocker):
     # Capture converted program
     edge_program: ExportedProgram = converter_spy.call_args.args[1]
 
-    np.random.seed(23)
     input_data = (np.random.random(input_shape).astype(np.float32) * 50).astype(np.int8)
 
     convert_run_compare(edge_program, input_data, tfl_model=tflite_flatbuffers_model, atol=1.,

backends/nxp/tests/ir/converter/node_converter/test_relu_converter.py

@@ -11,7 +11,7 @@
 
 @pytest.fixture(autouse=True)
 def reseed_model_per_test_run():
-    torch.seed()
+    torch.manual_seed(23)
     np.random.seed(23)
 
 

backends/nxp/tests/ir/converter/node_converter/test_softmax_converter.py

@@ -1,33 +1,34 @@
+import numpy as np
 import pytest
 import torch
 
-from executorch import exir
 from executorch.backends.nxp.backend.edge_program_converter import EdgeProgramToIRConverter
 from executorch.backends.nxp.backend.ir.conversion_config import ConversionConfig
+from executorch.backends.nxp.tests.executorch_pipeline import to_edge_program
 from executorch.backends.nxp.tests.executors import convert_run_compare
 from executorch.backends.nxp.tests.models import SoftmaxModule, SoftmaxConvModule
 
+
 @pytest.fixture(autouse=True)
 def reseed_model_per_test_run():
-    torch.seed()
+    torch.manual_seed(23)
+    np.random.seed(23)
+
 
 @pytest.mark.parametrize("input_shape,dim", [
     pytest.param((10,), -1, id="1D,dim=-1"),
     pytest.param((10,), 0, id="1D,dim=0"),
     pytest.param((10, 32), -1, id="2D,dim=-1"),
     pytest.param((10, 32), 1, id="2D,dim=1"),
 ])
-def test_softmax_conversion__formatless_input(input_shape, dim):
+def test_softmax_conversion__formatless_input(input_shape, dim: int):
     model = SoftmaxModule(dim)
 
-    example_input = (torch.ones(input_shape),)
-    exir_program = torch.export.export(model, example_input)
-    edge_program_manager = exir.to_edge(exir_program)
+    edge_program = to_edge_program(model, input_shape).exported_program()
 
-    torch.manual_seed(23)
-    input_data = torch.randn(input_shape, dtype=torch.float32).detach().numpy()
+    input_data = np.random.random(input_shape).astype(np.float32)
 
-    convert_run_compare(edge_program_manager.exported_program(), input_data=input_data)
+    convert_run_compare(edge_program, input_data=input_data)
 
 
 @pytest.mark.parametrize("input_shape,dim", [
@@ -38,39 +39,33 @@ def test_softmax_conversion__formatless_input(input_shape, dim):
     pytest.param((10, 32, 32, 8, 8), -1, id="5D,dim=-1"),
     pytest.param((10, 32, 32, 8, 8), 4, id="5D,dim=4"),
 ])
-def test_softmax_conversion__unknown_input_format(input_shape, dim):
+def test_softmax_conversion__unknown_input_format(input_shape, dim: int):
     model = SoftmaxModule(dim)
 
-    example_input = (torch.ones(input_shape),)
-    exir_program = torch.export.export(model, example_input)
-    edge_program_manager = exir.to_edge(exir_program)
+    edge_program = to_edge_program(model, input_shape).exported_program()
 
     # Currently this test not pass because the convertibility checker doesn't use tensor formats.
     with pytest.raises(AssertionError, match='`aten__softmax_default` is not convertible'):
-        EdgeProgramToIRConverter().convert_program(edge_program_manager.exported_program(), ConversionConfig())
+        EdgeProgramToIRConverter().convert_program(edge_program, ConversionConfig())
 
-    # torch.manual_seed(23)
-    # input_data = torch.randn(input_shape, dtype=torch.float32).detach().numpy()
+    # input_data = np.random.random(input_shape).astype(np.float32)
     # convert_run_compare(edge_program_manager.exported_program(), input_data=input_data, atol=5e-7)
 
 
 @pytest.mark.parametrize("input_shape,dim", [
     pytest.param((10, 4, 32, 32), 1, id="4D,dim=1"),
     pytest.param((10, 4, 16, 16), -3, id="4D,dim=-3"),
 ])
-def test_softmax_conversion_channel_last(input_shape, dim):
+def test_softmax_conversion_channel_last(input_shape, dim: int):
     model = SoftmaxConvModule(dim)
 
-    example_input = (torch.ones(input_shape),)
-    exir_program = torch.export.export(model, example_input)
-    edge_program_manager = exir.to_edge(exir_program)
+    edge_program = to_edge_program(model, input_shape).exported_program()
 
     # Currently this test not pass because the convertibility checker doesn't use tensor formats.
     with pytest.raises(AssertionError, match='`aten__softmax_default` is not convertible'):
-        EdgeProgramToIRConverter().convert_program(edge_program_manager.exported_program(), ConversionConfig())
+        EdgeProgramToIRConverter().convert_program(edge_program, ConversionConfig())
 
-    # torch.manual_seed(23)
-    # input_data = torch.randn(input_shape, dtype=torch.float32).detach().numpy()
+    # input_data = np.random.random(input_shape).astype(np.float32)
     # convert_run_compare(edge_program_manager.exported_program(), tflite_input_preprocess=ToNHWCPreprocess(),
     #                     tflite_output_preprocess=ToNCHWPreprocess(), input_data=input_data, atol=5e-7)
 
@@ -82,12 +77,10 @@ def test_softmax_conversion_channel_last(input_shape, dim):
     pytest.param((10, 32, 32, 8, 8), 3, id="5D,dim=3"),
     pytest.param((10, 32, 32, 8, 8), 2, id="5D,dim=2"),
 ])
-def test_softmax_conversion_unsupported_dims(input_shape, dim):
+def test_softmax_conversion_unsupported_dims(input_shape, dim: int):
     model = SoftmaxModule(dim)
 
-    example_input = (torch.ones(input_shape),)
-    exir_program = torch.export.export(model, example_input)
-    edge_program_manager = exir.to_edge(exir_program)
+    edge_program = to_edge_program(model, input_shape).exported_program()
 
     with pytest.raises(AssertionError, match="`aten__softmax_default` is not convertible"):
-        EdgeProgramToIRConverter().convert_program(edge_program_manager.exported_program(), ConversionConfig())
+        EdgeProgramToIRConverter().convert_program(edge_program, ConversionConfig())

backends/nxp/tests/ir/converter/node_converter/test_view_copy_converter.py

@@ -13,9 +13,12 @@
 from executorch.backends.nxp.tests.executors import convert_run_compare, ToNHWCPreprocess, ToNCHWPreprocess
 from torch.export import ExportedProgram
 
+
 @pytest.fixture(autouse=True)
 def reseed_model_per_test_run():
-    torch.seed()
+    torch.manual_seed(23)
+    np.random.seed(23)
+
 
 class FormatlessToChannelsFirstModule(nn.Module):
     def __init__(self, channels: int, new_shape: Sequence[int]):
@@ -69,8 +72,7 @@ def test__channels_first_to_2d(mocker):
     torch_model = ConvReshapeModule(channels=input_shape[1], new_shape=new_shape)
     edge_program = to_edge_program(torch_model, input_shape).exported_program()
 
-    torch.manual_seed(23)
-    input_data = np.random.random(input_shape).astype('float32')
+    input_data = np.random.random(input_shape).astype(np.float32)
 
     converter_spy = mocker.spy(ModelBuilder, "finish")
 
@@ -91,8 +93,7 @@ def test__channels_first_to_4d(mocker):
     torch_model = ConvReshapeModule(channels=input_shape[1], new_shape=new_shape)
     edge_program = to_edge_program(torch_model, input_shape).exported_program()
 
-    torch.manual_seed(23)
-    input_data = np.random.random(input_shape).astype('float32')
+    input_data = np.random.random(input_shape).astype(np.float32)
 
     converter_spy = mocker.spy(ModelBuilder, "finish")
 
@@ -113,8 +114,7 @@ def test__formatless_to_channels_first(mocker):
     torch_model = FormatlessToChannelsFirstModule(channels=new_shape[1], new_shape=new_shape)
     edge_program = to_edge_program(torch_model, input_shape).exported_program()
 
-    torch.manual_seed(23)
-    input_data = np.random.random(input_shape).astype('float32')
+    input_data = np.random.random(input_shape).astype(np.float32)
 
     converter_spy = mocker.spy(ModelBuilder, "finish")
 
@@ -135,8 +135,7 @@ def test__formatless_to_formatless(mocker):
     torch_model = FormatlessToFormatlessModule(new_shape=new_shape)
     edge_program = to_edge_program(torch_model, input_shape).exported_program()
 
-    torch.manual_seed(23)
-    input_data = np.random.random(input_shape).astype('float32')
+    input_data = np.random.random(input_shape).astype(np.float32)
 
     converter_spy = mocker.spy(ModelBuilder, "finish")
 
@@ -163,7 +162,6 @@ def test_view_copy_w_linear_quant_conversion(mocker, input_shape, new_shape):
     # Capture converted program
     edge_program: ExportedProgram = converter_spy.call_args.args[1]
 
-    np.random.seed(23)
     input_data = (np.random.random(input_shape).astype(np.float32) * 50).astype(np.int8)
 
     convert_run_compare(edge_program, input_data, tfl_model=tflite_flatbuffers_model, atol=1.)
@@ -185,7 +183,6 @@ def test_view_copy_w_conv_quant_conversion(mocker, input_shape, new_shape):
     # Capture converted program
     edge_program: ExportedProgram = converter_spy.call_args.args[1]
 
-    np.random.seed(23)
     input_data = (np.random.random(input_shape).astype(np.float32) * 50).astype(np.int8)
 
     convert_run_compare(edge_program, input_data, tflite_input_preprocess=ToNHWCPreprocess(),