Add float models e2e test

The original float models are tested by executing aot_compiler script.
Additionally, unify the outputs to tuple format in base tester.

Co-authored-by: chong-chen <[email protected]>

Author: Jiseong-oh

.github/workflows/pull.yml

@@ -929,12 +929,6 @@ jobs:
         export SAMSUNG_AI_LITECORE_KEY=$SECRET_SAMSUNG_AI_LITECORE_KEY
         source .ci/scripts/setup-samsung-linux-deps.sh
 
-        # Test models serially
-        models="mv2 ic3 resnet18 resnet50 mv3 ic4 dl3 edsr vit w2l"
-        for model in $models; do
-          python -m executorch.examples.samsung.aot_compiler --model_name=$model -c E9955
-        done
-
         # Test quant models
         model_scripts="deeplab_v3 edsr inception_v3 inception_v4 mobilenet_v2 mobilenet_v3 resnet18 resnet50 vit wav2letter"
         for m_script in $model_scripts; do
@@ -943,6 +937,8 @@ jobs:
 
         # Test ops
         python -m unittest discover -s backends/samsung/test/ops -p "test_*.py"
+        # Test models
+        python -m unittest discover -s backends/samsung/test/models -p "test_*.py"
 
 
   test-vulkan-models-linux:

backends/samsung/test/models/test_deeplab_v3.py

@@ -0,0 +1,28 @@
+# Copyright (c) Samsung Electronics Co. LTD
+# All rights reserved
+#
+# Licensed under the BSD License (the "License"); you may not use this file
+# except in compliance with the License. See the license file in the root
+# directory of this source tree for more details.
+import unittest
+
+from executorch.backends.samsung.serialization.compile_options import (
+    gen_samsung_backend_compile_spec,
+)
+from executorch.backends.samsung.test.tester import SamsungTester
+from executorch.examples.models.deeplab_v3 import DeepLabV3ResNet50Model
+
+
+class TestMilestoneDeepLabV3(unittest.TestCase):
+    def test_dl3_fp16(self):
+        model = DeepLabV3ResNet50Model().get_eager_model()
+        example_input = DeepLabV3ResNet50Model().get_example_inputs()
+        tester = SamsungTester(
+            model, example_input, [gen_samsung_backend_compile_spec("E9955")]
+        )
+        (
+            tester.export()
+            .to_edge_transform_and_lower()
+            .to_executorch()
+            .run_method_and_compare_outputs(inputs=example_input, atol=0.009)
+        )

backends/samsung/test/models/test_edsr.py

@@ -8,26 +8,23 @@
 
 import unittest
 
-import torch
-
 from executorch.backends.samsung.serialization.compile_options import (
     gen_samsung_backend_compile_spec,
 )
 from executorch.backends.samsung.test.tester import SamsungTester
 from executorch.examples.models.edsr import EdsrModel
 
 
-class Test_Milestone_Edsr(unittest.TestCase):
+class TestMilestoneEdsr(unittest.TestCase):
     def test_edsr_fp16(self):
         model = EdsrModel().get_eager_model()
         example_input = EdsrModel().get_example_inputs()
-
+        tester = SamsungTester(
+            model, example_input, [gen_samsung_backend_compile_spec("E9955")]
+        )
         (
-            SamsungTester(model,
-                          example_input,
-                          [gen_samsung_backend_compile_spec("E9955")],)
-            .export()
+            tester.export()
             .to_edge_transform_and_lower()
             .to_executorch()
-            .run_method_and_compare_outputs(atol=0.02, rtol=0.02)
+            .run_method_and_compare_outputs(inputs=example_input, atol=0.02)
         )

backends/samsung/test/models/test_inception_v3.py

@@ -8,26 +8,23 @@
 
 import unittest
 
-import torch
-
 from executorch.backends.samsung.serialization.compile_options import (
     gen_samsung_backend_compile_spec,
 )
 from executorch.backends.samsung.test.tester import SamsungTester
 from executorch.examples.models.inception_v3 import InceptionV3Model
 
 
-class Test_Inception_V3(unittest.TestCase):
+class TestInceptionV3(unittest.TestCase):
     def test_inception_v3_fp16(self):
         model = InceptionV3Model().get_eager_model()
         example_input = InceptionV3Model().get_example_inputs()
-
+        tester = SamsungTester(
+            model, example_input, [gen_samsung_backend_compile_spec("E9955")]
+        )
         (
-            SamsungTester(model,
-                          example_input,
-                          [gen_samsung_backend_compile_spec("E9955")],)
-            .export()
+            tester.export()
             .to_edge_transform_and_lower()
             .to_executorch()
-            .run_method_and_compare_outputs(atol=0.02, rtol=0.02)
+            .run_method_and_compare_outputs(inputs=example_input, atol=0.02, rtol=0.02)
         )

backends/samsung/test/models/test_inception_v4.py

@@ -8,26 +8,23 @@
 
 import unittest
 
-import torch
-
 from executorch.backends.samsung.serialization.compile_options import (
     gen_samsung_backend_compile_spec,
 )
 from executorch.backends.samsung.test.tester import SamsungTester
 from executorch.examples.models.inception_v4 import InceptionV4Model
 
 
-class Test_Milestone_InceptionV4(unittest.TestCase):
+class TestMilestoneInceptionV4(unittest.TestCase):
     def test_inception_v4_fp16(self):
         model = InceptionV4Model().get_eager_model()
         example_input = InceptionV4Model().get_example_inputs()
-
+        tester = SamsungTester(
+            model, example_input, [gen_samsung_backend_compile_spec("E9955")]
+        )
         (
-            SamsungTester(model,
-                          example_input,
-                          [gen_samsung_backend_compile_spec("E9955")],)
-            .export()
+            tester.export()
             .to_edge_transform_and_lower()
             .to_executorch()
-            .run_method_and_compare_outputs(atol=0.02, rtol=0.02)
+            .run_method_and_compare_outputs(inputs=example_input, atol=0.02, rtol=0.02)
         )

backends/samsung/test/models/test_mobilenet_v2.py

@@ -0,0 +1,28 @@
+# Copyright (c) Samsung Electronics Co. LTD
+# All rights reserved
+#
+# Licensed under the BSD License (the "License"); you may not use this file
+# except in compliance with the License. See the license file in the root
+# directory of this source tree for more details.
+import unittest
+
+from executorch.backends.samsung.serialization.compile_options import (
+    gen_samsung_backend_compile_spec,
+)
+from executorch.backends.samsung.test.tester import SamsungTester
+from executorch.examples.models.mobilenet_v2 import MV2Model
+
+
+class TestMilestoneMobilenetV2(unittest.TestCase):
+    def test_mv2_fp16(self):
+        model = MV2Model().get_eager_model()
+        example_input = MV2Model().get_example_inputs()
+        tester = SamsungTester(
+            model, example_input, [gen_samsung_backend_compile_spec("E9955")]
+        )
+        (
+            tester.export()
+            .to_edge_transform_and_lower()
+            .to_executorch()
+            .run_method_and_compare_outputs(inputs=example_input, atol=0.02)
+        )

backends/samsung/test/models/test_mobilenet_v3.py

@@ -17,18 +17,17 @@
 from executorch.examples.models.mobilenet_v3 import MV3Model
 
 
-class Test_Milestone_MobilenetV3(unittest.TestCase):
+class TestMilestoneMobilenetV3(unittest.TestCase):
     def test_mv3_fp16(self):
+        torch.manual_seed(8)
         model = MV3Model().get_eager_model()
         example_input = MV3Model().get_example_inputs()
-
+        tester = SamsungTester(
+            model, example_input, [gen_samsung_backend_compile_spec("E9955")]
+        )
         (
-            SamsungTester(model,
-                          example_input,
-                          [gen_samsung_backend_compile_spec("E9955")],)
-            .export()
+            tester.export()
             .to_edge_transform_and_lower()
             .to_executorch()
-            .run_method_and_compare_outputs(atol=0.06, rtol=0.06)
-            # TODO: theshold value should be updated after fixing accuracy issue
+            .run_method_and_compare_outputs(inputs=example_input, atol=0.07, rtol=0.07)
         )

backends/samsung/test/models/test_resnet18.py

@@ -8,26 +8,23 @@
 
 import unittest
 
-import torch
-
 from executorch.backends.samsung.serialization.compile_options import (
     gen_samsung_backend_compile_spec,
 )
 from executorch.backends.samsung.test.tester import SamsungTester
 from executorch.examples.models.resnet import ResNet18Model
 
 
-class Test_Milestone_ResNet18(unittest.TestCase):
+class TestMilestoneResNet18(unittest.TestCase):
     def test_resnet18_fp16(self):
         model = ResNet18Model().get_eager_model()
         example_input = ResNet18Model().get_example_inputs()
-
+        tester = SamsungTester(
+            model, example_input, [gen_samsung_backend_compile_spec("E9955")]
+        )
         (
-            SamsungTester(model,
-                          example_input,
-                          [gen_samsung_backend_compile_spec("E9955")],)
-            .export()
+            tester.export()
             .to_edge_transform_and_lower()
             .to_executorch()
-            .run_method_and_compare_outputs(atol=0.02, rtol=0.02)
+            .run_method_and_compare_outputs(inputs=example_input, atol=0.02, rtol=0.02)
         )

backends/samsung/test/models/test_resnet50.py

@@ -8,26 +8,23 @@
 
 import unittest
 
-import torch
-
 from executorch.backends.samsung.serialization.compile_options import (
     gen_samsung_backend_compile_spec,
 )
 from executorch.backends.samsung.test.tester import SamsungTester
 from executorch.examples.models.resnet import ResNet50Model
 
 
-class Test_Milestone_ResNet50(unittest.TestCase):
+class TestMilestoneResNet50(unittest.TestCase):
     def test_resnet50_fp16(self):
         model = ResNet50Model().get_eager_model()
         example_input = ResNet50Model().get_example_inputs()
-
+        tester = SamsungTester(
+            model, example_input, [gen_samsung_backend_compile_spec("E9955")]
+        )
         (
-            SamsungTester(model,
-                          example_input,
-                          [gen_samsung_backend_compile_spec("E9955")],)
-            .export()
+            tester.export()
             .to_edge_transform_and_lower()
             .to_executorch()
-            .run_method_and_compare_outputs(atol=0.02, rtol=0.02)
+            .run_method_and_compare_outputs(inputs=example_input, atol=0.02, rtol=0.02)
         )

backends/samsung/test/models/test_torchvision_vit.py

@@ -0,0 +1,32 @@
+# Copyright (c) Samsung Electronics Co. LTD
+# All rights reserved
+#
+# Licensed under the BSD License (the "License"); you may not use this file
+# except in compliance with the License. See the license file in the root
+# directory of this source tree for more details.
+import unittest
+
+import torch
+from executorch.backends.samsung.serialization.compile_options import (
+    gen_samsung_backend_compile_spec,
+)
+from executorch.backends.samsung.test.tester import SamsungTester
+from executorch.examples.models.torchvision_vit import TorchVisionViTModel
+
+
+class TestMilestoneTorchVisionViT(unittest.TestCase):
+    def test_torchvision_vit_fp16(self):
+        torch.manual_seed(8)
+        model = TorchVisionViTModel().get_eager_model()
+        example_input = TorchVisionViTModel().get_example_inputs()
+        tester = SamsungTester(
+            model, example_input, [gen_samsung_backend_compile_spec("E9955")]
+        )
+        (
+            tester.export()
+            .to_edge_transform_and_lower()
+            .to_executorch()
+            .run_method_and_compare_outputs(
+                inputs=example_input, atol=0.005, rtol=0.005
+            )
+        )