[Backend Tester] Add error message field to CSV report

backends/test/suite/generate_markdown_summary.py

@@ -12,6 +12,25 @@
 #
 
 
+def escape_for_markdown(text: str) -> str:
+    """
+    Modify a string to properly display in a markdown table cell.
+    """
+    if not text:
+        return text
+
+    # Replace newlines with <br /> tags
+    escaped = text.replace('\n', '<br />')
+
+    # Escape backslashes.
+    escaped = escaped.replace('\\', '\\\\')
+
+    # Escape pipe characters that would break table structure
+    escaped = escaped.replace('|', '\\|')
+
+    return escaped
+
+
 def generate_markdown(csv_path: str, exit_code: int = 0):  # noqa (C901)
     # Print warning if exit code is non-zero
     if exit_code != 0:
@@ -46,7 +65,7 @@ def generate_markdown(csv_path: str, exit_code: int = 0):  # noqa (C901)
 
     for row in data_rows:
         # Make a copy of the row to avoid modifying the original
-        processed_row = row.copy()
+        processed_row = [escape_for_markdown(cell) for cell in row]
 
         # Count results and collect failed tests
         if result_column_index is not None and result_column_index < len(row):
@@ -96,7 +115,8 @@ def generate_markdown(csv_path: str, exit_code: int = 0):  # noqa (C901)
     # Generate Failed Tests section
     print("# Failed Tests\n")
     if failed_tests:
-        print("| " + " | ".join(header) + " |")
+        escaped_header = [escape_for_markdown(col) for col in header]
+        print("| " + " | ".join(escaped_header) + " |")
         print("|" + "|".join(["---"] * len(header)) + "|")
         for row in failed_tests:
             print("| " + " | ".join(row) + " |")

backends/test/suite/models/__init__.py

@@ -22,7 +22,6 @@
 DTYPES: list[torch.dtype] = [
     torch.float16,
     torch.float32,
-    torch.float64,
 ]
 
 

backends/test/suite/operators/test_abs.py

@@ -8,6 +8,7 @@
 
 
 import torch
+import unittest
 from executorch.backends.test.suite.flow import TestFlow
 
 from executorch.backends.test.suite.operators import (
@@ -45,6 +46,7 @@ def test_abs_shapes(self, flow: TestFlow) -> None:
         # 3D tensor
         self._test_op(AbsModel(), (torch.randn(3, 4, 5),), flow)
 
+    @unittest.skip("NaN and Inf are not enforced for backends.")
     def test_abs_edge_cases(self, flow: TestFlow) -> None:
         # Test edge cases
 

backends/test/suite/operators/test_amax.py

@@ -9,6 +9,7 @@
 from typing import List, Optional, Tuple, Union
 
 import torch
+import unittest
 from executorch.backends.test.suite.flow import TestFlow
 
 from executorch.backends.test.suite.operators import (
@@ -201,6 +202,7 @@ def test_amax_shapes(self, flow: TestFlow) -> None:
             flow,
         )
 
+    @unittest.skip("NaN and Inf are not enforced for backends.")
     def test_amax_edge_cases(self, flow: TestFlow) -> None:
         x = torch.tensor([[1.0, float("inf"), 3.0], [4.0, 5.0, float("inf")]])
         self._test_op(

backends/test/suite/operators/test_amin.py

@@ -9,6 +9,7 @@
 from typing import List, Optional, Tuple, Union
 
 import torch
+import unittest
 from executorch.backends.test.suite.flow import TestFlow
 
 from executorch.backends.test.suite.operators import (
@@ -203,6 +204,7 @@ def test_amin_shapes(self, flow: TestFlow) -> None:
             flow,
         )
 
+    @unittest.skip("NaN and Inf are not enforced for backends.")
     def test_amin_edge_cases(self, flow: TestFlow) -> None:
         x = torch.tensor([[1.0, float("-inf"), 3.0], [4.0, 5.0, float("-inf")]])
         self._test_op(

backends/test/suite/operators/test_argmax.py

@@ -9,6 +9,7 @@
 from typing import Optional
 
 import torch
+import unittest
 from executorch.backends.test.suite.flow import TestFlow
 
 from executorch.backends.test.suite.operators import (
@@ -143,6 +144,7 @@ def test_argmax_shapes(self, flow: TestFlow) -> None:
             flow,
         )
 
+    @unittest.skip("NaN and Inf are not enforced for backends.")
     def test_argmax_edge_cases(self, flow: TestFlow) -> None:
         x = torch.tensor([[1.0, float("inf"), 3.0], [4.0, 5.0, float("inf")]])
         self._test_op(

backends/test/suite/operators/test_argmin.py

@@ -9,6 +9,7 @@
 from typing import Optional
 
 import torch
+import unittest
 from executorch.backends.test.suite.flow import TestFlow
 
 from executorch.backends.test.suite.operators import (
@@ -143,6 +144,7 @@ def test_argmin_shapes(self, flow: TestFlow) -> None:
             flow,
         )
 
+    @unittest.skip("NaN and Inf are not enforced for backends.")
     def test_argmin_edge_cases(self, flow: TestFlow) -> None:
         x = torch.tensor([[1.0, float("-inf"), 3.0], [4.0, 5.0, float("-inf")]])
         self._test_op(

backends/test/suite/operators/test_ceil.py

@@ -8,6 +8,7 @@
 
 
 import torch
+import unittest
 from executorch.backends.test.suite.flow import TestFlow
 
 from executorch.backends.test.suite.operators import (
@@ -45,6 +46,7 @@ def test_ceil_shapes(self, flow: TestFlow) -> None:
         # 3D tensor
         self._test_op(CeilModel(), (torch.randn(3, 4, 5),), flow)
 
+    @unittest.skip("NaN and Inf are not enforced for backends.")
     def test_ceil_edge_cases(self, flow: TestFlow) -> None:
         # Test edge cases
 

backends/test/suite/operators/test_clamp.py

@@ -8,6 +8,7 @@
 
 
 import torch
+import unittest
 from executorch.backends.test.suite.flow import TestFlow
 
 from executorch.backends.test.suite.operators import (
@@ -56,6 +57,7 @@ def test_clamp_shapes(self, flow: TestFlow) -> None:
         # 3D tensor
         self._test_op(model, (torch.randn(3, 4, 5),), flow)
 
+    @unittest.skip("NaN and Inf are not enforced for backends.")
     def test_clamp_edge_cases(self, flow: TestFlow) -> None:
         # Test edge cases
 

backends/test/suite/operators/test_elu.py

@@ -8,6 +8,7 @@
 
 
 import torch
+import unittest
 from executorch.backends.test.suite.flow import TestFlow
 
 from executorch.backends.test.suite.operators import (
@@ -42,5 +43,6 @@ def test_elu_f32_multi_dim(self, flow: TestFlow) -> None:
     def test_elu_f32_alpha(self, flow: TestFlow) -> None:
         self._test_op(Model(alpha=0.5), (torch.randn(3, 4, 5),), flow)
 
+    @unittest.skip("In place activations aren't properly defunctionalized yet.")
     def test_elu_f32_inplace(self, flow: TestFlow) -> None:
         self._test_op(Model(inplace=True), (torch.randn(3, 4, 5),), flow)

backends/test/suite/operators/test_exp.py

@@ -8,6 +8,7 @@
 
 
 import torch
+import unittest
 from executorch.backends.test.suite.flow import TestFlow
 
 from executorch.backends.test.suite.operators import (
@@ -46,6 +47,7 @@ def test_exp_shapes(self, flow: TestFlow) -> None:
         # 3D tensor
         self._test_op(ExpModel(), (torch.randn(3, 4, 5),), flow)
 
+    @unittest.skip("NaN and Inf are not enforced for backends.")
     def test_exp_edge_cases(self, flow: TestFlow) -> None:
         # Test edge cases
 

backends/test/suite/operators/test_floor.py

@@ -8,6 +8,7 @@
 
 
 import torch
+import unittest
 from executorch.backends.test.suite.flow import TestFlow
 
 from executorch.backends.test.suite.operators import (
@@ -42,6 +43,7 @@ def test_floor_shapes(self, flow: TestFlow) -> None:
         # 3D tensor
         self._test_op(FloorModel(), (torch.randn(3, 4, 5),), flow)
 
+    @unittest.skip("NaN and Inf are not enforced for backends.")
     def test_floor_edge_cases(self, flow: TestFlow) -> None:
         # Test edge cases
 

backends/test/suite/operators/test_floor_divide.py

@@ -7,6 +7,7 @@
 # pyre-unsafe
 
 import torch
+import unittest
 from executorch.backends.test.suite.flow import TestFlow
 
 from executorch.backends.test.suite.operators import (
@@ -178,6 +179,7 @@ def test_floor_divide_values(self, flow: TestFlow) -> None:
         y = torch.tensor([-2.0]).expand_as(x).clone()
         self._test_op(model, (x, y), flow, generate_random_test_inputs=False)
 
+    @unittest.skip("NaN and Inf are not enforced for backends.")
     def test_floor_divide_edge_cases(self, flow: TestFlow) -> None:
         # Test edge cases
         model = FloorDivideModel()

backends/test/suite/operators/test_hardsigmoid.py

@@ -8,6 +8,7 @@
 
 
 import torch
+import unittest
 from executorch.backends.test.suite.flow import TestFlow
 
 from executorch.backends.test.suite.operators import (
@@ -38,6 +39,7 @@ def test_hardsigmoid_f32_single_dim(self, flow: TestFlow) -> None:
     def test_hardsigmoid_f32_multi_dim(self, flow: TestFlow) -> None:
         self._test_op(Model(), (torch.randn(2, 3, 4, 5),), flow)
 
+    @unittest.skip("In place activations aren't properly defunctionalized yet.")
     def test_hardsigmoid_f32_inplace(self, flow: TestFlow) -> None:
         self._test_op(Model(inplace=True), (torch.randn(3, 4, 5),), flow)
 

backends/test/suite/operators/test_hardswish.py

@@ -8,6 +8,7 @@
 
 
 import torch
+import unittest
 from executorch.backends.test.suite.flow import TestFlow
 
 from executorch.backends.test.suite.operators import (
@@ -38,6 +39,7 @@ def test_hardswish_f32_single_dim(self, flow: TestFlow) -> None:
     def test_hardswish_f32_multi_dim(self, flow: TestFlow) -> None:
         self._test_op(Model(), (torch.randn(2, 3, 4, 5),), flow)
 
+    @unittest.skip("In place activations aren't properly defunctionalized yet.")
     def test_hardswish_f32_inplace(self, flow: TestFlow) -> None:
         self._test_op(Model(inplace=True), (torch.randn(3, 4, 5),), flow)
 

backends/test/suite/operators/test_hardtanh.py

@@ -8,6 +8,7 @@
 
 
 import torch
+import unittest
 from executorch.backends.test.suite.flow import TestFlow
 
 from executorch.backends.test.suite.operators import (
@@ -45,6 +46,7 @@ def test_hardtanh_f32_multi_dim(self, flow: TestFlow) -> None:
     def test_hardtanh_f32_custom_range(self, flow: TestFlow) -> None:
         self._test_op(Model(min_val=-2.0, max_val=2.0), (torch.randn(3, 4, 5),), flow)
 
+    @unittest.skip("In place activations aren't properly defunctionalized yet.")
     def test_hardtanh_f32_inplace(self, flow: TestFlow) -> None:
         self._test_op(Model(inplace=True), (torch.randn(3, 4, 5),), flow)
 

backends/test/suite/operators/test_leaky_relu.py

@@ -8,6 +8,7 @@
 
 
 import torch
+import unittest
 from executorch.backends.test.suite.flow import TestFlow
 
 from executorch.backends.test.suite.operators import (
@@ -44,6 +45,7 @@ def test_leaky_relu_f32_multi_dim(self, flow: TestFlow) -> None:
     def test_leaky_relu_f32_custom_slope(self, flow: TestFlow) -> None:
         self._test_op(Model(negative_slope=0.1), (torch.randn(3, 4, 5),), flow)
 
+    @unittest.skip("In place activations aren't properly defunctionalized yet.")
     def test_leaky_relu_f32_inplace(self, flow: TestFlow) -> None:
         self._test_op(Model(inplace=True), (torch.randn(3, 4, 5),), flow)
 

backends/test/suite/operators/test_log.py

@@ -8,6 +8,7 @@
 
 
 import torch
+import unittest
 from executorch.backends.test.suite.flow import TestFlow
 
 from executorch.backends.test.suite.operators import (
@@ -46,6 +47,7 @@ def test_log_shapes(self, flow: TestFlow) -> None:
         # 3D tensor
         self._test_op(LogModel(), (torch.rand(3, 4, 5) + 0.01,), flow)
 
+    @unittest.skip("NaN and Inf are not enforced for backends.")
     def test_log_edge_cases(self, flow: TestFlow) -> None:
         # Test edge cases
         # Tensor with infinity

backends/test/suite/operators/test_log10.py

@@ -8,6 +8,7 @@
 
 
 import torch
+import unittest
 from executorch.backends.test.suite.flow import TestFlow
 
 from executorch.backends.test.suite.operators import (
@@ -46,6 +47,7 @@ def test_log10_shapes(self, flow: TestFlow) -> None:
         # 3D tensor
         self._test_op(Log10Model(), (torch.rand(3, 4, 5) + 0.01,), flow)
 
+    @unittest.skip("NaN and Inf are not enforced for backends.")
     def test_log10_edge_cases(self, flow: TestFlow) -> None:
         # Test edge cases
         # Tensor with infinity

backends/test/suite/operators/test_log1p.py

@@ -8,6 +8,7 @@
 
 
 import torch
+import unittest
 from executorch.backends.test.suite.flow import TestFlow
 
 from executorch.backends.test.suite.operators import (
@@ -46,6 +47,7 @@ def test_log1p_shapes(self, flow: TestFlow) -> None:
         # 3D tensor
         self._test_op(Log1pModel(), (torch.rand(3, 4, 5) * 2 - 0.5,), flow)
 
+    @unittest.skip("NaN and Inf are not enforced for backends.")
     def test_log1p_edge_cases(self, flow: TestFlow) -> None:
         # Test edge cases
         # Tensor with infinity

backends/test/suite/operators/test_log2.py

@@ -8,6 +8,7 @@
 
 
 import torch
+import unittest
 from executorch.backends.test.suite.flow import TestFlow
 
 from executorch.backends.test.suite.operators import (
@@ -46,6 +47,7 @@ def test_log2_shapes(self, flow: TestFlow) -> None:
         # 3D tensor
         self._test_op(Log2Model(), (torch.rand(3, 4, 5) + 0.01,), flow)
 
+    @unittest.skip("NaN and Inf are not enforced for backends.")
     def test_log2_edge_cases(self, flow: TestFlow) -> None:
         # Test edge cases
         # Tensor with infinity

backends/test/suite/operators/test_mean.py

@@ -9,6 +9,7 @@
 from typing import List, Optional, Tuple, Union
 
 import torch
+import unittest
 from executorch.backends.test.suite.flow import TestFlow
 
 from executorch.backends.test.suite.operators import (
@@ -229,6 +230,7 @@ def test_mean_shapes(self, flow: TestFlow) -> None:
             flow,
         )
 
+    @unittest.skip("NaN and Inf are not enforced for backends.")
     def test_mean_edge_cases(self, flow: TestFlow) -> None:
         x = torch.tensor([[1.0, float("inf"), 3.0], [4.0, 5.0, float("inf")]])
         self._test_op(

backends/test/suite/operators/test_median.py

@@ -9,6 +9,7 @@
 from typing import Optional
 
 import torch
+import unittest
 from executorch.backends.test.suite.flow import TestFlow
 
 from executorch.backends.test.suite.operators import (
@@ -167,6 +168,7 @@ def test_median_shapes(self, flow: TestFlow) -> None:
         # 5D tensor
         self._test_op(MedianValueOnlyModel(), (torch.randn(2, 2, 3, 4, 5),), flow)
 
+    @unittest.skip("NaN and Inf are not enforced for backends.")
     def test_median_edge_cases(self, flow: TestFlow) -> None:
         # Tensor with NaN (NaN should be propagated)
         x = torch.tensor([[1.0, float("nan"), 3.0], [4.0, 5.0, float("nan")]])