add support for regular expression types (re.search, re.match, re.fullmatch) in status check and metrics definition

buildtest/builders/base.py

@@ -949,8 +949,13 @@ def add_metrics(self):
             if regex:
                 stream = regex.get("stream")
                 content = self._output if stream == "stdout" else self._error
-                match = re.search(regex["exp"], content, re.MULTILINE)
 
+                if regex.get("re") == "re.match":
+                    match = re.match(regex["exp"], content, re.MULTILINE)
+                elif regex.get("re") == "re.fullmatch":
+                    match = re.fullmatch(regex["exp"], content, re.MULTILINE)
+                else:
+                    match = re.search(regex["exp"], content, re.MULTILINE)
                 if match:
                     try:
                         self.metadata["metrics"][key] = match.group(

buildtest/buildsystem/checks.py

@@ -101,7 +101,6 @@ def file_regex_check(builder):
 
     Args:
         builder (buildtest.builders.base.BuilderBase): An instance of BuilderBase class used for printing the builder name
-
     Returns:
         bool: Returns True if there is a regex match otherwise returns False.
     """
@@ -110,6 +109,8 @@ def file_regex_check(builder):
 
     for file_check in builder.status["file_regex"]:
         fname = file_check["file"]
+        regex_type = file_check.get("re")
+        pattern = file_check["exp"]
         resolved_fname = resolve_path(fname)
         if not resolved_fname:
             msg = f"[blue]{builder}[/]: Unable to resolve file path: {fname}"
@@ -127,13 +128,22 @@ def file_regex_check(builder):
 
         # read file and apply regex
         content = read_file(resolved_fname)
-        regex = re.search(file_check["exp"], content)
+        content = content.strip()
+        match = None
+
+        if regex_type == "re.match":
+            match = re.match(pattern, content, re.MULTILINE)
+        elif regex_type == "re.fullmatch":
+            match = re.fullmatch(pattern, content, re.MULTILINE)
+        else:
+            match = re.search(pattern, content, re.MULTILINE)
+
         console.print(
-            f"[blue]{builder}[/]: Performing regex expression '{file_check['exp']}' on file {resolved_fname}"
+            f"[blue]{builder}[/]: Performing regex expression '{pattern}' on file {resolved_fname}"
         )
 
-        if not regex:
-            msg = f"[blue]{builder}[/]: Regular expression: '{file_check['exp']}' is not found in file: {resolved_fname}"
+        if not match:
+            msg = f"[blue]{builder}[/]: Regular expression: '{pattern}' not found in file: {resolved_fname}"
             logger.error(msg)
             console.print(msg, style="red")
             assert_file_regex.append(False)
@@ -166,6 +176,8 @@ def regex_check(builder):
     """
 
     file_stream = None
+    regex_type = builder.status["regex"].get("re")
+    pattern = builder.status["regex"]["exp"]
     if builder.status["regex"]["stream"] == "stdout":
         logger.debug(
             f"Detected regex stream 'stdout' so reading output file: {builder.metadata['outfile']}"
@@ -182,14 +194,20 @@ def regex_check(builder):
 
         file_stream = builder.metadata["errfile"]
 
-    logger.debug(f"Applying re.search with exp: {builder.status['regex']['exp']}")
-
-    regex = re.search(builder.status["regex"]["exp"], content)
+    logger.debug(f"Applying re.search with exp: {pattern}")
+    # remove any new lines
+    content = content.strip()
+    if regex_type == "re.match":
+        match = re.match(pattern, content, re.MULTILINE)
+    elif regex_type == "re.fullmatch":
+        match = re.fullmatch(pattern, content, re.MULTILINE)
+    else:
+        match = re.search(pattern, content, re.MULTILINE)
 
     console.print(
-        f"[blue]{builder}[/]: performing regular expression - '{builder.status['regex']['exp']}' on file: {file_stream}"
+        f"[blue]{builder}[/]: performing regular expression - '{pattern}' on file: {file_stream}"
     )
-    if not regex:
+    if not match:
         console.print(f"[blue]{builder}[/]: Regular Expression Match - [red]Failed![/]")
         return False
 

buildtest/schemas/definitions.schema.json

@@ -70,6 +70,20 @@
           "exp": {
             "type": "string",
             "description": "Specify a regular expression to run on the selected file name"
+          },
+          "item": {
+            "type": "integer",
+            "minimum": 0,
+            "description": "Specify the item number used to index element in `match.group() <https://docs.python.org/3/library/re.html#match-objects>`_"
+          },
+          "re": {
+            "type": "string",
+            "description": "Specify the regular expression type, it can be either re.search, re.match, or re.fullmatch. By default it uses re.search",
+            "enum": [
+                "re.search",
+                "re.match",
+                "re.fullmatch"
+            ]
           }
         }
       }
@@ -95,6 +109,15 @@
           "type": "integer",
           "minimum": 0,
           "description": "Specify the item number used to index element in `match.group() <https://docs.python.org/3/library/re.html#match-objects>`_"
+        },
+        "re": {
+          "type": "string",
+          "description": "Specify the regular expression type, it can be either re.search, re.match, or re.fullmatch. By default it uses re.search",
+          "enum": [
+              "re.search",
+              "re.match",
+              "re.fullmatch"
+          ]
         }
       }
     },
@@ -123,6 +146,15 @@
           "type": "integer",
           "minimum": 0,
           "description": "Specify the item number used to index element in `match.group() <https://docs.python.org/3/library/re.html#match-objects>`_"
+        },
+        "re": {
+          "type": "string",
+          "description": "Specify the regular expression type, it can be either re.search, re.match, or re.fullmatch. By default it uses re.search",
+          "enum": [
+              "re.search",
+              "re.match",
+              "re.fullmatch"
+          ]
         }
       }
     },

buildtest/schemas/examples/script.schema.json/invalid/invalid_re_value.yml

@@ -0,0 +1,11 @@
+buildspecs:
+  invalid_re_value:
+    type: script
+    executor: generic.local.bash
+    description: The "re" value is invalid
+    run: echo "world"
+    status:
+      regex:
+        stream: stdout
+        exp: "world$"
+        re: "search"

buildtest/schemas/examples/script.schema.json/valid/metrics_with_regex_type.yml

@@ -0,0 +1 @@
+../../../../../tutorials/metrics/metrics_with_regex_type.yml

buildtest/schemas/examples/script.schema.json/valid/specify_regex_type.yml

@@ -0,0 +1 @@
+../../../../../tutorials/test_status/specify_regex_type.yml

docs/writing_buildspecs/performance_checks.rst

@@ -52,6 +52,37 @@ join them together into a single string. Shown below is the metrics for the prev
    .. command-output:: buildtest report --filter buildspec=tutorials/metrics/metrics_regex.yml --format name,metrics
 
 
+Metrics with Regex Type via 're'
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Building on the previous example, we will use the ``re`` property specify the regular expression type to use. By default, buildtest will
+use `re.search <https://docs.python.org/3/library/re.html#re.search>`_ if **re** is not specified; however you can specify **re** to use `re.match <https://docs.python.org/3/library/re.html#re.match>`_,
+`re.fullmatch <https://docs.python.org/3/library/re.html#re.fullmatch>`_, or `re.search <https://docs.python.org/3/library/re.html#re.search>`_.
+
+In this example, we will define 4 metrics **hpcg_text**, **hpcg_result**, **hpcg_file_text**, **hpcg_file_result**. The first two
+metrics will capture from stdout using the ``regex`` property while the last two will capture from a file using the ``file_regex`` property.
+The ``re.match`` will be used to capture the text **HPCG result is VALID** and **HPCG result is INVALID** from stdout and file, whereas
+the ``re.search`` will be used to capture the test result **63.6515** and **28.1215** from stdout and file.
+
+Finally, we will use the comparison operator :ref:`assert_eq` to compare the metrics with reference value.
+
+.. literalinclude:: ../tutorials/metrics/metrics_with_regex_type.yml
+    :language: yaml
+    :emphasize-lines: 7-45
+
+Let's attempt to build this test
+
+.. dropdown:: ``buildtest build -b tutorials/metrics/metrics_with_regex_type.yml``
+
+   .. command-output:: buildtest build -b tutorials/metrics/metrics_with_regex_type.yml
+
+Upon completion, lets take a look at the metrics for this test, we can see this by running ``buildtest inspect query``
+which shows the name of captured metrics and its corresponding values.
+
+.. dropdown:: ``buildtest inspect query metric_regex_example_with_re``
+
+   .. command-output:: buildtest inspect query metric_regex_example_with_re
+
 Invalid Metrics
 ~~~~~~~~~~~~~~~~~
 

docs/writing_buildspecs/status_check.rst

@@ -87,3 +87,33 @@ We will simply try validating this buildspec and you will see the error message
 
     .. command-output:: buildtest buildspec validate -b tutorials/test_status/file_linecount_invalid.yml
        :returncode: 1
+
+.. _re:
+
+Using 're' property to specify regular expression type
+------------------------------------------------------
+
+The ``re`` property can be used to select the type of regular expression to use with :ref:`regex <regex>` or :ref:`file_regex <file_regex>`
+which can be `re.search <https://docs.python.org/3/library/re.html#re.search>`_, `re.match <https://docs.python.org/3/library/re.html#re.match>`_
+or `re.fullmatch <https://docs.python.org/3/library/re.html#re.fullmatch>`_. The ``re`` property is a string type which is used to select the
+regular expression function to use. In this next example, we will demonstrate the use of this feature with both ``regex`` and ``file_regex``.
+The ``re`` property is optional and if not specified it defaults to **re.search**.
+
+Since **re.search** will search for text at any position in the string, the first test ``re.search.stdout`` will match the
+string **is** with the output. In the second test ``re.match.stdout`` we use **re.match** which matches from beginning of
+string with input pattern **is** with output. We expect this match to **FAIL** since the output starts with **This is ...**.
+
+In the third test ``re.fullmatch.stdout`` we set ``re: re.fullmatch`` which will match the entire string with the pattern.
+We expect this match to **PASS** since the output and pattern are exactly the same. In the fourth test ``match_on_file_regex`` we have
+have three regular expression, one for each type **re.search**, **re.match** and **re.fullmatch**. All of these expressions will find a match
+and this test will **PASS**.
+
+.. literalinclude:: ../tutorials/test_status/specify_regex_type.yml
+   :language: yaml
+   :emphasize-lines: 6,8-11,17,19-22,28,30-33,39-41,43-52
+
+Let's try running this test example and see the generated output, all test should pass with exception of ``re.match.stdout``.
+
+.. dropdown:: ``buildtest build -b tutorials/test_status/specify_regex_type.yml``
+
+    .. command-output:: buildtest build -b tutorials/test_status/specify_regex_type.yml

tests/builders/metrics_with_regex_type.yml

@@ -0,0 +1 @@
+../../tutorials/metrics/metrics_with_regex_type.yml

tests/builders/specify_regex_type.yml

@@ -0,0 +1 @@
+../../tutorials/test_status/specify_regex_type.yml

tests/builders/test_builders.py

@@ -178,6 +178,16 @@ def test_file_regex():
     cmd.build()
 
 
+def test_regex_type():
+    """This test will perform status check with different regular expression type using ``re`` property that can be "re.match", "re.search", "re.fullmatch" """
+    cmd = BuildTest(
+        buildspecs=[os.path.join(here, "specify_regex_type.yml")],
+        buildtest_system=system,
+        configuration=config,
+    )
+    cmd.build()
+
+
 def test_runtime_check():
     """This test will perform status check with runtime"""
     cmd = BuildTest(
@@ -217,3 +227,13 @@ def test_file_linecount():
     )
     with pytest.raises(SystemExit):
         cmd.build()
+
+
+def test_metrics_with_regex_type():
+    """This test will perform status check with regular expression type and metrics"""
+    cmd = BuildTest(
+        buildspecs=[os.path.join(here, "metrics_with_regex_type.yml")],
+        buildtest_system=system,
+        configuration=config,
+    )
+    cmd.build()

tutorials/metrics/metrics_with_regex_type.yml

@@ -0,0 +1,45 @@
+buildspecs:
+  metric_regex_example_with_re:
+    executor: generic.local.bash
+    type: script
+    description: capture metric with different regex types
+    tags: tutorials
+    run: |
+      echo "HPCG result is VALID with a GFLOP/s rating of=63.6515"
+      echo "HPCG result is INVALID with a GFLOP/s rating of=28.1215" > hpcg.txt
+    metrics:
+      hpcg_result:
+        type: float
+        regex:
+          re: "re.search"
+          exp: '(\d+\.\d+)$'
+          stream: stdout
+      hpcg_text:
+        type: str
+        regex:
+          re: "re.match"
+          exp: '^HPCG result is VALID'
+          stream: stdout
+      hpcg_file_text:
+        type: str
+        file_regex:
+          re: "re.match"
+          exp: '^HPCG result is INVALID'
+          file: hpcg.txt
+      hpcg_file_result:
+        type: float
+        file_regex:
+          re: "re.search"
+          exp: '(\d+\.\d+)$'
+          file: hpcg.txt
+    status:
+      assert_eq:
+        comparisons:
+        - name: hpcg_text
+          ref: "HPCG result is VALID"
+        - name: hpcg_result
+          ref: 63.6515
+        - name: hpcg_file_text
+          ref: "HPCG result is INVALID"
+        - name: hpcg_file_result
+          ref: 28.1215

tutorials/test_status/specify_regex_type.yml

@@ -0,0 +1,52 @@
+buildspecs:
+  re.search.stdout:
+    executor: generic.local.bash
+    type: script
+    description: Test re.search on stdout
+    run: echo "This is a string"
+    status:
+      regex:
+        stream: stdout
+        exp: 'is'
+        re: "re.search"
+
+  re.match.stdout:
+    executor: generic.local.bash
+    type: script
+    description: Test re.match on stdout
+    run: echo "This is a string"
+    status:
+      regex:
+        stream: stdout
+        exp: 'is'
+        re: "re.match"
+
+  re.fullmatch.stdout:
+    executor: generic.local.bash
+    type: script
+    description: Test re.fullmatch on stdout
+    run: echo "This is a string"
+    status:
+      regex:
+        stream: stdout
+        exp: 'This is a string'
+        re: "re.fullmatch"
+
+  re.match_on_file_regex:
+    executor: generic.local.bash
+    type: script
+    description: Test re.match on file regex
+    run: |
+      echo "This is a string" > file.txt
+      echo "Hello World" > hello.txt
+    status:
+      file_regex:
+        - file: file.txt
+          exp: 'string'
+          re: "re.search"
+        - file: hello.txt
+          exp: 'Hello'
+          re: "re.match"
+        - file: hello.txt
+          exp: 'Hello World'
+          re: "re.fullmatch"