[P4Testgen] Remove untested/unreliable P4Testgen features. (#4165)

* Remove untested/unreliable P4Testgen features.

* Replace mention of "statements" with "nodes".

backends/p4tools/modules/testgen/CMakeLists.txt

@@ -25,8 +25,7 @@ set(
   core/symbolic_executor/depth_first.cpp
   core/symbolic_executor/selected_branches.cpp
   core/symbolic_executor/random_backtrack.cpp
-  core/symbolic_executor/greedy_stmt_cov.cpp
-  core/symbolic_executor/max_stmt_cov.cpp
+  core/symbolic_executor/greedy_node_cov.cpp
   core/symbolic_executor/symbolic_executor.cpp
   core/target.cpp
 

backends/p4tools/modules/testgen/README.md

@@ -74,7 +74,7 @@ To access the possible options for `p4testgen` use `p4testgen --help`. To genera
 Where `ARCH` specifies the P4 architecture (e.g., v1model.p4) and `TARGET` represents the targeted network device (e.g., BMv2). Choosing `0` as the option for max-tests will cause P4Testgen to generate tests until it has exhausted all possible paths.
 
 ### Coverage
-P4Testgen is able to track the (source code) coverage of the program it is generating tests for. With each test, P4Testgen can emit the cumulative program coverage it has achieved so far. Test 1 may have covered 2 out 10 P4 statements, test 2 5 out of 10 P4 statements, and so on. To enable program coverage, P4Testgen provides the `--track-coverage [NODE_TYPE]` option where `NODE_TYPE` refers to a particular P4 source node. Currently, `STATEMENTS` for P4 program statements and `TABLE_ENTRIES` for constant P4 table entries are supported. Multiple uses of `--track-coverage` are possible.
+P4Testgen is able to track the (source code) coverage of the program it is generating tests for. With each test, P4Testgen can emit the cumulative program coverage it has achieved so far. Test 1 may have covered 2 out 10 P4 nodes, test 2 5 out of 10 P4 nodes, and so on. To enable program coverage, P4Testgen provides the `--track-coverage [NODE_TYPE]` option where `NODE_TYPE` refers to a particular P4 source node. Currently, `STATEMENTS` for P4 program statements and `TABLE_ENTRIES` for constant P4 table entries are supported. Multiple uses of `--track-coverage` are possible.
 
 The option `--stop-metric MAX_NODE_COVERAGE` makes P4Testgen stop once it has hit 100% coverage as determined by `--track-coverage`.
 

backends/p4tools/modules/testgen/benchmarks/test_coverage.py

@@ -181,12 +181,12 @@ def parse_coverage_and_timestamps(test_files, parse_type):
                         datestr = datestr.replace("\n", "")
                     datestr = datestr.strip()
                     datestrs.append(datestr)
-                if "Current statement coverage:" in line:
+                if "Current node coverage:" in line:
                     if parse_type == "PROTOBUF":
-                        covstr = line.replace('metadata: "Current statement coverage: ', "")
+                        covstr = line.replace('metadata: "Current node coverage: ', "")
                         covstr = covstr.replace('"\n', "")
                     else:
-                        covstr = line.replace("# Current statement coverage: ", "")
+                        covstr = line.replace("# Current node coverage: ", "")
                     covstr = covstr.replace("\n", "")
                     cov_percentages.append(float(covstr))
     return cov_percentages, datestrs
@@ -230,15 +230,15 @@ def run_strategies_for_max_tests(options, test_args):
     testutils.exec_process(cmd, env=custom_env, capture_output=True, timeout=3600)
     end_timestamp = datetime.datetime.now()
 
-    statements_cov, timestamps = collect_data_from_folder(test_args.test_dir, options.test_backend)
-    if not statements_cov:
+    nodes_cov, timestamps = collect_data_from_folder(test_args.test_dir, options.test_backend)
+    if not nodes_cov:
         print("No errors found!")
         return [], [], []
     num_tests = len(timestamps)
-    final_cov = str(statements_cov[-1])
+    final_cov = str(nodes_cov[-1])
     time_needed = (end_timestamp - start_timestamp).total_seconds()
     print(
-        f"Pct Statements Covered: {final_cov} Number of tests: {num_tests} Time needed:"
+        f"Pct Nodes Covered: {final_cov} Number of tests: {num_tests} Time needed:"
         f" {time_needed}"
     )
 
@@ -252,7 +252,7 @@ def run_strategies_for_max_tests(options, test_args):
         perf["Percentage"]["step"],
         perf["Percentage"]["backend"],
     ]
-    return summarized_data, statements_cov, timestamps
+    return summarized_data, nodes_cov, timestamps
 
 
 def plot_coverage(out_dir, coverage_pairs):
@@ -362,15 +362,15 @@ def main(args, extra_args):
             test_args.extra_args = config[strategy]
             if options.extra_args:
                 test_args.extra_args += " " + " ".join(options.extra_args[1:])
-            summarized_data, statements_cov, timestamps = run_strategies_for_max_tests(
+            summarized_data, nodes_cov, timestamps = run_strategies_for_max_tests(
                 options, test_args
             )
             data_row.extend(summarized_data)
             summary_frame.loc[len(summary_frame)] = data_row
-            coverage_pairs[str(seed)] = statements_cov
+            coverage_pairs[str(seed)] = nodes_cov
             sub_frame = pd.DataFrame()
-            sub_frame["Seed"] = [seed] * len(statements_cov)
-            sub_frame["Coverage"] = statements_cov
+            sub_frame["Seed"] = [seed] * len(nodes_cov)
+            sub_frame["Coverage"] = nodes_cov
             sub_frame["Time"] = convert_timestamps_to_timedelta(timestamps)
             sub_frame["Time"] = sub_frame["Time"] / pd.Timedelta(milliseconds=1)
 

backends/p4tools/modules/testgen/core/small_step/cmd_stepper.cpp

@@ -232,7 +232,7 @@ bool CmdStepper::preorder(const IR::IfStatement *ifStatement) {
         nextState.replaceTopBody(&cmds);
 
         // Some path selection strategies depend on looking ahead and collecting potential
-        // statements. If that is the case, apply the CoverableNodesScanner visitor.
+        // nodes. If that is the case, apply the CoverableNodesScanner visitor.
         P4::Coverage::CoverageSet coveredNodes;
         if (requiresLookahead(TestgenOptions::get().pathSelectionPolicy)) {
             auto collector = CoverableNodesScanner(state);
@@ -250,7 +250,7 @@ bool CmdStepper::preorder(const IR::IfStatement *ifStatement) {
         nextState.replaceTopBody((ifStatement->ifFalse == nullptr) ? new IR::BlockStatement()
                                                                    : ifStatement->ifFalse);
         // Some path selection strategies depend on looking ahead and collecting potential
-        // statements. If that is the case, apply the CoverableNodesScanner visitor.
+        // nodes. If that is the case, apply the CoverableNodesScanner visitor.
         P4::Coverage::CoverageSet coveredNodes;
         if (requiresLookahead(TestgenOptions::get().pathSelectionPolicy)) {
             auto collector = CoverableNodesScanner(state);
@@ -554,7 +554,7 @@ bool CmdStepper::preorder(const IR::SwitchStatement *switchStatement) {
         for (const auto *switchCase : switchStatement->cases) {
             if (requiresLookahead(TestgenOptions::get().pathSelectionPolicy)) {
                 // Some path selection strategies depend on looking ahead and collecting potential
-                // statements. If that is the case, apply the CoverableNodesScanner visitor.
+                // nodes. If that is the case, apply the CoverableNodesScanner visitor.
                 auto collector = CoverableNodesScanner(state);
                 collector.updateNodeCoverage(switchCase->statement, coveredNodes);
             }

backends/p4tools/modules/testgen/core/small_step/expr_stepper.cpp

@@ -234,7 +234,7 @@ bool ExprStepper::preorder(const IR::Mux *mux) {
 
         auto &nextState = state.clone();
         // Some path selection strategies depend on looking ahead and collecting potential
-        // statements. If that is the case, apply the CoverableNodesScanner visitor.
+        // nodes. If that is the case, apply the CoverableNodesScanner visitor.
         P4::Coverage::CoverageSet coveredNodes;
         if (requiresLookahead(TestgenOptions::get().pathSelectionPolicy)) {
             auto collector = CoverableNodesScanner(state);
@@ -386,7 +386,7 @@ bool ExprStepper::preorder(const IR::SelectExpression *selectExpression) {
         const auto *decl = state.findDecl(selectCase->state)->getNode();
         nextState.replaceTopBody(Continuation::Return(decl));
         // Some path selection strategies depend on looking ahead and collecting potential
-        // statements. If that is the case, apply the CoverableNodesScanner visitor.
+        // nodes. If that is the case, apply the CoverableNodesScanner visitor.
         P4::Coverage::CoverageSet coveredNodes;
         if (requiresLookahead(TestgenOptions::get().pathSelectionPolicy)) {
             auto collector = CoverableNodesScanner(state);

backends/p4tools/modules/testgen/core/small_step/small_step.cpp

@@ -50,10 +50,10 @@ SmallStepEvaluator::Branch::Branch(std::optional<const Constraint *> c,
 
 SmallStepEvaluator::Branch::Branch(std::optional<const Constraint *> c,
                                    const ExecutionState &prevState, ExecutionState &nextState,
-                                   P4::Coverage::CoverageSet potentialStatements)
+                                   P4::Coverage::CoverageSet potentialNodes)
     : constraint(IR::getBoolLiteral(true)),
       nextState(nextState),
-      potentialStatements(std::move(potentialStatements)) {
+      potentialNodes(std::move(potentialNodes)) {
     if (c) {
         // Evaluate the branch constraint in the current state of symbolic environment.
         // Substitutes all variables to their symbolic value (expression on the program's initial

backends/p4tools/modules/testgen/core/small_step/small_step.h

@@ -30,7 +30,7 @@ class SmallStepEvaluator {
 
         ExecutionStateReference nextState;
 
-        P4::Coverage::CoverageSet potentialStatements;
+        P4::Coverage::CoverageSet potentialNodes;
 
         /// Simple branch without any constraint.
         explicit Branch(ExecutionState &nextState);
@@ -43,7 +43,7 @@ class SmallStepEvaluator {
         /// Branch constrained by a condition. prevState is the state in which the condition
         /// is later evaluated.
         Branch(std::optional<const Constraint *> c, const ExecutionState &prevState,
-               ExecutionState &nextState, P4::Coverage::CoverageSet potentialStatements);
+               ExecutionState &nextState, P4::Coverage::CoverageSet potentialNodes);
     };
 
     using Result = std::vector<Branch> *;

backends/p4tools/modules/testgen/core/small_step/table_stepper.cpp

@@ -220,7 +220,7 @@ const IR::Expression *TableStepper::evalTableConstEntries() {
         nextState.set(getTableHitVar(table), IR::getBoolLiteral(true));
         nextState.set(getTableReachedVar(table), IR::getBoolLiteral(true));
         // Some path selection strategies depend on looking ahead and collecting potential
-        // statements. If that is the case, apply the CoverableNodesScanner visitor.
+        // nodes. If that is the case, apply the CoverableNodesScanner visitor.
         P4::Coverage::CoverageSet coveredNodes;
         if (requiresLookahead(TestgenOptions::get().pathSelectionPolicy)) {
             auto collector = CoverableNodesScanner(stepper->state);
@@ -314,7 +314,7 @@ void TableStepper::setTableDefaultEntries(
         replacements.emplace_back(
             new IR::MethodCallStatement(Util::SourceInfo(), synthesizedAction));
         // Some path selection strategies depend on looking ahead and collecting potential
-        // statements. If that is the case, apply the CoverableNodesScanner visitor.
+        // nodes. If that is the case, apply the CoverableNodesScanner visitor.
         P4::Coverage::CoverageSet coveredNodes;
         if (requiresLookahead(TestgenOptions::get().pathSelectionPolicy)) {
             auto collector = CoverableNodesScanner(stepper->state);
@@ -389,7 +389,7 @@ void TableStepper::evalTableControlEntries(
         replacements.emplace_back(
             new IR::MethodCallStatement(Util::SourceInfo(), synthesizedAction));
         // Some path selection strategies depend on looking ahead and collecting potential
-        // statements. If that is the case, apply the CoverableNodesScanner visitor.
+        // nodes. If that is the case, apply the CoverableNodesScanner visitor.
         P4::Coverage::CoverageSet coveredNodes;
         if (requiresLookahead(TestgenOptions::get().pathSelectionPolicy)) {
             auto collector = CoverableNodesScanner(stepper->state);
@@ -546,7 +546,7 @@ void TableStepper::addDefaultAction(std::optional<const IR::Expression *> tableM
     nextState.add(*new TraceEvents::Generic(tableStream.str()));
     replacements.emplace_back(new IR::MethodCallStatement(Util::SourceInfo(), tableAction));
     // Some path selection strategies depend on looking ahead and collecting potential
-    // statements.
+    // nodes.
     P4::Coverage::CoverageSet coveredNodes;
     if (requiresLookahead(TestgenOptions::get().pathSelectionPolicy)) {
         auto collector = CoverableNodesScanner(stepper->state);

backends/p4tools/modules/testgen/core/symbolic_executor/greedy_stmt_cov.cpp → backends/p4tools/modules/testgen/core/symbolic_executor/greedy_node_cov.cpp

@@ -1,4 +1,4 @@
-#include "backends/p4tools/modules/testgen/core/symbolic_executor/greedy_stmt_cov.h"
+#include "backends/p4tools/modules/testgen/core/symbolic_executor/greedy_node_cov.h"
 
 #include <cstddef>
 #include <functional>
@@ -18,26 +18,26 @@
 
 namespace P4Tools::P4Testgen {
 
-GreedyStmtSelection::GreedyStmtSelection(AbstractSolver &solver, const ProgramInfo &programInfo)
+GreedyNodeSelection::GreedyNodeSelection(AbstractSolver &solver, const ProgramInfo &programInfo)
     : SymbolicExecutor(solver, programInfo) {}
 
-std::optional<SymbolicExecutor::Branch> GreedyStmtSelection::popPotentialBranch(
-    const P4::Coverage::CoverageSet &coveredStatements,
+std::optional<SymbolicExecutor::Branch> GreedyNodeSelection::popPotentialBranch(
+    const P4::Coverage::CoverageSet &coveredNodes,
     std::vector<SymbolicExecutor::Branch> &candidateBranches) {
     for (size_t idx = 0; idx < candidateBranches.size(); ++idx) {
         auto branch = candidateBranches.at(idx);
-        // First check all the potential set of statements we can cover by looking ahead.
-        for (const auto &stmt : branch.potentialStatements) {
-            if (coveredStatements.count(stmt) == 0U) {
+        // First check all the potential set of nodes we can cover by looking ahead.
+        for (const auto &stmt : branch.potentialNodes) {
+            if (coveredNodes.count(stmt) == 0U) {
                 candidateBranches[idx] = candidateBranches.back();
                 candidateBranches.pop_back();
                 return branch;
             }
         }
-        // If we did not find anything, check whether this state covers any new statements
+        // If we did not find anything, check whether this state covers any new nodes
         // already.
         for (const auto &stmt : branch.nextState.get().getVisited()) {
-            if (coveredStatements.count(stmt) == 0U) {
+            if (coveredNodes.count(stmt) == 0U) {
                 candidateBranches[idx] = candidateBranches.back();
                 candidateBranches.pop_back();
                 return branch;
@@ -47,7 +47,7 @@ std::optional<SymbolicExecutor::Branch> GreedyStmtSelection::popPotentialBranch(
     return std::nullopt;
 }
 
-std::optional<ExecutionStateReference> GreedyStmtSelection::pickSuccessor(StepResult successors) {
+std::optional<ExecutionStateReference> GreedyNodeSelection::pickSuccessor(StepResult successors) {
     if (successors->empty()) {
         return std::nullopt;
     }
@@ -60,7 +60,7 @@ std::optional<ExecutionStateReference> GreedyStmtSelection::pickSuccessor(StepRe
     // Only perform a greedy search if we are still producing tests consistently.
     // This guard is necessary to avoid getting caught in parser loops.
     if (stepsWithoutTest < MAX_STEPS_WITHOUT_TEST) {
-        // Try to find a branch that covers new statements.
+        // Try to find a branch that covers new nodes.
         auto branch = popPotentialBranch(getVisitedNodes(), *successors);
         // If we succeed, pick the branch and add the remainder to the list of
         // potential branches.
@@ -78,7 +78,7 @@ std::optional<ExecutionStateReference> GreedyStmtSelection::pickSuccessor(StepRe
     return nextState;
 }
 
-void GreedyStmtSelection::runImpl(const Callback &callBack,
+void GreedyNodeSelection::runImpl(const Callback &callBack,
                                   ExecutionStateReference executionState) {
     while (true) {
         try {
@@ -130,7 +130,7 @@ void GreedyStmtSelection::runImpl(const Callback &callBack,
         unexploredBranches.insert(unexploredBranches.end(), potentialBranches.begin(),
                                   potentialBranches.end());
         potentialBranches.clear();
-        // If we did not find any new statements, fall back to random.
+        // If we did not find any new nodes, fall back to random.
         executionState = popRandomBranch(unexploredBranches).nextState;
     }
 }