diff --git a/backends/p4tools/benchmarks/test_coverage.py b/backends/p4tools/benchmarks/test_coverage.py new file mode 100755 index 0000000000..84823648ab --- /dev/null +++ b/backends/p4tools/benchmarks/test_coverage.py @@ -0,0 +1,218 @@ +#!/usr/bin/env python3 + +import argparse +import subprocess +import random +import csv +import sys +import re +import tempfile +from pathlib import Path +import datetime + +FILE_DIR = Path(__file__).parent.resolve() +TOOLS_PATH = FILE_DIR.joinpath("../../../tools") +sys.path.append(str(TOOLS_PATH)) +import testutils + +TESTGEN_BIN = FILE_DIR.joinpath("../../../build/p4testgen") +OUTPUT_DIR = FILE_DIR.joinpath("../../../build/results") +# generate random seeds, increase the number for extra sampling +ITERATIONS = 1 +MAX_TESTS = 0 + +PARSER = argparse.ArgumentParser() +PARSER.add_argument( + "-p", + "--p4-program", + dest="p4_program", + required=True, + help="The P4 file to measure coverage on.", +) +PARSER.add_argument( + "-o", + "--out-dir", + dest="out_dir", + default=OUTPUT_DIR, + help="The output folder where all tests are dumped.", +) +PARSER.add_argument( + "-i", + "--iterations", + dest="iterations", + default=ITERATIONS, + type=int, + help="How many iterations to run.", +) +PARSER.add_argument( + "-m", + "--max-tests", + dest="max_tests", + default=MAX_TESTS, + type=int, + help="How many tests to run for each test.", +) +PARSER.add_argument( + "-s", + "--seed", + dest="seed", + default=0, + type=int, + help="The random seed for the tests.", +) +PARSER.add_argument( + "-b", + "--testgen-bin", + dest="p4testgen_bin", + default=TESTGEN_BIN, + help="Specifies the testgen binary.", +) + + +class Options: + def __init__(self): + self.p4testgen_bin = None # The P4Testgen binary. + self.p4_program = None # P4 Program that is being measured. + self.out_dir = None # The output directory. + self.seed = None # Program seed. + self.max_tests = None # The max tests parameter. + + +class TestArgs: + def __init__(self): + self.seed = None # The seed for this particular run. + self.extra_args = None # Extra arguments for P4Testgen execution. + self.test_dir = None # The testing directory associated with this test run. + self.strategy = None # The exploration strategy to execute. + + +def get_test_files(input_dir, extension): + test_files = input_dir.glob(extension) + + def natsort(s): + return [int(t) if t.isdigit() else t.lower() for t in re.split("(\d+)", str(s))] + + test_files = sorted(test_files, key=natsort) + return test_files + + +def parse_stmt_cov(test_file): + with test_file.open("r") as file_handle: + for line in file_handle.readlines(): + if "Current statement coverage:" in line: + covstr = line.replace('metadata: "Current statement coverage: ', "") + covstr = covstr.replace('"\n', "") + return covstr + return None + + +def parse_timestamp(test_file): + with test_file.open("r") as file_handle: + for line in file_handle.readlines(): + if "Date generated" in line: + datestr = line.replace('metadata: "Date generated: ', "") + datestr = datestr.replace('"\n', "") + datestr = datestr.strip() + return datestr + return None + + +def run_strategies_for_max_tests(data_row, options, test_args): + + cmd = ( + f"{options.p4testgen_bin} --target bmv2 --arch v1model --std p4-16" + f" -I/p4/p4c/build/p4include --test-backend PROTOBUF --seed {test_args.seed} " + f"--max-tests {options.max_tests} --out-dir {test_args.test_dir}" + f" --exploration-strategy {test_args.strategy} --stop-metric MAX_STATEMENT_COVERAGE " + f" {test_args.extra_args} {options.p4_program}" + ) + start_timestamp = datetime.datetime.now() + try: + # TODO: Use result + _ = subprocess.run( + cmd, + shell=True, + universal_newlines=True, + capture_output=True, + check=True, + ) + except subprocess.CalledProcessError as e: + returncode = e.returncode + print(f"Error executing {e.cmd} {returncode}:\n{e.stdout}\n{e.stderr}") + sys.exit(1) + end_timestamp = datetime.datetime.now() + + test_files = get_test_files(test_args.test_dir, "*.proto") + # Get the statement coverage of the last test generated. + last_test = test_files[-1] + statements_cov = parse_stmt_cov(last_test) + + time_needed = (end_timestamp - start_timestamp).total_seconds() + num_tests = len(test_files) + data_row.append(statements_cov) + data_row.append(str(num_tests)) + data_row.append(time_needed) + print( + f"Pct Statements Covered: {statements_cov} Number of tests: {num_tests} Time needed: {time_needed}" + ) + return data_row + + +def main(args): + + options = Options() + options.p4_program = Path(testutils.check_if_file(args.p4_program)) + options.max_tests = args.max_tests + options.out_dir = Path(args.out_dir).absolute() + options.seed = args.seed + options.p4testgen_bin = Path(testutils.check_if_file(args.p4testgen_bin)) + + # 7189 is an example of a good seed, which gets cov 1 with less than 100 tests + # in random access stack. + random.seed(options.seed) + seeds = [] + for _ in range(args.iterations): + seed = random.randint(0, sys.maxsize) + seeds.append(seed) + print(f"Chosen seeds: {seeds}") + p4_program_name = options.p4_program.stem + + header = [ + "seed", + "coverage", + "num_tests", + "time (s)", + ] + + strategies = ["RANDOM_ACCESS_STACK", "RANDOM_ACCESS_MAX_COVERAGE", "INCREMENTAL_STACK"] + config = { + "INCREMENTAL_STACK": "", + "RANDOM_ACCESS_STACK": "--pop-level 3", + "RANDOM_ACCESS_MAX_COVERAGE": "--saddle-point 5", + } + # csv results file path + for strategy in strategies: + test_dir = options.out_dir.joinpath(f"{strategy}") + testutils.check_and_create_dir(test_dir) + results_path_max = test_dir.joinpath(f"coverage_results_{p4_program_name}_{strategy}.csv") + with open(results_path_max, "w", encoding="utf-8") as f: + writer = csv.writer(f) + writer.writerow(header) + for seed in seeds: + data_row = [seed] + print( + f"Seed {seed} Generating metrics for {strategy} up to {options.max_tests} tests" + ) + test_args = TestArgs() + test_args.seed = seed + test_args.test_dir = Path(tempfile.mkdtemp(dir=test_args.test_dir)) + test_args.strategy = strategy + test_args.extra_args = config[strategy] + data_row = run_strategies_for_max_tests(data_row, options, test_args) + writer.writerow(data_row) + + +if __name__ == "__main__": + # Parse options and process argv + arguments, argv = PARSER.parse_known_args() + main(arguments) diff --git a/backends/p4tools/modules/testgen/targets/bmv2/program_info.cpp b/backends/p4tools/modules/testgen/targets/bmv2/program_info.cpp index bdb2829713..f33feee7ad 100644 --- a/backends/p4tools/modules/testgen/targets/bmv2/program_info.cpp +++ b/backends/p4tools/modules/testgen/targets/bmv2/program_info.cpp @@ -142,11 +142,13 @@ std::vector BMv2_V1ModelProgramInfo::processDeclaration( auto *egressPortVar = new IR::Member(IR::getBitType(TestgenTarget::getPortNumWidth_bits()), new IR::PathExpression("*standard_metadata"), "egress_port"); - /// Set the restriction on the output port, - /// this is necessary since ptf tests use ports from 0 to 7 - cmds.emplace_back(Continuation::Guard(getPortConstraint(getTargetOutputPortVar()))); auto *portStmt = new IR::AssignmentStatement(egressPortVar, getTargetOutputPortVar()); cmds.emplace_back(portStmt); + if (TestgenOptions::get().testBackend == "PTF") { + /// Set the restriction on the output port, + /// this is necessary since ptf tests use ports from 0 to 7 + cmds.emplace_back(Continuation::Guard(getPortConstraint(getTargetOutputPortVar()))); + } // TODO: We have not implemented multi cast yet. // Drop the packet if the multicast group is set. const IR::Expression *mcastGroupVar = new IR::Member( diff --git a/backends/p4tools/modules/testgen/targets/bmv2/test/BMV2ProtobufXfail.cmake b/backends/p4tools/modules/testgen/targets/bmv2/test/BMV2ProtobufXfail.cmake index 38f1c85be3..77602dd5bd 100644 --- a/backends/p4tools/modules/testgen/targets/bmv2/test/BMV2ProtobufXfail.cmake +++ b/backends/p4tools/modules/testgen/targets/bmv2/test/BMV2ProtobufXfail.cmake @@ -134,4 +134,5 @@ p4tools_add_xfail_reason( issue2345-multiple_dependencies.p4 issue2345-with_nested_if.p4 issue2345.p4 + up4.p4 ) diff --git a/backends/p4tools/modules/testgen/targets/bmv2/test/BMV2Xfail.cmake b/backends/p4tools/modules/testgen/targets/bmv2/test/BMV2Xfail.cmake index 8809259da7..5ba0dadf81 100644 --- a/backends/p4tools/modules/testgen/targets/bmv2/test/BMV2Xfail.cmake +++ b/backends/p4tools/modules/testgen/targets/bmv2/test/BMV2Xfail.cmake @@ -21,7 +21,6 @@ p4tools_add_xfail_reason( # It turns out that h.h.a matters more than the size of the array bmv2_hs1.p4 control-hs-index-test1.p4 - control-hs-index-test2.p4 # terminate called after throwing an instance of 'boost::wrapexcept' # Conversion from negative integer to an unsigned type results in undefined behaviour @@ -54,6 +53,7 @@ p4tools_add_xfail_reason( "Exception" # Running simple_switch_CLI: Exception Unexpected key field & match-on-exprs2-bmv2.p4 + v1model-special-ops-bmv2.p4 ) p4tools_add_xfail_reason( @@ -82,7 +82,6 @@ p4tools_add_xfail_reason( p4tools_add_xfail_reason( "testgen-p4c-bmv2" "Match type range not implemented for table keys" - up4.p4 ) p4tools_add_xfail_reason( @@ -96,7 +95,6 @@ p4tools_add_xfail_reason( p4tools_add_xfail_reason( "testgen-p4c-bmv2" "differs|Expected ([0-9]+) packets on port ([0-9]+) got ([0-9]+)" - v1model-special-ops-bmv2.p4 ) p4tools_add_xfail_reason( @@ -264,4 +262,5 @@ p4tools_add_xfail_reason( issue2345-multiple_dependencies.p4 issue2345-with_nested_if.p4 issue2345.p4 + up4.p4 ) diff --git a/tools/testutils.py b/tools/testutils.py index 6d07d9e9aa..6e41d79003 100644 --- a/tools/testutils.py +++ b/tools/testutils.py @@ -144,7 +144,7 @@ def run_timeout(verbose, args, timeout, outputs, errmsg): def check_root(): """ This function returns False if the user does not have root privileges. Caution: Only works on Unix systems """ - return (os.getuid() == 0) + return os.getuid() == 0 class PathError(RuntimeError): @@ -183,6 +183,18 @@ def check_if_dir(input_path): return Path(input_path.absolute()) +def check_and_create_dir(directory): + # create the folder if it does not exit + if not directory == "" and not os.path.exists(directory): + report_output(sys.stdout, f"Folder {directory} does not exist! Creating...") + directory.mkdir(parents=True, exist_ok=True) + +def del_dir(directory): + try: + shutil.rmtree(directory, ignore_errors=True) + except OSError as e: + report_err(sys.stderr, f"Could not delete directory, reason:\n{e.filename} - {e.strerror}.") + def copy_file(src, dst): try: if isinstance(src, list):