Deprecate .txt, support in favour of .txtpb. (#4352)

* Deprecate .txt, support .txtpb

* Reference files.

* Fix up the DPDK and PSA back end, too.

backends/dpdk/dpdkAsmOpt.cpp

@@ -619,7 +619,7 @@ bool EmitDpdkTableConfig::isAllKeysDefaultExpression(const IR::ListExpression *k
 void EmitDpdkTableConfig::postorder(const IR::DpdkTable *table) {
     auto entriesList = table->getEntries();
     if (entriesList == nullptr) return;
-    dpdkTableConfigFile.open(table->name + ".txt");
+    dpdkTableConfigFile.open(table->name + ".txtpb");
     auto needsPriority = tableNeedsPriority(table, refMap);
     int entryPriority = entriesList->entries.size();
     for (auto e : entriesList->entries) {

backends/dpdk/run-dpdk-ptf-test.py

@@ -288,7 +288,7 @@ def run_test(options: Options) -> int:
 
     # Define the test environment and compile the P4 target
     test_name = Path(options.p4_file.name)
-    info_name = options.testdir.joinpath("p4Info.txt")
+    info_name = options.testdir.joinpath("p4Info.txtpb")
     bf_rt_schema = options.testdir.joinpath("bf-rt.json")
     conf_bin = options.testdir.joinpath(test_name.with_suffix(".pb.bin"))
     # Files needed by the pipeline

backends/dpdk/run-dpdk-test.py

@@ -281,8 +281,8 @@ def process_file(options, argv):
         print("Writing temporary files into ", tmpdir)
     stderr = os.path.join(tmpdir, basename + "-error")
     spec = os.path.join(tmpdir, basename + ".spec")
-    p4runtimeFile = os.path.join(tmpdir, basename + ".p4info.txt")
-    p4runtimeEntriesFile = os.path.join(tmpdir, basename + ".entries.txt")
+    p4runtimeFile = os.path.join(tmpdir, basename + ".p4info.txtpb")
+    p4runtimeEntriesFile = os.path.join(tmpdir, basename + ".entries.txtpb")
     bfRtSchemaFile = os.path.join(tmpdir, basename + ".bfrt.json")
 
     def getArch(path):

backends/ebpf/ebpfParser.cpp

@@ -326,7 +326,7 @@ void StateTranslationVisitor::compileExtractField(const IR::Expression *expr,
             //   0x112233445566778809
             //   0x112233445566778890
             // To correctly insert that padding, the length of field must be known, but tools like
-            // nikss-ctl (and the nikss library) don't consume P4info.txt to have such knowledge.
+            // nikss-ctl (and the nikss library) don't consume P4info.txtpb to have such knowledge.
             // There is also a bug in (de)parser causing such fields to be deparsed incorrectly.
             ::error(ErrorType::ERR_UNSUPPORTED_ON_TARGET,
                     "%1%: fields wider than 64 bits must have a size multiple of 8 bits (1 byte) "

backends/ebpf/run-ebpf-test.py

@@ -190,9 +190,15 @@ def run_test(options, argv):
                 result = testutils.SUCCESS
             else:
                 result = run_model(ebpf, testfile)
+        elif options.replace:
+            exp_dir = os.path.dirname(options.p4filename).replace("_errors", "_errors_outputs", 1)
+            exp_file = options.p4filename.replace("_errors/", "_errors_outputs/", 1) + "-stderr"
+            testutils.check_and_create_dir(Path(exp_dir))
+            with open(exp_file, mode="w+", encoding="utf-8") as error_file:
+                error_file.write(out)
         else:  # Expected error, check output matches
             exp_file = options.p4filename.replace("_errors/", "_errors_outputs/", 1) + "-stderr"
-            with open(exp_file, 'r') as f:
+            with open(exp_file, 'r', encoding="utf-8") as f:
                 exp_stderr = f.read().strip()
             # Output will have full path which differs
             out = out.replace(options.p4filename, basename).strip()
@@ -234,6 +240,8 @@ def run_test(options, argv):
     options.compiler = testutils.check_if_file(args.compiler).as_posix()
     options.p4filename = testutils.check_if_file(args.p4filename).as_posix()
     options.replace = args.replace
+    if "P4TEST_REPLACE" in os.environ:
+        options.replace = True
     options.cleanupTmp = args.nocleanup
     if args.testfile:
         options.testfile = testutils.check_if_file(args.testfile).as_posix()

backends/ebpf/tests/ptf/bng.py

@@ -118,7 +118,7 @@ def pkt_decrement_ttl(pkt):
 
 class BNGTest(P4EbpfTest):
     p4_file_path = "../psa/examples/bng.p4"
-    p4info_reference_file_path = "../psa/examples/bng.p4info.txt"
+    p4info_reference_file_path = "../psa/examples/bng.p4info.txtpb"
     session_installed = False
 
     def setUp(self):

backends/ebpf/tests/ptf/common.py

@@ -130,7 +130,7 @@ def setUp(self):
             p4args += " --xdp"
 
         if self.p4info_reference_file_path:
-            self.p4info_p4c_generated = os.path.join("ptf_out", filename + ".p4info.txt")
+            self.p4info_p4c_generated = os.path.join("ptf_out", filename + ".p4info.txtpb")
             p4args += " --p4runtime-files {}".format(self.p4info_p4c_generated)
 
         p4args = p4args + " " + self.p4c_additional_args

backends/ebpf/tests/ptf/l2l3_acl.py

@@ -38,7 +38,7 @@ def pkt_add_vlan(pkt, vlan_vid=10, vlan_pcp=0, dl_vlan_cfi=0):
 
 class L2L3SwitchTest(P4EbpfTest):
     p4_file_path = "../psa/examples/l2l3-acl.p4"
-    p4info_reference_file_path = "../psa/examples/l2l3-acl.p4info.txt"
+    p4info_reference_file_path = "../psa/examples/l2l3-acl.p4info.txtpb"
 
     def configure_port(self, port_id, vlan_id=None):
         if vlan_id is None:

backends/ebpf/tests/ptf/meters.py

@@ -123,7 +123,7 @@ class MeterPSATest(P4EbpfTest):
     """
 
     p4_file_path = "p4testdata/meters.p4"
-    p4info_reference_file_path = "p4testdata/meters.p4info.txt"
+    p4info_reference_file_path = "p4testdata/meters.p4info.txtpb"
 
     def runTest(self):
         pkt = testutils.simple_ip_packet()
@@ -237,7 +237,7 @@ class DirectMeterPSATest(P4EbpfTest):
     """
 
     p4_file_path = "p4testdata/meters-direct.p4"
-    p4info_reference_file_path = "p4testdata/meters-direct.p4info.txt"
+    p4info_reference_file_path = "p4testdata/meters-direct.p4info.txtpb"
 
     def runTest(self):
         pkt = testutils.simple_ip_packet()

backends/ebpf/tests/ptf/register.py

@@ -66,7 +66,7 @@ class RegisterApplyPSATest(P4EbpfTest):
     """
 
     p4_file_path = "p4testdata/register-apply.p4"
-    p4info_reference_file_path = "p4testdata/register-apply.p4info.txt"
+    p4info_reference_file_path = "p4testdata/register-apply.p4info.txtpb"
 
     def runTest(self):
         pkt = testutils.simple_ip_packet()

backends/ebpf/tests/ptf/table_implementation.py

@@ -54,7 +54,7 @@ class ActionProfileTwoTablesSameInstancePSATest(P4EbpfTest):
     """
 
     p4_file_path = "p4testdata/action-profile2.p4"
-    p4info_reference_file_path = "p4testdata/action-profile2.p4info.txt"
+    p4info_reference_file_path = "p4testdata/action-profile2.p4info.txtpb"
 
     def runTest(self):
         ref = self.action_profile_add_action(ap="MyIC_ap", action=2, data=[0x1122])
@@ -236,7 +236,7 @@ class ActionSelectorTwoTablesSameInstancePSATest(ActionSelectorTest):
     """
 
     p4_file_path = "p4testdata/action-selector2.p4"
-    p4info_reference_file_path = "p4testdata/action-selector2.p4info.txt"
+    p4info_reference_file_path = "p4testdata/action-selector2.p4info.txtpb"
 
     def runTest(self):
         self.create_default_rule_set(table="MyIC_tbl", selector="MyIC_as")

backends/ebpf/tests/ptf/test.py

@@ -366,7 +366,7 @@ def runTest(self):
 
 class DigestPSATest(P4EbpfTest):
     p4_file_path = "p4testdata/digest.p4"
-    p4info_reference_file_path = "p4testdata/digest.p4info.txt"
+    p4info_reference_file_path = "p4testdata/digest.p4info.txtpb"
 
     def runTest(self):
         pkt = testutils.simple_ip_packet(eth_src="fa:fb:fc:fd:fe:f0")
@@ -409,7 +409,7 @@ def runTest(self):
 
 class CountersPSATest(P4EbpfTest):
     p4_file_path = "p4testdata/counters.p4"
-    p4info_reference_file_path = "p4testdata/counters.p4info.txt"
+    p4info_reference_file_path = "p4testdata/counters.p4info.txtpb"
 
     def runTest(self):
         pkt = testutils.simple_ip_packet(
@@ -481,7 +481,7 @@ class ParserValueSetPSATest(P4EbpfTest):
     """
 
     p4_file_path = "p4testdata/pvs.p4"
-    p4info_reference_file_path = "p4testdata/pvs.p4info.txt"
+    p4info_reference_file_path = "p4testdata/pvs.p4info.txtpb"
 
     def runTest(self):
         pkt = testutils.simple_udp_packet(ip_dst="10.0.0.1", udp_dport=80)

backends/ebpf/tests/ptf/upf.py

@@ -79,7 +79,7 @@ def pkt_route(pkt, mac_src, mac_dst):
 
 class UPFTest(P4EbpfTest):
     p4_file_path = "../psa/examples/upf.p4"
-    p4info_reference_file_path = "../psa/examples/upf.p4info.txt"
+    p4info_reference_file_path = "../psa/examples/upf.p4info.txtpb"
 
     def setup_pfcp_session(self, seid, teid, ue_ip):
         self.table_add(

backends/p4test/run-p4-sample.py

@@ -259,8 +259,8 @@ def process_file(options, argv):
     ppfile = tmpdir + "/" + basename  # after parsing
     referenceOutputs = ",".join(list(rename.keys()))
     stderr = tmpdir + "/" + basename + "-stderr"
-    p4runtimeFile = tmpdir + "/" + basename + ".p4info.txt"
-    p4runtimeEntriesFile = tmpdir + "/" + basename + ".entries.txt"
+    p4runtimeFile = tmpdir + "/" + basename + ".p4info.txtpb"
+    p4runtimeEntriesFile = tmpdir + "/" + basename + ".entries.txtpb"
 
     # Create the `json_outputs` directory if it doesn't already exist. There's a
     # race here since multiple tests may run this code in parallel, so we can't

control-plane/p4RuntimeSerializer.cpp

@@ -22,16 +22,16 @@ limitations under the License.
 #include <google/protobuf/util/json_util.h>
 #pragma GCC diagnostic pop
 
-#include <algorithm>
 #include <iostream>
 #include <iterator>
 #include <optional>
 #include <set>
-#include <typeinfo>
 #include <unordered_map>
 #include <utility>
 #include <vector>
 
+#include "lib/error.h"
+
 #pragma GCC diagnostic push
 #pragma GCC diagnostic ignored "-Wunused-parameter"
 #pragma GCC diagnostic ignored "-Wpedantic"
@@ -53,14 +53,11 @@ limitations under the License.
 #include "frontends/p4/externInstance.h"
 #include "frontends/p4/fromv1.0/v1model.h"
 #include "frontends/p4/methodInstance.h"
-#include "frontends/p4/parseAnnotations.h"
-#include "frontends/p4/simplify.h"
 #include "frontends/p4/typeChecking/typeChecker.h"
 #include "frontends/p4/typeMap.h"
 #include "ir/ir.h"
 #include "lib/log.h"
 #include "lib/nullstream.h"
-#include "lib/ordered_set.h"
 #include "p4RuntimeAnnotations.h"
 #include "p4RuntimeArchHandler.h"
 #include "p4RuntimeArchStandard.h"
@@ -1469,6 +1466,7 @@ void P4RuntimeAPI::serializeP4InfoTo(std::ostream *destination, P4RuntimeFormat
         case P4RuntimeFormat::JSON:
             success = writers::writeJsonTo(*p4Info, destination);
             break;
+        case P4RuntimeFormat::TEXT_PROTOBUF:
         case P4RuntimeFormat::TEXT:
             success = writers::writeTextTo(*p4Info, destination);
             break;
@@ -1488,6 +1486,7 @@ void P4RuntimeAPI::serializeEntriesTo(std::ostream *destination, P4RuntimeFormat
         case P4RuntimeFormat::JSON:
             success = writers::writeJsonTo(*entries, destination);
             break;
+        case P4RuntimeFormat::TEXT_PROTOBUF:
         case P4RuntimeFormat::TEXT:
             success = writers::writeTextTo(*entries, destination);
             break;
@@ -1515,7 +1514,11 @@ static bool parseFileNames(cstring fileNameVector, std::vector<cstring> &files,
                 formats.push_back(P4::P4RuntimeFormat::JSON);
             } else if (suffix == ".bin") {
                 formats.push_back(P4::P4RuntimeFormat::BINARY);
+            } else if (suffix == ".txtpb") {
+                formats.push_back(P4::P4RuntimeFormat::TEXT_PROTOBUF);
             } else if (suffix == ".txt") {
+                ::warning(ErrorType::WARN_DEPRECATED,
+                          ".txt format is being deprecated; use .txtpb instead");
                 formats.push_back(P4::P4RuntimeFormat::TEXT);
             } else {
                 ::error(ErrorType::ERR_UNKNOWN,
@@ -1525,7 +1528,8 @@ static bool parseFileNames(cstring fileNameVector, std::vector<cstring> &files,
             }
         } else {
             ::error(ErrorType::ERR_UNKNOWN,
-                    "%1%: unknown file kind; known suffixes are .bin, .txt, .json", name);
+                    "%1%: unknown file kind; known suffixes are .bin, .txt, .json, and .txtpb",
+                    name);
             return false;
         }
     }

control-plane/p4RuntimeSerializer.h

@@ -42,7 +42,7 @@ class CompilerOptions;
 namespace P4 {
 
 /// P4Runtime serialization formats.
-enum class P4RuntimeFormat { BINARY, JSON, TEXT };
+enum class P4RuntimeFormat { BINARY, JSON, TEXT, TEXT_PROTOBUF };
 
 /// A P4 program's control-plane API, represented in terms of P4Runtime's data
 /// structures. Can be inspected or serialized.