Add initial support for BEP (Build Event Protocol) (#961)

Add proto definitions for BEP and create `BepServer`
which implements the `publish_lifecycle_event` and
`publish_build_tool_event_stream` rpcs

Closes #925

Co-authored-by: Nathan (Blaise) Bruer <[email protected]>

nativelink-config/src/cas_server.rs

@@ -197,6 +197,14 @@ pub struct HealthConfig {
     pub path: String,
 }
 
+#[derive(Deserialize, Debug)]
+pub struct BepConfig {
+    /// The store to publish build events to.
+    /// The store name referenced in the `stores` map in the main config.
+    #[serde(deserialize_with = "convert_string_with_shellexpand")]
+    pub store: StoreRefName,
+}
+
 #[derive(Deserialize, Debug)]
 #[serde(deny_unknown_fields)]
 pub struct ServicesConfig {
@@ -234,6 +242,11 @@ pub struct ServicesConfig {
     /// that makes the remote execution/cache requests.
     pub worker_api: Option<WorkerApiConfig>,
 
+    /// Experimental - Build Event Protocol (BEP) configuration. This is
+    /// the service that will consume build events from the client and
+    /// publish them to a store for processing by an external service.
+    pub experimental_bep: Option<BepConfig>,
+
     /// Experimental - Prometheus metrics configuration. Metrics are gathered
     /// as a singleton but may be served on multiple endpoints.
     pub experimental_prometheus: Option<PrometheusConfig>,

nativelink-proto/BUILD.bazel

@@ -6,6 +6,7 @@ PROTO_NAMES = [
     "com.github.trace_machina.nativelink.remote_execution",
     "google.api",
     "google.bytestream",
+    "google.devtools.build.v1",
     "google.longrunning",
     "google.rpc",
 ]
@@ -28,8 +29,12 @@ genrule(
         "com/github/trace_machina/nativelink/remote_execution/worker_api.proto",
         "google/api/annotations.proto",
         "google/api/client.proto",
+        "google/api/field_behavior.proto",
         "google/api/http.proto",
         "google/bytestream/bytestream.proto",
+        "google/devtools/build/v1/build_events.proto",
+        "google/devtools/build/v1/build_status.proto",
+        "google/devtools/build/v1/publish_build_event.proto",
         "google/longrunning/operations.proto",
         "google/protobuf/any.proto",
         "google/protobuf/descriptor.proto",

nativelink-proto/genproto/google.api.pb.rs

@@ -380,3 +380,93 @@ pub struct CustomHttpPattern {
     #[prost(string, tag = "2")]
     pub path: ::prost::alloc::string::String,
 }
+/// An indicator of the behavior of a given field (for example, that a field
+/// is required in requests, or given as output but ignored as input).
+/// This **does not** change the behavior in protocol buffers itself; it only
+/// denotes the behavior and may affect how API tooling handles the field.
+///
+/// Note: This enum **may** receive new values in the future.
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)]
+#[repr(i32)]
+pub enum FieldBehavior {
+    /// Conventional default for enums. Do not use this.
+    Unspecified = 0,
+    /// Specifically denotes a field as optional.
+    /// While all fields in protocol buffers are optional, this may be specified
+    /// for emphasis if appropriate.
+    Optional = 1,
+    /// Denotes a field as required.
+    /// This indicates that the field **must** be provided as part of the request,
+    /// and failure to do so will cause an error (usually `INVALID_ARGUMENT`).
+    Required = 2,
+    /// Denotes a field as output only.
+    /// This indicates that the field is provided in responses, but including the
+    /// field in a request does nothing (the server *must* ignore it and
+    /// *must not* throw an error as a result of the field's presence).
+    OutputOnly = 3,
+    /// Denotes a field as input only.
+    /// This indicates that the field is provided in requests, and the
+    /// corresponding field is not included in output.
+    InputOnly = 4,
+    /// Denotes a field as immutable.
+    /// This indicates that the field may be set once in a request to create a
+    /// resource, but may not be changed thereafter.
+    Immutable = 5,
+    /// Denotes that a (repeated) field is an unordered list.
+    /// This indicates that the service may provide the elements of the list
+    /// in any arbitrary  order, rather than the order the user originally
+    /// provided. Additionally, the list's order may or may not be stable.
+    UnorderedList = 6,
+    /// Denotes that this field returns a non-empty default value if not set.
+    /// This indicates that if the user provides the empty value in a request,
+    /// a non-empty value will be returned. The user will not be aware of what
+    /// non-empty value to expect.
+    NonEmptyDefault = 7,
+    /// Denotes that the field in a resource (a message annotated with
+    /// google.api.resource) is used in the resource name to uniquely identify the
+    /// resource. For AIP-compliant APIs, this should only be applied to the
+    /// `name` field on the resource.
+    ///
+    /// This behavior should not be applied to references to other resources within
+    /// the message.
+    ///
+    /// The identifier field of resources often have different field behavior
+    /// depending on the request it is embedded in (e.g. for Create methods name
+    /// is optional and unused, while for Update methods it is required). Instead
+    /// of method-specific annotations, only `IDENTIFIER` is required.
+    Identifier = 8,
+}
+impl FieldBehavior {
+    /// String value of the enum field names used in the ProtoBuf definition.
+    ///
+    /// The values are not transformed in any way and thus are considered stable
+    /// (if the ProtoBuf definition does not change) and safe for programmatic use.
+    pub fn as_str_name(&self) -> &'static str {
+        match self {
+            FieldBehavior::Unspecified => "FIELD_BEHAVIOR_UNSPECIFIED",
+            FieldBehavior::Optional => "OPTIONAL",
+            FieldBehavior::Required => "REQUIRED",
+            FieldBehavior::OutputOnly => "OUTPUT_ONLY",
+            FieldBehavior::InputOnly => "INPUT_ONLY",
+            FieldBehavior::Immutable => "IMMUTABLE",
+            FieldBehavior::UnorderedList => "UNORDERED_LIST",
+            FieldBehavior::NonEmptyDefault => "NON_EMPTY_DEFAULT",
+            FieldBehavior::Identifier => "IDENTIFIER",
+        }
+    }
+    /// Creates an enum from field names used in the ProtoBuf definition.
+    pub fn from_str_name(value: &str) -> ::core::option::Option<Self> {
+        match value {
+            "FIELD_BEHAVIOR_UNSPECIFIED" => Some(Self::Unspecified),
+            "OPTIONAL" => Some(Self::Optional),
+            "REQUIRED" => Some(Self::Required),
+            "OUTPUT_ONLY" => Some(Self::OutputOnly),
+            "INPUT_ONLY" => Some(Self::InputOnly),
+            "IMMUTABLE" => Some(Self::Immutable),
+            "UNORDERED_LIST" => Some(Self::UnorderedList),
+            "NON_EMPTY_DEFAULT" => Some(Self::NonEmptyDefault),
+            "IDENTIFIER" => Some(Self::Identifier),
+            _ => None,
+        }
+    }
+}