Add beacon.watch (#3362)

> This is currently a WIP and all features are subject to alteration or removal at any time.

## Overview

The successor to #2873.

Contains the backbone of `beacon.watch` including syncing code, the initial API, and several core database tables.

See `watch/README.md` for more information, requirements and usage.

Author: macladson

Cargo.lock

@@ -87,6 +87,8 @@ members = [
 
     "validator_client",
     "validator_client/slashing_protection",
+
+    "watch",
 ]
 resolver = "2"
 

Cargo.toml

@@ -38,15 +38,15 @@ system_health = { path = "../../common/system_health" }
 directory = { path = "../../common/directory" }
 eth2_serde_utils = "0.1.1"
 operation_pool = { path = "../operation_pool" }
+sensitive_url = { path = "../../common/sensitive_url" }
+unused_port = {path = "../../common/unused_port"}
+logging = { path = "../../common/logging" }
+store = { path = "../store" }
 
 [dev-dependencies]
-store = { path = "../store" }
 environment = { path = "../../lighthouse/environment" }
-sensitive_url = { path = "../../common/sensitive_url" }
-logging = { path = "../../common/logging" }
 serde_json = "1.0.58"
 proto_array = { path = "../../consensus/proto_array" }
-unused_port = {path = "../../common/unused_port"}
 genesis = { path = "../genesis" }
 
 [[test]]

beacon_node/http_api/Cargo.toml

@@ -18,6 +18,7 @@ mod standard_block_rewards;
 mod state_id;
 mod sync_committee_rewards;
 mod sync_committees;
+pub mod test_utils;
 mod ui;
 mod validator_inclusion;
 mod version;

beacon_node/http_api/src/lib.rs

@@ -1,3 +1,4 @@
+use crate::{Config, Context};
 use beacon_chain::{
     test_utils::{
         BeaconChainHarness, BoxedMutator, Builder as HarnessBuilder, EphemeralHarnessType,
@@ -6,7 +7,6 @@ use beacon_chain::{
 };
 use directory::DEFAULT_ROOT_DIR;
 use eth2::{BeaconNodeHttpClient, Timeouts};
-use http_api::{Config, Context};
 use lighthouse_network::{
     discv5::enr::{CombinedKey, EnrBuilder},
     libp2p::{
@@ -182,7 +182,7 @@ pub async fn create_api_server_on_port<T: BeaconChainTypes>(
     let eth1_service =
         eth1::Service::new(eth1::Config::default(), log.clone(), chain.spec.clone()).unwrap();
 
-    let context = Arc::new(Context {
+    let ctx = Arc::new(Context {
         config: Config {
             enabled: true,
             listen_addr: IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)),
@@ -193,19 +193,19 @@ pub async fn create_api_server_on_port<T: BeaconChainTypes>(
             data_dir: std::path::PathBuf::from(DEFAULT_ROOT_DIR),
             spec_fork_name: None,
         },
-        chain: Some(chain.clone()),
+        chain: Some(chain),
         network_senders: Some(network_senders),
         network_globals: Some(network_globals),
         eth1_service: Some(eth1_service),
         log,
     });
-    let ctx = context.clone();
+
     let (shutdown_tx, shutdown_rx) = oneshot::channel();
     let server_shutdown = async {
         // It's not really interesting why this triggered, just that it happened.
         let _ = shutdown_rx.await;
     };
-    let (listening_socket, server) = http_api::serve(ctx, server_shutdown).unwrap();
+    let (listening_socket, server) = crate::serve(ctx, server_shutdown).unwrap();
 
     ApiServer {
         server,

beacon_node/http_api/tests/common.rs renamed to beacon_node/http_api/src/test_utils.rs

@@ -1,11 +1,11 @@
 //! Tests for API behaviour across fork boundaries.
-use crate::common::*;
 use beacon_chain::{
     test_utils::{RelativeSyncCommittee, DEFAULT_ETH1_BLOCK_HASH, HARNESS_GENESIS_TIME},
     StateSkipConfig,
 };
 use eth2::types::{IndexedErrorMessage, StateId, SyncSubcommittee};
 use genesis::{bls_withdrawal_credentials, interop_genesis_state_with_withdrawal_credentials};
+use http_api::test_utils::*;
 use std::collections::HashSet;
 use types::{
     test_utils::{generate_deterministic_keypair, generate_deterministic_keypairs},

beacon_node/http_api/tests/fork_tests.rs

@@ -1,11 +1,11 @@
 //! Generic tests that make use of the (newer) `InteractiveApiTester`
-use crate::common::*;
 use beacon_chain::{
     chain_config::ReOrgThreshold,
     test_utils::{AttestationStrategy, BlockStrategy, SyncCommitteeStrategy},
 };
 use eth2::types::DepositContractData;
 use execution_layer::{ForkchoiceState, PayloadAttributes};
+use http_api::test_utils::InteractiveTester;
 use parking_lot::Mutex;
 use slot_clock::SlotClock;
 use state_processing::{

beacon_node/http_api/tests/interactive_tests.rs

@@ -1,6 +1,5 @@
 #![cfg(not(debug_assertions))] // Tests are too slow in debug.
 
-pub mod common;
 pub mod fork_tests;
 pub mod interactive_tests;
 pub mod tests;

beacon_node/http_api/tests/main.rs

@@ -1,4 +1,3 @@
-use crate::common::{create_api_server, create_api_server_on_port, ApiServer};
 use beacon_chain::test_utils::RelativeSyncCommittee;
 use beacon_chain::{
     test_utils::{AttestationStrategy, BeaconChainHarness, BlockStrategy, EphemeralHarnessType},
@@ -18,7 +17,10 @@ use execution_layer::test_utils::{
 };
 use futures::stream::{Stream, StreamExt};
 use futures::FutureExt;
-use http_api::{BlockId, StateId};
+use http_api::{
+    test_utils::{create_api_server, create_api_server_on_port, ApiServer},
+    BlockId, StateId,
+};
 use lighthouse_network::{Enr, EnrExt, PeerId};
 use network::NetworkReceivers;
 use proto_array::ExecutionStatus;

beacon_node/http_api/tests/tests.rs

@@ -22,7 +22,7 @@ use lighthouse_network::PeerId;
 pub use reqwest;
 use reqwest::{IntoUrl, RequestBuilder, Response};
 pub use reqwest::{StatusCode, Url};
-pub use sensitive_url::SensitiveUrl;
+pub use sensitive_url::{SensitiveError, SensitiveUrl};
 use serde::{de::DeserializeOwned, Serialize};
 use std::convert::TryFrom;
 use std::fmt;