From eaa766d711553282914970b36db97904ed30075e Mon Sep 17 00:00:00 2001 From: b0xtch Date: Fri, 10 Feb 2023 15:02:44 -0500 Subject: [PATCH 1/4] manually provide ip addresses to index provider --- README.md | 3 +- crates/ursa-index-provider/src/engine.rs | 48 ++++++--------------- crates/ursa-index-provider/src/provider.rs | 26 ++++++++--- crates/ursa-index-provider/src/tests/mod.rs | 8 +--- crates/ursa-rpc-service/src/config.rs | 12 +++--- crates/ursa-rpc-service/src/tests/mod.rs | 5 +-- crates/ursa/src/config.rs | 10 +---- crates/ursa/src/main.rs | 10 +---- 8 files changed, 45 insertions(+), 77 deletions(-) diff --git a/README.md b/README.md index f85826b0..0c881c55 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,8 @@ keystore_path = "~/.ursa/keystore" identity = "default" [provider_config] -domain = "example.domain" +# Public IP address of the node +addresses = ["/ip4/127.0.0.1"] indexer_url = "https://dev.cid.contact" database_path = "~/.ursa/data/index_provider_db" diff --git a/crates/ursa-index-provider/src/engine.rs b/crates/ursa-index-provider/src/engine.rs index 63f678bb..756401f0 100644 --- a/crates/ursa-index-provider/src/engine.rs +++ b/crates/ursa-index-provider/src/engine.rs @@ -20,7 +20,7 @@ use axum::{body::Body, extract::Path, response::Response, routing::get, Extensio use crate::provider::ProviderError; use fvm_ipld_blockstore::Blockstore; use libipld::Cid; -use libp2p::{gossipsub::TopicHash, identity::Keypair, multiaddr::Protocol, Multiaddr, PeerId}; +use libp2p::{gossipsub::TopicHash, identity::Keypair, Multiaddr, PeerId}; use std::{collections::VecDeque, str::FromStr, sync::Arc}; use tracing::{error, info, warn}; use ursa_store::UrsaStore; @@ -91,9 +91,8 @@ pub struct ProviderEngine { command_receiver: Receiver, /// network command sender for communication with libp2p node network_command_sender: Sender, - /// Server from which advertised content is retrievable. - server_address: Multiaddr, - domain: Multiaddr, + /// List of addresses to submit to indexer. + addresses: Vec, } impl ProviderEngine @@ -106,8 +105,7 @@ where provider_store: Arc>, config: ProviderConfig, network_command_sender: Sender, - server_address: Multiaddr, - domain: Multiaddr, + addresses: Vec, ) -> Self { let (command_sender, command_receiver) = unbounded_channel(); ProviderEngine { @@ -117,8 +115,7 @@ where network_command_sender, provider: Provider::new(keypair, provider_store), store, - server_address, - domain, + addresses, } } pub fn command_sender(&self) -> Sender { @@ -166,7 +163,7 @@ where } else { match self .provider - .create_announce_message(peer_id, self.domain.clone()) + .create_announce_message(peer_id, &mut self.addresses) { Ok(announce_message) => { if let Err(e) = self @@ -192,40 +189,19 @@ where } pub async fn publish_local(&mut self, root_cid: Cid, file_size: u64) -> Result<()> { - let (listener_addresses_sender, listener_addresses_receiver) = oneshot::channel(); - self.network_command_sender - .send(NetworkCommand::GetListenerAddresses { - sender: listener_addresses_sender, - })?; - let context_id = root_cid.to_bytes(); info!( "Creating advertisement for cids under root cid: {:?}.", root_cid ); let peer_id = PeerId::from(self.provider.keypair().public()); + let addresses = self + .addresses + .iter() + .map(|address| address.to_string()) + .collect(); - let listener_addresses = listener_addresses_receiver.await?; - let mut addresses = vec![self.server_address.to_string()]; - for la in listener_addresses { - let mut address = Multiaddr::empty(); - for protocol in la.into_iter() { - match protocol { - Protocol::Ip6(ip) => address.push(Protocol::Ip6(ip)), - Protocol::Ip4(ip) => address.push(Protocol::Ip4(ip)), - Protocol::Tcp(port) => address.push(Protocol::Tcp(port)), - _ => {} - } - } - addresses.push(address.to_string()) - } - let advertisement = Advertisement::new( - context_id.clone(), - peer_id, - addresses.clone(), - false, - file_size, - ); + let advertisement = Advertisement::new(context_id, peer_id, addresses, false, file_size); let provider_id = self.provider.create(advertisement)?; let dag = self.store.dag_traversal(&(root_cid))?; diff --git a/crates/ursa-index-provider/src/provider.rs b/crates/ursa-index-provider/src/provider.rs index 254fbe55..e5a128b9 100644 --- a/crates/ursa-index-provider/src/provider.rs +++ b/crates/ursa-index-provider/src/provider.rs @@ -103,7 +103,11 @@ pub trait ProviderInterface: Sync + Send + 'static { fn create(&mut self, ad: Advertisement) -> Result; fn add_chunk(&mut self, bytes: Vec, id: usize) -> Result<()>; fn publish(&mut self, id: usize) -> Result; - fn create_announce_message(&mut self, peer_id: PeerId, domain: Multiaddr) -> Result>; + fn create_announce_message( + &mut self, + peer_id: PeerId, + addresses: &mut Vec, + ) -> Result>; } impl ProviderInterface for Provider @@ -157,15 +161,24 @@ where Err(anyhow!("ad not found")) } - fn create_announce_message(&mut self, peer_id: PeerId, domain: Multiaddr) -> Result> { - let mut multiaddr = domain; - multiaddr.push(Protocol::Http); - multiaddr.push(Protocol::P2p(peer_id.into())); + fn create_announce_message( + &mut self, + peer_id: PeerId, + addresses: &mut Vec, + ) -> Result> { + let multiaddrs = addresses + .iter_mut() + .map(|address| { + address.push(Protocol::Http); + address.push(Protocol::P2p(peer_id.into())); + address.clone() + }) + .collect::>(); if let Some(head_cid) = *self.head.read().unwrap() { let message = Message { Cid: head_cid, - Addrs: vec![multiaddr], + Addrs: multiaddrs, ExtraData: *b"", }; @@ -184,6 +197,7 @@ pub struct Message { pub Addrs: Vec, pub ExtraData: [u8; 0], } + impl Cbor for Message { fn marshal_cbor(&self) -> Result, fvm_ipld_encoding::Error> { const MESSAGE_BUFFER_LENGTH: [u8; 1] = [131]; diff --git a/crates/ursa-index-provider/src/tests/mod.rs b/crates/ursa-index-provider/src/tests/mod.rs index 60f7b5b9..d124f25e 100644 --- a/crates/ursa-index-provider/src/tests/mod.rs +++ b/crates/ursa-index-provider/src/tests/mod.rs @@ -5,7 +5,7 @@ use std::sync::Arc; use crate::{config::ProviderConfig, engine::ProviderEngine}; use db::MemoryDB; -use libp2p::{identity::Keypair, Multiaddr, PeerId}; +use libp2p::{identity::Keypair, PeerId}; use simple_logger::SimpleLogger; use tokio::task; use tracing::{info, log::LevelFilter}; @@ -44,17 +44,13 @@ pub fn provider_engine_init( let index_store = get_store(); let service = UrsaService::new(keypair.clone(), &network_config, Arc::clone(&store))?; - - let server_address = Multiaddr::try_from("/ip4/0.0.0.0/tcp/0").unwrap(); - let provider_engine = ProviderEngine::new( keypair, store, index_store, ProviderConfig::default(), service.command_sender(), - server_address, - "/ip4/127.0.0.1/tcp/4069".parse().unwrap(), + vec!["/ip4/127.0.0.1".parse().unwrap()], ); let router = provider_engine.router(); diff --git a/crates/ursa-rpc-service/src/config.rs b/crates/ursa-rpc-service/src/config.rs index 98f6433f..dc7e7194 100644 --- a/crates/ursa-rpc-service/src/config.rs +++ b/crates/ursa-rpc-service/src/config.rs @@ -3,9 +3,9 @@ use serde::{Deserialize, Serialize}; #[derive(Deserialize, Serialize, Debug)] pub struct ServerConfig { - /// Domain Multiaddress of the node, eg. `/dns/test-node.ursa.earth` - #[serde(default = "ServerConfig::default_domain")] - pub domain: Multiaddr, + /// Public IP address of the node, eg. `/ip4/127.0.0.1` + #[serde(default = "ServerConfig::default_addresses")] + pub addresss: Vec, /// Port to listen on #[serde(default = "ServerConfig::default_port")] pub port: u16, @@ -17,8 +17,8 @@ pub struct ServerConfig { } impl ServerConfig { - fn default_domain() -> Multiaddr { - "/ip4/127.0.0.1/tcp/4069".parse().unwrap() + fn default_addresses() -> Vec { + vec!["/ip4/127.0.0.1".parse().unwrap()] } fn default_port() -> u16 { 4069 @@ -31,7 +31,7 @@ impl ServerConfig { impl Default for ServerConfig { fn default() -> Self { Self { - domain: Self::default_domain(), + addresss: Self::default_addresses(), port: Self::default_port(), addr: Self::default_addr(), origin: Default::default(), diff --git a/crates/ursa-rpc-service/src/tests/mod.rs b/crates/ursa-rpc-service/src/tests/mod.rs index 79b08697..8ded546f 100644 --- a/crates/ursa-rpc-service/src/tests/mod.rs +++ b/crates/ursa-rpc-service/src/tests/mod.rs @@ -5,7 +5,6 @@ use anyhow::Result; use axum::{headers::HeaderMap, routing::get, Router}; use db::MemoryDB; use libp2p::identity::Keypair; -use libp2p::Multiaddr; use simple_logger::SimpleLogger; use std::sync::Arc; use tracing::{log::LevelFilter, warn}; @@ -44,7 +43,6 @@ pub fn init() -> InitResult { }; let keypair = Keypair::generate_ed25519(); let service = UrsaService::new(keypair.clone(), &network_config, Arc::clone(&store))?; - let server_address = Multiaddr::try_from("/ip4/0.0.0.0/tcp/0").unwrap(); let provider_engine = ProviderEngine::new( keypair, @@ -52,8 +50,7 @@ pub fn init() -> InitResult { get_store(), ProviderConfig::default(), service.command_sender(), - server_address, - "/ip4/127.0.0.1/tcp/4069".parse().unwrap(), + vec!["/ip4/127.0.0.1".parse().unwrap()], ); Ok((service, provider_engine, store)) diff --git a/crates/ursa/src/config.rs b/crates/ursa/src/config.rs index edfe7323..e9e8a3aa 100644 --- a/crates/ursa/src/config.rs +++ b/crates/ursa/src/config.rs @@ -34,15 +34,7 @@ impl UrsaConfig { let mut raw = String::new(); file.read_to_string(&mut raw)?; - let mut config: UrsaConfig = - toml::from_str(&raw).context("Failed to parse config file")?; - - // TEMP: remove this after some time - if let Some(domain) = config.provider_config.domain.clone() { - warn!("Config `provider_config.domain` depreciated, moving value to `server_config.domain`"); - config.server_config.domain = domain.parse().unwrap(); - config.provider_config.domain = None; - } + let config: UrsaConfig = toml::from_str(&raw).context("Failed to parse config file")?; // check if we modified the config at all let config_str = toml::to_string(&config)?; diff --git a/crates/ursa/src/main.rs b/crates/ursa/src/main.rs index cf8c1624..3f2f61e1 100644 --- a/crates/ursa/src/main.rs +++ b/crates/ursa/src/main.rs @@ -3,7 +3,6 @@ use anyhow::Result; use db::{rocks::RocksDb, rocks_config::RocksDbConfig}; use dotenv::dotenv; use libp2p::multiaddr::Protocol; -use libp2p::Multiaddr; use resolve_path::PathResolveExt; use std::env; use std::sync::Arc; @@ -94,12 +93,6 @@ async fn main() -> Result<()> { ) .expect("Opening provider RocksDB must succeed"); - let server_address = Multiaddr::try_from(format!( - "/ip4/{}/tcp/{}", - server_config.addr, server_config.port - )) - .expect("Server to have a valid address"); - let index_store = Arc::new(UrsaStore::new(Arc::clone(&Arc::new(provider_db)))); let index_provider_engine = ProviderEngine::new( keypair, @@ -107,8 +100,7 @@ async fn main() -> Result<()> { index_store, provider_config, service.command_sender(), - server_address, - server_config.domain.clone(), + server_config.addresss.clone(), ); let index_provider_router = index_provider_engine.router(); From cb131650cb53324f99160ab419285e207213a945 Mon Sep 17 00:00:00 2001 From: b0xtch Date: Fri, 10 Feb 2023 15:15:34 -0500 Subject: [PATCH 2/4] add port to examples of addresses to provide to indexer --- README.md | 2 +- crates/ursa-index-provider/src/tests/mod.rs | 2 +- crates/ursa-rpc-service/src/config.rs | 2 +- crates/ursa-rpc-service/src/tests/mod.rs | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 0c881c55..171eb4f9 100644 --- a/README.md +++ b/README.md @@ -64,7 +64,7 @@ identity = "default" [provider_config] # Public IP address of the node -addresses = ["/ip4/127.0.0.1"] +addresses = ["/ip4/127.0.0.1/tcp/4069"] indexer_url = "https://dev.cid.contact" database_path = "~/.ursa/data/index_provider_db" diff --git a/crates/ursa-index-provider/src/tests/mod.rs b/crates/ursa-index-provider/src/tests/mod.rs index d124f25e..bdcb56c6 100644 --- a/crates/ursa-index-provider/src/tests/mod.rs +++ b/crates/ursa-index-provider/src/tests/mod.rs @@ -50,7 +50,7 @@ pub fn provider_engine_init( index_store, ProviderConfig::default(), service.command_sender(), - vec!["/ip4/127.0.0.1".parse().unwrap()], + vec!["/ip4/127.0.0.1/tcp/4069".parse().unwrap()], ); let router = provider_engine.router(); diff --git a/crates/ursa-rpc-service/src/config.rs b/crates/ursa-rpc-service/src/config.rs index dc7e7194..a3ebcff9 100644 --- a/crates/ursa-rpc-service/src/config.rs +++ b/crates/ursa-rpc-service/src/config.rs @@ -18,7 +18,7 @@ pub struct ServerConfig { impl ServerConfig { fn default_addresses() -> Vec { - vec!["/ip4/127.0.0.1".parse().unwrap()] + vec!["/ip4/127.0.0.1/tcp/4069".parse().unwrap()] } fn default_port() -> u16 { 4069 diff --git a/crates/ursa-rpc-service/src/tests/mod.rs b/crates/ursa-rpc-service/src/tests/mod.rs index 8ded546f..e43fb4ca 100644 --- a/crates/ursa-rpc-service/src/tests/mod.rs +++ b/crates/ursa-rpc-service/src/tests/mod.rs @@ -50,7 +50,7 @@ pub fn init() -> InitResult { get_store(), ProviderConfig::default(), service.command_sender(), - vec!["/ip4/127.0.0.1".parse().unwrap()], + vec!["/ip4/127.0.0.1/tcp/4069".parse().unwrap()], ); Ok((service, provider_engine, store)) From a1bfd4a97a01b75ba66ff064a972af6fd04f9a85 Mon Sep 17 00:00:00 2001 From: b0xtch Date: Fri, 10 Feb 2023 15:24:14 -0500 Subject: [PATCH 3/4] use dev cid contact as default --- crates/ursa-gateway/src/config.rs | 2 +- crates/ursa-rpc-service/src/server.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/ursa-gateway/src/config.rs b/crates/ursa-gateway/src/config.rs index 555f099b..4267bc08 100644 --- a/crates/ursa-gateway/src/config.rs +++ b/crates/ursa-gateway/src/config.rs @@ -103,7 +103,7 @@ impl Default for GatewayConfig { stream_buf: 2_000_000, // 2MB }, indexer: IndexerConfig { - cid_url: "https://cid.contact/cid".into(), + cid_url: "https://dev.cid.contact/cid".into(), }, } } diff --git a/crates/ursa-rpc-service/src/server.rs b/crates/ursa-rpc-service/src/server.rs index 396d9bf2..13e415e4 100644 --- a/crates/ursa-rpc-service/src/server.rs +++ b/crates/ursa-rpc-service/src/server.rs @@ -8,7 +8,7 @@ use crate::{ api::NodeNetworkInterface, config::ServerConfig, http, - rpc::{routes, RpcServer}, + rpc::{self, RpcServer}, service::MultiplexService, }; use tracing::info; @@ -56,7 +56,7 @@ where pub fn rpc_app(&self) -> Router { Router::new() - .merge(routes::network::init()) + .merge(rpc::routes::network::init()) .layer(Extension(self.rpc_server.clone())) } From ca12776ff501248d1ee5f44c61e07c5e3118968f Mon Sep 17 00:00:00 2001 From: b0xtch Date: Fri, 10 Feb 2023 15:56:23 -0500 Subject: [PATCH 4/4] fix spelling --- crates/ursa-rpc-service/src/config.rs | 4 ++-- crates/ursa/src/main.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/ursa-rpc-service/src/config.rs b/crates/ursa-rpc-service/src/config.rs index a3ebcff9..986b8f52 100644 --- a/crates/ursa-rpc-service/src/config.rs +++ b/crates/ursa-rpc-service/src/config.rs @@ -5,7 +5,7 @@ use serde::{Deserialize, Serialize}; pub struct ServerConfig { /// Public IP address of the node, eg. `/ip4/127.0.0.1` #[serde(default = "ServerConfig::default_addresses")] - pub addresss: Vec, + pub addresses: Vec, /// Port to listen on #[serde(default = "ServerConfig::default_port")] pub port: u16, @@ -31,7 +31,7 @@ impl ServerConfig { impl Default for ServerConfig { fn default() -> Self { Self { - addresss: Self::default_addresses(), + addresses: Self::default_addresses(), port: Self::default_port(), addr: Self::default_addr(), origin: Default::default(), diff --git a/crates/ursa/src/main.rs b/crates/ursa/src/main.rs index 3f2f61e1..855ac929 100644 --- a/crates/ursa/src/main.rs +++ b/crates/ursa/src/main.rs @@ -100,7 +100,7 @@ async fn main() -> Result<()> { index_store, provider_config, service.command_sender(), - server_config.addresss.clone(), + server_config.addresses.clone(), ); let index_provider_router = index_provider_engine.router();