diff --git a/Cargo.toml b/Cargo.toml index b9460c0..122548b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,6 +7,7 @@ members = [ "conf", "testing", "tests", + "gossip_map", ] resolver = "2" diff --git a/Makefile b/Makefile index 0b69518..47d3123 100644 --- a/Makefile +++ b/Makefile @@ -1,22 +1,46 @@ CC=cargo FMT=fmt -OPTIONS= +# Please keep me update with the https://github.com/ElementsProject/lightning/blob/master/Makefile#L29 +BOLTDIR := /tmp/ +OPTIONS := "" default: fmt $(CC) build --all-features @make example -fmt: +fmt: ## Format the file $(CC) fmt --all -check: +check: ## Run all the tests inside the workspace @make default $(CC) test -- --show-output -example: +example: # build the examples $(CC) build --example foo_plugin $(CC) build --example macros_ex -clean: +clean: ## Clean the file $(CC) clean + +# FIXME: we should apply the diff over the csv file. +genfile: check_compiler ## Generate the file from a new main version of the spec, in addition apply potential patch + cd "$(BOLTDIR)" && git clone https://github.com/lightning/bolts.git + cd /tmp && git clone https://github.com/ElementsProject/lightning.git + cd $(BOLTDIR)/bolts && tools/extract-formats.py 07-routing-gossip.md > bolt7.csv + cp /tmp/lightning/gossipd/gossip_store_wire.csv gossip_map/spec/gossip_store_wire.csv + cp /tmp/bolts/bolt7.csv gossip_map/spec/bolt7.csv + patch gossip_map/spec/bolt7.csv gossip_map/spec/bolt7.diff + @make update-genfile + +update-genfile: ## update the local generated file without fetch a new specification, just the generated file. + lncodegen-cli -l rust generate -b gossip_map/spec/gossip_store_wire.csv gossip_map/src/gossip_stor_wiregen.rs + lncodegen-cli -l rust generate -b gossip_map/spec/bolt7.csv gossip_map/src/bolt7.rs + + +check_compiler: ## Check if the lncodegen exist or need to be installed (required the rust toolchain). + @command -v lncodegen-cli --help >/dev/null 2>&1 || (echo "`lncodegen-cli` not found, installing..." && cargo install lncodegen-cli --git https://github.com/lnspec-tools/lncodegen.git) + +help: ## Show Help + @grep --no-filename -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | \ + awk 'BEGIN {FS = ":.*?## "}; {printf "\033[32m%-15s\033[0m %s\n", $$1, $$2}' diff --git a/contrib/gossip_store b/contrib/gossip_store new file mode 100644 index 0000000..14d0ebb Binary files /dev/null and b/contrib/gossip_store differ diff --git a/gossip_map/Cargo.toml b/gossip_map/Cargo.toml new file mode 100644 index 0000000..15516f3 --- /dev/null +++ b/gossip_map/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "clightningrpc_gossip_map" +version = "0.1.0" +edition = "2021" + +[dependencies] +anyhow = "1" +log = "0.4.21" +byteorder = "1.4.3" +hex = "0.4.3" +bitcoin = { version = "0.30.0" } +fundamentals = { git = "https://github.com/lnspec-tools/ln-fundamentals.git", branch = "macros/fix_fixed_read" } +fundamentals-derive = { git = "https://github.com/lnspec-tools/ln-fundamentals.git", branch = "macros/fix_fixed_read" } + +[dev-dependencies] +anyhow = "1.0.70" +colored = "1.9" +log = { version = "0.4", features = ["std"] } +chrono = { version = "0.4", features = ["std"], default-features = false } diff --git a/gossip_map/spec/bolt7.csv b/gossip_map/spec/bolt7.csv new file mode 100644 index 0000000..865a3fa --- /dev/null +++ b/gossip_map/spec/bolt7.csv @@ -0,0 +1,84 @@ +msgtype,announcement_signatures,259 +msgdata,announcement_signatures,channel_id,channel_id, +msgdata,announcement_signatures,short_channel_id,short_channel_id, +msgdata,announcement_signatures,node_signature,signature, +msgdata,announcement_signatures,bitcoin_signature,signature, +msgtype,channel_announcement,256 +msgdata,channel_announcement,node_signature_1,signature, +msgdata,channel_announcement,node_signature_2,signature, +msgdata,channel_announcement,bitcoin_signature_1,signature, +msgdata,channel_announcement,bitcoin_signature_2,signature, +msgdata,channel_announcement,len,u16, +msgdata,channel_announcement,features,byte,len +msgdata,channel_announcement,chain_hash,chain_hash, +msgdata,channel_announcement,short_channel_id,short_channel_id, +msgdata,channel_announcement,node_id_1,point, +msgdata,channel_announcement,node_id_2,point, +msgdata,channel_announcement,bitcoin_key_1,point, +msgdata,channel_announcement,bitcoin_key_2,point, +msgtype,node_announcement,257 +msgdata,node_announcement,signature,signature, +msgdata,node_announcement,flen,u16, +msgdata,node_announcement,features,byte,flen +msgdata,node_announcement,timestamp,u32, +msgdata,node_announcement,node_id,point, +msgdata,node_announcement,rgb_color,byte,3 +msgdata,node_announcement,alias,byte,32 +msgdata,node_announcement,addrlen,u16, +msgdata,node_announcement,addresses,byte,addrlen +msgdata,node_announcement,tlvs,node_ann_tlvs, +tlvtype,node_ann_tlvs,option_will_fund,1 +tlvdata,node_ann_tlvs,option_will_fund,lease_rates,lease_rates, +msgtype,channel_update,258 +msgdata,channel_update,signature,signature, +msgdata,channel_update,chain_hash,chain_hash, +msgdata,channel_update,short_channel_id,short_channel_id, +msgdata,channel_update,timestamp,u32, +msgdata,channel_update,message_flags,byte, +msgdata,channel_update,channel_flags,byte, +msgdata,channel_update,cltv_expiry_delta,u16, +msgdata,channel_update,htlc_minimum_msat,u64, +msgdata,channel_update,fee_base_msat,u32, +msgdata,channel_update,fee_proportional_millionths,u32, +msgdata,channel_update,htlc_maximum_msat,u64, +msgtype,query_short_channel_ids,261 +msgdata,query_short_channel_ids,chain_hash,chain_hash, +msgdata,query_short_channel_ids,len,u16, +msgdata,query_short_channel_ids,encoded_short_ids,byte,len +msgdata,query_short_channel_ids,tlvs,query_short_channel_ids_tlvs, +tlvtype,query_short_channel_ids_tlvs,query_flags,1 +tlvdata,query_short_channel_ids_tlvs,query_flags,encoding_type,byte, +tlvdata,query_short_channel_ids_tlvs,query_flags,encoded_query_flags,byte,... +msgtype,reply_short_channel_ids_end,262 +msgdata,reply_short_channel_ids_end,chain_hash,chain_hash, +msgdata,reply_short_channel_ids_end,full_information,byte, +msgtype,query_channel_range,263 +msgdata,query_channel_range,chain_hash,chain_hash, +msgdata,query_channel_range,first_blocknum,u32, +msgdata,query_channel_range,number_of_blocks,u32, +msgdata,query_channel_range,tlvs,query_channel_range_tlvs, +tlvtype,query_channel_range_tlvs,query_option,1 +tlvdata,query_channel_range_tlvs,query_option,query_option_flags,bigsize, +msgtype,reply_channel_range,264 +msgdata,reply_channel_range,chain_hash,chain_hash, +msgdata,reply_channel_range,first_blocknum,u32, +msgdata,reply_channel_range,number_of_blocks,u32, +msgdata,reply_channel_range,sync_complete,byte, +msgdata,reply_channel_range,len,u16, +msgdata,reply_channel_range,encoded_short_ids,byte,len +msgdata,reply_channel_range,tlvs,reply_channel_range_tlvs, +tlvtype,reply_channel_range_tlvs,timestamps_tlv,1 +tlvdata,reply_channel_range_tlvs,timestamps_tlv,encoding_type,byte, +tlvdata,reply_channel_range_tlvs,timestamps_tlv,encoded_timestamps,byte,... +tlvtype,reply_channel_range_tlvs,checksums_tlv,3 +tlvdata,reply_channel_range_tlvs,checksums_tlv,checksums,channel_update_checksums,... +subtype,channel_update_timestamps +subtypedata,channel_update_timestamps,timestamp_node_id_1,u32, +subtypedata,channel_update_timestamps,timestamp_node_id_2,u32, +subtype,channel_update_checksums +subtypedata,channel_update_checksums,checksum_node_id_1,u32, +subtypedata,channel_update_checksums,checksum_node_id_2,u32, +msgtype,gossip_timestamp_filter,265 +msgdata,gossip_timestamp_filter,chain_hash,chain_hash, +msgdata,gossip_timestamp_filter,first_timestamp,u32, +msgdata,gossip_timestamp_filter,timestamp_range,u32, diff --git a/gossip_map/spec/bolt7.diff b/gossip_map/spec/bolt7.diff new file mode 100644 index 0000000..8c72922 --- /dev/null +++ b/gossip_map/spec/bolt7.diff @@ -0,0 +1,14 @@ +diff --git a/gossip_map/spec/bolt7.csv b/gossip_map/spec/bolt7.csv +index 35a2dc6..865a3fa 100644 +--- a/gossip_map/spec/bolt7.csv ++++ b/gossip_map/spec/bolt7.csv +@@ -26,6 +26,9 @@ msgdata,node_announcement,rgb_color,byte,3 + msgdata,node_announcement,alias,byte,32 + msgdata,node_announcement,addrlen,u16, + msgdata,node_announcement,addresses,byte,addrlen ++msgdata,node_announcement,tlvs,node_ann_tlvs, ++tlvtype,node_ann_tlvs,option_will_fund,1 ++tlvdata,node_ann_tlvs,option_will_fund,lease_rates,lease_rates, + msgtype,channel_update,258 + msgdata,channel_update,signature,signature, + msgdata,channel_update,chain_hash,chain_hash, diff --git a/gossip_map/spec/gossip_store_wire.csv b/gossip_map/spec/gossip_store_wire.csv new file mode 100644 index 0000000..b0f81c3 --- /dev/null +++ b/gossip_map/spec/gossip_store_wire.csv @@ -0,0 +1,29 @@ +# gossip_store messages: messages persisted in the gossip_store +# We store raw messages here, so these numbers must not overlap with +# 256/257/258. +#include +#include + +# This always follows the channel_announce / private_announce +msgtype,gossip_store_channel_amount,4101 +msgdata,gossip_store_channel_amount,satoshis,amount_sat, + +# Mimics a channel_announce, except signatures are all-zero +msgtype,gossip_store_private_channel_obs,4104 +msgdata,gossip_store_private_channel_obs,satoshis,amount_sat, +msgdata,gossip_store_private_channel_obs,len,u16, +msgdata,gossip_store_private_channel_obs,announcement,u8,len + +msgtype,gossip_store_private_update_obs,4102 +msgdata,gossip_store_private_update_obs,len,u16, +msgdata,gossip_store_private_update_obs,update,u8,len + +msgtype,gossip_store_delete_chan,4103 +msgdata,gossip_store_delete_chan,scid,short_channel_id, + +msgtype,gossip_store_ended,4105 +msgdata,gossip_store_ended,equivalent_offset,u64, + +msgtype,gossip_store_chan_dying,4106 +msgdata,gossip_store_chan_dying,scid,short_channel_id, +msgdata,gossip_store_chan_dying,blockheight,u32, diff --git a/gossip_map/src/bolt7.rs b/gossip_map/src/bolt7.rs new file mode 100644 index 0000000..782a693 --- /dev/null +++ b/gossip_map/src/bolt7.rs @@ -0,0 +1,135 @@ +// code generated with the lngen, please not edit this file. +use std::io::{Read, Write}; + +use fundamentals_derive::{DecodeWire, EncodeWire}; + +use crate::core::{FromWire, ToWire}; +use crate::prelude::*; + +#[derive(DecodeWire, EncodeWire, Debug, Clone)] +pub struct AnnouncementSignatures { + #[msg_type = 259] + pub ty: u16, + pub channel_id: ChannelId, + pub short_channel_id: ShortChannelId, + pub node_signature: Signature, + pub bitcoin_signature: Signature, +} + +#[derive(DecodeWire, EncodeWire, Debug, Clone)] +pub struct ChannelAnnouncement { + #[msg_type = 256] + pub ty: u16, + pub node_signature_1: Signature, + pub node_signature_2: Signature, + pub bitcoin_signature_1: Signature, + pub bitcoin_signature_2: Signature, + pub features: BitFlag, + pub chain_hash: ChainHash, + pub short_channel_id: ShortChannelId, + pub node_id_1: Point, + pub node_id_2: Point, + pub bitcoin_key_1: Point, + pub bitcoin_key_2: Point, +} + +#[derive(DecodeWire, EncodeWire, Debug, Clone)] +pub struct ChannelUpdate { + #[msg_type = 258] + pub ty: u16, + pub signature: Signature, + pub chain_hash: ChainHash, + pub short_channel_id: ShortChannelId, + pub timestamp: u32, + // FIXME: these are u8 but the codegen will decode it to BitFlag + pub message_flags: u8, + pub channel_flags: u8, + pub cltv_expiry_delta: u16, + pub htlc_minimum_msat: u64, + pub fee_base_msat: u32, + pub fee_proportional_millionths: u32, + pub htlc_maximum_msat: u64, +} + +#[derive(DecodeWire, EncodeWire, Debug, Clone)] +pub struct GossipTimestampFilter { + #[msg_type = 265] + pub ty: u16, + pub chain_hash: ChainHash, + pub first_timestamp: u32, + pub timestamp_range: u32, +} + +macro_rules! to_wire_type_with_size { + ($ty: ty, $size: expr) => { + impl ToWire for $ty { + fn to_wire(&self, buff: &mut W) -> std::io::Result<()> { + buff.write_all(self) + } + } + + impl FromWire for $ty { + fn from_wire(reader: &mut R) -> std::io::Result { + let mut buff = [0; $size]; + reader.read_exact(&mut buff)?; + Ok(buff) + } + } + }; +} + +pub type Alias = [u8; 32]; +pub type Rgb = [u8; 3]; + +#[derive(DecodeWire, EncodeWire, Debug, Clone)] +pub struct NodeAnnouncement { + #[msg_type = 257] + pub ty: u16, + pub signature: Signature, + pub features: BitFlag, + pub timestamp: u32, + pub node_id: Point, + pub rgb_color: Rgb, + pub alias: Alias, + pub addresses: BitFlag, + pub node_ann_tlvs: Stream, +} + +#[derive(DecodeWire, EncodeWire, Debug, Clone)] +pub struct QueryChannelRange { + #[msg_type = 263] + pub ty: u16, + pub chain_hash: ChainHash, + pub first_blocknum: u32, + pub number_of_blocks: u32, + pub query_channel_range_tlvs: Stream, +} + +#[derive(DecodeWire, EncodeWire, Debug, Clone)] +pub struct QueryShortChannelIds { + #[msg_type = 261] + pub ty: u16, + pub chain_hash: ChainHash, + pub encoded_short_ids: BitFlag, + pub query_short_channel_ids_tlvs: Stream, +} + +#[derive(DecodeWire, EncodeWire, Debug, Clone)] +pub struct ReplyChannelRange { + #[msg_type = 264] + pub ty: u16, + pub chain_hash: ChainHash, + pub first_blocknum: u32, + pub number_of_blocks: u32, + pub sync_complete: BitFlag, + pub encoded_short_ids: BitFlag, + pub reply_channel_range_tlvs: Stream, +} + +#[derive(DecodeWire, EncodeWire, Debug, Clone)] +pub struct ReplyShortChannelIdsEnd { + #[msg_type = 262] + pub ty: u16, + pub chain_hash: ChainHash, + pub full_information: BitFlag, +} diff --git a/gossip_map/src/flags.rs b/gossip_map/src/flags.rs new file mode 100644 index 0000000..89724cf --- /dev/null +++ b/gossip_map/src/flags.rs @@ -0,0 +1,20 @@ +//! Flag implementation for the gossip map types + +pub static GOSSIP_STORE_MAJOR_VERSION: u16 = 0 << 5; +pub static GOSSIP_STORE_MAJOR_VERSION_MASK: u16 = 0xE0; + +/// Deleted fields should be ignored: on restart, they will be removed as the gossip_store is rewritten. +pub const GOSSIP_STORE_LEN_DELETED_BIT: u16 = 0x8000; +/// The push flag indicates gossip which is generated locally: this is important for gossip timestamp filtering, +/// where peers request gossip and we always send our own gossip messages even if the timestamp wasn't within their +pub const GOSSIP_STORE_LEN_PUSH_BIT: u16 = 0x4000; +/// The ratelimit flag indicates that this gossip message came too fast. +/// The message are corded in the gossip map, but don't relay it to peers. +pub const GOSSIP_STORE_LEN_RATELIMIT_BIT: u16 = 0x2000; + +// These duplicate constants in lightning/gossipd/gossip_store_wiregen.h +pub const WIRE_GOSSIP_STORE_PRIVATE_CHANNEL: u16 = 4104; +pub const WIRE_GOSSIP_STORE_PRIVATE_UPDATE: u16 = 4102; +pub const WIRE_GOSSIP_STORE_DELETE_CHAN: u16 = 4103; +pub const WIRE_GOSSIP_STORE_ENDED: u16 = 4105; +pub const WIRE_GOSSIP_STORE_CHANNEL_AMOUNT: u16 = 4101; diff --git a/gossip_map/src/gossip_stor_wiregen.rs b/gossip_map/src/gossip_stor_wiregen.rs new file mode 100644 index 0000000..304370e --- /dev/null +++ b/gossip_map/src/gossip_stor_wiregen.rs @@ -0,0 +1,51 @@ +// code generated with the lngen, please not edit this file. +use std::io::{Read, Write}; + +use fundamentals_derive::{DecodeWire, EncodeWire}; + +use crate::core::{FromWire, ToWire}; +use crate::prelude::*; + +#[derive(DecodeWire, EncodeWire, Debug, Clone)] +pub struct GossipStoreChanDying { + #[msg_type = 4106] + pub ty: u16, + pub scid: ShortChannelId, + pub blockheight: u32, +} + +#[derive(DecodeWire, EncodeWire, Debug, Clone)] +pub struct GossipStoreChannelAmount { + #[msg_type = 4101] + pub ty: u16, + pub satoshis: u64, +} + +#[derive(DecodeWire, EncodeWire, Debug, Clone)] +pub struct GossipStoreDeleteChan { + #[msg_type = 4103] + pub ty: u16, + pub scid: ShortChannelId, +} + +#[derive(DecodeWire, EncodeWire, Debug, Clone)] +pub struct GossipStoreEnded { + #[msg_type = 4105] + pub ty: u16, + pub equivalent_offset: u64, +} + +#[derive(DecodeWire, EncodeWire, Debug, Clone)] +pub struct GossipStorePrivateChannelObs { + #[msg_type = 4104] + pub ty: u16, + pub satoshis: u64, + pub announcement: BitFlag, +} + +#[derive(DecodeWire, EncodeWire, Debug, Clone)] +pub struct GossipStorePrivateUpdateObs { + #[msg_type = 4102] + pub ty: u16, + pub update: BitFlag, +} diff --git a/gossip_map/src/gossip_types.rs b/gossip_map/src/gossip_types.rs new file mode 100644 index 0000000..fe3c8ea --- /dev/null +++ b/gossip_map/src/gossip_types.rs @@ -0,0 +1,168 @@ +//! Gossip map types implementations. +use std::fmt::Debug; +use std::io::Read; +use std::{collections::HashMap, io::BufRead, str::Bytes, vec::Vec}; + +use bitcoin::PublicKey; +use fundamentals::core::FromWire; +use fundamentals::types::ShortChannelId; +use fundamentals_derive::DecodeWire; + +use crate::bolt7::{ChannelAnnouncement, ChannelUpdate, NodeAnnouncement}; +use crate::gossip_stor_wiregen::GossipStoreChannelAmount; + +trait GossipType { + /// Decode the gossip message from a sequence of bytes. + fn decode(stream: &mut dyn BufRead) -> Result + where + Self: Sized; + + /// Encode the gossip message in a sequence of bytes. + fn encode(&self) -> Bytes; +} + +/// Node Id encoded for the gossip map +#[derive(Eq, Hash, PartialEq, Debug, Clone)] +pub struct GossipNodeId { + pub(crate) node_id: String, +} + +impl From<&str> for GossipNodeId { + fn from(value: &str) -> Self { + Self { + node_id: value.to_owned(), + } + } +} + +impl GossipNodeId { + pub(crate) fn from_bytes(buff: &[u8]) -> std::io::Result { + Ok(GossipNodeId { + node_id: PublicKey::from_slice(buff).unwrap().to_string(), + }) + } +} + +#[derive(Clone)] +pub struct GossipNode { + node_id: GossipNodeId, + announced: bool, + raw_message: Option, + channels: Vec, +} + +impl Debug for GossipNode { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + writeln!(f, "node_id: {:?}", self.node_id)?; + writeln!(f, "announced: {:?}", self.announced) + } +} + +impl GossipNode { + pub fn new(node_id: GossipNodeId, inner: Option) -> Self { + Self { + node_id, + // FIXME: this can be optional right? for + // private channel we do not have one. + raw_message: inner, + channels: vec![], + announced: true, + } + } +} + +impl GossipNode { + /// add a gossip channel inside the gossip map. + pub fn add_channel(&mut self, channel: &GossipChannel) { + self.channels.push(channel.clone()); + } +} + +/// Channel Information stored inside the Gossip Map. +#[derive(Clone)] +pub struct GossipChannel { + inner: ChannelAnnouncement, + annound_offset: u32, + scid: ShortChannelId, + node_one: GossipNodeId, + node_two: GossipNodeId, + update_fields: Vec>, + update_offset: Vec, + satoshi: Option, + half_channels: HashMap, + private: bool, +} + +impl Debug for GossipChannel { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + writeln!(f, "node_id_1: {:?}", self.node_one)?; + writeln!(f, "node_id_2: {:?}", self.node_two) + } +} + +impl GossipChannel { + pub fn new( + inner: ChannelAnnouncement, + node_one: &GossipNodeId, + node_two: &GossipNodeId, + ) -> Self { + GossipChannel { + inner: inner.clone(), + annound_offset: 0, + scid: inner.short_channel_id, + // FIXME: I can store only the ID? + node_one: node_one.clone(), + node_two: node_two.clone(), + update_fields: vec![], + update_offset: vec![], + satoshi: None, + half_channels: HashMap::new(), + private: false, + } + } + + pub fn channel_update(&mut self, channel_update: &ChannelUpdate) { + // FIXME: check how to normalize the BitFlag + let direction = 1; + self.half_channels.insert( + direction, + GossipPartialChannel::new(channel_update.to_owned()), + ); + } + + pub fn set_amount(&mut self, amount: GossipStoreChannelAmount) { + self.satoshi = Some(amount.satoshis.into()); + } + + pub fn set_private(&mut self, private: bool) { + self.private = private; + } +} + +/// One direction gossip map channel +#[derive(Debug, Clone)] +pub struct GossipPartialChannel { + pub inner: ChannelUpdate, +} + +impl GossipPartialChannel { + pub fn new(inner: ChannelUpdate) -> Self { + Self { inner } + } +} + +/// Gossip map header, that contains the version +/// of the gossip map. +#[derive(DecodeWire, Debug)] +pub struct GossipStoredHeader { + flags: u16, + pub len: u16, + crc: u32, + timestamp: u32, +} + +impl GossipStoredHeader { + pub fn flag(&self) -> u16 { + self.flags + } +} diff --git a/gossip_map/src/lib.rs b/gossip_map/src/lib.rs new file mode 100644 index 0000000..6a59ad2 --- /dev/null +++ b/gossip_map/src/lib.rs @@ -0,0 +1,252 @@ +use std::collections::HashMap; +use std::fs::File; +use std::io::{BufReader, Read, Seek, SeekFrom}; + +use fundamentals::core::FromWire; +use fundamentals::types::ShortChannelId; +// Reexport types +pub use fundamentals::*; + +mod bolt7; +mod flags; +mod gossip_stor_wiregen; +mod gossip_types; + +use crate::bolt7::{ChannelAnnouncement, ChannelUpdate, NodeAnnouncement}; +use crate::flags::{ + GOSSIP_STORE_MAJOR_VERSION, GOSSIP_STORE_MAJOR_VERSION_MASK, WIRE_GOSSIP_STORE_CHANNEL_AMOUNT, + WIRE_GOSSIP_STORE_DELETE_CHAN, WIRE_GOSSIP_STORE_ENDED, WIRE_GOSSIP_STORE_PRIVATE_CHANNEL, + WIRE_GOSSIP_STORE_PRIVATE_UPDATE, +}; +use crate::gossip_stor_wiregen::{ + GossipStoreChannelAmount, GossipStoreDeleteChan, GossipStoreEnded, +}; +use crate::gossip_types::{GossipChannel, GossipNode, GossipNodeId, GossipStoredHeader}; + +/// Gossip map implementation, that allow you to manage the gossip_store +/// written by core lightning. +#[derive(Debug)] +struct GossipMap { + // FIXME: make this optional + path: String, + version: u8, + stream: Option>, + nodes: HashMap, + channels: HashMap, + orphan_channel_updates: HashMap, +} + +impl GossipMap { + // Create a new instance of the gossip map. + pub fn new(version: u8) -> Self { + log::debug!("gossip map version `{version}`"); + GossipMap { + path: "".to_owned(), + version, + stream: None, + nodes: HashMap::new(), + channels: HashMap::new(), + orphan_channel_updates: HashMap::new(), + } + } + + pub fn from_file(file_name: &str) -> anyhow::Result { + log::debug!("Loading gossip map from file `{file_name}`"); + let gossip_store = File::open(file_name)?; + let stream = BufReader::new(gossip_store); + let mut gossip_map = GossipMap { + path: file_name.to_owned(), + version: 0, + stream: Some(stream), + nodes: HashMap::new(), + channels: HashMap::new(), + orphan_channel_updates: HashMap::new(), + }; + gossip_map.refresh()?; + Ok(gossip_map) + } + + pub fn get_channel(&self, short_chananel_id: &str) -> Option<&GossipChannel> { + self.channels.get(short_chananel_id.as_bytes()) + } + + pub fn get_node(&self, node_id: &str) -> Option<&GossipNode> { + let node_id = GossipNodeId::from(node_id); + self.nodes.get(&node_id) + } + + /// add a node announcement message inside the gossip map + fn add_node_announcement(&mut self, node_announce: NodeAnnouncement) {} + + /// add a channel announcement message inside the gossip map. + fn add_channel_announcement(&mut self, channel_announce: ChannelAnnouncement) {} + + fn refresh(&mut self) -> anyhow::Result<()> { + let gossip_store = File::open(self.path.clone())?; + let mut stream = BufReader::new(gossip_store); + + let version = u8::from_wire(&mut stream)? as u16; + if (version & GOSSIP_STORE_MAJOR_VERSION_MASK) != GOSSIP_STORE_MAJOR_VERSION { + anyhow::bail!("Invalid gossip store version {version}"); + } + self.version = version as u8; + log::info!("Gossip map version: v{}", self.version); + let mut last_short_channel_id: Option = None; + + while let Ok(header) = GossipStoredHeader::from_wire(&mut stream) { + log::debug!("header {:?}", header); + if (header.flag() & flags::GOSSIP_STORE_LEN_DELETED_BIT) != 0 { + log::warn!("flags::GOSSIP_STORE_LEN_DELETED_BIT"); + // Consume the buffer + let mut inner_stream: Vec = vec![0; header.len.into()]; + stream.read_exact(&mut inner_stream)?; + continue; + } + + let typmsg = u16::from_wire(&mut stream)?; + // fake lookup, because the message will decode the type. + stream.seek(SeekFrom::Current(-2))?; + log::info!("type: {typmsg}"); + match typmsg { + // channel announcement! + 256 => { + log::info!("channel announcement"); + let mut inner_stream: Vec = vec![0; header.len.into()]; + stream.read_exact(&mut inner_stream)?; + let mut inner_stream = inner_stream.as_slice(); + let channel_announcement = ChannelAnnouncement::from_wire(&mut inner_stream)?; + let node_one = + GossipNodeId::from_bytes(&channel_announcement.node_id_1.to_vec()).unwrap(); + let node_two = + GossipNodeId::from_bytes(&channel_announcement.node_id_2.to_vec()).unwrap(); + if !self.nodes.contains_key(&node_one) { + let node = GossipNode::new(node_one.clone(), None); + self.nodes.insert(node_one.clone(), node); + } + + if !self.nodes.contains_key(&node_two) { + let node = GossipNode::new(node_two.clone(), None); + self.nodes.insert(node_two.clone(), node); + } + last_short_channel_id = Some(channel_announcement.short_channel_id); + let channel = GossipChannel::new(channel_announcement, &node_one, &node_two); + // SAFETY: It is safe to unwrap because the node is always present, due the + // previous checks. + let node_one = self.nodes.get_mut(&node_one).unwrap(); + node_one.add_channel(&channel.clone()); + let node_two = self.nodes.get_mut(&node_two).unwrap(); + node_two.add_channel(&channel.clone()); + self.channels + .insert(last_short_channel_id.unwrap(), channel); + } + WIRE_GOSSIP_STORE_PRIVATE_CHANNEL => { + log::info!("private channel announcement"); + unimplemented!(); + } + WIRE_GOSSIP_STORE_CHANNEL_AMOUNT => { + log::info!("gossip store amount"); + let mut inner_stream: Vec = vec![0; header.len.into()]; + stream.read_exact(&mut inner_stream)?; + let mut inner_stream = inner_stream.as_slice(); + let channel_amount = GossipStoreChannelAmount::from_wire(&mut inner_stream)?; + //FIXME: remove the unwrap(). + let channel = self + .channels + .get_mut(&last_short_channel_id.unwrap()) + .unwrap(); + channel.set_amount(channel_amount); + } + WIRE_GOSSIP_STORE_PRIVATE_UPDATE => { + log::info!("private update for channel"); + unimplemented!() + } + WIRE_GOSSIP_STORE_DELETE_CHAN => { + log::info!("delte channel from gossip"); + let _ = GossipStoreDeleteChan::from_wire(&mut stream)?; + unimplemented!() + } + WIRE_GOSSIP_STORE_ENDED => { + log::info!("end of the gossip store"); + let _ = GossipStoreEnded::from_wire(&mut stream)?; + break; + } + 257 => { + let mut inner_stream: Vec = vec![0; header.len.into()]; + stream.read_exact(&mut inner_stream)?; + let mut inner_stream = inner_stream.as_slice(); + log::info!( + "buffer len {} vs expected {}", + inner_stream.len(), + header.len + ); + let node_announcement = NodeAnnouncement::from_wire(&mut inner_stream).unwrap(); + log::trace!("{:?}", node_announcement); + let node_id = GossipNodeId::from_bytes(&node_announcement.node_id)?; + if !self.nodes.contains_key(&node_id) { + let node = GossipNode::new(node_id.clone(), Some(node_announcement)); + self.nodes.insert(node_id, node); + } + } + 258 => { + log::info!("found channel update"); + let mut inner_stream: Vec = vec![0; header.len.into()]; + stream.read_exact(&mut inner_stream)?; + let mut inner_stream = inner_stream.as_slice(); + let channel_update = ChannelUpdate::from_wire(&mut inner_stream)?; + if let Some(channel) = self.channels.get_mut(&channel_update.short_channel_id) { + log::info!( + "found channel with short id `{}`", + hex::encode(channel_update.short_channel_id) + ); + channel.channel_update(&channel_update) + } else { + self.orphan_channel_updates + .insert(channel_update.short_channel_id, channel_update); + } + } + _ => { + log::error!( + "Unexpected message with type `{typmsg}`, breaking: size of nodes {}", + self.nodes.len() + ); + let mut inner_stream: Vec = vec![0; header.len.into()]; + stream.read_exact(&mut inner_stream)?; + continue; + } + } + } + Ok(()) + } +} + +#[cfg(test)] +mod logger; + +#[cfg(test)] +mod tests { + use std::sync::Once; + + use super::*; + + static INIT: Once = Once::new(); + + fn init() { + INIT.call_once(|| { + logger::init(log::Level::Trace).expect("initializing logger for the first time"); + }); + } + + #[test] + fn read_gossipmap_from_file() { + init(); + let path = concat!(env!("CARGO_MANIFEST_DIR"), "/../contrib/gossip_store"); + let pubkey = "03e2408a49f07d2f4083a47344138ef89e7617e63919202c92aa8d49b574a560ae"; + let map = GossipMap::from_file(path); + assert!(map.is_ok(), "{:?}", map); + let map = map.unwrap(); + assert!( + map.get_node(pubkey).is_some(), + "node with id `{pubkey}` not found!" + ); + } +} diff --git a/gossip_map/src/logger.rs b/gossip_map/src/logger.rs new file mode 100644 index 0000000..526d1b5 --- /dev/null +++ b/gossip_map/src/logger.rs @@ -0,0 +1,63 @@ +//! Logging module. +/// +/// Credit to https://github.com/vincenzopalazzo/nakamoto/blob/master/node/src/logger.rs +use std::{io, time::SystemTime}; + +use chrono::prelude::*; +use colored::*; +pub use log::{Level, Log, Metadata, Record, SetLoggerError}; + +struct Logger { + level: Level, +} + +impl Log for Logger { + fn enabled(&self, metadata: &Metadata) -> bool { + metadata.level() <= self.level + } + + fn log(&self, record: &Record) { + if self.enabled(record.metadata()) { + let target = record.target(); + + if record.level() == Level::Error { + write(record, target, io::stderr()); + } else { + write(record, target, io::stdout()); + } + + fn write(record: &log::Record, target: &str, mut stream: impl io::Write) { + let message = format!("{} {} {}", record.level(), target.bold(), record.args()); + let message = match record.level() { + Level::Error => message.red(), + Level::Warn => message.yellow(), + Level::Info => message.normal(), + Level::Debug => message.bright_cyan(), + Level::Trace => message.bright_blue().dimmed(), + }; + + writeln!( + stream, + "{} {}", + DateTime::from(SystemTime::now()) + .to_rfc3339_opts(SecondsFormat::Millis, true) + .white(), + message, + ) + .expect("write shouldn't fail"); + } + } + } + + fn flush(&self) {} +} + +/// Initialize a new logger. +pub fn init(level: Level) -> Result<(), SetLoggerError> { + let logger = Logger { level }; + + log::set_boxed_logger(Box::new(logger))?; + log::set_max_level(level.to_level_filter()); + + Ok(()) +}