feat(gossip_map): add gossip_map implementation

Cargo.toml

@@ -7,6 +7,7 @@ members = [
         "conf",
         "testing",
         "tests",
+        "gossip_map",
 ]
 resolver = "2"
 

Makefile

@@ -1,22 +1,46 @@
 CC=cargo
 FMT=fmt
 
-OPTIONS=
+# Please keep me update with the https://github.com/ElementsProject/lightning/blob/master/Makefile#L29
+BOLTDIR := /tmp/
+OPTIONS := ""
 
 default: fmt
 	$(CC) build --all-features
 	@make example
 
-fmt:
+fmt: ## Format the file
 	$(CC) fmt --all
 
-check:
+check: ## Run all the tests inside the workspace
 	@make default
 	$(CC) test -- --show-output
 
-example:
+example: # build the examples
 	$(CC) build --example foo_plugin
 	$(CC) build --example macros_ex
 
-clean:
+clean: ## Clean the file
 	$(CC) clean
+
+# FIXME: we should apply the diff over the csv file.
+genfile: check_compiler ## Generate the file from a new main version of the spec, in addition apply potential patch
+	cd "$(BOLTDIR)" && git clone https://github.com/lightning/bolts.git
+	cd /tmp && git clone https://github.com/ElementsProject/lightning.git
+	cd $(BOLTDIR)/bolts && tools/extract-formats.py 07-routing-gossip.md > bolt7.csv
+	cp /tmp/lightning/gossipd/gossip_store_wire.csv gossip_map/spec/gossip_store_wire.csv
+	cp /tmp/bolts/bolt7.csv gossip_map/spec/bolt7.csv
+	patch gossip_map/spec/bolt7.csv gossip_map/spec/bolt7.diff
+	@make update-genfile
+
+update-genfile: ## update the local generated file without fetch a new specification, just the generated file.
+	lncodegen-cli -l rust generate -b gossip_map/spec/gossip_store_wire.csv gossip_map/src/gossip_stor_wiregen.rs
+	lncodegen-cli -l rust generate -b gossip_map/spec/bolt7.csv gossip_map/src/bolt7.rs
+
+
+check_compiler: ## Check if the lncodegen exist or need to be installed (required the rust toolchain).
+	@command -v lncodegen-cli --help >/dev/null 2>&1 || (echo "`lncodegen-cli` not found, installing..." && cargo install lncodegen-cli --git https://github.com/lnspec-tools/lncodegen.git)
+
+help: ## Show Help
+	@grep --no-filename -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | \
+	awk 'BEGIN {FS = ":.*?## "}; {printf "\033[32m%-15s\033[0m %s\n", $$1, $$2}'

gossip_map/Cargo.toml

@@ -0,0 +1,19 @@
+[package]
+name = "clightningrpc_gossip_map"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+anyhow = "1"
+log = "0.4.21"
+byteorder = "1.4.3"
+hex = "0.4.3"
+bitcoin = { version = "0.30.0" }
+fundamentals = { git = "https://github.com/lnspec-tools/ln-fundamentals.git", branch = "macros/fix_fixed_read" }
+fundamentals-derive = { git = "https://github.com/lnspec-tools/ln-fundamentals.git", branch = "macros/fix_fixed_read" }
+
+[dev-dependencies]
+anyhow = "1.0.70"
+colored = "1.9"
+log = { version = "0.4", features = ["std"] }
+chrono = { version = "0.4", features = ["std"], default-features = false }

gossip_map/spec/bolt7.csv

@@ -0,0 +1,84 @@
+msgtype,announcement_signatures,259
+msgdata,announcement_signatures,channel_id,channel_id,
+msgdata,announcement_signatures,short_channel_id,short_channel_id,
+msgdata,announcement_signatures,node_signature,signature,
+msgdata,announcement_signatures,bitcoin_signature,signature,
+msgtype,channel_announcement,256
+msgdata,channel_announcement,node_signature_1,signature,
+msgdata,channel_announcement,node_signature_2,signature,
+msgdata,channel_announcement,bitcoin_signature_1,signature,
+msgdata,channel_announcement,bitcoin_signature_2,signature,
+msgdata,channel_announcement,len,u16,
+msgdata,channel_announcement,features,byte,len
+msgdata,channel_announcement,chain_hash,chain_hash,
+msgdata,channel_announcement,short_channel_id,short_channel_id,
+msgdata,channel_announcement,node_id_1,point,
+msgdata,channel_announcement,node_id_2,point,
+msgdata,channel_announcement,bitcoin_key_1,point,
+msgdata,channel_announcement,bitcoin_key_2,point,
+msgtype,node_announcement,257
+msgdata,node_announcement,signature,signature,
+msgdata,node_announcement,flen,u16,
+msgdata,node_announcement,features,byte,flen
+msgdata,node_announcement,timestamp,u32,
+msgdata,node_announcement,node_id,point,
+msgdata,node_announcement,rgb_color,byte,3
+msgdata,node_announcement,alias,byte,32
+msgdata,node_announcement,addrlen,u16,
+msgdata,node_announcement,addresses,byte,addrlen
+msgdata,node_announcement,tlvs,node_ann_tlvs,
+tlvtype,node_ann_tlvs,option_will_fund,1
+tlvdata,node_ann_tlvs,option_will_fund,lease_rates,lease_rates,
+msgtype,channel_update,258
+msgdata,channel_update,signature,signature,
+msgdata,channel_update,chain_hash,chain_hash,
+msgdata,channel_update,short_channel_id,short_channel_id,
+msgdata,channel_update,timestamp,u32,
+msgdata,channel_update,message_flags,byte,
+msgdata,channel_update,channel_flags,byte,
+msgdata,channel_update,cltv_expiry_delta,u16,
+msgdata,channel_update,htlc_minimum_msat,u64,
+msgdata,channel_update,fee_base_msat,u32,
+msgdata,channel_update,fee_proportional_millionths,u32,
+msgdata,channel_update,htlc_maximum_msat,u64,
+msgtype,query_short_channel_ids,261
+msgdata,query_short_channel_ids,chain_hash,chain_hash,
+msgdata,query_short_channel_ids,len,u16,
+msgdata,query_short_channel_ids,encoded_short_ids,byte,len
+msgdata,query_short_channel_ids,tlvs,query_short_channel_ids_tlvs,
+tlvtype,query_short_channel_ids_tlvs,query_flags,1
+tlvdata,query_short_channel_ids_tlvs,query_flags,encoding_type,byte,
+tlvdata,query_short_channel_ids_tlvs,query_flags,encoded_query_flags,byte,...
+msgtype,reply_short_channel_ids_end,262
+msgdata,reply_short_channel_ids_end,chain_hash,chain_hash,
+msgdata,reply_short_channel_ids_end,full_information,byte,
+msgtype,query_channel_range,263
+msgdata,query_channel_range,chain_hash,chain_hash,
+msgdata,query_channel_range,first_blocknum,u32,
+msgdata,query_channel_range,number_of_blocks,u32,
+msgdata,query_channel_range,tlvs,query_channel_range_tlvs,
+tlvtype,query_channel_range_tlvs,query_option,1
+tlvdata,query_channel_range_tlvs,query_option,query_option_flags,bigsize,
+msgtype,reply_channel_range,264
+msgdata,reply_channel_range,chain_hash,chain_hash,
+msgdata,reply_channel_range,first_blocknum,u32,
+msgdata,reply_channel_range,number_of_blocks,u32,
+msgdata,reply_channel_range,sync_complete,byte,
+msgdata,reply_channel_range,len,u16,
+msgdata,reply_channel_range,encoded_short_ids,byte,len
+msgdata,reply_channel_range,tlvs,reply_channel_range_tlvs,
+tlvtype,reply_channel_range_tlvs,timestamps_tlv,1
+tlvdata,reply_channel_range_tlvs,timestamps_tlv,encoding_type,byte,
+tlvdata,reply_channel_range_tlvs,timestamps_tlv,encoded_timestamps,byte,...
+tlvtype,reply_channel_range_tlvs,checksums_tlv,3
+tlvdata,reply_channel_range_tlvs,checksums_tlv,checksums,channel_update_checksums,...
+subtype,channel_update_timestamps
+subtypedata,channel_update_timestamps,timestamp_node_id_1,u32,
+subtypedata,channel_update_timestamps,timestamp_node_id_2,u32,
+subtype,channel_update_checksums
+subtypedata,channel_update_checksums,checksum_node_id_1,u32,
+subtypedata,channel_update_checksums,checksum_node_id_2,u32,
+msgtype,gossip_timestamp_filter,265
+msgdata,gossip_timestamp_filter,chain_hash,chain_hash,
+msgdata,gossip_timestamp_filter,first_timestamp,u32,
+msgdata,gossip_timestamp_filter,timestamp_range,u32,

gossip_map/spec/bolt7.diff

@@ -0,0 +1,14 @@
+diff --git a/gossip_map/spec/bolt7.csv b/gossip_map/spec/bolt7.csv
+index 35a2dc6..865a3fa 100644
+--- a/gossip_map/spec/bolt7.csv
++++ b/gossip_map/spec/bolt7.csv
+@@ -26,6 +26,9 @@ msgdata,node_announcement,rgb_color,byte,3
+ msgdata,node_announcement,alias,byte,32
+ msgdata,node_announcement,addrlen,u16,
+ msgdata,node_announcement,addresses,byte,addrlen
++msgdata,node_announcement,tlvs,node_ann_tlvs,
++tlvtype,node_ann_tlvs,option_will_fund,1
++tlvdata,node_ann_tlvs,option_will_fund,lease_rates,lease_rates,
+ msgtype,channel_update,258
+ msgdata,channel_update,signature,signature,
+ msgdata,channel_update,chain_hash,chain_hash,

gossip_map/spec/gossip_store_wire.csv

@@ -0,0 +1,29 @@
+# gossip_store messages: messages persisted in the gossip_store
+# We store raw messages here, so these numbers must not overlap with
+# 256/257/258.
+#include <common/amount.h>
+#include <common/node_id.h>
+
+# This always follows the channel_announce / private_announce
+msgtype,gossip_store_channel_amount,4101
+msgdata,gossip_store_channel_amount,satoshis,amount_sat,
+
+# Mimics a channel_announce, except signatures are all-zero
+msgtype,gossip_store_private_channel_obs,4104
+msgdata,gossip_store_private_channel_obs,satoshis,amount_sat,
+msgdata,gossip_store_private_channel_obs,len,u16,
+msgdata,gossip_store_private_channel_obs,announcement,u8,len
+
+msgtype,gossip_store_private_update_obs,4102
+msgdata,gossip_store_private_update_obs,len,u16,
+msgdata,gossip_store_private_update_obs,update,u8,len
+
+msgtype,gossip_store_delete_chan,4103
+msgdata,gossip_store_delete_chan,scid,short_channel_id,
+
+msgtype,gossip_store_ended,4105
+msgdata,gossip_store_ended,equivalent_offset,u64,
+
+msgtype,gossip_store_chan_dying,4106
+msgdata,gossip_store_chan_dying,scid,short_channel_id,
+msgdata,gossip_store_chan_dying,blockheight,u32,

gossip_map/src/bolt7.rs

@@ -0,0 +1,135 @@
+// code generated with the lngen, please not edit this file.
+use std::io::{Read, Write};
+
+use fundamentals_derive::{DecodeWire, EncodeWire};
+
+use crate::core::{FromWire, ToWire};
+use crate::prelude::*;
+
+#[derive(DecodeWire, EncodeWire, Debug, Clone)]
+pub struct AnnouncementSignatures {
+    #[msg_type = 259]
+    pub ty: u16,
+    pub channel_id: ChannelId,
+    pub short_channel_id: ShortChannelId,
+    pub node_signature: Signature,
+    pub bitcoin_signature: Signature,
+}
+
+#[derive(DecodeWire, EncodeWire, Debug, Clone)]
+pub struct ChannelAnnouncement {
+    #[msg_type = 256]
+    pub ty: u16,
+    pub node_signature_1: Signature,
+    pub node_signature_2: Signature,
+    pub bitcoin_signature_1: Signature,
+    pub bitcoin_signature_2: Signature,
+    pub features: BitFlag,
+    pub chain_hash: ChainHash,
+    pub short_channel_id: ShortChannelId,
+    pub node_id_1: Point,
+    pub node_id_2: Point,
+    pub bitcoin_key_1: Point,
+    pub bitcoin_key_2: Point,
+}
+
+#[derive(DecodeWire, EncodeWire, Debug, Clone)]
+pub struct ChannelUpdate {
+    #[msg_type = 258]
+    pub ty: u16,
+    pub signature: Signature,
+    pub chain_hash: ChainHash,
+    pub short_channel_id: ShortChannelId,
+    pub timestamp: u32,
+    // FIXME: these are u8 but the codegen will decode it to BitFlag
+    pub message_flags: u8,
+    pub channel_flags: u8,
+    pub cltv_expiry_delta: u16,
+    pub htlc_minimum_msat: u64,
+    pub fee_base_msat: u32,
+    pub fee_proportional_millionths: u32,
+    pub htlc_maximum_msat: u64,
+}
+
+#[derive(DecodeWire, EncodeWire, Debug, Clone)]
+pub struct GossipTimestampFilter {
+    #[msg_type = 265]
+    pub ty: u16,
+    pub chain_hash: ChainHash,
+    pub first_timestamp: u32,
+    pub timestamp_range: u32,
+}
+
+macro_rules! to_wire_type_with_size {
+    ($ty: ty, $size: expr) => {
+        impl ToWire for $ty {
+            fn to_wire<W: Write>(&self, buff: &mut W) -> std::io::Result<()> {
+                buff.write_all(self)
+            }
+        }
+
+        impl FromWire for $ty {
+            fn from_wire<R: Read>(reader: &mut R) -> std::io::Result<Self> {
+                let mut buff = [0; $size];
+                reader.read_exact(&mut buff)?;
+                Ok(buff)
+            }
+        }
+    };
+}
+
+pub type Alias = [u8; 32];
+pub type Rgb = [u8; 3];
+
+#[derive(DecodeWire, EncodeWire, Debug, Clone)]
+pub struct NodeAnnouncement {
+    #[msg_type = 257]
+    pub ty: u16,
+    pub signature: Signature,
+    pub features: BitFlag,
+    pub timestamp: u32,
+    pub node_id: Point,
+    pub rgb_color: Rgb,
+    pub alias: Alias,
+    pub addresses: BitFlag,
+    pub node_ann_tlvs: Stream,
+}
+
+#[derive(DecodeWire, EncodeWire, Debug, Clone)]
+pub struct QueryChannelRange {
+    #[msg_type = 263]
+    pub ty: u16,
+    pub chain_hash: ChainHash,
+    pub first_blocknum: u32,
+    pub number_of_blocks: u32,
+    pub query_channel_range_tlvs: Stream,
+}
+
+#[derive(DecodeWire, EncodeWire, Debug, Clone)]
+pub struct QueryShortChannelIds {
+    #[msg_type = 261]
+    pub ty: u16,
+    pub chain_hash: ChainHash,
+    pub encoded_short_ids: BitFlag,
+    pub query_short_channel_ids_tlvs: Stream,
+}
+
+#[derive(DecodeWire, EncodeWire, Debug, Clone)]
+pub struct ReplyChannelRange {
+    #[msg_type = 264]
+    pub ty: u16,
+    pub chain_hash: ChainHash,
+    pub first_blocknum: u32,
+    pub number_of_blocks: u32,
+    pub sync_complete: BitFlag,
+    pub encoded_short_ids: BitFlag,
+    pub reply_channel_range_tlvs: Stream,
+}
+
+#[derive(DecodeWire, EncodeWire, Debug, Clone)]
+pub struct ReplyShortChannelIdsEnd {
+    #[msg_type = 262]
+    pub ty: u16,
+    pub chain_hash: ChainHash,
+    pub full_information: BitFlag,
+}

gossip_map/src/flags.rs

@@ -0,0 +1,20 @@
+//! Flag implementation for the gossip map types
+
+pub static GOSSIP_STORE_MAJOR_VERSION: u16 = 0 << 5;
+pub static GOSSIP_STORE_MAJOR_VERSION_MASK: u16 = 0xE0;
+
+/// Deleted fields should be ignored: on restart, they will be removed as the gossip_store is rewritten.
+pub const GOSSIP_STORE_LEN_DELETED_BIT: u16 = 0x8000;
+/// The push flag indicates gossip which is generated locally: this is important for gossip timestamp filtering,
+/// where peers request gossip and we always send our own gossip messages even if the timestamp wasn't within their
+pub const GOSSIP_STORE_LEN_PUSH_BIT: u16 = 0x4000;
+/// The ratelimit flag indicates that this gossip message came too fast.
+/// The message are corded in the gossip map, but don't relay it to peers.
+pub const GOSSIP_STORE_LEN_RATELIMIT_BIT: u16 = 0x2000;
+
+// These duplicate constants in lightning/gossipd/gossip_store_wiregen.h
+pub const WIRE_GOSSIP_STORE_PRIVATE_CHANNEL: u16 = 4104;
+pub const WIRE_GOSSIP_STORE_PRIVATE_UPDATE: u16 = 4102;
+pub const WIRE_GOSSIP_STORE_DELETE_CHAN: u16 = 4103;
+pub const WIRE_GOSSIP_STORE_ENDED: u16 = 4105;
+pub const WIRE_GOSSIP_STORE_CHANNEL_AMOUNT: u16 = 4101;