remove txid table if txindex is enabled

Cargo.toml

@@ -9,7 +9,7 @@ repository = "https://github.com/romanz/electrs"
 keywords = ["bitcoin", "electrum", "server", "index", "database"]
 documentation = "https://docs.rs/electrs/"
 readme = "README.md"
-edition = "2018"
+edition = "2021"
 build = "build.rs"
 
 [features]
@@ -27,11 +27,11 @@ bitcoincore-rpc = "0.17.0"
 configure_me = "0.4"
 crossbeam-channel = "0.5"
 dirs-next = "2.0"
-env_logger = "0.9"
+env_logger = "0.10"
 log = "0.4"
-parking_lot = "0.11"
+parking_lot = "0.12"
 prometheus = { version = "0.13", optional = true }
-rayon = "1.5"
+rayon = "1.7"
 serde = "1.0"
 serde_derive = "1.0"
 serde_json = "1.0"
@@ -40,17 +40,15 @@ tiny_http = { version = "0.12", optional = true }
 hex_lit = "0.1.1"
 
 [dependencies.electrs-rocksdb]
-# Workaround the following issues:
-# - https://github.com/romanz/electrs/issues/403 (support building on ARM 32-bit)
-# - https://github.com/romanz/electrs/issues/469 (dynamic linking on Debian 11)
-version = "0.15.0-e3"
+version = "0.19.0-e1"
+
 default-features = false
 # ZSTD is used for data compression
 # Snappy is only for checking old DB
 features = ["zstd", "snappy"]
 
 [build-dependencies]
-configure_me_codegen = { version = "0.4.3", default-features = false }
+configure_me_codegen = { version = "0.4.4", default-features = false }
 
 [dev-dependencies]
 tempfile = "3.5"

Dockerfile

@@ -2,7 +2,7 @@
 # The maintainers of electrs are not deeply familiar with Docker, so you should DYOR.
 # If you are not familiar with Docker either it's probably be safer to NOT use it.
 
-FROM debian:bullseye-slim as base
+FROM debian:testing-slim as base
 RUN apt-get update -qqy
 RUN apt-get install -qqy librocksdb-dev curl
 

Dockerfile.ci

@@ -2,7 +2,7 @@
 # The maintainers of electrs are not deeply familiar with Docker, so you should DYOR.
 # If you are not familiar with Docker either it's probably be safer to NOT use it.
 
-FROM debian:bullseye-slim as base
+FROM debian:testing-slim as base
 RUN apt-get update -qqy
 RUN apt-get install -qqy librocksdb-dev wget
 
@@ -36,9 +36,11 @@ RUN bitcoind -version && bitcoin-cli -version
 ### Electrum ###
 # Clone latest Electrum wallet and a few test tools
 WORKDIR /build/
-RUN apt-get install -qqy git libsecp256k1-0 python3-cryptography python3-setuptools python3-pip jq curl
+RUN apt-get install -qqy git libsecp256k1-1 python3-cryptography python3-setuptools python3-venv python3-pip jq curl
 RUN git clone --recurse-submodules https://github.com/spesmilo/electrum/ && cd electrum/ && git log -1
-RUN python3 -m pip install -e electrum/
+RUN python3 -m venv --system-site-packages venv && \
+	venv/bin/pip install -e electrum/ && \
+	ln /build/venv/bin/electrum /usr/bin/electrum
 
 RUN electrum version --offline
 WORKDIR /

doc/install.md

@@ -25,7 +25,7 @@ $ ./target/release/electrs --version  # should print the latest version
 Note for Raspberry Pi 4 owners: the old versions of OS/toolchains produce broken binaries.
 Make sure to use latest OS! (see #226)
 
-Install [recent Rust](https://rustup.rs/) (1.48.0+, `apt install cargo` is preferred for Debian 11),
+Install [recent Rust](https://rustup.rs/) (1.63.0+, `apt install cargo` is preferred for Debian 12),
 [latest Bitcoin Core](https://bitcoincore.org/en/download/) (0.21+)
 and [latest Electrum wallet](https://electrum.org/#download) (4.0+).
 
@@ -52,18 +52,18 @@ The advantages of dynamic linking:
 * Cross compilation is more reliable
 * If another application is also using `rocksdb`, you don't store it on disk and in RAM twice
 
-If you decided to use dynamic linking, you will also need to install the library ([6.11.4 release](https://github.com/facebook/rocksdb/releases/tag/v6.11.4) is required).
-On [Debian 11 (bullseye)](https://packages.debian.org/bullseye/librocksdb-dev) and [Ubuntu 21.04 (hirsute)](https://packages.ubuntu.com/hirsute/librocksdb-dev):
+If you decided to use dynamic linking, you will also need to install the library ([7.8.3 release](https://github.com/facebook/rocksdb/releases/tag/v7.8.3) is required).
+On [Debian 12 (bookworm)](https://packages.debian.org/bookworm/librocksdb-dev) and [Ubuntu 23.04 (lunar)](https://packages.ubuntu.com/lunar/librocksdb-dev):
 
 ```bash
-$ sudo apt install librocksdb-dev=6.11.4-3
+$ sudo apt install librocksdb-dev=7.8.3-2
 ```
 
 For other versions of Debian or Ubuntu, you can build librocksdb and install inside `/usr/local` directory using following command.
 
 ```bash
 $ sudo apt install -y libgflags-dev libsnappy-dev zlib1g-dev libbz2-dev liblz4-dev libzstd-dev
-$ git clone -b v6.11.4 --depth 1 https://github.com/facebook/rocksdb && cd rocksdb
+$ git clone -b v7.8.3 --depth 1 https://github.com/facebook/rocksdb && cd rocksdb
 $ make shared_lib -j $(nproc) && sudo make install-shared
 $ cd .. && rm -r rocksdb
 ```

rust-toolchain

@@ -1 +1 @@
-1.48.0
+1.63.0

src/daemon.rs

@@ -94,6 +94,7 @@ fn rpc_connect(config: &Config) -> Result<Client> {
 pub struct Daemon {
     p2p: Mutex<Connection>,
     rpc: Client,
+    txindex_enabled: bool,
 }
 
 impl Daemon {
@@ -131,13 +132,23 @@ impl Daemon {
             bail!("electrs requires non-pruned bitcoind node");
         }
 
+        let txindex_enabled = rpc.get_index_info()?.txindex.is_some();
+
         let p2p = Mutex::new(Connection::connect(
             config.network,
             config.daemon_p2p_addr,
             metrics,
             config.signet_magic,
         )?);
-        Ok(Self { p2p, rpc })
+        Ok(Self {
+            p2p,
+            rpc,
+            txindex_enabled,
+        })
+    }
+
+    pub fn txindex_enabled(&self) -> bool {
+        self.txindex_enabled
     }
 
     pub(crate) fn estimate_fee(&self, nblocks: u16) -> Result<Option<Amount>> {

src/db.rs

@@ -82,6 +82,8 @@ const DB_PROPERIES: &[&str] = &[
 struct Config {
     compacted: bool,
     format: u64,
+    #[serde(default)]
+    no_txid: bool,
 }
 
 const CURRENT_FORMAT: u64 = 0;
@@ -91,11 +93,15 @@ impl Default for Config {
         Config {
             compacted: false,
             format: CURRENT_FORMAT,
+            no_txid: false,
         }
     }
 }
 
 fn default_opts() -> rocksdb::Options {
+    let mut block_opts = rocksdb::BlockBasedOptions::default();
+    block_opts.set_checksum_type(rocksdb::ChecksumType::CRC32c);
+
     let mut opts = rocksdb::Options::default();
     opts.set_keep_log_file_num(10);
     opts.set_max_open_files(16);
@@ -106,6 +112,7 @@ fn default_opts() -> rocksdb::Options {
     opts.set_disable_auto_compactions(true); // for initial bulk load
     opts.set_advise_random_on_open(false); // bulk load uses sequential I/O
     opts.set_prefix_extractor(rocksdb::SliceTransform::create_fixed_prefix(8));
+    opts.set_block_based_table_factory(&block_opts);
     opts
 }
 
@@ -148,7 +155,7 @@ impl DBStore {
     }
 
     /// Opens a new RocksDB at the specified location.
-    pub fn open(path: &Path, auto_reindex: bool) -> Result<Self> {
+    pub fn open(path: &Path, auto_reindex: bool, no_txid: bool) -> Result<Self> {
         let mut store = Self::open_internal(path)?;
         let config = store.get_config();
         debug!("DB {:?}", config);
@@ -161,10 +168,12 @@ impl DBStore {
                 "unsupported format {} != {}",
                 config.format, CURRENT_FORMAT
             ))
+        } else if config.no_txid && !no_txid {
+            Some("txid column family needed if bitcoind has no txindex".to_owned())
         } else {
             None
         };
-        if let Some(cause) = reindex_cause {
+        if let Some(cause) = &reindex_cause {
             if !auto_reindex {
                 bail!("re-index required due to {}", cause);
             }
@@ -183,10 +192,31 @@ impl DBStore {
             })?;
             store = Self::open_internal(path)?;
             config = Config::default(); // re-init config after dropping DB
+            config.no_txid = no_txid;
         }
         if config.compacted {
             store.start_compactions();
         }
+        if no_txid != config.no_txid {
+            if no_txid {
+                if reindex_cause.is_none() {
+                    info!("Because Bitcoin Core's txindex was enabled, parts of the database are not needed anymore and will be removed");
+                    info!("The DB will shrink by about 25%");
+                }
+                // TODO 'partially indexed' store separate txid tip and 'other' tip?
+                store.db.drop_cf(TXID_CF)?;
+                store.db.create_cf(TXID_CF, &default_opts())?;
+            } /* else {
+                  if reindex_cause.is_none() {
+                      // TODO this path is currently unreachable
+                      // The whole database will be reindexed instead of only 'txid'
+                      info!("Because Bitcoin Core's txindex was disabled, blocks need to be partially reindexed");
+                      info!("The DB will grow by about 33%");
+                  }
+                  // todo!("reindex txid");
+              } */
+            config.no_txid = no_txid;
+        }
         store.set_config(config);
         Ok(store)
     }
@@ -233,15 +263,15 @@ impl DBStore {
         opts.set_prefix_same_as_start(true); // requires .set_prefix_extractor() above.
         self.db
             .iterator_cf_opt(cf, opts, mode)
-            .map(|(key, _value)| key) // values are empty in prefix-scanned CFs
+            .map(|row| row.expect("prefix iterator failed").0) // values are empty in prefix-scanned CFs
     }
 
     pub(crate) fn read_headers(&self) -> Vec<Row> {
         let mut opts = rocksdb::ReadOptions::default();
         opts.fill_cache(false);
         self.db
             .iterator_cf_opt(self.headers_cf(), opts, rocksdb::IteratorMode::Start)
-            .map(|(key, _)| key)
+            .map(|row| row.expect("header iterator failed").0) // extract key from row
             .filter(|key| &key[..] != TIP_KEY) // headers' rows are longer than TIP_KEY
             .collect()
     }
@@ -310,7 +340,7 @@ impl DBStore {
             DB_PROPERIES.iter().filter_map(move |property_name| {
                 let value = self
                     .db
-                    .property_int_value_cf(cf, property_name)
+                    .property_int_value_cf(cf, *property_name)
                     .expect("failed to get property");
                 Some((*cf_name, *property_name, value?))
             })
@@ -360,21 +390,24 @@ mod tests {
     fn test_reindex_new_format() {
         let dir = tempfile::tempdir().unwrap();
         {
-            let store = DBStore::open(dir.path(), false).unwrap();
+            let store = DBStore::open(dir.path(), false, false).unwrap();
             let mut config = store.get_config().unwrap();
             config.format += 1;
             store.set_config(config);
         };
         assert_eq!(
-            DBStore::open(dir.path(), false).err().unwrap().to_string(),
+            DBStore::open(dir.path(), false, false)
+                .err()
+                .unwrap()
+                .to_string(),
             format!(
                 "re-index required due to unsupported format {} != {}",
                 CURRENT_FORMAT + 1,
                 CURRENT_FORMAT
             )
         );
         {
-            let store = DBStore::open(dir.path(), true).unwrap();
+            let store = DBStore::open(dir.path(), true, false).unwrap();
             store.flush();
             let config = store.get_config().unwrap();
             assert_eq!(config.format, CURRENT_FORMAT);
@@ -392,11 +425,14 @@ mod tests {
             db.put(b"F", b"").unwrap(); // insert legacy DB compaction marker (in 'default' column family)
         };
         assert_eq!(
-            DBStore::open(dir.path(), false).err().unwrap().to_string(),
+            DBStore::open(dir.path(), false, false)
+                .err()
+                .unwrap()
+                .to_string(),
             format!("re-index required due to legacy format",)
         );
         {
-            let store = DBStore::open(dir.path(), true).unwrap();
+            let store = DBStore::open(dir.path(), true, false).unwrap();
             store.flush();
             let config = store.get_config().unwrap();
             assert_eq!(config.format, CURRENT_FORMAT);
@@ -406,7 +442,7 @@ mod tests {
     #[test]
     fn test_db_prefix_scan() {
         let dir = tempfile::tempdir().unwrap();
-        let store = DBStore::open(dir.path(), true).unwrap();
+        let store = DBStore::open(dir.path(), true, false).unwrap();
 
         let items: &[&[u8]] = &[
             b"ab",

src/electrum.rs

@@ -137,9 +137,9 @@ impl Rpc {
             metrics::default_duration_buckets(),
         );
 
-        let tracker = Tracker::new(config, metrics)?;
         let signal = Signal::new();
-        let daemon = Daemon::connect(config, signal.exit_flag(), tracker.metrics())?;
+        let daemon = Daemon::connect(config, signal.exit_flag(), &metrics)?;
+        let tracker = Tracker::new(config, metrics, daemon.txindex_enabled())?;
         let cache = Cache::new(tracker.metrics());
         Ok(Self {
             tracker,
@@ -368,7 +368,7 @@ impl Rpc {
             let blockhash = self
                 .tracker
                 .lookup_transaction(&self.daemon, txid)?
-                .map(|(blockhash, _tx)| blockhash);
+                .and_then(|(blockhash, _tx)| blockhash);
             return self.daemon.get_transaction_info(&txid, blockhash);
         }
         if let Some(tx) = self.cache.get_tx(&txid, |tx| serialize_hex(tx)) {
@@ -536,6 +536,7 @@ impl Rpc {
         self.rpc_duration.observe_duration(&call.method, || {
             if self.tracker.status().is_err() {
                 // Allow only a few RPC (for sync status notification) not requiring index DB being compacted.
+                // TODO 'partially indexed' without txid and txindex?
                 match &call.params {
                     Params::BlockHeader(_)
                     | Params::BlockHeaders(_)

src/index.rs

@@ -224,7 +224,7 @@ impl Index {
         daemon.for_blocks(blockhashes, |_blockhash, block| {
             let height = heights.next().expect("unexpected block");
             self.stats.observe_duration("block", || {
-                index_single_block(block, height).extend(&mut batch)
+                index_single_block(block, height, daemon.txindex_enabled()).extend(&mut batch)
             });
             self.stats.height.set("tip", height as f64);
         })?;
@@ -251,13 +251,15 @@ fn db_rows_size(rows: &[Row]) -> usize {
     rows.iter().map(|key| key.len()).sum()
 }
 
-fn index_single_block(block: Block, height: usize) -> IndexResult {
+fn index_single_block(block: Block, height: usize, no_txid: bool) -> IndexResult {
     let mut funding_rows = Vec::with_capacity(block.txdata.iter().map(|tx| tx.output.len()).sum());
     let mut spending_rows = Vec::with_capacity(block.txdata.iter().map(|tx| tx.input.len()).sum());
-    let mut txid_rows = Vec::with_capacity(block.txdata.len());
+    let mut txid_rows = Vec::with_capacity(block.txdata.len()); // capacity to 0 if no_txid?
 
     for tx in &block.txdata {
-        txid_rows.push(TxidRow::row(tx.txid(), height));
+        if !no_txid {
+            txid_rows.push(TxidRow::row(tx.txid(), height));
+        }
 
         funding_rows.extend(
             tx.output

src/status.rs

@@ -376,9 +376,8 @@ impl ScriptHashStatus {
                 let sorted_entries = entries_map
                     .into_iter()
                     .collect::<BTreeMap<usize, TxEntry>>()
-                    .into_iter()
-                    .map(|(_pos, entry)| entry)
-                    .collect::<Vec<TxEntry>>();
+                    .into_values()
+                    .collect();
                 (blockhash, sorted_entries)
             })
             .collect())
@@ -415,7 +414,7 @@ impl ScriptHashStatus {
                 .spent = spent_outpoints;
             cache.add_tx(entry.txid, || entry.tx.clone());
         }
-        result.into_iter().map(|(_txid, entry)| entry).collect()
+        result.into_values().collect()
     }
 
     /// Sync with currently confirmed txs and mempool, downloading non-cached transactions via p2p protocol.