bitcoind: support v28.0's new default "blk*.dat" xor'ing

src/daemon.rs

@@ -1,12 +1,13 @@
 use std::cell::OnceCell;
 use std::collections::{HashMap, HashSet};
-use std::env;
+use std::convert::TryFrom;
 use std::io::{BufRead, BufReader, Lines, Write};
 use std::net::{SocketAddr, TcpStream};
 use std::path::PathBuf;
 use std::str::FromStr;
 use std::sync::{Arc, Mutex};
 use std::time::Duration;
+use std::{env, fs, io};
 
 use base64::prelude::{Engine, BASE64_STANDARD};
 use error_chain::ChainedError;
@@ -393,6 +394,26 @@ impl Daemon {
         Ok(paths)
     }
 
+    /// bitcoind v28.0+ defaults to xor-ing all blk*.dat files with this key,
+    /// stored in the blocks dir.
+    /// See: <https://github.com/bitcoin/bitcoin/pull/28052>
+    pub fn read_blk_file_xor_key(&self) -> Result<Option<[u8; 8]>> {
+        // From: <https://github.com/bitcoin/bitcoin/blob/v28.0/src/node/blockstorage.cpp#L1160>
+        let path = self.blocks_dir.join("xor.dat");
+        let bytes = match fs::read(path) {
+            Ok(bytes) => bytes,
+            Err(err) if err.kind() == io::ErrorKind::NotFound => return Ok(None),
+            Err(err) => return Err(err).chain_err(|| "failed to read daemon xor.dat file"),
+        };
+        let xor_key: [u8; 8] = <[u8; 8]>::try_from(bytes.as_slice()).chain_err(|| {
+            format!(
+                "xor.dat unexpected length: actual: {}, expected: 8",
+                bytes.len()
+            )
+        })?;
+        Ok(Some(xor_key))
+    }
+
     pub fn magic(&self) -> u32 {
         self.network.magic()
     }

src/lib.rs

@@ -1,11 +1,5 @@
 #![recursion_limit = "1024"]
 
-// See https://github.com/romanz/electrs/issues/193 & https://github.com/rust-rocksdb/rust-rocksdb/issues/327
-#[cfg(not(feature = "oldcpu"))]
-extern crate rocksdb;
-#[cfg(feature = "oldcpu")]
-extern crate rocksdb_oldcpu as rocksdb;
-
 #[macro_use]
 extern crate clap;
 #[macro_use]

src/new_index/db.rs

@@ -163,10 +163,7 @@ impl DB {
         rows.sort_unstable_by(|a, b| a.key.cmp(&b.key));
         let mut batch = rocksdb::WriteBatch::default();
         for row in rows {
-            #[cfg(not(feature = "oldcpu"))]
             batch.put(&row.key, &row.value);
-            #[cfg(feature = "oldcpu")]
-            batch.put(&row.key, &row.value).unwrap();
         }
         let do_flush = match flush {
             DBFlush::Enable => true,

src/new_index/fetch.rs

@@ -111,14 +111,15 @@ fn blkfiles_fetcher(
 ) -> Result<Fetcher<Vec<BlockEntry>>> {
     let magic = daemon.magic();
     let blk_files = daemon.list_blk_files()?;
+    let xor_key = daemon.read_blk_file_xor_key()?;
 
     let chan = SyncChannel::new(1);
     let sender = chan.sender();
 
     let mut entry_map: HashMap<BlockHash, HeaderEntry> =
         new_headers.into_iter().map(|h| (*h.hash(), h)).collect();
 
-    let parser = blkfiles_parser(blkfiles_reader(blk_files), magic);
+    let parser = blkfiles_parser(blkfiles_reader(blk_files, xor_key), magic);
     Ok(Fetcher::from(
         chan.into_receiver(),
         spawn_thread("blkfiles_fetcher", move || {
@@ -151,7 +152,7 @@ fn blkfiles_fetcher(
     ))
 }
 
-fn blkfiles_reader(blk_files: Vec<PathBuf>) -> Fetcher<Vec<u8>> {
+fn blkfiles_reader(blk_files: Vec<PathBuf>, xor_key: Option<[u8; 8]>) -> Fetcher<Vec<u8>> {
     let chan = SyncChannel::new(1);
     let sender = chan.sender();
 
@@ -160,8 +161,11 @@ fn blkfiles_reader(blk_files: Vec<PathBuf>) -> Fetcher<Vec<u8>> {
         spawn_thread("blkfiles_reader", move || {
             for path in blk_files {
                 trace!("reading {:?}", path);
-                let blob = fs::read(&path)
+                let mut blob = fs::read(&path)
                     .unwrap_or_else(|e| panic!("failed to read {:?}: {:?}", path, e));
+                if let Some(xor_key) = xor_key {
+                    blkfile_apply_xor_key(xor_key, &mut blob);
+                }
                 sender
                     .send(blob)
                     .unwrap_or_else(|_| panic!("failed to send {:?} contents", path));
@@ -170,6 +174,14 @@ fn blkfiles_reader(blk_files: Vec<PathBuf>) -> Fetcher<Vec<u8>> {
     )
 }
 
+/// By default, bitcoind v28.0+ applies an 8-byte "xor key" over each "blk*.dat"
+/// file. We have xor again to undo this transformation.
+fn blkfile_apply_xor_key(xor_key: [u8; 8], blob: &mut [u8]) {
+    for (i, blob_i) in blob.iter_mut().enumerate() {
+        *blob_i ^= xor_key[i & 0x7];
+    }
+}
+
 fn blkfiles_parser(blobs: Fetcher<Vec<u8>>, magic: u32) -> Fetcher<Vec<SizedBlock>> {
     let chan = SyncChannel::new(1);
     let sender = chan.sender();