From 48451c17d959c4390f92f415666800afad6717bc Mon Sep 17 00:00:00 2001
From: Marc Nijdam <marc_nijdam@mac.com>
Date: Tue, 18 Jun 2024 18:38:56 -0500
Subject: [PATCH] Parse merkle tree account for canopy height (#377)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Not all merkle trees are represented in the known canopy height shadow drive. In case it’s not found we need to actually downlad the merkle tree, parse the header and compute the canopy height
---
 Cargo.lock                | 235 +++++++++++++++++++++++++++++++++-----
 helium-lib/Cargo.toml     |   1 +
 helium-lib/src/asset.rs   |  67 +++++++----
 helium-lib/src/hotspot.rs |   2 +-
 4 files changed, 254 insertions(+), 51 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 16688fa2..3340f628 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -156,6 +156,18 @@ dependencies = [
  "alloc-no-stdlib",
 ]
 
+[[package]]
+name = "anchor-attribute-access-control"
+version = "0.29.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e5f619f1d04f53621925ba8a2e633ba5a6081f2ae14758cbb67f38fd823e0a3e"
+dependencies = [
+ "anchor-syn 0.29.0",
+ "proc-macro2",
+ "quote",
+ "syn 1.0.109",
+]
+
 [[package]]
 name = "anchor-attribute-access-control"
 version = "0.30.0"
@@ -168,6 +180,19 @@ dependencies = [
  "syn 1.0.109",
 ]
 
+[[package]]
+name = "anchor-attribute-account"
+version = "0.29.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e7f2a3e1df4685f18d12a943a9f2a7456305401af21a07c9fe076ef9ecd6e400"
+dependencies = [
+ "anchor-syn 0.29.0",
+ "bs58 0.5.1",
+ "proc-macro2",
+ "quote",
+ "syn 1.0.109",
+]
+
 [[package]]
 name = "anchor-attribute-account"
 version = "0.30.0"
@@ -181,6 +206,17 @@ dependencies = [
  "syn 1.0.109",
 ]
 
+[[package]]
+name = "anchor-attribute-constant"
+version = "0.29.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9423945cb55627f0b30903288e78baf6f62c6c8ab28fb344b6b25f1ffee3dca7"
+dependencies = [
+ "anchor-syn 0.29.0",
+ "quote",
+ "syn 1.0.109",
+]
+
 [[package]]
 name = "anchor-attribute-constant"
 version = "0.30.0"
@@ -192,6 +228,17 @@ dependencies = [
  "syn 1.0.109",
 ]
 
+[[package]]
+name = "anchor-attribute-error"
+version = "0.29.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "93ed12720033cc3c3bf3cfa293349c2275cd5ab99936e33dd4bf283aaad3e241"
+dependencies = [
+ "anchor-syn 0.29.0",
+ "quote",
+ "syn 1.0.109",
+]
+
 [[package]]
 name = "anchor-attribute-error"
 version = "0.30.0"
@@ -203,6 +250,18 @@ dependencies = [
  "syn 1.0.109",
 ]
 
+[[package]]
+name = "anchor-attribute-event"
+version = "0.29.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "eef4dc0371eba2d8c8b54794b0b0eb786a234a559b77593d6f80825b6d2c77a2"
+dependencies = [
+ "anchor-syn 0.29.0",
+ "proc-macro2",
+ "quote",
+ "syn 1.0.109",
+]
+
 [[package]]
 name = "anchor-attribute-event"
 version = "0.30.0"
@@ -215,6 +274,17 @@ dependencies = [
  "syn 1.0.109",
 ]
 
+[[package]]
+name = "anchor-attribute-program"
+version = "0.29.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b18c4f191331e078d4a6a080954d1576241c29c56638783322a18d308ab27e4f"
+dependencies = [
+ "anchor-syn 0.29.0",
+ "quote",
+ "syn 1.0.109",
+]
+
 [[package]]
 name = "anchor-attribute-program"
 version = "0.30.0"
@@ -238,7 +308,7 @@ version = "0.30.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b4adc1b211826d72036dc2fcb679a8ef7fe5b9afda376b0b26debe19e28de3ea"
 dependencies = [
- "anchor-lang",
+ "anchor-lang 0.30.0",
  "anyhow",
  "futures",
  "regex",
@@ -251,6 +321,17 @@ dependencies = [
  "url",
 ]
 
+[[package]]
+name = "anchor-derive-accounts"
+version = "0.29.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5de10d6e9620d3bcea56c56151cad83c5992f50d5960b3a9bebc4a50390ddc3c"
+dependencies = [
+ "anchor-syn 0.29.0",
+ "quote",
+ "syn 1.0.109",
+]
+
 [[package]]
 name = "anchor-derive-accounts"
 version = "0.30.0"
@@ -262,6 +343,19 @@ dependencies = [
  "syn 1.0.109",
 ]
 
+[[package]]
+name = "anchor-derive-serde"
+version = "0.29.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f4e2e5be518ec6053d90a2a7f26843dbee607583c779e6c8395951b9739bdfbe"
+dependencies = [
+ "anchor-syn 0.29.0",
+ "borsh-derive-internal 0.10.3",
+ "proc-macro2",
+ "quote",
+ "syn 1.0.109",
+]
+
 [[package]]
 name = "anchor-derive-serde"
 version = "0.30.0"
@@ -275,6 +369,17 @@ dependencies = [
  "syn 1.0.109",
 ]
 
+[[package]]
+name = "anchor-derive-space"
+version = "0.29.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1ecc31d19fa54840e74b7a979d44bcea49d70459de846088a1d71e87ba53c419"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 1.0.109",
+]
+
 [[package]]
 name = "anchor-derive-space"
 version = "0.30.0"
@@ -332,21 +437,46 @@ dependencies = [
  "syn 1.0.109",
 ]
 
+[[package]]
+name = "anchor-lang"
+version = "0.29.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "35da4785497388af0553586d55ebdc08054a8b1724720ef2749d313494f2b8ad"
+dependencies = [
+ "anchor-attribute-access-control 0.29.0",
+ "anchor-attribute-account 0.29.0",
+ "anchor-attribute-constant 0.29.0",
+ "anchor-attribute-error 0.29.0",
+ "anchor-attribute-event 0.29.0",
+ "anchor-attribute-program 0.29.0",
+ "anchor-derive-accounts 0.29.0",
+ "anchor-derive-serde 0.29.0",
+ "anchor-derive-space 0.29.0",
+ "arrayref",
+ "base64 0.13.1",
+ "bincode",
+ "borsh 0.10.3",
+ "bytemuck",
+ "getrandom 0.2.15",
+ "solana-program",
+ "thiserror",
+]
+
 [[package]]
 name = "anchor-lang"
 version = "0.30.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e41feb9c1cd9f4b0fad1c004fc8f289183f3ce27e9db38fa6e434470c716fb1e"
 dependencies = [
- "anchor-attribute-access-control",
- "anchor-attribute-account",
- "anchor-attribute-constant",
- "anchor-attribute-error",
- "anchor-attribute-event",
- "anchor-attribute-program",
- "anchor-derive-accounts",
- "anchor-derive-serde",
- "anchor-derive-space",
+ "anchor-attribute-access-control 0.30.0",
+ "anchor-attribute-account 0.30.0",
+ "anchor-attribute-constant 0.30.0",
+ "anchor-attribute-error 0.30.0",
+ "anchor-attribute-event 0.30.0",
+ "anchor-attribute-program 0.30.0",
+ "anchor-derive-accounts 0.30.0",
+ "anchor-derive-serde 0.30.0",
+ "anchor-derive-space 0.30.0",
  "arrayref",
  "base64 0.21.7",
  "bincode",
@@ -374,7 +504,7 @@ version = "0.30.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0dcee54a30b27ea8317ca647759b5d9701a8c7caaaa0c922c6d3c306a7278a7a"
 dependencies = [
- "anchor-lang",
+ "anchor-lang 0.30.0",
  "spl-associated-token-account 3.0.2",
  "spl-pod 0.2.2",
  "spl-token",
@@ -402,6 +532,24 @@ dependencies = [
  "thiserror",
 ]
 
+[[package]]
+name = "anchor-syn"
+version = "0.29.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d9101b84702fed2ea57bd22992f75065da5648017135b844283a2f6d74f27825"
+dependencies = [
+ "anyhow",
+ "bs58 0.5.1",
+ "heck 0.3.3",
+ "proc-macro2",
+ "quote",
+ "serde",
+ "serde_json",
+ "sha2 0.10.8",
+ "syn 1.0.109",
+ "thiserror",
+]
+
 [[package]]
 name = "anchor-syn"
 version = "0.30.0"
@@ -924,7 +1072,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4114279215a005bc675e386011e594e1d9b800918cea18fcadadcce864a2046b"
 dependencies = [
  "borsh-derive 0.10.3",
- "hashbrown 0.12.3",
+ "hashbrown 0.13.2",
 ]
 
 [[package]]
@@ -1222,7 +1370,7 @@ version = "0.1.0"
 source = "git+https://github.com/helium/helium-anchor-gen.git#fe60ed1d49e9255bd779a99bdd7928f278c07256"
 dependencies = [
  "anchor-gen",
- "anchor-lang",
+ "anchor-lang 0.30.0",
 ]
 
 [[package]]
@@ -1703,7 +1851,7 @@ version = "0.2.1"
 source = "git+https://github.com/helium/helium-anchor-gen.git#fe60ed1d49e9255bd779a99bdd7928f278c07256"
 dependencies = [
  "anchor-gen",
- "anchor-lang",
+ "anchor-lang 0.30.0",
 ]
 
 [[package]]
@@ -2019,7 +2167,7 @@ version = "0.1.0"
 source = "git+https://github.com/helium/helium-anchor-gen.git#fe60ed1d49e9255bd779a99bdd7928f278c07256"
 dependencies = [
  "anchor-gen",
- "anchor-lang",
+ "anchor-lang 0.30.0",
 ]
 
 [[package]]
@@ -2370,7 +2518,7 @@ version = "0.1.0"
 source = "git+https://github.com/helium/helium-anchor-gen.git#fe60ed1d49e9255bd779a99bdd7928f278c07256"
 dependencies = [
  "anchor-gen",
- "anchor-lang",
+ "anchor-lang 0.30.0",
  "circuit-breaker",
  "data-credits",
  "fanout",
@@ -2416,7 +2564,7 @@ version = "0.3.1"
 source = "git+https://github.com/helium/helium-anchor-gen.git#fe60ed1d49e9255bd779a99bdd7928f278c07256"
 dependencies = [
  "anchor-gen",
- "anchor-lang",
+ "anchor-lang 0.30.0",
 ]
 
 [[package]]
@@ -2451,6 +2599,7 @@ dependencies = [
  "solana-program",
  "solana-sdk",
  "solana-transaction-status",
+ "spl-account-compression",
  "spl-associated-token-account 3.0.2",
  "thiserror",
  "tracing",
@@ -2486,7 +2635,7 @@ version = "0.1.5"
 source = "git+https://github.com/helium/helium-anchor-gen.git#fe60ed1d49e9255bd779a99bdd7928f278c07256"
 dependencies = [
  "anchor-gen",
- "anchor-lang",
+ "anchor-lang 0.30.0",
 ]
 
 [[package]]
@@ -2545,7 +2694,7 @@ version = "0.0.5"
 source = "git+https://github.com/helium/helium-anchor-gen.git#fe60ed1d49e9255bd779a99bdd7928f278c07256"
 dependencies = [
  "anchor-gen",
- "anchor-lang",
+ "anchor-lang 0.30.0",
 ]
 
 [[package]]
@@ -2908,7 +3057,7 @@ version = "0.1.0"
 source = "git+https://github.com/helium/helium-anchor-gen.git#fe60ed1d49e9255bd779a99bdd7928f278c07256"
 dependencies = [
  "anchor-gen",
- "anchor-lang",
+ "anchor-lang 0.30.0",
 ]
 
 [[package]]
@@ -2917,7 +3066,7 @@ version = "0.2.0"
 source = "git+https://github.com/helium/helium-anchor-gen.git#fe60ed1d49e9255bd779a99bdd7928f278c07256"
 dependencies = [
  "anchor-gen",
- "anchor-lang",
+ "anchor-lang 0.30.0",
 ]
 
 [[package]]
@@ -3125,7 +3274,7 @@ version = "0.1.2"
 source = "git+https://github.com/helium/helium-anchor-gen.git#fe60ed1d49e9255bd779a99bdd7928f278c07256"
 dependencies = [
  "anchor-gen",
- "anchor-lang",
+ "anchor-lang 0.30.0",
 ]
 
 [[package]]
@@ -3380,7 +3529,7 @@ version = "0.7.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "681030a937600a36906c185595136d26abfebb4aa9c65701cefcaf8578bb982b"
 dependencies = [
- "proc-macro-crate 1.1.3",
+ "proc-macro-crate 3.1.0",
  "proc-macro2",
  "quote",
  "syn 2.0.66",
@@ -3651,7 +3800,7 @@ version = "0.2.1"
 source = "git+https://github.com/helium/helium-anchor-gen.git#fe60ed1d49e9255bd779a99bdd7928f278c07256"
 dependencies = [
  "anchor-gen",
- "anchor-lang",
+ "anchor-lang 0.30.0",
 ]
 
 [[package]]
@@ -4142,7 +4291,7 @@ version = "0.2.0"
 source = "git+https://github.com/helium/helium-anchor-gen.git#fe60ed1d49e9255bd779a99bdd7928f278c07256"
 dependencies = [
  "anchor-gen",
- "anchor-lang",
+ "anchor-lang 0.30.0",
 ]
 
 [[package]]
@@ -5559,6 +5708,18 @@ dependencies = [
  "der",
 ]
 
+[[package]]
+name = "spl-account-compression"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "85c43bd4455d9fb29b9e4f83c087ccffa2f6f41fecfc0549932ae391d00f3378"
+dependencies = [
+ "anchor-lang 0.29.0",
+ "bytemuck",
+ "spl-concurrent-merkle-tree",
+ "spl-noop",
+]
+
 [[package]]
 name = "spl-associated-token-account"
 version = "2.3.0"
@@ -5591,6 +5752,17 @@ dependencies = [
  "thiserror",
 ]
 
+[[package]]
+name = "spl-concurrent-merkle-tree"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "141eaea58588beae81b71d101373a53f096737739873de42d6b1368bc2b8fc30"
+dependencies = [
+ "bytemuck",
+ "solana-program",
+ "thiserror",
+]
+
 [[package]]
 name = "spl-discriminator"
 version = "0.1.1"
@@ -5670,6 +5842,15 @@ dependencies = [
  "solana-program",
 ]
 
+[[package]]
+name = "spl-noop"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6dd67ea3d0070a12ff141f5da46f9695f49384a03bce1203a5608f5739437950"
+dependencies = [
+ "solana-program",
+]
+
 [[package]]
 name = "spl-pod"
 version = "0.1.1"
@@ -6341,7 +6522,7 @@ version = "0.2.0"
 source = "git+https://github.com/helium/helium-anchor-gen.git#fe60ed1d49e9255bd779a99bdd7928f278c07256"
 dependencies = [
  "anchor-gen",
- "anchor-lang",
+ "anchor-lang 0.30.0",
 ]
 
 [[package]]
@@ -6527,7 +6708,7 @@ version = "0.3.1"
 source = "git+https://github.com/helium/helium-anchor-gen.git#fe60ed1d49e9255bd779a99bdd7928f278c07256"
 dependencies = [
  "anchor-gen",
- "anchor-lang",
+ "anchor-lang 0.30.0",
 ]
 
 [[package]]
diff --git a/helium-lib/Cargo.toml b/helium-lib/Cargo.toml
index ea5b490a..818ed118 100644
--- a/helium-lib/Cargo.toml
+++ b/helium-lib/Cargo.toml
@@ -31,6 +31,7 @@ reqwest = { version = "0", default-features = false, features = [
 ] }
 helium-anchor-gen = {git = "https://github.com/helium/helium-anchor-gen.git"}
 spl-associated-token-account = { version = "*", features = ["no-entrypoint"] }
+spl-account-compression = { version = "0.3", features = ["no-entrypoint"] }
 mpl-bubblegum = "1"
 solana-program = "*"
 solana-transaction-status = "*"
diff --git a/helium-lib/src/asset.rs b/helium-lib/src/asset.rs
index 25722e01..e860411b 100644
--- a/helium-lib/src/asset.rs
+++ b/helium-lib/src/asset.rs
@@ -38,24 +38,47 @@ pub async fn for_kta_with_proof(
     Ok((asset, asset_proof))
 }
 
-pub async fn get_canopy_heights() -> Result<HashMap<Pubkey, usize>> {
-    const KNOWN_CANOPY_HEIGHT_URL: &str = "https://shdw-drive.genesysgo.net/6tcnBSybPG7piEDShBcrVtYJDPSvGrDbVvXmXKpzBvWP/merkles.json";
-    let client = reqwest::Client::new();
-    let map: HashMap<String, usize> = client
-        .get(KNOWN_CANOPY_HEIGHT_URL)
-        .send()
-        .await?
-        .error_for_status()?
-        .json()
-        .await?;
-
-    map.into_iter()
-        .map(|(str, value)| {
-            Pubkey::from_str(str.as_str())
-                .map_err(|err| DecodeError::from(err).into())
-                .map(|key| (key, value))
-        })
-        .try_collect()
+pub mod canopy {
+    use super::*;
+    use spl_account_compression::state::{merkle_tree_get_size, ConcurrentMerkleTreeHeader};
+
+    async fn get_heights() -> Result<HashMap<Pubkey, usize>> {
+        const KNOWN_CANOPY_HEIGHT_URL: &str = "https://shdw-drive.genesysgo.net/6tcnBSybPG7piEDShBcrVtYJDPSvGrDbVvXmXKpzBvWP/merkles.json";
+        let client = reqwest::Client::new();
+        let map: HashMap<String, usize> = client
+            .get(KNOWN_CANOPY_HEIGHT_URL)
+            .send()
+            .await?
+            .error_for_status()?
+            .json()
+            .await?;
+
+        map.into_iter()
+            .map(|(str, value)| {
+                Pubkey::from_str(str.as_str())
+                    .map_err(|err| DecodeError::from(err).into())
+                    .map(|key| (key, value))
+            })
+            .try_collect()
+    }
+
+    pub async fn height_for_tree(settings: &Settings, tree: &Pubkey) -> Result<usize> {
+        use helium_anchor_gen::anchor_lang::AnchorDeserialize;
+        if let Some(height) = get_heights().await?.get(tree) {
+            return Ok(*height);
+        }
+        let solana_client = settings.mk_solana_client()?;
+        let tree_account = solana_client.get_account(tree).await?;
+        let header = ConcurrentMerkleTreeHeader::deserialize(&mut &tree_account.data[..])
+            .map_err(|_| DecodeError::other("invalid merkle tree header"))?;
+        let merkle_tree_size = merkle_tree_get_size(&header)
+            .map_err(|_| DecodeError::other("invalid merkle tree header"))?;
+        let canopy_size = tree_account.data.len()
+            - std::mem::size_of::<ConcurrentMerkleTreeHeader>()
+            - merkle_tree_size;
+        let canopy_depth = (canopy_size / 32 + 1).ilog2();
+        Ok(canopy_depth as usize)
+    }
 }
 
 pub mod proof {
@@ -220,13 +243,11 @@ impl AssetProof {
 
     pub async fn proof_for_tree(
         &self,
+        settings: &Settings,
         tree: &Pubkey,
     ) -> Result<Vec<solana_program::instruction::AccountMeta>> {
-        let canopy_heights = get_canopy_heights().await?;
-        let height = canopy_heights
-            .get(tree)
-            .ok_or_else(Error::account_not_found)?;
-        self.proof(Some(*height))
+        let height = canopy::height_for_tree(settings, tree).await?;
+        self.proof(Some(height))
     }
 }
 
diff --git a/helium-lib/src/hotspot.rs b/helium-lib/src/hotspot.rs
index c985956f..a0561151 100644
--- a/helium-lib/src/hotspot.rs
+++ b/helium-lib/src/hotspot.rs
@@ -407,7 +407,7 @@ pub async fn transfer_transaction(
 
     let leaf_delegate = asset.ownership.delegate.unwrap_or(asset.ownership.owner);
     let merkle_tree = asset_proof.tree_id;
-    let remaining_accounts = asset_proof.proof_for_tree(&merkle_tree).await?;
+    let remaining_accounts = asset_proof.proof_for_tree(settings, &merkle_tree).await?;
 
     let transfer = mpl_bubblegum::instructions::Transfer {
         leaf_owner: (asset.ownership.owner, false),