diff --git a/.gitignore b/.gitignore index abb984a..3e37bdb 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ CLAUDE.md .deploy /apps/*/out /apps/*/cache +/persistence diff --git a/Cargo.lock b/Cargo.lock index dafe99b..2d777c8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4212,6 +4212,18 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "migrate_persistence" +version = "0.1.0" +dependencies = [ + "clap", + "eyre", + "index-maker", + "serde", + "serde_json", + "symm-core", +] + [[package]] name = "mime" version = "0.3.17" diff --git a/Cargo.toml b/Cargo.toml index 8551085..7f8a9ee 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,6 +16,7 @@ members = [ "apps/anvil_orchestrator", "apps/anvil_provisioner", "apps/index_deployer", + "apps/migrate_persistence", "apps/tracker", "_deprecated/liquidity_tracker", "_deprecated/alloy-evm-connector", diff --git a/apps/migrate_persistence/Cargo.toml b/apps/migrate_persistence/Cargo.toml new file mode 100644 index 0000000..bb8a9e2 --- /dev/null +++ b/apps/migrate_persistence/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "migrate_persistence" +version = "0.1.0" +edition = "2021" + +[dependencies] +eyre = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +clap = { workspace = true} +index-maker = { workspace = true } +symm-core = { workspace = true } diff --git a/apps/migrate_persistence/README.md b/apps/migrate_persistence/README.md new file mode 100644 index 0000000..eb97eec --- /dev/null +++ b/apps/migrate_persistence/README.md @@ -0,0 +1,2 @@ +### usage +cargo run --release -- --file path_to_file/old_invoice.json \ No newline at end of file diff --git a/apps/migrate_persistence/src/main.rs b/apps/migrate_persistence/src/main.rs new file mode 100644 index 0000000..e1af729 --- /dev/null +++ b/apps/migrate_persistence/src/main.rs @@ -0,0 +1,114 @@ +use clap::Parser; +use eyre::Result; +use std::path::PathBuf; +use std::sync::Arc; + +// Import from your index-maker project +use index_maker::solver::mint_invoice_manager::MintInvoiceManager; +use symm_core::core::persistence::util::JsonFilePersistence; + +/// Migrate MintInvoiceManager from old format to new composite persistence format +#[derive(Parser, Debug)] +#[command(author, version, about, long_about = None)] +struct Args { + /// Path to the InvoiceManager.json file + #[arg(short, long)] + file: PathBuf, + + /// Dry run - don't write any files + #[arg(short, long, default_value_t = false)] + dry_run: bool, +} + +fn main() -> Result<()> { + let args = Args::parse(); + + println!("=== MintInvoiceManager Migration Tool ===\n"); + println!("File: {:?}", args.file); + + if args.dry_run { + println!("Mode: DRY RUN (no files will be written)\n"); + } else { + println!("Mode: LIVE (files will be written)\n"); + } + + // Check if file exists + if !args.file.exists() { + eprintln!("Error: File not found: {:?}", args.file); + std::process::exit(1); + } + + // Read the file + println!("Reading file..."); + let file_content = std::fs::read_to_string(&args.file)?; + let root_value: serde_json::Value = serde_json::from_str(&file_content)?; + + // Check if already migrated + if root_value.get("metadata").is_some() { + println!("✓ File is already in new format (has 'metadata' key)."); + println!("✓ Nothing to do."); + return Ok(()); + } + + // Parse old format + let invoices_array = match root_value.get("invoices") { + Some(inv) => inv, + None => { + eprintln!("Error: No 'invoices' key found. Unknown format."); + std::process::exit(1); + } + }; + + use index_maker::solver::mint_invoice::MintInvoice; + use symm_core::core::bits::Address; + + let old_invoices: Vec<(u32, Address, MintInvoice)> = + serde_json::from_value(invoices_array.clone())?; + + println!("Found {} invoices in old format\n", old_invoices.len()); + + if args.dry_run { + // Just show what would happen + for (i, (chain_id, address, invoice)) in old_invoices.iter().enumerate() { + println!("[{}/{}] Would process: {}", + i + 1, + old_invoices.len(), + invoice.client_order_id + ); + println!(" -> Chain: {}, Address: {}", chain_id, address); + } + + println!("\n Run without --dry-run to apply changes"); + return Ok(()); + } + + // Create backup + let backup_path = args.file.with_extension("json.backup"); + println!("Creating backup: {:?}", backup_path); + std::fs::copy(&args.file, &backup_path)?; + println!("✓ Backup created\n"); + + // Use MintInvoiceManager to do the migration + // This automatically uses the correct key format and child path logic + let persistence = Arc::new(JsonFilePersistence::new(&args.file)); + let mut manager = MintInvoiceManager::new(persistence); + + // Add each invoice - this creates child files automatically + for (i, (chain_id, address, invoice)) in old_invoices.into_iter().enumerate() { + // This will: + // 1. Create child file with full invoice data + // 2. Add metadata entry + // 3. Update root file + manager.add_invoice(chain_id, address, invoice)?; + + println!(" Done"); + } + + println!(); + println!("=== Migration Summary ==="); + println!(" Updated root file with metadata"); + println!(" Backup saved as: {:?}", backup_path); + println!("\n Migration complete!"); + + Ok(()) +} \ No newline at end of file diff --git a/libs/symm-core/src/core/persistence.rs b/libs/symm-core/src/core/persistence.rs index 7be2432..4aa2217 100644 --- a/libs/symm-core/src/core/persistence.rs +++ b/libs/symm-core/src/core/persistence.rs @@ -9,10 +9,11 @@ pub trait Persist { pub trait Persistence { fn load_value(&self) -> Result>; fn store_value(&self, value: Value) -> Result<()>; + fn child(&self, key: String) -> Result>; // To allow nested persistence } pub mod util { - use std::{fs, path::PathBuf}; + use std::{collections::HashMap, fs, path::PathBuf, sync::Arc}; use eyre::{Context, Result}; use parking_lot::RwLock; @@ -23,20 +24,35 @@ pub mod util { ; pub struct InMemoryPersistence { - data: RwLock>, + data: Arc>>, // Shared storage with keys + prefix: String, // Namespace for this instance } impl InMemoryPersistence { pub fn new() -> Self { Self { - data: RwLock::new(None), + data: Arc::new(RwLock::new(HashMap::new())), + prefix: String::new(), + } + } + + fn new_with_data(data: Arc>>, prefix: String) -> Self { + Self { data, prefix } + } + + fn get_key(&self) -> String { + if self.prefix.is_empty() { + "root".to_string() + } else { + self.prefix.clone() } } } impl Persistence for InMemoryPersistence { fn load_value(&self) -> Result> { - if let Some(json_string) = self.data.read().as_ref() { + let key = self.get_key(); + if let Some(json_string) = self.data.read().get(&key) { Ok(Some( serde_json::from_str(json_string).context("Failed to deserialize")?, )) @@ -46,10 +62,23 @@ pub mod util { } fn store_value(&self, value: Value) -> Result<()> { + let key = self.get_key(); let json_string = serde_json::to_string_pretty(&value)?; - *self.data.write() = Some(json_string); + self.data.write().insert(key, json_string); Ok(()) } + + fn child(&self, key: String) -> Result> { + let child_prefix = if self.prefix.is_empty() { + key + } else { + format!("{}/{}", self.prefix, key) + }; + Ok(Box::new(InMemoryPersistence::new_with_data( + Arc::clone(&self.data), + child_prefix, + ))) + } } pub struct JsonFilePersistence { @@ -60,6 +89,19 @@ pub mod util { pub fn new(path: impl Into) -> Self { JsonFilePersistence { path: path.into() } } + + fn create_child_path(&self, key: &str) -> PathBuf { + let mut parent_path = self.path.clone(); + + // If parent has .json extension, remove it to use as directory + if parent_path.extension().is_some() { + parent_path.set_extension(""); + } + + // Add key as filename with .json extension + parent_path.push(format!("{}.json", key)); + parent_path + } } impl Persistence for JsonFilePersistence { @@ -85,5 +127,10 @@ pub mod util { let json_string = serde_json::to_string_pretty(&value).context("Failed to serialize")?; fs::write(&self.path, &json_string).context("Failed to write json file") } + + fn child(&self, key: String) -> Result> { + let child_path = self.create_child_path(&key); + Ok(Box::new(JsonFilePersistence::new(child_path))) + } } } diff --git a/src/solver/mint_invoice_manager.rs b/src/solver/mint_invoice_manager.rs index 0444227..1f926bf 100644 --- a/src/solver/mint_invoice_manager.rs +++ b/src/solver/mint_invoice_manager.rs @@ -1,12 +1,7 @@ -use std::{ - collections::{HashMap, VecDeque}, - ops::Deref, - sync::Arc, -}; +use std::sync::Arc; use alloy_primitives::U256; use chrono::{DateTime, Utc}; -use itertools::Itertools; use serde::{Deserialize, Serialize}; use serde_json::json; use symm_core::core::{ @@ -14,9 +9,49 @@ use symm_core::core::{ persistence::{Persist, Persistence}, }; -use crate::{ - collateral::collateral_position::CollateralPosition, solver::mint_invoice::MintInvoice, -}; +use crate::solver::mint_invoice::MintInvoice; + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct InvoiceMetadata { + pub chain_id: u32, + pub address: Address, + pub client_order_id: ClientOrderId, + pub payment_id: PaymentId, + pub seq_num: U256, + pub symbol: Symbol, + pub filled_quantity: Amount, + pub total_amount: Amount, + pub amount_paid: Amount, + pub amount_remaining: Amount, + pub management_fee: Amount, + pub assets_value: Amount, + pub exchange_fee: Amount, + pub fill_rate: Amount, + pub timestamp: DateTime, + // NOTE: No lots, no position - those are stored in individual files +} + +impl From<&MintInvoice> for InvoiceMetadata { + fn from(invoice: &MintInvoice) -> Self { + Self { + chain_id: 0, // Will be set by add_invoice + address: Address::default(), // Will be set by add_invoice + client_order_id: invoice.client_order_id.clone(), + payment_id: invoice.payment_id.clone(), + seq_num: invoice.seq_num, + symbol: invoice.symbol.clone(), + filled_quantity: invoice.filled_quantity.clone(), + total_amount: invoice.total_amount, + amount_paid: invoice.amount_paid, + amount_remaining: invoice.amount_remaining, + management_fee: invoice.management_fee, + assets_value: invoice.assets_value, + exchange_fee: invoice.exchange_fee, + fill_rate: invoice.fill_rate, + timestamp: invoice.timestamp, + } + } +} #[derive(Serialize, Deserialize)] pub struct GetInvoiceData { @@ -46,117 +81,360 @@ pub struct GetInvoicesData { pub struct MintInvoiceManager { persistence: Arc, - user_invoices: HashMap<(u32, Address), VecDeque>>, - all_invoices: VecDeque<(u32, Address, Arc)>, + + // Store ALL metadata in memory (loaded from root json) + // This is lightweight - no lots, no positions + metadata: Vec, } impl MintInvoiceManager { pub fn new(persistence: Arc) -> Self { Self { persistence, - user_invoices: HashMap::new(), - all_invoices: VecDeque::new(), + metadata: Vec::new(), } } + /// Create a unique key for an invoice + fn create_invoice_key( + chain_id: u32, + address: &Address, + client_order_id: &ClientOrderId, + ) -> String { + format!( + "chain_{}/address_{}/order_{}", + chain_id, + address, + client_order_id + ) + } + + /// Add new invoice pub fn add_invoice( &mut self, chain_id: u32, address: Address, invoice: MintInvoice, ) -> eyre::Result<()> { - let invoices = self.user_invoices.entry((chain_id, address)).or_default(); - let invoice = Arc::new(invoice); - invoices.push_back(invoice.clone()); - self.all_invoices.push_back((chain_id, address, invoice)); + // 1. Store full invoice (with lots & positions) in individual file + let key = Self::create_invoice_key(chain_id, &address, &invoice.client_order_id); + let child_persistence = self.persistence.child(key)?; + + let invoice_value = serde_json::to_value(&invoice)?; + child_persistence.store_value(invoice_value)?; + + // 2. Add metadata (without lots & positions) to root + let mut meta = InvoiceMetadata::from(&invoice); + meta.chain_id = chain_id; + meta.address = address; + self.metadata.push(meta); + + // 3. Save root json (small - just metadata) + self.store()?; + Ok(()) } + /// Get specific invoice (loads full data including lots & positions) pub fn get_invoice( &self, chain_id: u32, address: Address, client_order_id: &ClientOrderId, ) -> eyre::Result> { - let Some(invoices) = self.user_invoices.get(&(chain_id, address)) else { - return Ok(None); - }; + // Check if invoice exists in metadata + let exists = self.metadata.iter().any(|m| + m.chain_id == chain_id + && m.address == address + && m.client_order_id == *client_order_id + ); - let Some(invoice) = invoices - .iter() - .find(|x| x.client_order_id.eq(client_order_id)) - else { + if !exists { return Ok(None); - }; + } - Ok(Some(GetInvoiceData { - chain_id, - address, - invoice: invoice.deref().clone(), - })) + // Load full invoice from individual file + let key = Self::create_invoice_key(chain_id, &address, client_order_id); + let child_persistence = self.persistence.child(key)?; + + if let Some(invoice_value) = child_persistence.load_value()? { + let invoice: MintInvoice = serde_json::from_value(invoice_value)?; + + Ok(Some(GetInvoiceData { + chain_id, + address, + invoice, + })) + } else { + Ok(None) + } } + /// Query invoices by date range (uses metadata only - no file loading!) pub fn get_invoices_in_date_range( &self, from_date: DateTime, to_date: DateTime, ) -> eyre::Result> { - let invoices = self - .all_invoices + // Filter metadata only - no file I/O needed! + let results = self + .metadata .iter() - .rev() - .take_while_inclusive(|(.., x)| from_date <= x.timestamp) - .map(|(chain_id, address, invoice)| (chain_id, address, invoice)) - .collect_vec(); - - let invoices = invoices - .into_iter() - .rev() - .take_while_inclusive(|(.., x)| x.timestamp <= to_date) - .map(|(chain_id, address, invoice)| GetInvoicesData { - chain_id: *chain_id, - address: *address, - client_order_id: invoice.client_order_id.clone(), - symbol: invoice.symbol.clone(), - payment_id: invoice.payment_id.clone(), - seq_num: invoice.seq_num, - filled_quantity: invoice.filled_quantity.clone(), - total_amount: invoice.total_amount, - amount_paid: invoice.amount_paid, - amount_remaining: invoice.amount_remaining, - management_fee: invoice.management_fee, - assets_value: invoice.assets_value, - exchange_fee: invoice.exchange_fee, - fill_rate: invoice.fill_rate, - timestamp: invoice.timestamp, + .filter(|m| m.timestamp >= from_date && m.timestamp <= to_date) + .map(|m| GetInvoicesData { + chain_id: m.chain_id, + address: m.address, + client_order_id: m.client_order_id.clone(), + payment_id: m.payment_id.clone(), + seq_num: m.seq_num, + symbol: m.symbol.clone(), + filled_quantity: m.filled_quantity.clone(), + total_amount: m.total_amount, + amount_paid: m.amount_paid, + amount_remaining: m.amount_remaining, + management_fee: m.management_fee, + assets_value: m.assets_value, + exchange_fee: m.exchange_fee, + fill_rate: m.fill_rate, + timestamp: m.timestamp, }) - .collect_vec(); + .collect(); - Ok(invoices) + Ok(results) } } impl Persist for MintInvoiceManager { + /// Load metadata from root json fn load(&mut self) -> eyre::Result<()> { - self.user_invoices = HashMap::new(); - self.all_invoices = VecDeque::new(); - + self.metadata = Vec::new(); if let Some(mut value) = self.persistence.load_value()? { - if let Some(invoices) = value.get_mut("invoices") { - let all_invoices: Vec<(u32, Address, MintInvoice)> = - serde_json::from_value(invoices.take())?; - - for (chain_id, address, invoice) in all_invoices { - self.add_invoice(chain_id, address, invoice)? - } + if let Some(metadata) = value.get_mut("metadata") { + self.metadata = serde_json::from_value(metadata.take())?; } } Ok(()) } + /// Store metadata to root json fn store(&self) -> eyre::Result<()> { + // Store only metadata (no lots, no positions) self.persistence - .store_value(json!({"invoices": self.all_invoices}))?; + .store_value(json!({ "metadata": self.metadata }))?; + Ok(()) } } + + +#[cfg(test)] +mod tests { + use super::*; + use std::path::PathBuf; + use symm_core::core::persistence::util::JsonFilePersistence; + + #[test] + fn test_load_new_format() -> eyre::Result<()> { + let file_path = PathBuf::from("persistence/InvoiceManager.json"); + + if !file_path.exists() { + println!("Skipping test - InvoiceManager.json not found"); + return Ok(()); + } + + let persistence = Arc::new(JsonFilePersistence::new(&file_path)); + let mut manager = MintInvoiceManager::new(persistence); + + manager.load()?; + + println!("Loaded {} invoices", manager.metadata.len()); + + assert!(manager.metadata.len() >= 3, "Should have loaded at least 3 invoices"); + + for meta in &manager.metadata { + assert_eq!(meta.chain_id, 8453); + println!(" - {}: {}", meta.client_order_id, meta.symbol); + } + + Ok(()) + } + + #[test] + fn test_get_invoice_with_full_data() -> eyre::Result<()> { + let file_path = PathBuf::from("persistence/InvoiceManager.json"); + + if !file_path.exists() { + println!("Skipping test - InvoiceManager.json not found"); + return Ok(()); + } + + let persistence = Arc::new(JsonFilePersistence::new(&file_path)); + let mut manager = MintInvoiceManager::new(persistence); + manager.load()?; + + if let Some(first_meta) = manager.metadata.first() { + println!("Retrieving invoice: {}", first_meta.client_order_id); + + let result = manager.get_invoice( + first_meta.chain_id, + first_meta.address, + &first_meta.client_order_id + )?; + + assert!(result.is_some(), "Should retrieve invoice"); + + let invoice_data = result.unwrap(); + assert_eq!(invoice_data.invoice.client_order_id, first_meta.client_order_id); + assert_eq!(invoice_data.chain_id, first_meta.chain_id); + + println!(" Invoice: {}", invoice_data.invoice.client_order_id); + println!(" Symbol: {}", invoice_data.invoice.symbol); + println!(" Amount: {}", invoice_data.invoice.total_amount); + println!(" Lots: {}", invoice_data.invoice.lots.len()); + } + + Ok(()) + } + + #[test] + fn test_query_by_date_range() -> eyre::Result<()> { + let file_path = PathBuf::from("persistence/InvoiceManager.json"); + + if !file_path.exists() { + println!("Skipping test - InvoiceManager.json not found"); + return Ok(()); + } + + let persistence = Arc::new(JsonFilePersistence::new(&file_path)); + let mut manager = MintInvoiceManager::new(persistence); + manager.load()?; + + let from = chrono::DateTime::parse_from_rfc3339("2025-11-01T00:00:00Z") + .unwrap() + .with_timezone(&Utc); + let to = chrono::DateTime::parse_from_rfc3339("2025-12-01T00:00:00Z") + .unwrap() + .with_timezone(&Utc); + + let results = manager.get_invoices_in_date_range(from, to)?; + + println!("Found {} invoices in November 2025", results.len()); + assert!(results.len() >= 3); + + for invoice_data in &results { + println!(" - {} ({}) at {}", + invoice_data.client_order_id, + invoice_data.symbol, + invoice_data.timestamp + ); + } + + Ok(()) + } + + #[test] + fn test_get_specific_invoice() -> eyre::Result<()> { + let file_path = PathBuf::from("persistence/InvoiceManager.json"); + + if !file_path.exists() { + println!("Skipping test - InvoiceManager.json not found"); + return Ok(()); + } + + let persistence = Arc::new(JsonFilePersistence::new(&file_path)); + let mut manager = MintInvoiceManager::new(persistence); + manager.load()?; + + // Get known invoice + let chain_id = 8453u32; + let address: Address = "0xc0d3c9e530ca6d71469bb678e6592274154d9cad".parse()?; + let client_order_id: ClientOrderId = "JYA-NIF-EYJ-9644".into(); + + let result = manager.get_invoice(chain_id, address, &client_order_id)?; + + assert!(result.is_some(), "Should find invoice JYA-NIF-EYJ-9644"); + + let invoice_data = result.unwrap(); + assert_eq!(invoice_data.invoice.symbol.to_string(), "SY100"); + assert_eq!(invoice_data.invoice.client_order_id, "JYA-NIF-EYJ-9644".into()); + + println!(" Found invoice JYA-NIF-EYJ-9644"); + println!(" Payment ID: {}", invoice_data.invoice.payment_id); + println!(" Lots: {}", invoice_data.invoice.lots.len()); + + Ok(()) + } + + #[test] + fn test_all_invoices_have_lots() -> eyre::Result<()> { + let file_path = PathBuf::from("persistence/InvoiceManager.json"); + + if !file_path.exists() { + println!("Skipping test - InvoiceManager.json not found"); + return Ok(()); + } + + let persistence = Arc::new(JsonFilePersistence::new(&file_path)); + let mut manager = MintInvoiceManager::new(persistence); + manager.load()?; + + // Get all invoices and verify they have lots + for meta in manager.metadata.clone() { + let result = manager.get_invoice( + meta.chain_id, + meta.address, + &meta.client_order_id + )?; + + if let Some(invoice_data) = result { + assert!(invoice_data.invoice.lots.len() > 0, + "Invoice {} should have lots", meta.client_order_id); + + println!("✓ {}: {} lots", + invoice_data.invoice.client_order_id, + invoice_data.invoice.lots.len() + ); + } + } + + Ok(()) + } + + #[test] + fn test_child_file_structure() -> eyre::Result<()> { + let file_path = PathBuf::from("persistence/InvoiceManager.json"); + + if !file_path.exists() { + println!("Skipping test - InvoiceManager.json not found"); + return Ok(()); + } + + let persistence = Arc::new(JsonFilePersistence::new(&file_path)); + let mut manager = MintInvoiceManager::new(persistence); + manager.load()?; + + // Verify child files exist + if let Some(first_meta) = manager.metadata.first() { + let key = MintInvoiceManager::create_invoice_key( + first_meta.chain_id, + &first_meta.address, + &first_meta.client_order_id + ); + + // Expected path format + let expected_path = format!("persistence/InvoiceManager/{}.json", key); + println!("Expected child file: {}", expected_path); + + // Verify we can load it + let result = manager.get_invoice( + first_meta.chain_id, + first_meta.address, + &first_meta.client_order_id + )?; + + assert!(result.is_some(), "Child file should exist and be loadable"); + } + + Ok(()) + } +} \ No newline at end of file