thread 'three_node_cluster_fencing' panicked at fog/ingest/server/tests/three_node_cluster.rs:200:20:
recovery_db.get_highest_known_block_index: R2d2(Error(None))
stack backtrace:
   0: rust_begin_unwind
             at /rustc/ca62d2c445628587660ae48013f460b08b1f5552/library/std/src/panicking.rs:597:5
   1: core::panicking::panic_fmt
             at /rustc/ca62d2c445628587660ae48013f460b08b1f5552/library/core/src/panicking.rs:72:14
   2: core::result::unwrap_failed
             at /rustc/ca62d2c445628587660ae48013f460b08b1f5552/library/core/src/result.rs:1652:5
   3: mc_fog_ingest_server_test_utils::IngestServerTestHelper::wait_till_recovery_db_in_sync_up_to
   4: mc_fog_ingest_server_test_utils::IngestServerTestHelper::wait_till_recovery_db_in_sync
   5: three_node_cluster::__wrapped_three_node_cluster_fencing
   6: core::ops::function::FnOnce::call_once
   7: core::ops::function::FnOnce::call_once
             at /rustc/ca62d2c445628587660ae48013f460b08b1f5552/library/core/src/ops/function.rs:250:5
note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace.
2024-09-26 20:14:38.265859590 UTC DEBG Now creating ingest invocation id, mc.node_id: 8, mc.test_name: three_node_cluster::three_node_cluster_fencing, mc.module: mc_fog_ingest_server::controller, mc.src: fog/ingest/server/src/controller.rs:497
2024-09-26 20:14:38.643485490 UTC INFO publishing report to DB, mc.node_id: 9, mc.test_name: three_node_cluster::three_node_cluster_fencing, mc.module: mc_fog_ingest_server::controller, mc.src: fog/ingest/server/src/controller.rs:1250
2024-09-26 20:14:41.675023944 UTC DEBG Ingesting block #7: 4/4 txs ingested, mc.node_id: 7, mc.test_name: three_node_cluster::three_node_cluster_fencing, mc.module: mc_fog_ingest_server::controller, mc.src: fog/ingest/server/src/controller.rs:587
2024-09-26 20:14:41.675093103 UTC INFO add_block_data, mc.node_id: 7, mc.test_name: three_node_cluster::three_node_cluster_fencing, mc.module: mc_fog_ingest_server::controller, mc.src: fog/ingest/server/src/controller.rs:596
2024-09-26 20:14:42.669498854 UTC DEBG Checking up on peers..., mc.node_id: 7, mc.test_name: three_node_cluster::three_node_cluster_fencing, mc.module: mc_fog_ingest_server::worker, mc.src: fog/ingest/server/src/worker.rs:235
2024-09-26 20:14:47.669886370 UTC DEBG Checking up on peers..., mc.node_id: 7, mc.test_name: three_node_cluster::three_node_cluster_fencing, mc.module: mc_fog_ingest_server::worker, mc.src: fog/ingest/server/src/worker.rs:235
2024-09-26 20:14:50.623294789 UTC DEBG Ingesting block #7: 4/4 txs ingested, mc.node_id: 8, mc.test_name: three_node_cluster::three_node_cluster_fencing, mc.module: mc_fog_ingest_server::controller, mc.src: fog/ingest/server/src/controller.rs:587
2024-09-26 20:14:50.623357376 UTC INFO add_block_data, mc.node_id: 8, mc.test_name: three_node_cluster::three_node_cluster_fencing, mc.module: mc_fog_ingest_server::controller, mc.src: fog/ingest/server/src/controller.rs:596
2024-09-26 20:14:51.617835603 UTC DEBG Checking up on peers..., mc.node_id: 8, mc.test_name: three_node_cluster::three_node_cluster_fencing, mc.module: mc_fog_ingest_server::worker, mc.src: fog/ingest/server/src/worker.rs:235
2024-09-26 20:14:52.670292154 UTC DEBG Checking up on peers..., mc.node_id: 7, mc.test_name: three_node_cluster::three_node_cluster_fencing, mc.module: mc_fog_ingest_server::worker, mc.src: fog/ingest/server/src/worker.rs:235
2024-09-26 20:14:56.618369821 UTC DEBG Checking up on peers..., mc.node_id: 8, mc.test_name: three_node_cluster::three_node_cluster_fencing, mc.module: mc_fog_ingest_server::worker, mc.src: fog/ingest/server/src/worker.rs:235
2024-09-26 20:14:57.460875714 UTC INFO Succeeded writing block 7 to the database, invocation id Some(IngestInvocationId(13)) ingress key CompressedRistrettoPublic(46a86cb69bcb161f50f55029cc31c6545a5bef7faeeca844e8d054ad7571813d), mc.node_id: 7, mc.test_name: three_node_cluster::three_node_cluster_fencing, mc.module: mc_fog_ingest_server::controller, mc.src: fog/ingest/server/src/controller.rs:636
2024-09-26 20:14:57.460948780 UTC DEBG Incremented next block index: 8, mc.node_id: 7, mc.test_name: three_node_cluster::three_node_cluster_fencing, mc.module: mc_fog_ingest_server::controller_state, mc.src: fog/ingest/server/src/controller_state.rs:155
2024-09-26 20:14:57.460999535 UTC DEBG Controller state: { mode: Active, next_block_index: 8, pubkey_expiry_window: 10, ingest_invocation_id: 13, peers: [insecure-igp://0.0.0.0:5038/] }, mc.node_id: 7, mc.test_name: three_node_cluster::three_node_cluster_fencing, mc.module: mc_fog_ingest_server::controller, mc.src: fog/ingest/server/src/controller.rs:647
2024-09-26 20:14:57.461049649 UTC INFO Finished ingesting block #7, mc.node_id: 7, mc.test_name: three_node_cluster::three_node_cluster_fencing, mc.module: mc_fog_ingest_server::controller, mc.src: fog/ingest/server/src/controller.rs:659
2024-09-26 20:14:57.461098099 UTC DEBG Writing state file to StateFile { file_path: "/tmp/.tmp6PE54H/mc-fog-ingest-state-7" }, mc.node_id: 7, mc.test_name: three_node_cluster::three_node_cluster_fencing, mc.module: mc_fog_ingest_server::controller, mc.src: fog/ingest/server/src/controller.rs:1314
2024-09-26 20:14:57.461655169 UTC INFO update was a no-op, mc.test_name: three_node_cluster::three_node_cluster_fencing, mc.module: mc_fog_sql_recovery_db, mc.src: fog/sql_recovery_db/src/lib.rs:1194
2024-09-26 20:14:57.461739647 UTC INFO check ingress key passed, mc.test_name: three_node_cluster::three_node_cluster_fencing, mc.module: mc_fog_sql_recovery_db, mc.src: fog/sql_recovery_db/src/lib.rs:1197
2024-09-26 20:14:57.461778499 UTC INFO Got status for key: IngressPublicKeyStatus { start_block: 7, pubkey_expiry: 17, retired: false, lost: false }, mc.test_name: three_node_cluster::three_node_cluster_fencing, mc.module: mc_fog_sql_recovery_db, mc.src: fog/sql_recovery_db/src/lib.rs:1214
2024-09-26 20:14:57.472329325 UTC DEBG Now creating ingest invocation id, mc.node_id: 9, mc.test_name: three_node_cluster::three_node_cluster_fencing, mc.module: mc_fog_ingest_server::controller, mc.src: fog/ingest/server/src/controller.rs:497
2024-09-26 20:14:57.472988906 UTC INFO Unique constraint violated when adding block 7 for ingest invocation id 14: "duplicate key value violates unique constraint \"ingested_blocks_ingress_public_key_block_number_key\"", mc.test_name: three_node_cluster::three_node_cluster_fencing, mc.module: mc_fog_sql_recovery_db, mc.src: fog/sql_recovery_db/src/lib.rs:627
2024-09-26 20:14:57.473034301 UTC INFO Another active server did work for block 7, we should become idle and back off, mc.node_id: 8, mc.test_name: three_node_cluster::three_node_cluster_fencing, mc.module: mc_fog_ingest_server::controller, mc.src: fog/ingest/server/src/controller.rs:628
2024-09-26 20:14:57.473073124 UTC INFO Server moved to idle from active, was scanning at block index 7, pubkey_expiry_window = 10, mc.node_id: 8, mc.test_name: three_node_cluster::three_node_cluster_fencing, mc.module: mc_fog_ingest_server::controller_state, mc.src: fog/ingest/server/src/controller_state.rs:129
2024-09-26 20:14:57.473111665 UTC INFO Mode switching from Active to Idle, mc.node_id: 8, mc.test_name: three_node_cluster::three_node_cluster_fencing, mc.module: mc_fog_ingest_server::controller_state, mc.src: fog/ingest/server/src/controller_state.rs:254
2024-09-26 20:14:57.487538366 UTC DEBG Ingesting block #7: 4/4 txs ingested, mc.node_id: 9, mc.test_name: three_node_cluster::three_node_cluster_fencing, mc.module: mc_fog_ingest_server::controller, mc.src: fog/ingest/server/src/controller.rs:587
2024-09-26 20:14:57.487601394 UTC INFO add_block_data, mc.node_id: 9, mc.test_name: three_node_cluster::three_node_cluster_fencing, mc.module: mc_fog_ingest_server::controller, mc.src: fog/ingest/server/src/controller.rs:596
2024-09-26 20:14:57.491370039 UTC INFO Unique constraint violated when adding block 7 for ingest invocation id 15: "duplicate key value violates unique constraint \"ingested_blocks_ingress_public_key_block_number_key\"", mc.test_name: three_node_cluster::three_node_cluster_fencing, mc.module: mc_fog_sql_recovery_db, mc.src: fog/sql_recovery_db/src/lib.rs:627
2024-09-26 20:14:57.491431534 UTC INFO Another active server did work for block 7, we should become idle and back off, mc.node_id: 9, mc.test_name: three_node_cluster::three_node_cluster_fencing, mc.module: mc_fog_ingest_server::controller, mc.src: fog/ingest/server/src/controller.rs:628
2024-09-26 20:14:57.491483811 UTC INFO Server moved to idle from active, was scanning at block index 7, pubkey_expiry_window = 10, mc.node_id: 9, mc.test_name: three_node_cluster::three_node_cluster_fencing, mc.module: mc_fog_ingest_server::controller_state, mc.src: fog/ingest/server/src/controller_state.rs:129
2024-09-26 20:14:57.491530258 UTC INFO Mode switching from Active to Idle, mc.node_id: 9, mc.test_name: three_node_cluster::three_node_cluster_fencing, mc.module: mc_fog_ingest_server::controller_state, mc.src: fog/ingest/server/src/controller_state.rs:254
2024-09-26 20:14:57.492114178 UTC DEBG Writing state file to StateFile { file_path: "/tmp/.tmpRpMdhc/mc-fog-ingest-state-8" }, mc.node_id: 8, mc.test_name: three_node_cluster::three_node_cluster_fencing, mc.module: mc_fog_ingest_server::controller, mc.src: fog/ingest/server/src/controller.rs:1314
2024-09-26 20:14:57.500775577 UTC DEBG Controller state: { mode: Idle, next_block_index: 7, pubkey_expiry_window: 10, peers: [insecure-igp://0.0.0.0:5043/] }, mc.node_id: 8, mc.test_name: three_node_cluster::three_node_cluster_fencing, mc.module: mc_fog_ingest_server::controller, mc.src: fog/ingest/server/src/controller.rs:647
2024-09-26 20:14:57.500843634 UTC INFO Finished ingesting block #7, mc.node_id: 8, mc.test_name: three_node_cluster::three_node_cluster_fencing, mc.module: mc_fog_ingest_server::controller, mc.src: fog/ingest/server/src/controller.rs:659
2024-09-26 20:14:57.500896312 UTC DEBG Writing state file to StateFile { file_path: "/tmp/.tmpRpMdhc/mc-fog-ingest-state-8" }, mc.node_id: 8, mc.test_name: three_node_cluster::three_node_cluster_fencing, mc.module: mc_fog_ingest_server::controller, mc.src: fog/ingest/server/src/controller.rs:1314
2024-09-26 20:14:57.500947087 UTC DEBG Writing state file to StateFile { file_path: "/tmp/.tmpyoUFdJ/mc-fog-ingest-state-9" }, mc.node_id: 9, mc.test_name: three_node_cluster::three_node_cluster_fencing, mc.module: mc_fog_ingest_server::controller, mc.src: fog/ingest/server/src/controller.rs:1314
2024-09-26 20:14:57.511176111 UTC DEBG Controller state: { mode: Idle, next_block_index: 7, pubkey_expiry_window: 10, peers: [insecure-igp://0.0.0.0:5048/] }, mc.node_id: 9, mc.test_name: three_node_cluster::three_node_cluster_fencing, mc.module: mc_fog_ingest_server::controller, mc.src: fog/ingest/server/src/controller.rs:647
2024-09-26 20:14:57.511223089 UTC INFO Finished ingesting block #7, mc.node_id: 9, mc.test_name: three_node_cluster::three_node_cluster_fencing, mc.module: mc_fog_ingest_server::controller, mc.src: fog/ingest/server/src/controller.rs:659
2024-09-26 20:14:57.511357310 UTC DEBG Writing state file to StateFile { file_path: "/tmp/.tmpyoUFdJ/mc-fog-ingest-state-9" }, mc.node_id: 9, mc.test_name: three_node_cluster::three_node_cluster_fencing, mc.module: mc_fog_ingest_server::controller, mc.src: fog/ingest/server/src/controller.rs:1314
2024-09-26 20:14:57.525168141 UTC INFO Stop Requested: Polling loop stopped at block number 7, is_idle true, mc.node_id: 9, mc.test_name: three_node_cluster::three_node_cluster_fencing, mc.module: mc_fog_ingest_server::worker, mc.src: fog/ingest/server/src/worker.rs:75
2024-09-26 20:14:57.670784538 UTC DEBG Checking up on peers..., mc.node_id: 7, mc.test_name: three_node_cluster::three_node_cluster_fencing, mc.module: mc_fog_ingest_server::worker, mc.src: fog/ingest/server/src/worker.rs:235
2024-09-26 20:14:58.482127237 UTC DEBG Peer checkup thread stop requested., mc.node_id: 9, mc.test_name: three_node_cluster::three_node_cluster_fencing, mc.module: mc_fog_ingest_server::worker, mc.src: fog/ingest/server/src/worker.rs:229
2024-09-26 20:14:58.791757525 UTC DEBG Report cache thread stop requested., mc.node_id: 9, mc.test_name: three_node_cluster::three_node_cluster_fencing, mc.module: mc_fog_ingest_server::worker, mc.src: fog/ingest/server/src/worker.rs:289
2024-09-26 20:14:59.011597208 UTC INFO Stop Requested: Polling loop stopped at block number 7, is_idle true, mc.node_id: 8, mc.test_name: three_node_cluster::three_node_cluster_fencing, mc.module: mc_fog_ingest_server::worker, mc.src: fog/ingest/server/src/worker.rs:75
2024-09-26 20:14:59.618637533 UTC DEBG Peer checkup thread stop requested., mc.node_id: 8, mc.test_name: three_node_cluster::three_node_cluster_fencing, mc.module: mc_fog_ingest_server::worker, mc.src: fog/ingest/server/src/worker.rs:229
2024-09-26 20:15:00.376104556 UTC DEBG Report cache thread stop requested., mc.node_id: 8, mc.test_name: three_node_cluster::three_node_cluster_fencing, mc.module: mc_fog_ingest_server::worker, mc.src: fog/ingest/server/src/worker.rs:289
2024-09-26 20:15:00.480418132 UTC INFO Stop Requested: Polling loop stopped at block number 8, is_idle false, mc.node_id: 7, mc.test_name: three_node_cluster::three_node_cluster_fencing, mc.module: mc_fog_ingest_server::worker, mc.src: fog/ingest/server/src/worker.rs:75
2024-09-26 20:15:00.671043993 UTC DEBG Peer checkup thread stop requested., mc.node_id: 7, mc.test_name: three_node_cluster::three_node_cluster_fencing, mc.module: mc_fog_ingest_server::worker, mc.src: fog/ingest/server/src/worker.rs:229
2024-09-26 20:15:00.948601017 UTC DEBG Report cache thread stop requested., mc.node_id: 7, mc.test_name: three_node_cluster::three_node_cluster_fencing, mc.module: mc_fog_ingest_server::worker, mc.src: fog/ingest/server/src/worker.rs:289