Merge branch 'master' into gdb_fixing

.rgignore

@@ -0,0 +1 @@
+tests/*

Cargo.toml

@@ -17,7 +17,7 @@ name = "humility"
 #
 # Be sure to check in and push all of the files that change.  Happy versioning!
 #
-version = "0.11.9"
+version = "0.11.11"
 authors = ["Bryan Cantrill <bryan@oxide.computer>"]
 edition = "2018"
 license = "MPL-2.0"

cmd/auxflash/src/lib.rs

@@ -254,6 +254,7 @@ impl<'a> AuxFlashHandler<'a> {
             bar.set_position(offset as u64);
         }
         bar.set_position(out.len() as u64);
+        bar.finish_and_clear();
 
         Ok(out)
     }
@@ -349,6 +350,7 @@ impl<'a> AuxFlashHandler<'a> {
             bar.set_position(offset as u64);
         }
         bar.set_position(data.len() as u64);
+        bar.finish_and_clear();
         humility::msg!("done");
         Ok(())
     }

cmd/dump/src/lib.rs

@@ -74,7 +74,7 @@ use humility_cli::{ExecutionContext, Subcommand};
 use humility_cmd::{Archive, Attach, Command, CommandKind, Validate};
 use humility_dump_agent::{
     task_areas, DumpAgent, DumpAgentCore, DumpAgentExt, DumpArea,
-    HiffyDumpAgent, UdpDumpAgent,
+    DumpBreakdown, HiffyDumpAgent, UdpDumpAgent,
 };
 use humpty::DumpTask;
 use indicatif::{HumanBytes, HumanDuration, ProgressBar, ProgressStyle};
@@ -97,7 +97,7 @@ struct DumpArgs {
     timeout: u32,
 
     /// show dump agent status
-    #[clap(long, conflicts_with_all = &["simulation", "task", "all"])]
+    #[clap(long, conflicts_with_all = &["simulation", "task", "extract-all"])]
     dump_agent_status: bool,
 
     /// force use of the dump agent when directly attached with a debug probe
@@ -111,25 +111,25 @@ struct DumpArgs {
     /// force manual initiation, leaving target halted
     #[clap(
         long, requires = "force-dump-agent",
-        conflicts_with_all = &["all", "task"]
+        conflicts_with_all = &["extract-all", "task"]
     )]
     force_manual_initiation: bool,
 
     /// force existing in situ dump to be read
-    #[clap(long, conflicts_with_all = &["simulation", "task", "all"])]
+    #[clap(long, conflicts_with_all = &["simulation", "task", "extract-all"])]
     force_read: bool,
 
     /// initialize dump state, clearing any dump at the dump agent
     #[clap(long, conflicts_with = "simulation")]
     initialize_dump_agent: bool,
 
     /// retain dump state after a system dump
-    #[clap(long, conflicts_with_all = &["task", "list", "area"])]
+    #[clap(long, conflicts_with_all = &["task", "list"])]
     retain_state: bool,
 
     /// overwrite any dump state as part of taking dump
     #[clap(
-        long, short = 'F',
+        long,
         conflicts_with_all = &["initialize-dump-agent", "simulate-dumper"],
     )]
     force_overwrite: bool,
@@ -141,7 +141,7 @@ struct DumpArgs {
     /// in addition to simulating the dumper, generate a stock dump
     #[clap(
         long, requires = "simulation",
-        conflicts_with_all = &["task", "all"],
+        conflicts_with_all = &["task", "extract-all"],
         value_name = "filename",
     )]
     stock_dumpfile: Option<String>,
@@ -154,7 +154,7 @@ struct DumpArgs {
     /// simulates a single-task dump
     #[clap(
         long,
-        conflicts_with_all = &["area", "stock-dumpfile", "list"],
+        conflicts_with_all = &["extract", "stock-dumpfile", "list"],
         requires = "simulate-dumper",
         value_name = "task",
     )]
@@ -163,28 +163,38 @@ struct DumpArgs {
     /// dumps a single task
     #[clap(
         long,
-        conflicts_with_all = &["simulation", "list", "area"],
+        conflicts_with_all = &["simulation", "list", "extract"],
         value_name = "task",
     )]
     task: Option<String>,
 
+    /// extracts the dump in the specified area
+    #[clap(
+        short = 'a', alias = "area", long, value_name = "area",
+        conflicts_with_all = &["simulation", "list"]
+    )]
+    extract: Option<usize>,
+
     /// extracts every available dump
     #[clap(
         long,
-        conflicts_with_all = &["simulation", "list", "area", "task"]
+        alias = "all",
+        conflicts_with_all = &["simulation", "list", "extract", "task"]
     )]
-    all: bool,
-
-    #[clap(short, long, conflicts_with_all = &["simulation", "list"])]
-    area: Option<usize>,
+    extract_all: bool,
 
     /// leave the target halted
     #[clap(long, conflicts_with = "simulation")]
     leave_halted: bool,
 
-    #[clap(long, short, conflicts_with_all = &["simulation", "area"])]
+    /// list all dump areas
+    #[clap(long, short, conflicts_with_all = &["simulation", "extract"])]
     list: bool,
 
+    /// print dump breakdown
+    #[clap(long)]
+    print_dump_breakdown: bool,
+
     dumpfile: Option<String>,
 }
 
@@ -208,7 +218,7 @@ fn emulate_dump(
 
     let mut rnum = 0;
 
-    let r = humpty::dump::<anyhow::Error, 2048, { humpty::DUMPER_EMULATED }>(
+    let r = humpty::dump::<anyhow::Error, 1024, { humpty::DUMPER_EMULATED }>(
         base,
         task,
         || {
@@ -327,14 +337,82 @@ fn get_dump_agent<'a>(
     }
 }
 
+fn read_dump<'a>(
+    agent: &mut Box<dyn DumpAgent + 'a>,
+    area: Option<DumpArea>,
+    out: &mut DumpAgentCore,
+    subargs: &DumpArgs,
+) -> Result<Option<DumpTask>> {
+    let (task, breakdown) = agent.read_dump(area, out, true)?;
+
+    if subargs.print_dump_breakdown {
+        print_dump_breakdown(&breakdown);
+    }
+
+    Ok(task)
+}
+
+fn print_dump_breakdown(breakdown: &DumpBreakdown) {
+    let w = 30;
+    let w2 = 10;
+
+    let mut total = 0;
+
+    humility::msg!("Dump breakdown:");
+
+    let print_val = |str, val| {
+        humility::msg!("  {str:<w$} => {val}");
+    };
+
+    let print_signed_val = |str, val| {
+        humility::msg!("  {str:<w$} => {val}");
+    };
+
+    let mut print_perc = |str, val, acct| {
+        let (label, perc) = if acct {
+            total += val;
+            ("consumed", (val as f32 / breakdown.used as f32) * 100.0)
+        } else {
+            ("total", (val as f32 / breakdown.total as f32) * 100.0)
+        };
+
+        humility::msg!("  {str:<w$} => {val:<w2$} {perc:>6.2}% of {label}");
+    };
+
+    print_val("total dump area", breakdown.total);
+    print_perc("dump area consumed", breakdown.used, false);
+    print_perc("dump headers", breakdown.headers, true);
+    print_perc("segment headers", breakdown.segment_headers, true);
+    print_perc("register headers + data", breakdown.registers, true);
+    print_perc("data headers", breakdown.data_headers, true);
+    print_perc("data", breakdown.compressed, true);
+    print_perc("data padding", breakdown.segment_padding, true);
+    print_perc("orphaned", breakdown.orphaned, true);
+
+    humility::msg!(
+        "  -------------------------------------\
+        --------------------------------"
+    );
+
+    let unaccounted = breakdown.used as i32 - total as i32;
+    print_val("total accounted", total);
+    print_signed_val("unaccounted dump area", unaccounted);
+
+    humility::msg!("");
+
+    print_val("uncompressed size", breakdown.uncompressed);
+    print_val("compressed size", breakdown.compressed);
+    print_val("data expansion", breakdown.inverted);
+}
+
 fn dump_via_agent(
     hubris: &HubrisArchive,
     core: &mut dyn Core,
     subargs: &DumpArgs,
 ) -> Result<()> {
     let mut out = DumpAgentCore::new(HubrisFlashMap::new(hubris)?);
     let started = Some(Instant::now());
-    let mut area = subargs.area.map(DumpArea::ByIndex);
+    let mut area = subargs.extract.map(DumpArea::ByIndex);
 
     //
     // Our task can come from a couple of different spots:  we can either
@@ -473,17 +551,15 @@ fn dump_via_agent(
         let mut agent = get_dump_agent(hubris, core, subargs)?;
         let header = agent.read_dump_header()?;
 
-        if !subargs.force_read && subargs.area.is_none() {
+        if !subargs.force_read && subargs.extract.is_none() {
             if header.dumper != humpty::DUMPER_NONE
                 && !subargs.initialize_dump_agent
                 && !subargs.force_overwrite
                 && task.is_none()
             {
                 bail!(
                     "there appears to already be one or more dumps in situ; \
-                    list them with --list, clear them with \
-                    --initialize-dump-agent, or force them to be overwritten \
-                    with --force-overwrite"
+                    list them with --list and extract them with --extract-all"
                 )
             }
 
@@ -533,7 +609,7 @@ fn dump_via_agent(
             emulate_dump(agent.core(), task, address, total)?;
             agent.core().run()?;
             humility::msg!("core resumed");
-        } else if !subargs.force_read && subargs.area.is_none() {
+        } else if !subargs.force_read && subargs.extract.is_none() {
             if subargs.force_manual_initiation {
                 agent.core().halt()?;
                 humility::msg!("leaving core halted");
@@ -559,15 +635,47 @@ fn dump_via_agent(
             agent.core().set_timeout(std::time::Duration::new(60, 0))?;
 
             //
-            // Tell the thing to take a dump
+            // Tell the thing to take a dump.
             //
-            agent.take_dump()?;
+            if let Err(err) = agent.take_dump() {
+                //
+                // If that fails, it may be because we ran out of space.  Check
+                // our dump headers; if all of them are consumed, assume
+                // that we ran out of space -- and if any of them are consumed,
+                // process whatever we find (some dump is better than none!) and
+                // warn accordingly.
+                //
+                if let Ok(all) = agent.read_dump_headers(true) {
+                    let c = all
+                        .iter()
+                        .filter(|&&(h, _)| h.dumper != humpty::DUMPER_NONE)
+                        .count();
+
+                    if c == all.len() {
+                        humility::warn!(
+                            "dump has indicated failure ({err:?}), but this is \
+                            likely due to space exhaustion; \
+                            dump will be extracted but may be incomplete!"
+                        );
+                    } else if c != 0 {
+                        humility::warn!(
+                            "dump has indicated failure ({err:?}), but some dump \
+                            contents appear to have been written; \
+                            dump will be extracted but may be incomplete!"
+                        );
+                    } else {
+                        return Err(err);
+                    }
+                } else {
+                    return Err(err);
+                }
+            }
         }
 
         //
         // If we're here, we have a dump in situ -- time to pull it.
         //
-        task = agent.read_dump(area, &mut out, true)?;
+        task = read_dump(&mut agent, area, &mut out, subargs)?;
 
         //
         // If this was a whole-system dump, we will leave our state initialized
@@ -610,10 +718,11 @@ fn dump_task_via_agent(
         bail!("cannot dump supervisor");
     }
     let area = agent.dump_task(ndx)?;
-    let task = agent.read_dump(
+    let task = read_dump(
+        &mut agent,
         Some(DumpArea::ByIndex(area as usize)),
         &mut out,
-        true,
+        subargs,
     )?;
     assert!(task.is_some());
     hubris.dump(&mut out, task, subargs.dumpfile.as_deref(), started)?;
@@ -710,11 +819,13 @@ fn dump_all(
 
             let mut out = DumpAgentCore::new(HubrisFlashMap::new(hubris)?);
             let started = Some(Instant::now());
-            let task = agent.read_dump(
+            let task = read_dump(
+                &mut agent,
                 Some(DumpArea::ByIndex(*area)),
                 &mut out,
-                true,
+                subargs,
             )?;
+
             assert!(task.is_some());
             hubris.dump(&mut out, task, Some(&dumpfile), started)?;
         }
@@ -752,7 +863,7 @@ fn dumpcmd(context: &mut ExecutionContext) -> Result<()> {
         bail!("can only force the dump agent when attached via debug probe");
     }
 
-    if subargs.all {
+    if subargs.extract_all {
         dump_all(hubris, core, &subargs)
     } else if subargs.list {
         dump_list(hubris, core, &subargs)
@@ -766,7 +877,7 @@ fn dumpcmd(context: &mut ExecutionContext) -> Result<()> {
     } else if core.is_net()
         || subargs.force_dump_agent
         || subargs.force_read
-        || subargs.area.is_some()
+        || subargs.extract.is_some()
     {
         dump_via_agent(hubris, core, &subargs)
     } else {
@@ -797,7 +908,7 @@ pub fn init() -> Command {
         run: dumpcmd,
         kind: CommandKind::Attached {
             archive: Archive::Required,
-            attach: Attach::LiveOnly,
+            attach: Attach::Any,
             validate: Validate::Match,
         },
     }

cmd/rebootleby/src/lib.rs

@@ -42,13 +42,20 @@ fn rebootleby(context: &mut ExecutionContext) -> Result<()> {
     let mut zip =
         ZipArchive::new(bundle_reader).context("opening bundle file as ZIP")?;
 
-    let img_bootleby = {
+    let img_bootleby = if zip.file_names().any(|x| x == "bootleby.bin") {
         let mut entry = zip
             .by_name("bootleby.bin")
             .context("can't find bootleby.bin in bundle")?;
         let mut data = vec![];
         entry.read_to_end(&mut data).context("reading bootleby.bin")?;
         data
+    } else {
+        let mut entry = zip
+            .by_name("img/final.bin")
+            .context("can't find bootleby.bin or img/final.bin in bundle")?;
+        let mut data = vec![];
+        entry.read_to_end(&mut data).context("reading bootleby.bin")?;
+        data
     };
     let img_cmpa = {
         let mut entry =