Skip to content

Commit

Permalink
Better backtraces from stack overflows (#509)
Browse files Browse the repository at this point in the history
This PR adds a new --guess argument to humility tasks, which tries to use the
saved frame pointer to get a stack trace under exceptional circumstances. I
deliberately implemented it as a suspicious-sounding flag instead of an
automatic fallback, because it may not be 100% reliable (but it looks fine in
all of the unit tests).
  • Loading branch information
mkeeter authored Oct 9, 2024
1 parent 88fcc93 commit ac2272f
Show file tree
Hide file tree
Showing 179 changed files with 473 additions and 139 deletions.
71 changes: 71 additions & 0 deletions cmd/tasks/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -135,6 +135,10 @@ struct TasksArgs {
#[clap(long, short)]
stack: bool,

/// use saved frame pointer to guess at stack trace
#[clap(long, requires = "stack")]
guess: bool,

/// show line number information with stack backtrace
#[clap(long, short, requires = "stack")]
line: bool,
Expand Down Expand Up @@ -193,6 +197,7 @@ fn tasks(context: &mut ExecutionContext) -> Result<()> {
subargs.registers,
subargs.stack,
subargs.line,
subargs.guess,
subargs.spin,
subargs.verbose,
subargs.task,
Expand All @@ -207,6 +212,7 @@ pub fn print_tasks(
registers: bool,
stack: bool,
line: bool,
guess: bool,
spin: bool,
verbose: bool,
task_arg: Option<String>,
Expand Down Expand Up @@ -426,6 +432,24 @@ pub fn print_tasks(
}
Err(e) => {
writeln!(w, " could not read registers: {e:?}")?;
if stack {
if guess {
match stack_guess(
w, core, hubris, t, task_value, &desc,
) {
Ok(stack) => printer.print(hubris, &stack),
Err(e) => writeln!(
w,
" stack unwind failed: {e:?}"
)?,
}
} else {
writeln!(
w,
" use `--guess` to guess at stack trace"
)?;
}
}
}
}
}
Expand Down Expand Up @@ -472,6 +496,53 @@ pub fn print_tasks(
Ok(())
}

fn stack_guess<'a>(
w: &'a mut dyn Write,
core: &'a mut dyn Core,
hubris: &'a HubrisArchive,
t: HubrisTask,
task_value: &'a reflect::Value,
desc: &'a TaskDesc,
) -> Result<Vec<HubrisStackFrame<'a>>> {
writeln!(w, " guessing at stack trace using saved frame pointer")?;
let reflect::Value::Struct(s) = &task_value else {
bail!("invalid type for task_value")
};
let Some(reflect::Value::Struct(save)) = s.get("save") else {
bail!("invalid type for save")
};
let Some(reflect::Value::Base(reflect::Base::U32(addr))) = save.get("r7")
else {
bail!("invalid type for r7")
};
let pc = core.read_word_32(*addr + 4)? & !1;

let mut regs = BTreeMap::new();
regs.insert(ARMRegister::R7, *addr);
regs.insert(ARMRegister::LR, *addr);

// Provide a dummy stack value to pick the
// correct memory region
regs.insert(ARMRegister::SP, desc.initial_stack);

// See if the previous instruction was a branch;
// if so, use that as a fake PC
regs.insert(
ARMRegister::PC,
if let Some(HubrisTarget::Call(t)) =
pc.checked_sub(4).and_then(|pc| hubris.instr_target(pc))
{
t + 4
} else {
pc
},
);

let initial = desc.initial_stack;

hubris.stack(core, t, initial, &regs)
}

#[derive(Copy, Clone, Debug)]
enum Deadline {
Absolute { t: u64, notif: u32 },
Expand Down
3 changes: 2 additions & 1 deletion cmd/test/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -306,7 +306,8 @@ fn test(context: &mut ExecutionContext) -> Result<()> {
writeln!(out, "==== Task state")?;

cmd_tasks::print_tasks(
&mut out, core, hubris, false, false, false, false, false, None,
&mut out, core, hubris, false, false, false, false, false, false,
None,
)?;
}
println!("Ran a total of {} cases", ran_cases);
Expand Down
2 changes: 1 addition & 1 deletion tests/cli_tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -190,7 +190,7 @@ fn make_all_tests() -> Result<()> {
Test::witharg(Kind::Postmortem, "sensors-read", "sensors", ""),
Test::basic(Kind::Postmortem, "stackmargin"),
Test::basic(Kind::Postmortem, "tasks"),
Test::witharg(Kind::Postmortem, "tasks-slvr", "tasks", "-slvr"),
Test::witharg(Kind::Postmortem, "tasks-slvr", "tasks", "-slvr --guess"),
Test::basic(Kind::Postmortem, "counters"),
Test::witharg(Kind::Postmortem, "counters-list", "counters", "list"),
Test::witharg(Kind::Postmortem, "counters-full", "counters", "--full"),
Expand Down
2 changes: 1 addition & 1 deletion tests/cmd/tasks-slvr/tasks-slvr.chilly.0.toml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion tests/cmd/tasks-slvr/tasks-slvr.counters.0.toml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion tests/cmd/tasks-slvr/tasks-slvr.extern-regions.toml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion tests/cmd/tasks-slvr/tasks-slvr.flash-ram-mismatch.0.toml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion tests/cmd/tasks-slvr/tasks-slvr.host-panic.0.toml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion tests/cmd/tasks-slvr/tasks-slvr.host-panic.1.toml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion tests/cmd/tasks-slvr/tasks-slvr.host-panic.2.toml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion tests/cmd/tasks-slvr/tasks-slvr.host-panic.3.toml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion tests/cmd/tasks-slvr/tasks-slvr.host-panic.4.toml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion tests/cmd/tasks-slvr/tasks-slvr.idol-returns-an-enum.toml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion tests/cmd/tasks-slvr/tasks-slvr.igor.0.toml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion tests/cmd/tasks-slvr/tasks-slvr.in_bootloader.0.toml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion tests/cmd/tasks-slvr/tasks-slvr.inheritance.0.toml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion tests/cmd/tasks-slvr/tasks-slvr.ipc-counts.0.toml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion tests/cmd/tasks-slvr/tasks-slvr.kernel-panic.0.toml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion tests/cmd/tasks-slvr/tasks-slvr.kernel-panic.1.toml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions tests/cmd/tasks-slvr/tasks-slvr.kiowa.0.stdout

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion tests/cmd/tasks-slvr/tasks-slvr.kiowa.0.toml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

5 changes: 5 additions & 0 deletions tests/cmd/tasks-slvr/tasks-slvr.kiowa.1.stdout

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion tests/cmd/tasks-slvr/tasks-slvr.kiowa.1.toml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion tests/cmd/tasks-slvr/tasks-slvr.kiowa.10.toml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion tests/cmd/tasks-slvr/tasks-slvr.kiowa.11.toml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion tests/cmd/tasks-slvr/tasks-slvr.kiowa.12.toml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion tests/cmd/tasks-slvr/tasks-slvr.kiowa.13.toml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion tests/cmd/tasks-slvr/tasks-slvr.kiowa.14.toml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion tests/cmd/tasks-slvr/tasks-slvr.kiowa.15.toml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion tests/cmd/tasks-slvr/tasks-slvr.kiowa.16.toml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion tests/cmd/tasks-slvr/tasks-slvr.kiowa.17.toml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion tests/cmd/tasks-slvr/tasks-slvr.kiowa.18.toml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion tests/cmd/tasks-slvr/tasks-slvr.kiowa.19.toml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

5 changes: 5 additions & 0 deletions tests/cmd/tasks-slvr/tasks-slvr.kiowa.2.stdout

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading

0 comments on commit ac2272f

Please sign in to comment.