pkg/terminal,service/debugger: Support to add a new suboption --follow-calls to trace subcommand (#3594)

* rebasing on master to implement --followcalls

* in progress changes to enable --followcalls

* rebase to master: modified function to add children to funcs array

* modify main traversal loop

* added tests to check different scenarios

* added tests to check different scenarios

* added tests to check different scenarios

* add test to check for overlapping regular expression

* modified type of strings array as a return only

* changed depth to a simple integer instead of a global map

* avoid calling traverse on recursive calls

* Added tests for various call graphs to test trace followfuncs

* Added tests for various call graphs to test trace followfuncs

* Added tests for various call graphs to test trace followfuncs

* made auxillary changes for build to go through for new option follow-calls

* Add support to print depth of the function calls as well

* Added two sample output files for checking

* Bypass morestack_noctxt in output for verification testing

* Corrected newline error by adding newlines only if the line does not match morestack_noctxt

* Added more tests

* Cleanup

* Updated documentation

* fixed error message in fmt.Errorf

* Fixed result of Errorf not used error

* Addressing review comments to fix depth reporting and other issues

* dont invoke stacktrace if tracefollowcalls is enabled, compute depth from main regex root symbol than main.main

* Addressing a part of review comments

* Added changes to allow deferred functions to be picked up for tracing

* Fix issue to avoid printing stack for a simple trace option

* Moving most tests to integration2_test.go and keeping only one in dlv_test.go

* Moving most tests to integration2_test.go and keeping only one in dlv_test.go

* Adding panic-defer test case

* Moved rest of the tests to integration2_test.go

* addressing review comments: folding Functions and FunctionsDeep, reducing branches by using depth prefix, wrap using %w and other comments

* Optimize traversal and parts of printing trace point function and modify trace output layout
and adjust tests accordingly

* Resolved error occurring due to staticcheck

* Implemented traversal algorithm using breadth first search

* Addressing review comments on the breadth first search implementation and
other comments

* Inline filterRuntimeFuncs and remove duplicate initialization

Author: archanaravindar

Documentation/cli/starlark.md

@@ -50,7 +50,7 @@ breakpoints(All) | Equivalent to API call [ListBreakpoints](https://godoc.org/gi
 checkpoints() | Equivalent to API call [ListCheckpoints](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.ListCheckpoints)
 dynamic_libraries() | Equivalent to API call [ListDynamicLibraries](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.ListDynamicLibraries)
 function_args(Scope, Cfg) | Equivalent to API call [ListFunctionArgs](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.ListFunctionArgs)
-functions(Filter) | Equivalent to API call [ListFunctions](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.ListFunctions)
+functions(Filter, FollowCalls) | Equivalent to API call [ListFunctions](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.ListFunctions)
 goroutines(Start, Count, Filters, GoroutineGroupingOptions, EvalScope) | Equivalent to API call [ListGoroutines](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.ListGoroutines)
 local_vars(Scope, Cfg) | Equivalent to API call [ListLocalVars](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.ListLocalVars)
 package_vars(Filter, Cfg) | Equivalent to API call [ListPackageVars](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.ListPackageVars)

Documentation/usage/dlv_trace.md

@@ -21,14 +21,15 @@ dlv trace [package] regexp [flags]
 ### Options
 
 ```
-      --ebpf            Trace using eBPF (experimental).
-  -e, --exec string     Binary file to exec and trace.
-  -h, --help            help for trace
-      --output string   Output path for the binary.
-  -p, --pid int         Pid to attach to.
-  -s, --stack int       Show stack trace with given depth. (Ignored with --ebpf)
-  -t, --test            Trace a test binary.
-      --timestamp       Show timestamp in the output
+      --ebpf               Trace using eBPF (experimental).
+  -e, --exec string        Binary file to exec and trace.
+      --follow-calls int   Trace all children of the function to the required depth
+  -h, --help               help for trace
+      --output string      Output path for the binary.
+  -p, --pid int            Pid to attach to.
+  -s, --stack int          Show stack trace with given depth. (Ignored with --ebpf)
+  -t, --test               Trace a test binary.
+      --timestamp          Show timestamp in the output
 ```
 
 ### Options inherited from parent commands

_fixtures/leaf4.go

@@ -0,0 +1,23 @@
+package main
+
+import "fmt"
+
+func D(i int) int {
+	return i * i * i
+}
+func C(i int) int {
+
+	return i + 20
+}
+func B(i int) int {
+	d := C(i) + 40
+	return d + D(i)
+}
+func A(i int) int {
+	return 10 + B(i)
+}
+func main() {
+	j := 0
+	j += A(2)
+	fmt.Println(j)
+}

_fixtures/leafcommon.go

@@ -0,0 +1,23 @@
+package main
+
+import "fmt"
+
+func D(i int) int {
+	return i * i * i
+}
+func C(i int) int {
+
+	return D(i+10) + 20
+}
+func B(i int) int {
+	return i * D(i)
+}
+func A(i int) int {
+	d := 10 + B(i)
+	return d + C(i)
+}
+func main() {
+	j := 0
+	j += A(2)
+	fmt.Println(j)
+}

_fixtures/leafindrec.go

@@ -0,0 +1,23 @@
+package main
+
+import "fmt"
+
+func B(i int) int {
+	if i > 0 {
+		return A(i - 1)
+	} else {
+		return 0
+	}
+}
+func A(n int) int {
+	if n <= 1 {
+		return n
+	} else {
+		return B(n - 3)
+	}
+}
+func main() {
+	j := 0
+	j += B(12)
+	fmt.Println(j)
+}

_fixtures/leafrec.go

@@ -0,0 +1,17 @@
+package main
+
+import "fmt"
+
+func A(i int, n int) int {
+	if n == 1 {
+		return i
+	} else {
+		n--
+		return (i * A(i-1, n))
+	}
+}
+func main() {
+	j := 0
+	j += A(5, 5)
+	fmt.Println(j)
+}

_fixtures/leafregex.go

@@ -0,0 +1,23 @@
+package main
+
+import "fmt"
+
+func callmed(i int) int {
+	return i * i * i
+}
+func callmee(i int) int {
+
+	return i + 20
+}
+func callme2(i int) int {
+	d := callmee(i) + 40
+	return d + callmed(i)
+}
+func callme(i int) int {
+	return 10 + callme2(i)
+}
+func main() {
+	j := 0
+	j += callme(2)
+	fmt.Println(j)
+}

_fixtures/panicex.go

@@ -0,0 +1,28 @@
+package main
+
+        func F0() {
+                defer func() {
+                        recover()
+                }()
+                F1()
+        }
+
+        func F1() {
+                F2()
+        }
+
+        func F2() {
+                F3()
+        }
+
+        func F3() {
+                F4()
+        }
+
+        func F4() {
+                panic("blah")
+        }
+
+        func main() {
+                F0()
+        }

_fixtures/testtracefns.go

@@ -0,0 +1,81 @@
+package main
+
+import "fmt"
+
+func D(i int) int {
+	return i * i * i
+}
+func C(i int) int {
+
+	return D(i+10) + 20
+}
+func B(i int) int {
+	return i * D(i)
+}
+func A(i int) int {
+	d := 10 + B(i)
+	return d + C(i)
+}
+func second(i int) int {
+	if i > 0 {
+		return first(i - 1)
+	} else {
+		return 0
+	}
+}
+func first(n int) int {
+	if n <= 1 {
+		return n
+	} else {
+		return second(n - 3)
+	}
+}
+
+func callmed(i int) int {
+	return i * i * i
+}
+func callmee(i int) int {
+
+	return i + 20
+}
+func callme2(i int) int {
+	d := callmee(i) + 40
+	return d + callmed(i)
+}
+func callme(i int) int {
+	return 10 + callme2(i)
+}
+
+func F0() {
+	defer func() {
+		recover()
+	}()
+	F1()
+}
+
+func F1() {
+	F2()
+}
+
+func F2() {
+	F3()
+}
+
+func F3() {
+	F4()
+}
+
+func F4() {
+	panic("blah")
+}
+
+func main() {
+	j := 0
+	j += A(2)
+
+	j += first(6)
+	j += callme(2)
+	fmt.Println(j)
+	F0()
+
+}

cmd/dlv/cmds/commands.go

@@ -88,6 +88,7 @@ var (
 	traceStackDepth    int
 	traceUseEBPF       bool
 	traceShowTimestamp bool
+	traceFollowCalls   int
 
 	// redirect specifications for target process
 	redirects []string
@@ -363,6 +364,7 @@ only see the output of the trace operations you can redirect stdout.`,
 	must(traceCommand.RegisterFlagCompletionFunc("stack", cobra.NoFileCompletions))
 	traceCommand.Flags().String("output", "", "Output path for the binary.")
 	must(traceCommand.MarkFlagFilename("output"))
+	traceCommand.Flags().IntVarP(&traceFollowCalls, "follow-calls", "", 0, "Trace all children of the function to the required depth")
 	rootCommand.AddCommand(traceCommand)
 
 	coreCommand := &cobra.Command{
@@ -702,6 +704,10 @@ func traceCmd(cmd *cobra.Command, args []string, conf *config.Config) int {
 
 			processArgs = append([]string{debugname}, targetArgs...)
 		}
+		if dlvArgsLen >= 3 && traceFollowCalls <= 0 {
+			fmt.Fprintln(os.Stderr, "Need to specify a trace depth of atleast 1")
+			return 1
+		}
 
 		// Make a local in-memory connection that client and server use to communicate
 		listener, clientConn := service.ListenerPipe()
@@ -738,8 +744,7 @@ func traceCmd(cmd *cobra.Command, args []string, conf *config.Config) int {
 			<-ch
 			client.Halt()
 		}()
-
-		funcs, err := client.ListFunctions(regexp)
+		funcs, err := client.ListFunctions(regexp, traceFollowCalls)
 		if err != nil {
 			fmt.Fprintln(os.Stderr, err)
 			return 1
@@ -755,13 +760,22 @@ func traceCmd(cmd *cobra.Command, args []string, conf *config.Config) int {
 				}
 			} else {
 				// Fall back to breakpoint based tracing if we get an error.
+				var stackdepth int
+				// Default size of stackdepth to trace function calls and descendants=20
+				stackdepth = traceStackDepth
+				if traceFollowCalls > 0 && stackdepth == 0 {
+					stackdepth = 20
+				}
 				_, err = client.CreateBreakpoint(&api.Breakpoint{
-					FunctionName: funcs[i],
-					Tracepoint:   true,
-					Line:         -1,
-					Stacktrace:   traceStackDepth,
-					LoadArgs:     &terminal.ShortLoadConfig,
+					FunctionName:     funcs[i],
+					Tracepoint:       true,
+					Line:             -1,
+					Stacktrace:       stackdepth,
+					LoadArgs:         &terminal.ShortLoadConfig,
+					TraceFollowCalls: traceFollowCalls,
+					RootFuncName:     regexp,
 				})
+
 				if err != nil && !isBreakpointExistsErr(err) {
 					fmt.Fprintf(os.Stderr, "unable to set tracepoint on function %s: %#v\n", funcs[i], err)
 					continue
@@ -775,11 +789,13 @@ func traceCmd(cmd *cobra.Command, args []string, conf *config.Config) int {
 				}
 				for i := range addrs {
 					_, err = client.CreateBreakpoint(&api.Breakpoint{
-						Addr:        addrs[i],
-						TraceReturn: true,
-						Stacktrace:  traceStackDepth,
-						Line:        -1,
-						LoadArgs:    &terminal.ShortLoadConfig,
+						Addr:             addrs[i],
+						TraceReturn:      true,
+						Stacktrace:       stackdepth,
+						Line:             -1,
+						LoadArgs:         &terminal.ShortLoadConfig,
+						TraceFollowCalls: traceFollowCalls,
+						RootFuncName:     regexp,
 					})
 					if err != nil && !isBreakpointExistsErr(err) {
 						fmt.Fprintf(os.Stderr, "unable to set tracepoint on function %s: %#v\n", funcs[i], err)

cmd/dlv/dlv_test.go

@@ -970,6 +970,39 @@ func TestTrace2(t *testing.T) {
 	assertNoError(cmd.Wait(), t, "cmd.Wait()")
 }
 
+func TestTraceDirRecursion(t *testing.T) {
+	dlvbin := getDlvBin(t)
+
+	expected := []byte("> goroutine(1):frame(1) main.A(5, 5)\n > goroutine(1):frame(2) main.A(4, 4)\n  > goroutine(1):frame(3) main.A(3, 3)\n   > goroutine(1):frame(4) main.A(2, 2)\n    > goroutine(1):frame(5) main.A(1, 1)\n    >> goroutine(1):frame(5) main.A => (1)\n   >> goroutine(1):frame(4) main.A => (2)\n  >> goroutine(1):frame(3) main.A => (6)\n >> goroutine(1):frame(2) main.A => (24)\n>> goroutine(1):frame(1) main.A => (120)\n")
+
+	fixtures := protest.FindFixturesDir()
+	cmd := exec.Command(dlvbin, "trace", "--output", filepath.Join(t.TempDir(), "__debug"), filepath.Join(fixtures, "leafrec.go"), "main.A", "--follow-calls", "4")
+	rdr, err := cmd.StderrPipe()
+	assertNoError(err, t, "stderr pipe")
+	defer rdr.Close()
+
+	cmd.Dir = filepath.Join(fixtures, "buildtest")
+
+	assertNoError(cmd.Start(), t, "running trace")
+	// Parse output to ignore calls to morestack_noctxt for comparison
+	scan := bufio.NewScanner(rdr)
+	text := ""
+	outputtext := ""
+	for scan.Scan() {
+		text = scan.Text()
+		if !strings.Contains(text, "morestack_noctxt") {
+			outputtext += text
+			outputtext += "\n"
+		}
+	}
+	output := []byte(outputtext)
+
+	if !bytes.Contains(output, expected) {
+		t.Fatalf("expected:\n%s\ngot:\n%s", string(expected), string(output))
+	}
+	assertNoError(cmd.Wait(), t, "cmd.Wait()")
+}
+
 func TestTraceMultipleGoroutines(t *testing.T) {
 	dlvbin := getDlvBin(t)