compiler: Add support for debug information in BEAM files

The new `beam_debug_info` compiler option will insert
`executable_line` instructions in the same way that `line_coverage`
does, but it will also maintain information about which variables the
BEAM registers contain at each `executable_line` instruction. This
information will be inserted into a "DbgB" chunk in the BEAM file.

Here is an example where the debug information translated to text has
been inserted as comments before the lines they apply to:

    sum(A, B, _Ignored) ->
	%% no stack frame; A in x0, B in x1, _Ignored in x2
	C = A + B,

	%% no stack frame; B in x1, C in x0
	io:format("~p\n", [C]),

	%% stack frame size is 1; C in y0
	D = 10 * C,

	%% stack frame size is 1; C in y0, D in x0
	{ok,D}.

Note that not all variables are available in the debug information.

For example, before the call to `io:format/2`, the sum of A and B have
overwritten the register that used to hold the value of A, and the value
for _Ignore was wiped out by the `+` operation.

The size of the current stack frame is also given at each executable
line to be able to safely find the beginning of the previous stack frame.
Scanning from the stack top is not safe, because not all slots (Y registers)
are guaranteed to be initialized when an `executable_line` instruction is
reached.

lib/compiler/src/beam_asm.erl

@@ -27,11 +27,14 @@
 
 -export_type([fail/0,label/0,src/0,module_code/0,function_name/0]).
 
--import(lists, [append/1,duplicate/2,map/2,member/2,keymember/3,splitwith/2]).
+-import(lists, [append/1,duplicate/2,keymember/3,last/1,map/2,
+                member/2,splitwith/2]).
 
 -include("beam_opcodes.hrl").
 -include("beam_asm.hrl").
 
+-define(BEAM_DEBUG_INFO_VERSION, 0).
+
 %% Common types for describing operands for BEAM instructions.
 -type src() :: beam_reg() |
 	       {'literal',term()} |
@@ -60,23 +63,24 @@
 -define(BEAMFILE_EXECUTABLE_LINE, 1).
 -define(BEAMFILE_FORCE_LINE_COUNTERS, 2).
 
--spec module(module_code(), [{binary(), binary()}], [{atom(),term()}], [compile:option()]) ->
-                    {'ok',binary()}.
-
-module(Code, ExtraChunks, CompileInfo, CompilerOpts) ->
-    {ok,assemble(Code, ExtraChunks, CompileInfo, CompilerOpts)}.
+-spec module(module_code(), [{binary(), binary()}],
+             [{atom(),term()}], [compile:option()]) ->
+          {'ok',binary()}.
 
-assemble({Mod,Exp0,Attr0,Asm0,NumLabels}, ExtraChunks, CompileInfo, CompilerOpts) ->
+module(Code0, ExtraChunks, CompileInfo, CompilerOpts) ->
+    {Mod,Exp0,Attr0,Asm0,NumLabels} = Code0,
     {1,Dict0} = beam_dict:atom(Mod, beam_dict:new()),
     {0,Dict1} = beam_dict:fname(atom_to_list(Mod) ++ ".erl", Dict0),
     {0,Dict2} = beam_dict:type(any, Dict1),
     Dict3 = reject_unsupported_versions(Dict2),
+
     NumFuncs = length(Asm0),
     {Asm,Attr} = on_load(Asm0, Attr0),
     Exp = sets:from_list(Exp0),
-    {Code,Dict} = assemble_1(Asm, Exp, Dict3, []),
-    build_file(Code, Attr, Dict, NumLabels, NumFuncs,
-               ExtraChunks, CompileInfo, CompilerOpts).
+    {Code,Dict} = assemble(Asm, Exp, Dict3, []),
+    Beam = build_file(Code, Attr, Dict, NumLabels, NumFuncs,
+                      ExtraChunks, CompileInfo, CompilerOpts),
+    {ok,Beam}.
 
 reject_unsupported_versions(Dict) ->
     %% Emit an instruction that was added in our lowest supported
@@ -106,16 +110,16 @@ insert_on_load_instruction(Is0, Entry) ->
 		  end, Is0),
     Bef ++ [El,on_load|Is].
 
-assemble_1([{function,Name,Arity,Entry,Asm}|T], Exp, Dict0, Acc) ->
+assemble([{function,Name,Arity,Entry,Asm}|T], Exp, Dict0, Acc) ->
     Dict1 = case sets:is_element({Name,Arity}, Exp) of
 		true ->
 		    beam_dict:export(Name, Arity, Entry, Dict0);
 		false ->
 		    beam_dict:local(Name, Arity, Entry, Dict0)
 	    end,
     {Code, Dict2} = assemble_function(Asm, Acc, Dict1),
-    assemble_1(T, Exp, Dict2, Code);
-assemble_1([], _Exp, Dict0, Acc) ->
+    assemble(T, Exp, Dict2, Code);
+assemble([], _Exp, Dict0, Acc) ->
     {IntCodeEnd,Dict1} = make_op(int_code_end, Dict0),
     {list_to_binary(lists:reverse(Acc, [IntCodeEnd])),Dict1}.
 
@@ -125,17 +129,23 @@ assemble_function([H|T], Acc, Dict0) ->
 assemble_function([], Code, Dict) ->
     {Code, Dict}.
 
-build_file(Code, Attr, Dict, NumLabels, NumFuncs, ExtraChunks0, CompileInfo, CompilerOpts) ->
+build_file(Code, Attr, Dict0, NumLabels, NumFuncs, ExtraChunks0,
+           CompileInfo, CompilerOpts) ->
     %% Create the code chunk.
 
     CodeChunk = chunk(<<"Code">>,
 		      <<16:32,
 		       (beam_opcodes:format_number()):32,
-		       (beam_dict:highest_opcode(Dict)):32,
+		       (beam_dict:highest_opcode(Dict0)):32,
 		       NumLabels:32,
 		       NumFuncs:32>>,
 		      Code),
 
+    %% Build the BEAM debug information chunk. It is important
+    %% to build it early, because it will add entries to the
+    %% atom and literal tables.
+    {ExtraChunks1,Dict} = build_beam_debug_info(ExtraChunks0, CompilerOpts, Dict0),
+
     %% Create the atom table chunk.
     {NumAtoms, AtomTab} = beam_dict:atom_table(Dict),
     AtomChunk = chunk(<<"AtU8">>, <<NumAtoms:32>>, AtomTab),
@@ -194,13 +204,14 @@ build_file(Code, Attr, Dict, NumLabels, NumFuncs, ExtraChunks0, CompileInfo, Com
                       TypeTab),
 
     %% Create the meta chunk
-    Meta = proplists:get_value(<<"Meta">>, ExtraChunks0, empty),
+    Meta = proplists:get_value(<<"Meta">>, ExtraChunks1, empty),
     MetaChunk = case Meta of
                     empty -> [];
                     Meta -> chunk(<<"Meta">>, Meta)
                 end,
+
     %% Remove Meta chunk from ExtraChunks since it is essential
-    ExtraChunks = ExtraChunks0 -- [{<<"Meta">>, Meta}],
+    ExtraChunks = ExtraChunks1 -- [{<<"Meta">>, Meta}],
 
     %% Create the attributes and compile info chunks.
 
@@ -352,6 +363,100 @@ filter_essentials([<<>>|T]) ->
     filter_essentials(T);
 filter_essentials([]) -> [].
 
+%%%
+%%% Build the BEAM debug information chunk.
+%%%
+
+build_beam_debug_info(ExtraChunks, CompilerOpts, Dict) ->
+    case member(beam_debug_info, CompilerOpts) of
+        true ->
+            build_beam_debug_info_1(ExtraChunks, Dict);
+        false ->
+            {ExtraChunks,Dict}
+    end.
+
+build_beam_debug_info_1(ExtraChunks0, Dict0) ->
+    DebugTab0 = beam_dict:debug_table(Dict0),
+    DebugTab1 = [{Index,Info} ||
+                    Index := Info <- maps:iterator(DebugTab0, ordered)],
+    DebugTab = build_bdi_fill_holes(DebugTab1),
+    NumVars = lists:sum([length(Vs) || {_,Vs} <- DebugTab]),
+    {Contents0,Dict} = build_bdi(DebugTab, Dict0),
+    NumItems = length(Contents0),
+    Contents1 = iolist_to_binary(Contents0),
+
+    0 = NumItems bsr 31,                        %Assertion.
+    0 = NumVars bsr 31,                         %Assertion.
+
+    Contents = <<?BEAM_DEBUG_INFO_VERSION:32,
+                 NumItems:32,
+                 NumVars:32,
+                 Contents1/binary>>,
+    ExtraChunks = [{~"DbgB",Contents}|ExtraChunks0],
+    {ExtraChunks,Dict}.
+
+build_bdi_fill_holes([{_,Item}]) ->
+    [Item];
+build_bdi_fill_holes([{I0,Item}|[{I1,_}|_]=T]) ->
+    case I0 + 1 of
+        I1 ->
+            [Item|build_bdi_fill_holes(T)];
+        Next ->
+            NewPair = {Next,{none,[]}},
+            [Item|build_bdi_fill_holes([NewPair|T])]
+    end.
+
+build_bdi([{FrameSize0,Vars0}|Items], Dict0) ->
+    %% The debug information utilizes the encoding machinery for BEAM
+    %% instructions. The debug information for `executable_line`
+    %% instruction is translated to:
+    %%
+    %%    {call,FrameSize,{list,[VariableName,Where,...]}}
+    %%
+    %% Where:
+    %%
+    %%    FrameSize := 'none' | 0..1023
+    %%    VariableName := binary()
+    %%    Where := {x,0..1023} | {y,0..1023} | {literal,_} |
+    %%             {integer,_} | {atom,_} | {float,_} | nil
+    %%
+    %% The only reason the `call` instruction is used is because it
+    %% has two operands.
+    %%
+    %% The following example:
+    %%
+    %%    {executable_line,[...],1,
+    %%       {4, [{'Args',[{y,3}]},
+    %%            {'Line',[{y,2}]},
+    %%            {'Live',[{x,0},{y,1}]}]}}.
+    %%
+    %% will be translated to the following instruction:
+    %%
+    %%     {call,4,{list,[{literal,<<"Args">>},{y,3},
+    %%                    {literal,<<"Line">>},{y,2},
+    %%                    {literal,<<"Live">>},{y,1}]}}
+    %%
+    %% Note that only one register is given for each variable. It
+    %% is always the last register listed.
+
+    FrameSize = case FrameSize0 of
+                    none -> nil;
+                    _ -> FrameSize0
+                end,
+    Vars1 = [[{literal,atom_to_binary(Name)},last(Regs)] ||
+                {Name,[_|_]=Regs} <- Vars0],
+    Vars = append(Vars1),
+    Instr0 = {call,FrameSize,{list,Vars}},
+    {Instr,Dict1} = make_op(Instr0, Dict0),
+    {Tail,Dict2} = build_bdi(Items, Dict1),
+    {[Instr|Tail],Dict2};
+build_bdi([], Dict) ->
+    {[],Dict}.
+
+%%%
+%%% Functions for assembling BEAM instruction.
+%%%
+
 bif_type(fnegate, 1) -> {op,fnegate};
 bif_type(fadd, 2)   -> {op,fadd};
 bif_type(fsub, 2)   -> {op,fsub};
@@ -368,6 +473,10 @@ make_op({line=Op,Location}, Dict0) ->
 make_op({executable_line=Op,Location,Index}, Dict0) ->
     {LocationIndex,Dict} = beam_dict:line(Location, Dict0, Op),
     encode_op(executable_line, [LocationIndex,Index], Dict);
+make_op({executable_line=Op,Location,Index,DebugInfo}, Dict0) ->
+    {LocationIndex,Dict1} = beam_dict:line(Location, Dict0, Op),
+    Dict = beam_dict:debug_info(Index, DebugInfo, Dict1),
+    encode_op(executable_line, [LocationIndex,Index], Dict);
 make_op({bif, Bif, {f,_}, [], Dest}, Dict) ->
     %% BIFs without arguments cannot fail.
     encode_op(bif0, [{extfunc, erlang, Bif, 0}, Dest], Dict);

lib/compiler/src/beam_block.erl

@@ -25,7 +25,7 @@
 -include("beam_asm.hrl").
 
 -export([module/2]).
--import(lists, [keysort/2,member/2,reverse/1,reverse/2,
+-import(lists, [flatmap/2,keysort/2,member/2,reverse/1,reverse/2,
                 splitwith/2,usort/1]).
 
 -spec module(beam_utils:module_code(), [compile:option()]) ->
@@ -172,12 +172,17 @@ collect({put_map,{f,0},Op,S,D,R,{list,Puts}}) ->
 collect({fmove,S,D})         -> {set,[D],[S],fmove};
 collect({fconv,S,D})         -> {set,[D],[S],fconv};
 collect({executable_line,_,_}=Line) -> {set,[],[],Line};
+collect({executable_line,_,_,_}=Line) -> collect_executable_line(Line);
 collect({swap,D1,D2})        ->
     Regs = [D1,D2],
     {set,Regs,Regs,swap};
 collect({make_fun3,F,I,U,D,{list,Ss}}) -> {set,[D],Ss,{make_fun3,F,I,U}};
 collect(_)                   -> error.
 
+collect_executable_line({executable_line,_Loc,_Index,{_,Vars}}=I) ->
+    Ss = flatmap(fun({_Name,Regs}) -> Regs end, Vars),
+    {set,[],Ss,I}.
+
 %% embed_lines([Instruction]) -> [Instruction]
 %%  Combine blocks that would be split by line/1 instructions.
 %%  Also move a line instruction before a block into the block,