From a3289bb4d02ad82cb587a16b3d127db396821ea4 Mon Sep 17 00:00:00 2001 From: Alex Alabuzhev Date: Sun, 19 Jan 2025 20:58:40 +0000 Subject: [PATCH] Compact map files in distibution to only include what we actually need --- appveyor.yml | 2 + far/changelog | 5 ++ far/map_file.cpp | 56 +++++++++++++++++++-- far/tools/stacktrace.py | 100 ++++++++++++++++++++++++++++++++++---- far/vbuild.m4 | 2 +- misc/nightly/installer.sh | 3 ++ 6 files changed, 154 insertions(+), 14 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index d87242f6e2..bdb6e2dd4f 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -138,6 +138,8 @@ after_build: - copy /Y enc\build\lua\macroapi_manual.pl\macroapi_manual.pl.chm ci\%configuration%.%platform_name%\Encyclopedia #copy misc docs and addons - xcopy /e /q /y /k extra\* ci\%configuration%.%platform_name%\ + #compact maps + - far\tools\stacktrace.py ci\%configuration%.%platform_name% #test & archive - cd ci\%configuration%.%platform_name% #Run macrotest.lua, on failure %ERRORLEVEL% is non zero diff --git a/far/changelog b/far/changelog index 5b26e6ab25..eefb5dc999 100644 --- a/far/changelog +++ b/far/changelog @@ -1,3 +1,8 @@ +-------------------------------------------------------------------------------- +drkns 2025-01-19 20:58:24+00:00 - build 6418 + +1. Compact map files in distibution to only include what we actually need. + -------------------------------------------------------------------------------- drkns 2025-01-17 14:41:31+00:00 - build 6417 diff --git a/far/map_file.cpp b/far/map_file.cpp index f0d3333424..6b4f160b89 100644 --- a/far/map_file.cpp +++ b/far/map_file.cpp @@ -150,6 +150,7 @@ map_file::info map_file::get(uintptr_t const Address) enum class map_format { unknown, + mini, msvc, clang, gcc, @@ -157,6 +158,9 @@ enum class map_format static auto determine_format(string_view const Str) { + if (Str == L"MINIMAP"sv) + return map_format::mini; + if (Str.starts_with(L' ')) return map_format::msvc; @@ -181,6 +185,49 @@ static auto determine_format(std::istream& Stream) return determine_format(Lines.begin()->Str); } +static void read_mini(std::istream& Stream, unordered_string_set& Files, std::map& Symbols) +{ + RegExp ReObject, ReSymbol; + ReObject.Compile(L"^(\\d+) (.+)$"sv, OP_OPTIMIZE); + ReSymbol.Compile(L"^([0-9A-Fa-f]+) (\\d+) (.+)$"sv, OP_OPTIMIZE); + + regex_match Match; + auto& m = Match.Matches; + m.reserve(4); + + std::unordered_map ObjNames; + bool FilesSeen = false, SymbolsSeen = false; + + for (const auto& i: enum_lines(Stream, CP_UTF8)) + { + if (i.Str.empty()) + continue; + + if (!FilesSeen) + FilesSeen = i.Str == L"FILES"sv; + + if (!SymbolsSeen) + SymbolsSeen = i.Str == L"SYMBOLS"sv; + + if (FilesSeen && !SymbolsSeen && ReObject.Search(i.Str, Match)) + { + ObjNames.try_emplace(from_string(get_match(i.Str, m[1])), get_match(i.Str, m[2])); + continue; + } + + if (SymbolsSeen && ReSymbol.Search(i.Str, Match)) + { + map_file::line Line; + Line.Name = get_match(i.Str, m[3]); + const auto FileIndex = from_string(get_match(i.Str, m[2])); + Line.File = std::to_address(Files.emplace(ObjNames.find(FileIndex)->second).first); + const auto Address = from_string(get_match(i.Str, m[1]), {}, 16); + Symbols.try_emplace(Address, std::move(Line)); + continue; + } + } +} + static void read_vc(std::istream& Stream, unordered_string_set& Files, std::map& Symbols) { RegExp ReBase, ReSymbol; @@ -189,7 +236,7 @@ static void read_vc(std::istream& Stream, unordered_string_set& Files, std::map< regex_match Match; auto& m = Match.Matches; - m.reserve(3); + m.reserve(4); uintptr_t BaseAddress{}; @@ -232,7 +279,7 @@ static void read_clang(std::istream& Stream, unordered_string_set& Files, std::m regex_match Match; auto& m = Match.Matches; - m.reserve(2); + m.reserve(3); string ObjName; @@ -268,7 +315,7 @@ static void read_gcc(std::istream& Stream, unordered_string_set& Files, std::map regex_match Match; auto& m = Match.Matches; - m.reserve(2); + m.reserve(3); const auto BaseAddress = 0x1000; @@ -304,6 +351,9 @@ void map_file::read(std::istream& Stream) { switch (determine_format(Stream)) { + case map_format::mini: + return read_mini(Stream, m_Files, m_Symbols); + case map_format::msvc: return read_vc(Stream, m_Files, m_Symbols); diff --git a/far/tools/stacktrace.py b/far/tools/stacktrace.py index 3e56ab5154..4a54ee3944 100644 --- a/far/tools/stacktrace.py +++ b/far/tools/stacktrace.py @@ -1,8 +1,10 @@ import bisect import ctypes +import os import re import sys + def undecorate(name): UNDNAME_NO_FUNCTION_RETURNS = 0x0004 UNDNAME_NO_ALLOCATION_MODEL = 0x0008 @@ -25,6 +27,38 @@ def undecorate(name): return str(out.value, "utf-8") if ctypes.windll.dbghelp.UnDecorateSymbolName(ctypes.create_string_buffer(bytes(name, "utf-8")), out, buffer_size, flags) != 0 else name +def parse_mini(map_file, map_data): + re_object = re.compile(r"^(\d+) (.+)\s+$") + re_symbol = re.compile(r"^([0-9A-Fa-f]+) (\d+) (.+)\s+$") + + obj_names = {} + files_seen = False + symbols_seen = False + + for line in map_file: + if len(line) == 0: + continue + + if not files_seen: + files_seen = line.startswith("FILES") + + if not symbols_seen: + symbols_seen = line.startswith("SYMBOLS") + + if files_seen and not symbols_seen: + m = re_object.search(line) + if m is not None: + obj_names.setdefault(int(m.group(1)), m.group(2)) + continue; + + if symbols_seen: + m = re_symbol.search(line) + if m is not None: + Address = int(m.group(1), 16); + file_name = obj_names[int(m.group(2))] + map_data.setdefault(Address, (m.group(3), file_name)) + + def parse_vc(map_file, map_data): re_base = re.compile(r"^ +Preferred load address is ([0-9A-Fa-f]+)\s+$") re_symbol = re.compile(r"^ +[0-9A-Fa-f]+:[0-9A-Fa-f]+ +([^ ]+) +([0-9A-Fa-f]+) .+ ([^ ]+)\s+$") @@ -42,18 +76,18 @@ def parse_vc(map_file, map_data): Address = int(m.group(2), 16); if Address >= BaseAddress: Address -= BaseAddress - map_data[Address] = (m.group(1), m.group(3)) + map_data.setdefault(Address, (m.group(1), m.group(3))) def parse_clang(map_file, map_data): - re_object = re.compile(r"^[0-9A-Fa-f]+ [0-9A-Fa-f]+ +[0-9]+ (.+)\s+$") + re_object = re.compile(r"^[0-9A-Fa-f]+ [0-9A-Fa-f]+ +[0-9]+ (.+):(.+)\s+$") re_symbol = re.compile(r"^([0-9A-Fa-f]+) [0-9A-Fa-f]+ 0 (.+)\s+$") objname = None for line in map_file: m = re_symbol.search(line) if m is not None: - map_data[int(m.group(1), 16)] = (m.group(2), objname) + map_data.setdefault(int(m.group(1), 16), (m.group(2), objname)) continue m = re_object.search(line) @@ -80,28 +114,34 @@ def parse_gcc(map_file, map_data): m = re_symbol.search(line) if m is not None: - map_data[int(m.group(1), 16) + BaseAddress] = (m.group(2), file_name) + map_data.setdefault(int(m.group(1), 16) + BaseAddress, (m.group(2), file_name)) continue last_line = line -def show_stack(): +def read_map(map_file_name): map_data = dict() - with open(sys.argv[1], encoding="utf-8") as map_file: + with open(map_file_name, 'r', encoding="utf-8") as map_file: header = map_file.readline() map_file.seek(0) - if header.startswith("Address Size Align Out In Symbol"): + if header.startswith("MINIMAP"): + parse_mini(map_file, map_data) + elif header.startswith("Address Size Align Out In Symbol"): parse_clang(map_file, map_data) elif len(header.strip()) == 0: parse_gcc(map_file, map_data) else: parse_vc(map_file, map_data) - keys = sorted(map_data.keys()) + return sorted(map_data.keys()), map_data - for i in sys.argv[2:]: + +def show_stack(map_file_name, frames): + keys, map_data = read_map(map_file_name) + + for i in frames: address = int(i, 16) key_index = bisect.bisect(keys, address) if keys[key_index] > address: @@ -110,5 +150,45 @@ def show_stack(): print("{0}: {1}+{2:X} ({3})".format(i, undecorate(data[0]), address - keys[key_index], data[1])) +def compact_map(map_file_name): + keys, map_data = read_map(map_file_name) + + tmp_name = map_file_name + ".tmp" + with open(tmp_name, 'w', encoding="utf-8") as out_file: + out_file.write("MINIMAP\n") + + files = set() + for key in keys: + data = map_data[key] + file = data[1] + files.add(file) + + out_file.write("\nFILES\n") + files_map = {} + for index, file in enumerate(sorted(files)): + files_map[file] = index + out_file.write(f"{index} {file}\n") + + out_file.write("\nSYMBOLS\n") + for key in keys: + data = map_data[key] + out_file.write(f"{key:X} {files_map[data[1]]} {data[0]}\n") + + os.replace(tmp_name, map_file_name) + + +def compact_maps(directory_name): + for root, dirnames, filenames in os.walk(directory_name): + for filename in filenames: + if filename.lower().endswith(".map"): + compact_map(os.path.join(root, filename)) + + +def main(): + if len(sys.argv) == 2: + compact_maps(sys.argv[1]) + else: + show_stack(sys.argv[1], sys.argv[2:]) + if __name__ == "__main__": - show_stack() + main() diff --git a/far/vbuild.m4 b/far/vbuild.m4 index d46c0e4db6..999f25d959 100644 --- a/far/vbuild.m4 +++ b/far/vbuild.m4 @@ -1 +1 @@ -6417 +6418 diff --git a/misc/nightly/installer.sh b/misc/nightly/installer.sh index d46f8d984d..6d49901f01 100755 --- a/misc/nightly/installer.sh +++ b/misc/nightly/installer.sh @@ -2,10 +2,13 @@ cd misc/msi-installer || exit 1 +python far/tools/stacktrace.py outfinalnew32 WINEDLLOVERRIDES="msi=n" wine cmd /c ../../installer.32.bat +python far/tools/stacktrace.py outfinalnew64 WINEDLLOVERRIDES="msi=n" wine cmd /c ../../installer.64.bat +python far/tools/stacktrace.py outfinalnewARM64 WINEDLLOVERRIDES="msi=n" wine cmd /c ../../installer.ARM64.bat cd ../..