Skip to content

Commit

Permalink
Ensure reproducible builds
Browse files Browse the repository at this point in the history
  • Loading branch information
szapp committed May 31, 2024
2 parents 6b5cb47 + b84a4ce commit 850cfb2
Show file tree
Hide file tree
Showing 4 changed files with 257 additions and 42 deletions.
110 changes: 71 additions & 39 deletions Makefile
Original file line number Diff line number Diff line change
@@ -1,22 +1,39 @@
# Platform specific definitions
ifdef OS
RM := del /Q
FixPath = $(subst /,\,$1)
mkdir = mkdir $(subst /,\,$(1)) > nul 2>&1 || (exit 0)
SCRIPTEXT := .bat
else ifeq ($(shell uname), Linux)
RM := rm -f
FixPath = $1
mkdir = mkdir -p $(1)
SCRIPTEXT := .sh
RM := del /Q
FixPath = $(subst /,\,$1)
mkdir = mkdir $(patsubst %\,%,$(subst /,\,$(1)))> nul 2>&1 || (exit 0)
SCRIPTEXT := .bat
# Force shell to CMD
ifdef COMSPEC
export SHELL := $(shell echo %COMSPEC%)
else
export SHELL := $(shell where cmd)
endif
else
# Note that *nix is no longer supported in other parts of the Makefile
RM := rm -f
FixPath = $1
mkdir = mkdir -p $(1)
SCRIPTEXT := .sh
$(error Only Windows is supported)
endif

# Meta data
# Meta data and escaped characters
META := metadata
oe := \xF6
c := \xA9

# Include meta data
include $(META)
export VERSION=$(VBASE).$(VMAJOR).$(VMINOR)
export VERSION := $(VBASE).$(VMAJOR).$(VMINOR)
export PATH := $(subst ;;,;,$(PATH);$(shell getGnuWin32Path))
export BUILD_TIME:=$(shell getCommitTime)
ifneq ($(BUILD_TIME),)
$(info Build time is set to $(BUILD_TIME))
else
$(info Warning: No build time is set. Build will not be reproducible!)
endif

# Directories
BUILDDIR := build/
Expand All @@ -39,20 +56,25 @@ RESEXT := .res
DLLEXT := .dll
OBJEXT := .obj
NSIEXT := .nsi
INIEXT := .ini

# Assemblers/builders
NSIS := makensis
NASM := nasm
GOLINK := golink
GORC := gorc
LINKER := golink
RCCOMP := gorc
REPRO := ducible
GETBINLIST := $(call FixPath,./getBinList)$(SCRIPTEXT)
EXTRACTSYM := $(call FixPath,./extractSymbols)$(SCRIPTEXT)
VERIFYSIZE := $(call FixPath,./verifySize)$(SCRIPTEXT)
PATCHREPRO := $(call FixPath,./patchBuildBytes)$(SCRIPTEXT)
SETFILETIME := $(call FixPath,./setTimestamps)$(SCRIPTEXT)

FLAGS_C := -I$(SRCDIR)
FLAGS_A := -f win32 $(FLAGS_C)
FLAGS_A := -f win32 --reproducible $(FLAGS_C)
FLAGS_L := /dll /entry DllMain /largeaddressaware /nxcompat /dynamicbase /ni
FLAGS_RC := /ni
FLAGS_N := -X"SetCompress force" -X"SetCompressor /FINAL /SOLID lzma" -X"SetCompressorDictSize 8" -X"SetDatablockOptimize on"

# TARGET
TARGET := $(BUILDDIR)Ninja$(DLLEXT)
Expand All @@ -62,18 +84,19 @@ RSC := $(DLLDIR)resource$(RESEXT)
SRCDLL := $(DLLDIR)Ninja$(ASMEXT)
IKLG := $(INCDIR)iklg.data

# WRAPPER
WRAPPER := $(BUILDDIR)BugslayerUtil$(DLLEXT)
WRAPPER_OBJ := $(BINDIR)BugslayerUtil$(OBJEXT)
WRAPPER_SRC := $(DLLDIR)BugslayerUtil$(ASMEXT)
# LOADER
LOADER := $(BUILDDIR)BugslayerUtil$(DLLEXT)
LOADER_OBJ := $(BINDIR)BugslayerUtil$(OBJEXT)
LOADER_SRC := $(DLLDIR)BugslayerUtil$(ASMEXT)

# SETUP
SETUP := $(BUILDDIR)Ninja-$(VERSION)$(EXEEXT)
SETUPSCR := $(SETUPDIR)Ninja$(NSIEXT)
SETUPINI := $(SETUPDIR)setup$(INIEXT)

# System dependencies
SYSDEP := User32$(DLLEXT) Kernel32$(DLLEXT) NtDll$(DLLEXT)
WRAPPER_SYSDEP := Kernel32$(DLLEXT)
SYSDEP := Kernel32$(DLLEXT) User32$(DLLEXT) NtDll$(DLLEXT)
LOADER_SYSDEP := Kernel32$(DLLEXT)

# Content
CONTENT := $(INCDIR)injections$(INCEXT)
Expand Down Expand Up @@ -193,6 +216,8 @@ DATA := $(DATA_BASE:%=$(DATADIR)%$(ASMEXT))
all : $(SETUP)

clean :
@$(call mkdir,$(BUILDDIR))
@$(call mkdir,$(BINDIR))
$(RM) $(call FixPath,$(BUILDDIR)*)
$(RM) $(call FixPath,$(BINDIR)*)
$(RM) $(call FixPath,$(CONTENT))
Expand All @@ -202,7 +227,7 @@ clean :

cleanDLL :
$(RM) $(call FixPath,$(TARGET))
$(RM) $(call FixPath,$(WRAPPER))
$(RM) $(call FixPath,$(LOADER))
$(RM) $(call FixPath,$(RC))

remake : clean all
Expand All @@ -213,27 +238,34 @@ relink : cleanDLL all


# Build dependencies
$(SETUP) : $(WRAPPER) $(TARGET) LICENSE $(SETUPSCR)
$(NSIS) /X"SetCompressor /FINAL lzma" $(SETUPSCR)
$(SETUP) : $(LOADER) $(TARGET) LICENSE $(SETUPSCR) $(SETUPINI)
$(SETFILETIME) $(call FixPath,$^)
$(NSIS) $(FLAGS_N) $(SETUPSCR)

$(WRAPPER) : $(WRAPPER_OBJ) $(TARGET)
$(LOADER) : $(LOADER_OBJ) $(TARGET)
@$(call mkdir,$(BUILDDIR))
golink $(FLAGS_L) /fo $(call FixPath,$@) $^ $(WRAPPER_SYSDEP)
$(LINKER) $(FLAGS_L) /fo $@ $^ $(LOADER_SYSDEP)
$(REPRO) $@
$(PATCHREPRO) $(call FixPath,$@)

$(TARGET) : $(OBJ) $(RSC)
@$(call mkdir,$(BUILDDIR))
golink $(FLAGS_L) /fo $(call FixPath,$@) $^ $(SYSDEP)
$(LINKER) $(FLAGS_L) /fo $@ $^ $(SYSDEP)
$(REPRO) $@
$(PATCHREPRO) $(call FixPath,$@)

$(WRAPPER_OBJ) : $(WRAPPER_SRC)
$(LOADER_OBJ) : $(LOADER_SRC)
@$(call mkdir,$(BINDIR))
$(NASM) $(FLAGS_A) -o $@ $<

$(OBJ) : $(SRCDLL) $(CONTENT) $(IKLG)
@$(call mkdir,$(BINDIR))
$(NASM) $(FLAGS_A) -o $@ $<

# Overwrite MemoryFlags (0x36 WORD) in the RES Header which sometimes varies across builds on different machines
$(RSC) : $(RC)
gorc $(FLAGS_RC) /fo $@ /r $^
$(RCCOMP) $(FLAGS_RC) /fo $@ /r $^
ECHO -n 0000 | xxd -r -p | dd of=$@ bs=1 seek=54 count=2 conv=notrunc > nul 2>&1

$(CONTENT) : $(BINARIES_G1) $(BINARIES_G112) $(BINARIES_G130) $(BINARIES_G2)
$(GETBINLIST) $(call FixPath,$@) $(SRCDIR)
Expand Down Expand Up @@ -272,20 +304,20 @@ $(RC) : $(META)
@ECHO {>> "$(call FixPath,$@)"
@ECHO BLOCK "StringFileInfo">> "$(call FixPath,$@)"
@ECHO {>> "$(call FixPath,$@)"
@ECHO BLOCK "000004B0">> "$(call FixPath,$@)"
@ECHO {>> "$(call FixPath,$@)"
@ECHO VALUE "FileDescription", "Ninja <$(NINJA_WEBSITE)>">> "$(call FixPath,$@)"
@ECHO VALUE "FileVersion", "$(VBASE).$(VMAJOR).$(VMINOR)">> "$(call FixPath,$@)"
@ECHO VALUE "InternalName", "Ninja">> "$(call FixPath,$@)"
@ECHO VALUE "LegalCopyright", "Copyright © $(RYEARS) Sören Zapp">> "$(call FixPath,$@)"
@ECHO VALUE "OriginalFilename", "Ninja.dll">> "$(call FixPath,$@)"
@ECHO VALUE "ProductName", "Ninja">> "$(call FixPath,$@)"
@ECHO VALUE "ProductVersion", "$(VBASE).$(VMAJOR).$(VMINOR)">> "$(call FixPath,$@)"
@ECHO }>> "$(call FixPath,$@)"
@ECHO BLOCK "000004E4">> "$(call FixPath,$@)"
@ECHO {>> "$(call FixPath,$@)"
@ECHO VALUE "FileDescription", "Ninja <$(NINJA_WEBSITE)>">> "$(call FixPath,$@)"
@ECHO VALUE "FileVersion", "$(VBASE).$(VMAJOR).$(VMINOR)">> "$(call FixPath,$@)"
@ECHO VALUE "InternalName", "Ninja">> "$(call FixPath,$@)"
@ECHO VALUE "LegalCopyright", "Copyright $(c) $(RYEARS) S$(oe)ren Zapp">> "$(call FixPath,$@)"
@ECHO VALUE "OriginalFilename", "Ninja.dll">> "$(call FixPath,$@)"
@ECHO VALUE "ProductName", "Ninja">> "$(call FixPath,$@)"
@ECHO VALUE "ProductVersion", "$(VBASE).$(VMAJOR).$(VMINOR)">> "$(call FixPath,$@)"
@ECHO }>> "$(call FixPath,$@)"
@ECHO }>> "$(call FixPath,$@)"
@ECHO/>> "$(call FixPath,$@)"
@ECHO BLOCK "VarFileInfo">> "$(call FixPath,$@)"
@ECHO {>> "$(call FixPath,$@)"
@ECHO VALUE "Translation", 0x0000 0x04B^0>> "$(call FixPath,$@)"
@ECHO VALUE "Translation", 0x0000 0x04E4>> "$(call FixPath,$@)"
@ECHO }>> "$(call FixPath,$@)"
@ECHO }>> "$(call FixPath,$@)"
6 changes: 3 additions & 3 deletions getBinList.bat
Original file line number Diff line number Diff line change
Expand Up @@ -33,13 +33,10 @@ COPY /Y NUL "%outfile%" >NUL
:: Process files
SET gothic=1
FOR %%N IN ("%indir%") DO CALL :fileloop %%N || EXIT /B 3
ECHO/>> %outfile%
SET gothic=112
FOR %%N IN ("%indir%") DO CALL :fileloop %%N || EXIT /B 3
ECHO/>> %outfile%
SET gothic=130
FOR %%N IN ("%indir%") DO CALL :fileloop %%N || EXIT /B 3
ECHO/>> %outfile%
SET gothic=2
FOR %%N IN ("%indir%") DO CALL :fileloop %%N || EXIT /B 3

Expand Down Expand Up @@ -71,6 +68,9 @@ IF NOT DEFINED address ECHO %filename%: %getAddress% failed to execute.&& EXIT /
:: Write to output file
ECHO add_inject_g%gothic% %address%,"../bin/%filebase%_g%gothic%">> "%outfile%"

:: Sort the lines in the file for reproducibility
"%SystemRoot%\System32\sort" /M 10240 "%outfile%" /o "%outfile%"

EXIT /B 0

:usage
Expand Down
154 changes: 154 additions & 0 deletions patchBuildBytes.bat
Original file line number Diff line number Diff line change
@@ -0,0 +1,154 @@
::
:: Patch build files to remove non-deterministic properties for reproducible builds
:: This batch file complements ducible which does not fix .rscr section timestamps of sub tables
:: and the import table ordinals and DLL name capitalization that differ between machine versions
::
:: Arguments: DLLFILE
::
@ECHO OFF
SETLOCAL ENABLEDELAYEDEXPANSION
SET LC_ALL=en_US.utf8

:: Sanity check
IF [%1] == [] GOTO usage
SET "file=%~1"
SET filebase=%~n1

:: Use objdump to get a summary of the file to later compare patched bytes
objdump -x %file% > %file%.txt

:: First: Zero out the hints and ordinal values in the import table

:: Find image base
TYPE %file%.txt ^
| grep -oiP "^ImageBase[[:blank:]]+[[:xdigit:]]+[[:space:]]?$" ^
> base.txt
FOR /F "tokens=2" %%a IN (base.txt) DO (
SET /A "image_base=0x%%a"
)
DEL /Q base.txt

ECHO ImageBase: %image_base%

:: Find VMA and file offset of .idata section
objdump -h -j .idata %file% | findstr "\.idata" > header.txt 2>nul
FOR /F "tokens=3,4,6" %%a IN (header.txt) DO (
SET /A "idata_size=0x%%a"
SET /A "idata_vma=0x%%b"
SET /A "idata_offset=0x%%c"
)
DEL /Q header.txt

:: Subtract image base from VMA for relative addresses
SET /A "idata_vma=idata_vma-image_base"

ECHO Import table: %idata_vma% at offset %idata_offset%

:: Find the addresses of IAT entries
TYPE %file%.txt ^
| grep -zoiP "(?<=The Import Tables)(.|\r|\n)*(?=(?:0{8}\s){5})" ^
| grep -aoiP "^[[:blank:]]+[[:xdigit:]]+[[:blank:]]+\d+[[:blank:]]+[\w\d@_]+[[:space:]]?$" ^
> iat.txt
FOR /F "tokens=1" %%A IN (iat.txt) DO (
:: Calculate the file offset from VMA
SET "hex=%%A"
SET /A "hex_dec=0x!hex!"
SET /A "addr=!hex_dec!-!idata_vma!+!idata_offset!"
:: Zero out ordinal number at address (WORD)
ECHO -n 0000 | xxd -r -p | dd of=%file% bs=1 seek=!addr! count=2 conv=notrunc > nul 2>&1
)
DEL /Q iat.txt

:: Next: Change DLL names upper case for consistency

:: First extract the list of DLL names
TYPE %file%.txt ^
| grep -zoiP "(?<=The Import Tables)(.|\r|\n)*(?=(?:0{8}\s){5})" ^
| grep -aoiP "^[[:blank:]]+Dll Name:[[:blank:]]\K.*[[:space:]]?$" ^
> dllnames.txt
SET "dllnames="
FOR /F "delims=" %%A IN (dllnames.txt) DO SET "dllnames=!dllnames! %%A"
DEL /Q dllnames.txt

:: Then extract the import tables
TYPE %file%.txt ^
| grep -zoiP "(?<=The Import Tables)(.|\r|\n)*(?=(?:0{8}\s){5})" ^
| grep -aoiP "^[[:blank:]]+([[:xdigit:]]{8}[[:blank:]]+){5}[[:xdigit:]]{8}[[:space:]]?$" ^
> dlltables.txt

SET /A dll_count=0
FOR /F "tokens=5" %%A IN (dlltables.txt) DO (
SET /A dll_count=dll_count+1
:: Calculate the file offset from VMA
SET "hex=%%A"
SET /A "hex_dec=0x!hex!"
SET /A "addr=!hex_dec!-!idata_vma!+!idata_offset!"
:: Find the length of the DLL name
SET /A name_idx=0
FOR %%N IN (%dllnames%) DO (
SET /A name_idx=name_idx+1
IF !name_idx! EQU !dll_count! SET "name=%%N"
)
CALL :strlen name len
:: Convert name to uppercase
dd if=%file% of=%file% bs=1 seek=!addr! skip=!addr! count=!len! conv=ucase,notrunc > nul 2>&1
)
DEL /Q dlltables.txt

:: Next: Remove time stamps from resource table

:: Find size and offset of .rsrc section
objdump -h -j .rsrc %file% | findstr "\.rsrc" > header.txt
FOR /F "tokens=3,6" %%a IN (header.txt) DO (
SET /A "rsrc_size=0x%%a"
SET /A "rsrc_offset=0x%%b"
)
DEL /Q header.txt

:: Better safe than sorry: Assert the section size
IF "%rsrc_size%" == "" GOTO :CLEANUP
IF "%rsrc_size%" NEQ "768" ECHO Warning: Resource directory sections is of unexpected size: %rsrc_size%. Cannot make build reproducible.&& GOTO :CLEANUP

:: CAUTION the following is hard-coded to match the offsets of the resource (see Makefile)

:: Type Table Time
SET /A "timestamp_offset=rsrc_offset + 4"
ECHO -n 003b3d4b | xxd -r -p | dd of=%file% bs=1 seek=%timestamp_offset% count=4 conv=notrunc > nul 2>&1

:: Name Table Time
SET /A "timestamp_offset=rsrc_offset + 28"
ECHO -n 003b3d4b | xxd -r -p | dd of=%file% bs=1 seek=%timestamp_offset% count=4 conv=notrunc > nul 2>&1

:: Language Table Time
SET /A "timestamp_offset=rsrc_offset + 52"
ECHO -n 003b3d4b | xxd -r -p | dd of=%file% bs=1 seek=%timestamp_offset% count=4 conv=notrunc > nul 2>&1

:CLEANUP

:: Show the differences
objdump -x %file% > %file%_patched.txt
FC %file%.txt %file%_patched.txt

:: Cleanup
DEL /Q %file%.txt
DEL /Q %file%_patched.txt

EXIT /B 0

:usage
ECHO Usage: %~nx0 DLLFILE
EXIT /B 0

:: Source: https://ss64.com/nt/syntax-strlen.html
:strlen StrVar [RtnVar]
SETLOCAL ENABLEDELAYEDEXPANSION
SET "s=#!%~1!"
SET "len=0"
FOR %%N IN (4096 2048 1024 512 256 128 64 32 16 8 4 2 1) DO (
IF "!s:~%%N,1!" NEQ "" (
SET /A "len+=%%N"
SET "s=!s:~%%N!"
)
)
ENDLOCAL&if "%~2" NEQ "" (SET %~2=%len%) ELSE ECHO %len%
EXIT /B
Loading

0 comments on commit 850cfb2

Please sign in to comment.