Merge pull request #103 from RevoSucks/libleo_boot2

decompile and implement libleo encryption.

Makefile

@@ -126,6 +126,8 @@ ASM_PROC_FLAGS  := --input-enc=utf-8 --output-enc=euc-jp --convert-statics=globa
 SPLAT           := $(PYTHON) -m splat split
 SPLAT_YAML      := $(TARGET)-$(VERSION).yaml
 
+ENCRYPT_LIBLEO  := $(PYTHON) tools/encrypt_libleo.py
+
 IINC := -Iinclude -Isrc -Iassets/$(VERSION) -I. -I$(BUILD_DIR)
 IINC += -Ilib/ultralib/include -Ilib/ultralib/include/PR -Ilib/ultralib/include/ido
 IINC += -Iinclude/
@@ -242,6 +244,12 @@ build/src/hal_libc.o: CFLAGS += -signed
 build/src/libleo/%.o: CC := $(ASM_PROC) $(ASM_PROC_FLAGS) $(CC_OLD) -- $(AS) $(ASFLAGS) --
 build/src/%.o: CC := $(ASM_PROC) $(ASM_PROC_FLAGS) $(CC) -- $(AS) $(ASFLAGS) --
 
+###################### Ugly hacksz #############################
+
+# This is unsanitary to do, -but-, because this file is encrypted we cant have splat decrypt it
+# on split. This requires us to include this manually, but it needs to be in the dependency
+# list. So we add it.
+O_FILES += build/src/libleo/bootstrap.s.o
 
 #### Main Targets ###
 
@@ -312,6 +320,8 @@ endif
 
 $(ROM): $(ELF)
 	$(OBJCOPY) -O binary --gap-fill=0xFF $< $@
+	$(ENCRYPT_LIBLEO) $@ $(MAP)
+
 # TODO: update rom header checksum
 
 # TODO: avoid using auto/undefined

src/libleo/bootdisk.c

@@ -0,0 +1,123 @@
+#include <ultra64.h>
+#include "libleo/internal.h"
+
+void LeoBootGame(void* entry);
+void __LeoBootGame2(void* entry);
+void __LeoBootGame3(void* entry);
+
+void LeoBootGame(void* entry) {
+    u8* ptr;
+    u8 key;
+    u32 i;
+
+    //Descramble __LeoBootGame2
+    ptr = (u8*)&__LeoBootGame2;
+    //Define difference key (a sum of each byte of __LeoBootGame2 address)
+    key = ((((u32)ptr & 0xFF000000) >> 0x18)
+         + (((u32)ptr & 0x00FF0000) >> 0x10)
+         + (((u32)ptr & 0x0000FF00) >> 0x08)
+         + (((u32)ptr & 0x000000FF))) & 0xFF;
+    for (i = 0; i < 0x324; i += 4) {
+        ptr[0];
+        ptr[1];
+        ptr[2] -= key;
+        ptr[3] += key;
+        ptr += 4;
+    }
+    osWritebackDCacheAll();
+    osInvalICache(__LeoBootGame2, 0x324);
+
+    //Descramble __LeoBootGame3
+    ptr = (u8*)&__LeoBootGame3;
+    for (i = 0; i < 0x60; i += 4) {
+        ptr[0];
+        ptr[1];
+        ptr[2] -= key;
+        ptr[3] += key;
+        ptr += 4;
+    }
+    osWritebackDCacheAll();
+    osInvalICache(__LeoBootGame3, 0x60);
+
+    __LeoBootGame2(entry);
+}
+
+// -----------------------------------------
+// Encrypted function
+// -----------------------------------------
+
+// This MUST be scrambled by the build process or they will not be executable
+// at runtime when descrambled.
+void __LeoBootGame2(void* entry) {
+    u8* entry2 = (u8*)entry;
+    u32 i;
+    s32 sp134;
+    u8 sp34[0x100];
+    vu32* var_v0;
+
+    sp134 = 0;
+    bzero(LeoBootGame, 0x13C);
+    osWritebackDCache(LeoBootGame, 0x13C);
+    __osSetSR(0x30000000);
+
+    *(u8*)0xA0000010 = (u32) (LEO_country_code & 0xFF000000) >> 0x18;
+
+    while (!IO_READ(0x4040010) & 1) {}
+
+    IO_WRITE(0x4040010, 0xA);
+
+    while (sp134 < 0x50) {
+        sp34[(sp134 * 0x11) % 256] = entry2[sp134];
+        sp134++;
+    }
+
+    while (IO_READ(0x4040018) & 1) {}
+
+    IO_WRITE(0x4600010, 3);
+
+    while (sp134 < 0xAA) {
+        sp34[(sp134 * 0x11) % 256] = entry2[sp134];
+        sp134++;
+    }
+
+    while (IO_READ(0x4400010) >= 11) {}
+
+    IO_WRITE(0x440000C, 0x3FF);
+    IO_WRITE(0x4400024, 0);
+    *(u32* )0xA4400010 = 0;
+    *(u8*)0xA0000190 = LEO_country_code & 0xFF;
+    IO_WRITE(0x4500000, 0);
+    IO_WRITE(0x4500004, 0);
+
+    while (sp134 < 0x100) {
+        sp34[(sp134 * 0x11) % 256] = entry2[sp134];
+        sp134++;
+    }
+
+    _bcopy(&sp34, entry2, 0x100U);
+    *(u8*)0xA0000110 = (u32) (LEO_country_code & 0xFF00) >> 8;
+    __osSetCause(0);
+    __osSetCount(0);
+    __osSetCompare(0);
+    IO_WRITE(0x4040010, 0xAAAAAE);
+    *(u8*)0xA0000090 = (u32) (LEO_country_code & 0xFF0000) >> 0x10;
+    IO_WRITE(0x430000C, 0);
+    IO_WRITE(0x4800018, 0);
+    IO_WRITE(0x450000C, 0);
+    IO_WRITE(0x4300000, 0x800);
+    IO_WRITE(0x4600010, 2);
+    osResetType = 2;
+
+    var_v0 = (vu32*)0xA4001000;
+    for (i = 0; i < 0x100; i++) {\
+        *var_v0++ = 0;
+    }
+
+    var_v0 = (vu32*)0xA4000000;
+    for (i = 0; i < 0x100; i++) {\
+        *var_v0++ = 0;
+    }
+    bzero(__LeoBootGame2, 0x2E4);
+    osWritebackDCache(__LeoBootGame2, 0x2E4);
+    __LeoBootGame3(entry2);
+}

tools/encrypt_libleo.py

@@ -0,0 +1,56 @@
+#!/usr/bin/env python3
+
+import sys
+import os
+
+# Libleo encryptor. v0.3
+# called via: encrypt_libleo.py [rom path] [map path]
+
+# Changelog:
+# v0.1: It exists. Please read the ROM in and do 1 write rather than 100s though
+# v0.2: Refactor to buffer ROM asm into 1 bytearray call for each encrypt for better I/O.
+# v0.3: Compact the encrypt function(s).
+
+# Return a virtual address for a given function.
+def get_address_from_map(Lines, name):
+    for line in Lines:
+        if name in line:
+            return line.split()[0]
+    return -1 # we didnt find anything. sad day.
+
+# Encrypt a function at a given location of the ROM via the expected libleo
+# obfuscation for an encrypted boot function.
+def encrypt_func(rom_file, vaddr, rom_addr, key, func_size):
+    i = 0
+    rom_file.seek(int(rom_addr, 16), os.SEEK_SET)
+    ba = bytearray(rom_file.read(func_size))
+    for i in range(i, func_size, 4):
+        ba[i + 2] = (ba[i + 2] + key) & 0xFF
+        ba[i + 3] = (ba[i + 3] - key) & 0xFF
+    rom_file.seek(int(rom_addr, 16), os.SEEK_SET)
+    rom_file.write(ba)
+    return 0
+
+# Open the 2 files
+rom_file = open(sys.argv[1], 'r+b')
+map_file = open(sys.argv[2], 'r')
+
+Lines = map_file.readlines()
+
+# Find the virtual addresses for each function to encrypt.
+game2_addr = hex(int(get_address_from_map(Lines, "__LeoBootGame2"), 16))
+game3_addr = hex(int(get_address_from_map(Lines, "__LeoBootGame3"), 16))
+
+game2_rom_addr = hex(int(game2_addr, 16) - 0x7FFFF400)
+game3_rom_addr = hex(int(game3_addr, 16) - 0x7FFFF400)
+
+game2_addr_int = int(game2_addr, 16)
+
+# Calculate the encryption key.
+key = ((game2_addr_int & 0xFF000000) >> 0x18) + ((game2_addr_int & 0x00FF0000) >> 0x10) + ((game2_addr_int & 0x0000FF00) >> 0x08) + ((game2_addr_int & 0x000000FF)) & 0xFF
+
+print(key)
+
+# TODO: The sizes are hardcoded for __LeoBootGame2 and __LeoBootGame3. TODO: Don't hardcode?
+encrypt_func(rom_file, game2_addr, game2_rom_addr, key, 804) # __LeoBootGame2
+encrypt_func(rom_file, game3_addr, game3_rom_addr, key, 96)  # __LeoBootGame3

yamls/us/rom.yaml

@@ -45,8 +45,8 @@
     - [0xC3D0, asm, exception_set] # exception functions
     - [0xC3F0, hasm, yay0] # Yay0 decoder
     - [0xC4A0, hasm, abs] # abs function
-    - [0xC4B0, asm, libleo_bootdisk]  # These are libleo's bootdisk and bootstrap files, but
-    - [0xC910, hasm, libleo/bootstrap]
+    - [0xC4B0, c, libleo/bootdisk]
+    - [0xC910, hasm, ../../src/libleo/bootstrap]
     - [0xC970, c]
     - [0xCE80, c] #
     - [0xD470, c] #