Merge pull request #81 from RevoSucks/format_util

Util.c and add clang-format

.clang-format

@@ -0,0 +1,23 @@
+IndentWidth: 4
+Language: Cpp
+UseTab: Never
+ColumnLimit: 120
+PointerAlignment: Left
+BreakBeforeBraces: Attach
+SpaceAfterCStyleCast: false
+Cpp11BracedListStyle: false
+IndentCaseLabels: true
+BinPackArguments: true
+BinPackParameters: true
+AlignAfterOpenBracket: Align
+AlignOperands: true
+BreakBeforeTernaryOperators: true
+BreakBeforeBinaryOperators: None
+AllowShortBlocksOnASingleLine: true
+AllowShortIfStatementsOnASingleLine: false
+AllowShortLoopsOnASingleLine: false
+AllowShortCaseLabelsOnASingleLine: false
+AllowShortFunctionsOnASingleLine: false
+AlignEscapedNewlines: Left
+AlignTrailingComments: true
+SortIncludes: false

.clang-tidy

@@ -0,0 +1,9 @@
+Checks: '-*,readability-braces-around-statements,readability-inconsistent-declaration-parameter-name'
+WarningsAsErrors: ''
+HeaderFilterRegex: '(src|include)\/.*\.h$'
+FormatStyle: 'file'
+CheckOptions:
+  # Require argument names to match exactly (instead of allowing a name to be a prefix/suffix of another)
+  # Note: 'true' is expected by clang-tidy 12+ but '1' is used for compatibility with older versions
+  - key: readability-inconsistent-declaration-parameter-name.Strict
+    value: 1

format.py

@@ -0,0 +1,190 @@
+#!/usr/bin/env python3
+
+import argparse
+import glob
+import multiprocessing
+import os
+import re
+import shutil
+import subprocess
+import sys
+import tempfile
+from functools import partial
+from typing import List
+
+# clang-format, clang-tidy and clang-apply-replacements default version
+# This specific version is used when available, for more consistency between contributors
+CLANG_VER = 14
+
+# Clang-Format options (see .clang-format for rules applied)
+FORMAT_OPTS = "-i -style=file"
+
+# Clang-Tidy options (see .clang-tidy for checks enabled)
+TIDY_OPTS = "-p ."
+TIDY_FIX_OPTS = "--fix --fix-errors"
+
+# Clang-Apply-Replacements options (used for multiprocessing)
+APPLY_OPTS = "--format --style=file"
+
+# Compiler options used with Clang-Tidy
+# Normal warnings are disabled with -Wno-everything to focus only on tidying
+INCLUDES = "-Iinclude -Isrc -Ibuild -I."
+DEFINES = "-D_LANGUAGE_C -DNON_MATCHING -D_MIPS_SZLONG=32"
+COMPILER_OPTS = f"-fno-builtin -std=gnu90 -m32 -Wno-everything {INCLUDES} {DEFINES}"
+
+
+def get_clang_executable(allowed_executables: List[str]):
+    for executable in allowed_executables:
+        try:
+            subprocess.check_call([executable, "--version"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+            return executable
+        except FileNotFoundError or subprocess.CalledProcessError:
+            pass
+    return None
+
+
+def get_tidy_version(tidy_executable: str):
+    tidy_version_run = subprocess.run([tidy_executable, "--version"], stdout=subprocess.PIPE, universal_newlines=True)
+    match = re.search(r"LLVM version ([0-9]+)", tidy_version_run.stdout)
+    return int(match.group(1))
+
+
+CLANG_FORMAT = get_clang_executable([f"clang-format-{CLANG_VER}", "clang-format"])
+if CLANG_FORMAT is None:
+    sys.exit(f"Error: neither clang-format nor clang-format-{CLANG_VER} found")
+
+CLANG_TIDY = get_clang_executable([f"clang-tidy-{CLANG_VER}", "clang-tidy"])
+if CLANG_TIDY is None:
+    sys.exit(f"Error: neither clang-tidy nor clang-tidy-{CLANG_VER} found")
+
+CLANG_APPLY_REPLACEMENTS = get_clang_executable([f"clang-apply-replacements-{CLANG_VER}", "clang-apply-replacements"])
+
+# Try to detect the clang-tidy version and add --fix-notes for version 13+
+# This is used to ensure all fixes are applied properly in recent versions
+if get_tidy_version(CLANG_TIDY) >= 13:
+    TIDY_FIX_OPTS += " --fix-notes"
+
+
+def list_chunks(list: List, chunk_length: int):
+    for i in range(0, len(list), chunk_length):
+        yield list[i : i + chunk_length]
+
+
+def run_clang_format(files: List[str]):
+    exec_str = f"{CLANG_FORMAT} {FORMAT_OPTS} {' '.join(files)}"
+    subprocess.run(exec_str, shell=True)
+
+
+def run_clang_tidy(files: List[str]):
+    exec_str = f"{CLANG_TIDY} {TIDY_OPTS} {TIDY_FIX_OPTS} {' '.join(files)} -- {COMPILER_OPTS}"
+    subprocess.run(exec_str, shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+
+
+def run_clang_tidy_with_export(tmp_dir: str, files: List[str]):
+    (handle, tmp_file) = tempfile.mkstemp(suffix=".yaml", dir=tmp_dir)
+    os.close(handle)
+
+    exec_str = f"{CLANG_TIDY} {TIDY_OPTS} --export-fixes={tmp_file} {' '.join(files)} -- {COMPILER_OPTS}"
+    subprocess.run(exec_str, shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+
+
+def run_clang_apply_replacements(tmp_dir: str):
+    exec_str = f"{CLANG_APPLY_REPLACEMENTS} {APPLY_OPTS} {tmp_dir}"
+    subprocess.run(exec_str, shell=True)
+
+
+def add_final_new_line(file: str):
+    # https://backreference.org/2010/05/23/sanitizing-files-with-no-trailing-newline/index.html
+    # "gets the last character of the file pipes it into read, which will exit with a nonzero exit
+    # code if it encounters EOF before newline (so, if the last character of the file isn't a newline).
+    # If read exits nonzero, then append a newline onto the file using echo (if read exits 0,
+    # that satisfies the ||, so the echo command isn't run)." (https://stackoverflow.com/a/34865616)
+    exec_str = f"tail -c1 {file} | read -r _ || echo >> {file}"
+    subprocess.run(exec_str, shell=True)
+
+
+def format_files(src_files: List[str], extra_files: List[str], nb_jobs: int):
+    if nb_jobs != 1:
+        print(f"Formatting files with {nb_jobs} jobs")
+    else:
+        print(f"Formatting files with a single job (consider using -j to make this faster)")
+
+    # Format files in chunks to improve performance while still utilizing jobs
+    file_chunks = list(list_chunks(src_files, (len(src_files) // nb_jobs) + 1))
+
+    print("Running clang-format...")
+    # clang-format only applies changes in the given files, so it's safe to run in parallel
+    with multiprocessing.get_context("fork").Pool(nb_jobs) as pool:
+        pool.map(run_clang_format, file_chunks)
+
+    print("Running clang-tidy...")
+    if nb_jobs > 1:
+        # clang-tidy may apply changes in #included files, so when running it in parallel we use --export-fixes
+        # then we call clang-apply-replacements to apply all suggested fixes at the end
+        tmp_dir = tempfile.mkdtemp()
+
+        try:
+            with multiprocessing.get_context("fork").Pool(nb_jobs) as pool:
+                pool.map(partial(run_clang_tidy_with_export, tmp_dir), file_chunks)
+
+            run_clang_apply_replacements(tmp_dir)
+        finally:
+            shutil.rmtree(tmp_dir)
+    else:
+        run_clang_tidy(src_files)
+
+    print("Adding missing final new lines...")
+    # Adding final new lines is safe to do in parallel and can be applied to all types of files
+    with multiprocessing.get_context("fork").Pool(nb_jobs) as pool:
+        pool.map(add_final_new_line, src_files + extra_files)
+
+    print("Done formatting files.")
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Format files in the codebase to enforce most style rules")
+    parser.add_argument(
+        "--show-paths",
+        dest="show_paths",
+        action="store_true",
+        help="Print the paths to the clang-* binaries used",
+    )
+    parser.add_argument("files", metavar="file", nargs="*")
+    parser.add_argument(
+        "-j",
+        dest="jobs",
+        type=int,
+        nargs="?",
+        default=1,
+        help="number of jobs to run (default: 1 without -j, number of cpus with -j)",
+    )
+    args = parser.parse_args()
+
+    if args.show_paths:
+        import shutil
+
+        print("CLANG_FORMAT             ->", shutil.which(CLANG_FORMAT))
+        print("CLANG_TIDY               ->", shutil.which(CLANG_TIDY))
+        print("CLANG_APPLY_REPLACEMENTS ->", shutil.which(CLANG_APPLY_REPLACEMENTS))
+
+    nb_jobs = args.jobs or multiprocessing.cpu_count()
+    if nb_jobs > 1:
+        if CLANG_APPLY_REPLACEMENTS is None:
+            sys.exit(
+                f"Error: neither clang-apply-replacements nor clang-apply-replacements-{CLANG_VER} found (required to use -j)"
+            )
+
+    if args.files:
+        files = args.files
+        extra_files = []
+    else:
+        # Ignore libultra files. They cause too many formatter issues so we just wont format libultra for now and only
+        # format the game specific files. We can format libleo however.
+        files = [file for file in glob.glob("src/**/*.c", recursive=True) if not file.startswith('src/libultra/')]
+        extra_files = glob.glob("assets/**/*.xml", recursive=True)
+
+    format_files(files, extra_files, nb_jobs)
+
+
+if __name__ == "__main__":
+    main()

include/PR/gu.h

@@ -191,7 +191,7 @@ extern float sinf(float angle);
 extern float cosf(float angle);
 extern signed short sins (unsigned short angle);
 extern signed short coss (unsigned short angle);
-extern float sqrtf(float value);
+extern float sqrtf(float f);
 
 /*
  *  Dump routines for low-level display lists

include/PR/leo.h

@@ -245,8 +245,8 @@ extern  u32 LeoDriveExist(void);
 
 /* Synchronous functions */
 extern	s32 LeoClearQueue(void);
-extern	s32 LeoByteToLBA(s32 startLBA, u32 nbytes, s32 *lbas);
-extern	s32 LeoLBAToByte(s32 startLBA, u32 nLBAs, s32 *bytes);
+extern s32 LeoByteToLBA(s32 startlba, u32 nbytes, s32* lba);
+extern s32 LeoLBAToByte(s32 startlba, u32 nlbas, s32* bytes);
 extern	s32 LeoReadCapacity(LEOCapacity *cap, s32 dir);
 extern	s32 LeoInquiry(LEOVersion *ver);
 extern	s32 LeoTestUnitReady(LEOStatus *status);

include/PR/leoappli.h

@@ -40,8 +40,8 @@
 /*-----------------------------------*/
 /*   LEO FUNCTION DEFINITIONS        */
 /*-----------------------------------*/
-extern void leoInitialize(OSPri PRI_WRK, OSPri PRI_INT, OSMesg *command_que_buf, u32 cmd_buff_size);
-extern void leoCommand(void *CDB);
+extern void leoInitialize(OSPri compri, OSPri intpri, OSMesg* command_que_buf, u32 cmd_buff_size);
+extern void leoCommand(void* cmd_blk_addr);
 extern void LeoReset(void);
 extern s32  LeoResetClear(void);
 

include/PR/sptask.h

@@ -187,7 +187,7 @@ typedef u32 OSYieldResult;
 /*
  * break this up into two steps for debugging.
  */
-extern void		osSpTaskLoad(OSTask *tp);
+extern void osSpTaskLoad(OSTask* intp);
 extern void		osSpTaskStartGo(OSTask *tp);
 
 extern void		osSpTaskYield(void);

include/PR/xstdio.h

@@ -1,6 +1,6 @@
 #ifndef _XSTDIO_H
 #define _XSTDIO_H
-#include <ultratypes.h>
+#include "ultratypes.h"
 #include <stdlib.h>
 #include <stdarg.h>
 

include/functions.h

@@ -9,12 +9,6 @@ extern s32 func_8005A990(OSPiHandle *);
 // bcopy.s
 extern void _bcopy(void *, void *, u32);
 
-// 3A80.c
-extern uintptr_t convert_addr_to_virt_addr(uintptr_t addr);
-extern void func_80002F58(void);
-extern void *func_80002FDC(s32);
-extern void func_80003004(void *);
-
 // 3FB0.s
 extern void func_80003B30(void *, s32, s32, s32); // types unknown
 

include/ultra64/controller.h

@@ -147,9 +147,9 @@ typedef struct
 
 extern s32 __osEepStatus(OSMesgQueue *, OSContStatus *);
 u16 __osSumcalc(u8 *ptr, int length);
-s32 __osIdCheckSum(u16 *ptr, u16 *csum, u16 *icsum);
+s32 __osIdCheckSum(u16* ptr, u16* checkSum, u16* idSum);
 s32 __osRepairPackId(OSPfs *pfs, __OSPackId *badid, __OSPackId *newid);
-s32 __osCheckPackId(OSPfs *pfs, __OSPackId *temp);
+s32 __osCheckPackId(OSPfs* pfs, __OSPackId* check);
 s32 __osGetId(OSPfs *pfs);
 s32 __osCheckId(OSPfs *pfs);
 s32 __osPfsRWInode(OSPfs *pfs, __OSInode *inode, u8 flag, u8 bank);

include/variables.h

@@ -3,6 +3,12 @@
 
 #include "ultra64.h"
 
+// thread pris
+#define THREAD_PRI_IDLE_INIT 100
+
+// thread IDs
+#define THREAD_ID_IDLE 1
+
 #define POOL_END_4MB 0x80400000
 #define POOL_END_6MB 0x80600000
 

splat.yaml

@@ -38,7 +38,7 @@ segments:
     - [0x28E0, c, memmap]
     - [0x2EC0, c, memory_main] # handles the main global pool
     - [0x3640, c, memory] # memory_pool
-    - [0x3A80, c]
+    - [0x3A80, c, util]
     - [0x3FB0, asm] # PRES-JPEG decoder
     - [0x5580, asm] # there's a split here according to PAL
     - [0x60A0, asm] # 
@@ -325,7 +325,7 @@ segments:
     - [0x68020, bin] # rest of rom part 1
 
     # .data is somewhere in here
-    - [0x69790, .data, 3A80]
+    - [0x69790, .data, util]
     - [0x697A0, bin, rom_data_697A0]
     - [0x6A1B0, .data, crash_screen]
     - [0x6A3B0, .data, profiler]
@@ -354,7 +354,7 @@ segments:
     # Start of .rodata
     - [0x7BAC0, .rodata, rsp]
     - [0x7BB10, rodata, rom_rodata_7BB10]
-    - [0x7BB20, .rodata, 3A80]
+    - [0x7BB20, .rodata, util]
     - [0x7BB50, rodata, rom_rodata_7BB50]
     - [0x7BBE0, .rodata, crash_screen]
     - [0x7BFA0, .rodata, profiler]