Updates for build system integration

- Use a config file as input to `dol split`
- Add depfile output
- Adjust splits config writing

Author: encounter

Cargo.lock

@@ -3,7 +3,7 @@ name = "decomp-toolkit"
 description = "Yet another GameCube/Wii decompilation toolkit."
 authors = ["Luke Street <luke@street.dev>"]
 license = "MIT OR Apache-2.0"
-version = "0.3.1"
+version = "0.3.2"
 edition = "2021"
 publish = false
 build = "build.rs"
@@ -49,6 +49,7 @@ petgraph = "0.6.3"
 ppc750cl = { git = "https://github.com/encounter/ppc750cl", rev = "5f6e991bf495388c4104f188d2e90c79da9f78de" }
 regex = "1.9.0"
 serde = "1.0.166"
+serde_json = "1.0.104"
 serde_repr = "0.1.14"
 serde_yaml = "0.9.22"
 sha-1 = "0.10.1"

Cargo.toml

@@ -269,9 +269,11 @@ $ dtk dol info input.dol
 Analyzes and splits a DOL file into relocatable objects based on user configuration.
 
 ```shell
-$ dtk dol split input.dol target -s config/symbols.txt -p config/splits.txt
+$ dtk dol split config.yml target
 ```
 
+TODO: document configuration file
+
 ### dwarf dump
 
 Dumps DWARF 1.1 information from an ELF file. (Does **not** support DWARF 2+)

README.md

@@ -8,6 +8,7 @@ use std::{
 
 use anyhow::{anyhow, bail, Context, Result};
 use argp::FromArgs;
+use serde::{Deserialize, Serialize};
 
 use crate::{
     analysis::{
@@ -24,6 +25,7 @@ use crate::{
     util::{
         asm::write_asm,
         config::{apply_splits, parse_symbol_line, write_splits, write_symbols},
+        dep::DepFile,
         dol::process_dol,
         elf::{process_elf, write_elf},
         file::{map_file, map_reader, touch},
@@ -60,25 +62,45 @@ pub struct InfoArgs {
 #[argp(subcommand, name = "split")]
 pub struct SplitArgs {
     #[argp(positional)]
-    /// input file
-    in_file: PathBuf,
+    /// input configuration file
+    config: PathBuf,
     #[argp(positional)]
     /// output directory
     out_dir: PathBuf,
-    #[argp(option, short = 's')]
-    /// path to symbols file
-    symbols_file: Option<PathBuf>,
-    #[argp(option, short = 'p')]
-    /// path to splits file
-    splits_file: Option<PathBuf>,
-    #[argp(option, short = 'e')]
-    /// ELF file to validate against (debugging only)
-    elf_file: Option<PathBuf>,
     #[argp(switch)]
     /// skip updating splits & symbol files (for build systems)
     no_update: bool,
 }
 
+#[inline]
+fn bool_true() -> bool { true }
+
+#[derive(Serialize, Deserialize, Debug, Clone)]
+pub struct ProjectConfig {
+    pub object: PathBuf,
+    pub splits: Option<PathBuf>,
+    pub symbols: Option<PathBuf>,
+    // Analysis options
+    #[serde(default = "bool_true")]
+    pub detect_objects: bool,
+    #[serde(default = "bool_true")]
+    pub detect_strings: bool,
+    #[serde(default = "bool_true")]
+    pub write_asm: bool,
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone)]
+pub struct OutputUnit {
+    pub object: PathBuf,
+    pub name: String,
+    pub autogenerated: bool,
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone, Default)]
+pub struct OutputConfig {
+    pub units: Vec<OutputUnit>,
+}
+
 pub fn run(args: Args) -> Result<()> {
     match args.command {
         SubCommand::Info(c_args) => info(c_args),
@@ -128,10 +150,20 @@ fn info(args: InfoArgs) -> Result<()> {
 }
 
 fn split(args: SplitArgs) -> Result<()> {
-    log::info!("Loading {}", args.in_file.display());
-    let mut obj = process_dol(&args.in_file)?;
+    log::info!("Loading {}", args.config.display());
+    let mut config_file = File::open(&args.config)
+        .with_context(|| format!("Failed to open config file '{}'", args.config.display()))?;
+    let config: ProjectConfig = serde_yaml::from_reader(&mut config_file)?;
 
-    if let Some(splits_path) = &args.splits_file {
+    let out_config_path = args.out_dir.join("config.json");
+    let mut dep = DepFile::new(out_config_path.clone());
+
+    log::info!("Loading {}", config.object.display());
+    let mut obj = process_dol(&config.object)?;
+    dep.push(config.object.clone());
+
+    if let Some(splits_path) = &config.splits {
+        dep.push(splits_path.clone());
         if splits_path.is_file() {
             let map = map_file(splits_path)?;
             apply_splits(map_reader(&map), &mut obj)?;
@@ -140,7 +172,8 @@ fn split(args: SplitArgs) -> Result<()> {
 
     let mut state = AnalyzerState::default();
 
-    if let Some(symbols_path) = &args.symbols_file {
+    if let Some(symbols_path) = &config.symbols {
+        dep.push(symbols_path.clone());
         if symbols_path.is_file() {
             let map = map_file(symbols_path)?;
             for result in map_reader(&map).lines() {
@@ -176,22 +209,29 @@ fn split(args: SplitArgs) -> Result<()> {
     log::info!("Applying relocations");
     tracker.apply(&mut obj, false)?;
 
-    log::info!("Detecting object boundaries");
-    detect_object_boundaries(&mut obj)?;
+    if config.detect_objects {
+        log::info!("Detecting object boundaries");
+        detect_object_boundaries(&mut obj)?;
+    }
 
-    log::info!("Detecting strings");
-    detect_strings(&mut obj)?;
+    if config.detect_strings {
+        log::info!("Detecting strings");
+        detect_strings(&mut obj)?;
+    }
+
+    log::info!("Adjusting splits");
+    update_splits(&mut obj)?;
 
     if !args.no_update {
-        if let Some(symbols_path) = &args.symbols_file {
+        if let Some(symbols_path) = &config.symbols {
             let mut symbols_writer = BufWriter::new(
                 File::create(symbols_path)
                     .with_context(|| format!("Failed to create '{}'", symbols_path.display()))?,
             );
             write_symbols(&mut symbols_writer, &obj)?;
         }
 
-        if let Some(splits_path) = &args.splits_file {
+        if let Some(splits_path) = &config.splits {
             let mut splits_writer = BufWriter::new(
                 File::create(splits_path)
                     .with_context(|| format!("Failed to create '{}'", splits_path.display()))?,
@@ -200,9 +240,6 @@ fn split(args: SplitArgs) -> Result<()> {
         }
     }
 
-    log::info!("Adjusting splits");
-    update_splits(&mut obj)?;
-
     log::info!("Splitting {} objects", obj.link_order.len());
     let split_objs = split_obj(&obj)?;
 
@@ -224,13 +261,17 @@ fn split(args: SplitArgs) -> Result<()> {
         };
     }
 
-    let mut units_file = BufWriter::new(File::create(args.out_dir.join("units.txt"))?);
+    let mut out_config = OutputConfig::default();
     for unit in &obj.link_order {
         let object = file_map
             .get(unit)
             .ok_or_else(|| anyhow!("Failed to find object file for unit '{unit}'"))?;
         let out_path = obj_dir.join(obj_path_for_unit(unit));
-        writeln!(units_file, "{}:{}", out_path.display(), unit)?;
+        out_config.units.push(OutputUnit {
+            object: out_path.clone(),
+            name: unit.clone(),
+            autogenerated: false,
+        });
         if let Some(parent) = out_path.parent() {
             DirBuilder::new().recursive(true).create(parent)?;
         }
@@ -239,7 +280,11 @@ fn split(args: SplitArgs) -> Result<()> {
         file.write_all(object)?;
         file.flush()?;
     }
-    units_file.flush()?;
+    {
+        let mut out_file = BufWriter::new(File::create(&out_config_path)?);
+        serde_json::to_writer_pretty(&mut out_file, &out_config)?;
+        out_file.flush()?;
+    }
 
     // Generate ldscript.lcf
     fs::write(args.out_dir.join("ldscript.lcf"), generate_ldscript(&obj)?)?;
@@ -256,14 +301,26 @@ fn split(args: SplitArgs) -> Result<()> {
         w.flush()?;
     }
 
-    // (debugging) validate against ELF
-    if let Some(file) = &args.elf_file {
-        validate(&obj, file, &state)?;
+    // Write dep file
+    {
+        let dep_path = args.out_dir.join("dep");
+        let mut dep_file = BufWriter::new(
+            File::create(&dep_path)
+                .with_context(|| format!("Failed to create dep file '{}'", dep_path.display()))?,
+        );
+        dep.write(&mut dep_file)?;
+        dep_file.flush()?;
     }
 
+    // (debugging) validate against ELF
+    // if let Some(file) = &args.elf_file {
+    //     validate(&obj, file, &state)?;
+    // }
+
     Ok(())
 }
 
+#[allow(dead_code)]
 fn validate<P: AsRef<Path>>(obj: &ObjInfo, elf_file: P, state: &AnalyzerState) -> Result<()> {
     let real_obj = process_elf(elf_file)?;
     for real_section in &real_obj.sections {

src/cmd/dol.rs

@@ -764,6 +764,12 @@ impl ObjInfo {
         self.splits.entry(address).or_default().push(split);
         Ok(())
     }
+
+    pub fn is_unit_autogenerated(&self, unit: &str) -> bool {
+        self.splits_for_range(..)
+            .filter(|(_, split)| split.unit == unit)
+            .all(|(_, split)| split.autogenerated)
+    }
 }
 
 impl ObjSection {

src/obj/mod.rs

@@ -402,8 +402,7 @@ fn resolve_link_order(obj: &ObjInfo) -> Result<Vec<String>> {
         unit_to_index_map.insert(split.unit.clone(), NodeIndex::new(0));
     }
     for (unit, index) in unit_to_index_map.iter_mut() {
-        let new_index = graph.add_node(unit.clone());
-        *index = new_index;
+        *index = graph.add_node(unit.clone());
     }
 
     for section in &obj.sections {

src/obj/split.rs

@@ -244,7 +244,14 @@ fn symbol_data_kind_from_str(s: &str) -> Option<ObjDataKind> {
 }
 
 pub fn write_splits<W: Write>(w: &mut W, obj: &ObjInfo) -> Result<()> {
-    for unit in &obj.link_order {
+    let mut begin = true;
+    for unit in obj.link_order.iter().filter(|unit| !obj.is_unit_autogenerated(unit)) {
+        if begin {
+            begin = false;
+        } else {
+            writeln!(w)?;
+        }
+
         writeln!(w, "{}:", unit)?;
         let mut split_iter = obj.splits_for_range(..).peekable();
         while let Some((addr, split)) = split_iter.next() {
@@ -260,7 +267,6 @@ pub fn write_splits<W: Write>(w: &mut W, obj: &ObjInfo) -> Result<()> {
             writeln!(w, "\t{:<11} start:{:#010X} end:{:#010X}", section.name, addr, end)?;
             // align:{}
         }
-        writeln!(w)?;
     }
     Ok(())
 }

src/util/config.rs

@@ -0,0 +1,20 @@
+use std::{io::Write, path::PathBuf};
+
+pub struct DepFile {
+    pub name: PathBuf,
+    pub dependencies: Vec<PathBuf>,
+}
+
+impl DepFile {
+    pub fn new(name: PathBuf) -> Self { Self { name, dependencies: vec![] } }
+
+    pub fn push(&mut self, dependency: PathBuf) { self.dependencies.push(dependency); }
+
+    pub fn write<W: Write>(&self, w: &mut W) -> std::io::Result<()> {
+        write!(w, "{}:", self.name.display())?;
+        for dep in &self.dependencies {
+            write!(w, " \\\n  {}", dep.display())?;
+        }
+        Ok(())
+    }
+}

src/util/dep.rs

@@ -1,6 +1,7 @@
 pub mod asm;
 pub mod comment;
 pub mod config;
+pub mod dep;
 pub mod dol;
 pub mod dwarf;
 pub mod elf;