From 87e8f729aad5d00d9989cd7a535f0eb509db4355 Mon Sep 17 00:00:00 2001 From: Ian MacLeod Date: Mon, 30 Mar 2020 17:18:18 -0700 Subject: [PATCH 1/5] go fmt --- cmd/class-tree.go | 7 ++++--- cmd/extract.go | 5 +++-- cmd/root.go | 3 ++- cmd/test.go | 5 +++-- parser/parser_types.go | 5 +++-- 5 files changed, 15 insertions(+), 10 deletions(-) diff --git a/cmd/class-tree.go b/cmd/class-tree.go index e911c1a..729cf26 100644 --- a/cmd/class-tree.go +++ b/cmd/class-tree.go @@ -3,13 +3,14 @@ package cmd import ( "encoding/json" "fmt" + "io/ioutil" + "os" + "path/filepath" + "github.com/Vilsol/ue4pak/parser" "github.com/fatih/color" "github.com/gobwas/glob" "github.com/spf13/cobra" - "io/ioutil" - "os" - "path/filepath" ) func init() { diff --git a/cmd/extract.go b/cmd/extract.go index 76983cc..4472765 100644 --- a/cmd/extract.go +++ b/cmd/extract.go @@ -3,12 +3,13 @@ package cmd import ( "encoding/json" "fmt" - "github.com/Vilsol/ue4pak/parser" - "github.com/fatih/color" "io/ioutil" "os" "path/filepath" + "github.com/Vilsol/ue4pak/parser" + "github.com/fatih/color" + "github.com/gobwas/glob" "github.com/spf13/cobra" ) diff --git a/cmd/root.go b/cmd/root.go index 2b6eb9d..091fbbb 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -2,11 +2,12 @@ package cmd import ( "fmt" + "os" + _ "github.com/Vilsol/ue4pak/parser/games/satisfactory" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" "github.com/spf13/viper" - "os" ) var PakFile string diff --git a/cmd/test.go b/cmd/test.go index cad7889..9450c43 100644 --- a/cmd/test.go +++ b/cmd/test.go @@ -2,12 +2,13 @@ package cmd import ( "fmt" - "github.com/Vilsol/ue4pak/parser" - "github.com/fatih/color" "os" "path/filepath" "strings" + "github.com/Vilsol/ue4pak/parser" + "github.com/fatih/color" + "github.com/spf13/cobra" ) diff --git a/parser/parser_types.go b/parser/parser_types.go index 379e66c..0b20d49 100644 --- a/parser/parser_types.go +++ b/parser/parser_types.go @@ -3,11 +3,12 @@ package parser import ( "encoding/binary" "fmt" + "math" + "strings" + "github.com/Vilsol/ue4pak/utils" log "github.com/sirupsen/logrus" "github.com/spf13/viper" - "math" - "strings" ) type PakParser struct { From 60d787a97f16c5f5ab0a099a46f5fb5ccb2a4ca9 Mon Sep 17 00:00:00 2001 From: Ian MacLeod Date: Mon, 30 Mar 2020 17:19:54 -0700 Subject: [PATCH 2/5] Refactor the parser into a streamable API --- cmd/class-tree.go | 8 +++----- cmd/extract.go | 10 ++++++---- cmd/test.go | 2 +- parser/parser_types.go | 18 ++++++++---------- 4 files changed, 18 insertions(+), 20 deletions(-) diff --git a/cmd/class-tree.go b/cmd/class-tree.go index 729cf26..d59e38e 100644 --- a/cmd/class-tree.go +++ b/cmd/class-tree.go @@ -56,16 +56,14 @@ var classTreeCmd = &cobra.Command{ } p := parser.NewParser(file) - things, _ := p.ProcessPak(nil) - - for _, thing := range things { - for _, export := range thing.Exports { + p.ProcessPak(nil, func(entry *parser.PakEntrySet, _ *parser.PakFile) { + for _, export := range entry.Exports { open.WriteString(fmt.Sprintf("Class: %s%s\n", trim(export.Export.ObjectName), BuildClassTree(export.Export.ClassIndex))) open.WriteString(fmt.Sprintf("Super: %s%s\n", trim(export.Export.ObjectName), BuildSuperTree(export.Export.SuperIndex))) open.WriteString(fmt.Sprintf("Templ: %s%s\n", trim(export.Export.ObjectName), BuildTemplateTree(export.Export.TemplateIndex))) open.WriteString(fmt.Sprintf("Outer: %s%s\n", trim(export.Export.ObjectName), BuildOuterTree(export.Export.OuterIndex))) } - } + }) // indent, _ := json.MarshalIndent(concreteRecipe.Exports, "", " ") // fmt.Println(string(indent)) diff --git a/cmd/extract.go b/cmd/extract.go index 4472765..c68aa71 100644 --- a/cmd/extract.go +++ b/cmd/extract.go @@ -55,8 +55,7 @@ var extractCmd = &cobra.Command{ panic(err) } - p := parser.NewParser(file) - entrySets, _ := p.ProcessPak(func(name string) bool { + shouldProcess := func(name string) bool { for _, pattern := range patterns { if pattern.Match(name) { return true @@ -64,9 +63,12 @@ var extractCmd = &cobra.Command{ } return false - }) + } - results = append(results, entrySets...) + p := parser.NewParser(file) + p.ProcessPak(shouldProcess, func(entry *parser.PakEntrySet, _ *parser.PakFile) { + results = append(results, entry) + }) } var resultBytes []byte diff --git a/cmd/test.go b/cmd/test.go index 9450c43..3f6d2eb 100644 --- a/cmd/test.go +++ b/cmd/test.go @@ -38,7 +38,7 @@ var testCmd = &cobra.Command{ } p := parser.NewParser(file) - p.ProcessPak(nil) + p.ProcessPak(nil, nil) /* f, err := os.OpenFile("dump.txt", os.O_WRONLY | os.O_CREATE, 0644) fmt.Println(err) diff --git a/parser/parser_types.go b/parser/parser_types.go index 0b20d49..1a4b54b 100644 --- a/parser/parser_types.go +++ b/parser/parser_types.go @@ -36,11 +36,9 @@ func NewParser(reader PakReader) *PakParser { } } -func (parser *PakParser) ProcessPak(parseFile func(string) bool) ([]*PakEntrySet, *PakFile) { +func (parser *PakParser) ProcessPak(parseFile func(string) bool, handleEntry func(*PakEntrySet, *PakFile)) { pak := parser.Parse() - results := make([]*PakEntrySet, 0) - summaries := make(map[string]*FPackageFileSummary, 0) // First pass, parse summaries @@ -96,15 +94,15 @@ func (parser *PakParser) ProcessPak(parseFile func(string) bool) ([]*PakEntrySet i++ } - results = append(results, &PakEntrySet{ - ExportRecord: record, - Summary: summary, - Exports: exportSet, - }) + if handleEntry != nil { + handleEntry(&PakEntrySet{ + ExportRecord: record, + Summary: summary, + Exports: exportSet, + }, pak) + } } } - - return results, pak } func (parser *PakParser) Parse() *PakFile { From dc91f2c51f18c468b16a81d75b60f15ddb09031a Mon Sep 17 00:00:00 2001 From: Ian MacLeod Date: Mon, 30 Mar 2020 17:25:20 -0700 Subject: [PATCH 3/5] Refactor formatting into a separate function --- cmd/extract.go | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/cmd/extract.go b/cmd/extract.go index c68aa71..74d2d86 100644 --- a/cmd/extract.go +++ b/cmd/extract.go @@ -71,26 +71,32 @@ var extractCmd = &cobra.Command{ }) } - var resultBytes []byte + resultBytes := formatResults(results) + err = ioutil.WriteFile(*output, resultBytes, 0644) - if *format == "json" { - if *pretty { - resultBytes, err = json.MarshalIndent(results, "", " ") - } else { - resultBytes, err = json.Marshal(results) - } + if err != nil { + panic(err) + } + }, +} - if err != nil { - panic(err) - } +func formatResults(result interface{}) []byte { + var resultBytes []byte + var err error + + if *format == "json" { + if *pretty { + resultBytes, err = json.MarshalIndent(result, "", " ") } else { - panic("Unknown output format: " + *format) + resultBytes, err = json.Marshal(result) } - err = ioutil.WriteFile(*output, resultBytes, 0644) - if err != nil { panic(err) } - }, + } else { + panic("Unknown output format: " + *format) + } + + return resultBytes } From d6c997b1125c314d65ae0fe143b73e5a4c833781 Mon Sep 17 00:00:00 2001 From: Ian MacLeod Date: Mon, 30 Mar 2020 17:31:42 -0700 Subject: [PATCH 4/5] --split --- cmd/class-tree.go | 2 +- cmd/extract.go | 24 +++++++++++++++++++----- cmd/shared.go | 1 + parser/parser_types.go | 4 ++-- 4 files changed, 23 insertions(+), 8 deletions(-) diff --git a/cmd/class-tree.go b/cmd/class-tree.go index d59e38e..6302f1f 100644 --- a/cmd/class-tree.go +++ b/cmd/class-tree.go @@ -56,7 +56,7 @@ var classTreeCmd = &cobra.Command{ } p := parser.NewParser(file) - p.ProcessPak(nil, func(entry *parser.PakEntrySet, _ *parser.PakFile) { + p.ProcessPak(nil, func(_ string, entry *parser.PakEntrySet, _ *parser.PakFile) { for _, export := range entry.Exports { open.WriteString(fmt.Sprintf("Class: %s%s\n", trim(export.Export.ObjectName), BuildClassTree(export.Export.ClassIndex))) open.WriteString(fmt.Sprintf("Super: %s%s\n", trim(export.Export.ObjectName), BuildSuperTree(export.Export.SuperIndex))) diff --git a/cmd/extract.go b/cmd/extract.go index 74d2d86..f13fc7f 100644 --- a/cmd/extract.go +++ b/cmd/extract.go @@ -19,7 +19,8 @@ var assets *[]string func init() { assets = extractCmd.Flags().StringSliceP("assets", "a", []string{}, "Comma-separated list of asset paths to extract. (supports glob) (required)") format = extractCmd.Flags().StringP("format", "f", "json", "Output format type") - output = extractCmd.Flags().StringP("output", "o", "extracted.json", "Output file") + output = extractCmd.Flags().StringP("output", "o", "extracted.json", "Output file (or directory if --split)") + split = extractCmd.Flags().Bool("split", false, "Whether output should be split into a file per asset") pretty = extractCmd.Flags().Bool("pretty", false, "Whether to output in a pretty format") extractCmd.MarkFlagRequired("assets") @@ -66,13 +67,26 @@ var extractCmd = &cobra.Command{ } p := parser.NewParser(file) - p.ProcessPak(shouldProcess, func(entry *parser.PakEntrySet, _ *parser.PakFile) { - results = append(results, entry) + p.ProcessPak(shouldProcess, func(name string, entry *parser.PakEntrySet, _ *parser.PakFile) { + if *split { + destination := filepath.Join(*output, name+"."+*format) + err := os.MkdirAll(filepath.Dir(destination), 0755) + if err != nil { + panic(err) + } + + resultBytes := formatResults(entry) + ioutil.WriteFile(destination, resultBytes, 0644) + } else { + results = append(results, entry) + } }) } - resultBytes := formatResults(results) - err = ioutil.WriteFile(*output, resultBytes, 0644) + if !*split { + resultBytes := formatResults(results) + err = ioutil.WriteFile(*output, resultBytes, 0644) + } if err != nil { panic(err) diff --git a/cmd/shared.go b/cmd/shared.go index 498f3f2..730a1fb 100644 --- a/cmd/shared.go +++ b/cmd/shared.go @@ -2,4 +2,5 @@ package cmd var format *string var output *string +var split *bool var pretty *bool diff --git a/parser/parser_types.go b/parser/parser_types.go index 1a4b54b..3527308 100644 --- a/parser/parser_types.go +++ b/parser/parser_types.go @@ -36,7 +36,7 @@ func NewParser(reader PakReader) *PakParser { } } -func (parser *PakParser) ProcessPak(parseFile func(string) bool, handleEntry func(*PakEntrySet, *PakFile)) { +func (parser *PakParser) ProcessPak(parseFile func(string) bool, handleEntry func(string, *PakEntrySet, *PakFile)) { pak := parser.Parse() summaries := make(map[string]*FPackageFileSummary, 0) @@ -95,7 +95,7 @@ func (parser *PakParser) ProcessPak(parseFile func(string) bool, handleEntry fun } if handleEntry != nil { - handleEntry(&PakEntrySet{ + handleEntry(trimmed, &PakEntrySet{ ExportRecord: record, Summary: summary, Exports: exportSet, From 3c73c705a5794fbd01b1d876ffeee5faf83a3f84 Mon Sep 17 00:00:00 2001 From: Ian MacLeod Date: Mon, 30 Mar 2020 21:19:42 -0700 Subject: [PATCH 5/5] Logging --- cmd/extract.go | 7 ++++++- parser/parser_types.go | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/cmd/extract.go b/cmd/extract.go index f13fc7f..9a4df74 100644 --- a/cmd/extract.go +++ b/cmd/extract.go @@ -9,6 +9,7 @@ import ( "github.com/Vilsol/ue4pak/parser" "github.com/fatih/color" + log "github.com/sirupsen/logrus" "github.com/gobwas/glob" "github.com/spf13/cobra" @@ -75,8 +76,12 @@ var extractCmd = &cobra.Command{ panic(err) } + log.Infof("Writing Result: %s\n", destination) resultBytes := formatResults(entry) - ioutil.WriteFile(destination, resultBytes, 0644) + err = ioutil.WriteFile(destination, resultBytes, 0644) + if err != nil { + panic(err) + } } else { results = append(results, entry) } diff --git a/parser/parser_types.go b/parser/parser_types.go index 3527308..e2ea165 100644 --- a/parser/parser_types.go +++ b/parser/parser_types.go @@ -53,7 +53,7 @@ func (parser *PakParser) ProcessPak(parseFile func(string) bool, handleEntry fun if strings.HasSuffix(trimmed, "uasset") { offset := record.FileOffset + pak.Footer.HeaderSize() - log.Infof("Reading Record: %d [%x-%x]: %s\n", j, offset, offset+record.FileSize, trimmed) + log.Infof("Reading Summary: %d [%x-%x]: %s\n", j, offset, offset+record.FileSize, trimmed) summaries[trimmed[0:strings.Index(trimmed, ".uasset")]] = record.ReadUAsset(pak, parser) summaries[trimmed[0:strings.Index(trimmed, ".uasset")]].Record = record }