Update to 4.25, add failsafe uexp reading

cmd/extract.go

@@ -3,6 +3,7 @@ package cmd
 import (
 	"encoding/json"
 	"fmt"
+	"github.com/spf13/viper"
 	"io/ioutil"
 	"os"
 	"path/filepath"
@@ -24,6 +25,12 @@ func init() {
 	split = extractCmd.Flags().Bool("split", false, "Whether output should be split into a file per asset")
 	pretty = extractCmd.Flags().Bool("pretty", false, "Whether to output in a pretty format")
 
+	extractCmd.Flags().Bool("with-index", false, "Whether to output FPackageIndex")
+	extractCmd.Flags().Bool("with-names", false, "Whether to output names")
+
+	_ = viper.BindPFlag("with-index", extractCmd.Flags().Lookup("with-index"))
+	_ = viper.BindPFlag("with-names", extractCmd.Flags().Lookup("with-names"))
+
 	extractCmd.MarkFlagRequired("assets")
 
 	rootCmd.AddCommand(extractCmd)
@@ -45,6 +52,7 @@ var extractCmd = &cobra.Command{
 		for i, asset := range *assets {
 			patterns[i] = glob.MustCompile(asset)
 		}
+		fmt.Println(patterns, *assets)
 
 		results := make([]*parser.PakEntrySet, 0)
 
@@ -88,6 +96,14 @@ var extractCmd = &cobra.Command{
 			})
 		}
 
+		/*
+			if c, ok := x.Reference.(*FObjectExport); ok {
+				if strings.Trim(c.ObjectName, "\x00") == "BPD_ResearchTreeNode_C" {
+					uAsset.ParseObject(parser, c, pak, record)
+				}
+			}
+		*/
+
 		if !*split {
 			resultBytes := formatResults(results)
 			err = ioutil.WriteFile(*output, resultBytes, 0644)

cmd/test.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"fmt"
+	"github.com/gobwas/glob"
 	"os"
 	"path/filepath"
 	"strings"
@@ -12,7 +13,11 @@ import (
 	"github.com/spf13/cobra"
 )
 
+var testAssets *[]string
+
 func init() {
+	testAssets = testCmd.Flags().StringSliceP("assets", "a", []string{}, "Comma-separated list of asset paths to extract. (supports glob)")
+
 	rootCmd.AddCommand(testCmd)
 }
 
@@ -24,6 +29,11 @@ var testCmd = &cobra.Command{
 
 		paks, err := filepath.Glob(cmd.Flag("pak").Value.String())
 
+		patterns := make([]glob.Glob, len(*testAssets))
+		for i, asset := range *testAssets {
+			patterns[i] = glob.MustCompile(asset)
+		}
+
 		if err != nil {
 			panic(err)
 		}
@@ -37,8 +47,22 @@ var testCmd = &cobra.Command{
 				panic(err)
 			}
 
+			shouldProcess := func(name string) bool {
+				if len(patterns) == 0 {
+					return true
+				}
+
+				for _, pattern := range patterns {
+					if pattern.Match(name) {
+						return true
+					}
+				}
+
+				return false
+			}
+
 			p := parser.NewParser(file)
-			p.ProcessPak(nil, nil)
+			p.ProcessPak(shouldProcess, nil)
 			/*
 				f, err := os.OpenFile("dump.txt", os.O_WRONLY | os.O_CREATE, 0644)
 				fmt.Println(err)

go.mod

@@ -3,10 +3,11 @@ module github.com/Vilsol/ue4pak
 go 1.14
 
 require (
-	github.com/davecgh/go-spew v1.1.1
-	github.com/fatih/color v1.9.0
+	github.com/fatih/color v1.10.0
 	github.com/gobwas/glob v0.2.3
-	github.com/sirupsen/logrus v1.4.2
-	github.com/spf13/cobra v0.0.5
-	github.com/spf13/viper v1.3.2
+	github.com/sirupsen/logrus v1.7.0
+	github.com/spate/glimage v0.0.0-20200505055513-fbdcc60a65e5
+	github.com/spf13/cobra v1.1.1
+	github.com/spf13/viper v1.7.1
+	github.com/x448/float16 v0.8.4
 )

parser/class.go

@@ -31,6 +31,48 @@ var classResolvers = map[string]ClassResolver{
 		parser.Read(24)
 		return parser.ReadFPackageIndex(uAsset.Imports, uAsset.Exports)
 	},
+	/*
+		"Texture2D": func(parser *PakParser, export *FObjectExport, size int32, uAsset *FPackageFileSummary) interface{} {
+			// TODO Figure out
+			parser.Read(4)
+
+			// Some unknown flags
+			parser.Read(2)
+			parser.Read(2)
+
+			cooked := parser.ReadUint32()
+			textures := make([]*FTexturePlatformData, 0)
+
+			if cooked != 1 {
+				// Uncooked asset. No idea how it can exist.
+				return textures
+			}
+
+			pixelFormat := parser.ReadFName(uAsset.Names)
+
+			for strings.Trim(pixelFormat, "\x00") != "None" {
+				// TODO Unknown
+				parser.ReadInt64()
+				textures = append(textures, parser.ReadFTexturePlatformData(0))
+				pixelFormat = parser.ReadFName(uAsset.Names)
+			}
+
+			texture := Texture2D{
+				Cooked:   cooked,
+				Textures: textures,
+			}
+
+			img := texture.ToImage()
+
+			if img != nil {
+				f, _ := os.Create("paks/" + strings.Trim(export.ObjectName, "\x00") + ".png")
+				defer f.Close()
+				_ = png.Encode(f, img)
+			}
+
+			return texture
+		},
+	*/
 }
 
 type ClassType struct {

parser/parser_asset.go

@@ -22,6 +22,11 @@ func (record *FPakEntry) ReadUAsset(pak *PakFile, parser *PakParser) *FPackageFi
 	// TODO custom_version_container: Vec<FCustomVersion>
 	parser.Read(4)
 
+	if pak.Footer.Version >= 9 {
+		// TODO Unknown bytes
+		parser.Read(3)
+	}
+
 	totalHeaderSize := parser.ReadInt32()
 	folderName := parser.ReadString()
 	packageFlags := parser.ReadUint32()
@@ -180,15 +185,26 @@ func (record *FPakEntry) ReadUExp(pak *PakFile, parser *PakParser, uAsset *FPack
 
 	exports := make(map[*FObjectExport]*ExportData)
 
+	// spew.Dump(uAsset.Names)
+
 	for _, export := range uAsset.Exports {
 		offset := headerSize + int64(record.FileOffset) + (export.SerialOffset - int64(uAsset.TotalHeaderSize))
 		log.Debugf("Reading export [%x]: %#v", offset, export.TemplateIndex.Reference)
 		parser.Seek(offset, 0)
 
+		// fmt.Println(utils.HexDump(parser.Read(int32(export.SerialSize))))
+		// parser.Seek(offset, 0)
+
 		tracker := parser.TrackRead()
 
+		if pak.Footer.Version >= 9 {
+			// TODO Unknown bytes
+			parser.Read(3)
+		}
+
 		properties := parser.ReadFPropertyTagLoop(uAsset)
 
+		parser.preload = nil
 		if int64(tracker.bytesRead) < export.SerialSize {
 			parser.Preload(int32(export.SerialSize - int64(tracker.bytesRead)))
 		}
@@ -205,6 +221,7 @@ func (record *FPakEntry) ReadUExp(pak *PakFile, parser *PakParser, uAsset *FPack
 
 				if !parsed {
 					if className := export.TemplateIndex.ClassName(); className != nil {
+						// fmt.Println(utils.HexDump(parser.preload))
 						log.Warningf("Unknown export class type (%s)[%d]: %s", strings.Trim(export.ObjectName, "\x00"), preloadSize, strings.Trim(*className, "\x00"))
 					}
 				}
@@ -368,6 +385,9 @@ func (parser *PakParser) ReadTag(size int32, uAsset *FPackageFileSummary, proper
 			case "UInt32Property":
 				values[i] = parser.ReadUint32()
 				break
+			case "UInt64Property":
+				values[i] = parser.ReadUint64()
+				break
 			case "TextProperty":
 				values[i] = parser.ReadFText()
 				break
@@ -397,7 +417,7 @@ func (parser *PakParser) ReadTag(size int32, uAsset *FPackageFileSummary, proper
 		break
 	case "StructProperty":
 		if tagData == nil {
-			log.Trace("%sReading Generic StructProperty", d(depth))
+			log.Tracef("%sReading Generic StructProperty", d(depth))
 		} else {
 			log.Tracef("%sReading StructProperty: %s", d(depth), strings.Trim(tagData.(*StructProperty).Type, "\x00"))
 
@@ -534,7 +554,7 @@ func (parser *PakParser) ReadTag(size int32, uAsset *FPackageFileSummary, proper
 
 		results := make([]*MapPropertyEntry, num)
 		for i := int32(0); i < num; i++ {
-			key := parser.ReadTag(-4, uAsset, keyType, keyData, nil, depth+1)
+			key := parser.ReadTag(8, uAsset, keyType, keyData, nil, depth+1)
 
 			if key == nil {
 				parser.Read(size - 8)

parser/parser_common.go

@@ -80,20 +80,20 @@ func (parser *PakParser) ReadFText() *FText {
 	flags := parser.ReadUint32()
 	historyType := int8(parser.Read(1)[0])
 
-	if historyType != 0 {
-		return &FText{
-			Flags:       flags,
-			HistoryType: historyType,
-		}
+	text := FText{
+		Flags:       flags,
+		HistoryType: historyType,
 	}
 
-	return &FText{
-		Flags:        flags,
-		HistoryType:  historyType,
-		Namespace:    parser.ReadString(),
-		Key:          parser.ReadString(),
-		SourceString: parser.ReadString(),
+	if historyType != 0 {
+		return &text
 	}
+
+	text.Namespace = parser.ReadString()
+	text.Key = parser.ReadString()
+	text.SourceString = parser.ReadString()
+
+	return &text
 }
 
 func (parser *PakParser) ReadFPropertyTagLoop(uAsset *FPackageFileSummary) []*FPropertyTag {

parser/parser_pak.go

@@ -89,6 +89,11 @@ func (parser *PakParser) Parse() *PakFile {
 		if pakFooter.Version == 4 {
 			// TODO ???
 		}
+
+		if pakFooter.Version >= 9 {
+			// TODO Unknown bytes
+			parser.Read(3)
+		}
 	}
 
 	return &PakFile{