Support new binary i64 data type from HOI4 1.13

This new data type can be seen in HOI4 1.13 saves as
```
manpower = { ratio = 81900 }
```

This datatype technically conflicts with some detected tokens in
Imperator and Vic3 (`playbackrate`), but I believe this token to be
unused and shouldn't pose too much of a problem.

I determined that it was signed 64bit by editing the binary to turn all
bits on and I saw the output as -1.

This is a breaking change as `BinaryToken` is receiving an addition.

Author: nickbabcock

src/bin/stats.rs

@@ -18,6 +18,7 @@ struct Stats {
     f64: u32,
     token: u32,
     rgb: u32,
+    i64: u32,
 }
 
 impl Stats {
@@ -31,6 +32,7 @@ impl Stats {
             BinaryToken::Bool(_) => self.bool += 1,
             BinaryToken::U32(_) => self.u32 += 1,
             BinaryToken::U64(_) => self.u64 += 1,
+            BinaryToken::I64(_) => self.i64 += 1,
             BinaryToken::I32(_) => self.i32 += 1,
             BinaryToken::Quoted(_) => self.quoted += 1,
             BinaryToken::Unquoted(_) => self.unquoted += 1,
@@ -56,7 +58,8 @@ impl std::fmt::Display for Stats {
             + self.f32
             + self.f64
             + self.token
-            + self.rgb;
+            + self.rgb
+            + self.i64;
 
         let total = total as f64;
 
@@ -168,6 +171,15 @@ impl std::fmt::Display for Stats {
             )?;
         }
 
+        if self.i64 != 0 {
+            writeln!(
+                f,
+                "i64:\t\t{:<8}({:.2}%)",
+                self.i64,
+                (self.i64 as f64) / total * 100.0
+            )?;
+        }
+
         writeln!(f, "total:\t\t{:<8}", total)?;
 
         Ok(())

src/binary/de.rs

@@ -69,6 +69,13 @@ impl<'data> OndemandParser<'data> {
         Ok(u64::from_le_bytes(head))
     }
 
+    #[inline]
+    fn read_i64(&mut self) -> Result<i64, Error> {
+        let (head, rest) = get_split::<8>(self.data).ok_or_else(Error::eof)?;
+        self.data = rest;
+        Ok(i64::from_le_bytes(head))
+    }
+
     #[inline]
     fn read_i32(&mut self) -> Result<i32, Error> {
         let (head, rest) = get_split::<4>(self.data).ok_or_else(Error::eof)?;
@@ -109,6 +116,10 @@ impl<'data> OndemandParser<'data> {
                 self.read_u64()?;
                 Ok(())
             }
+            I64 => {
+                self.read_i64()?;
+                Ok(())
+            }
             BOOL => {
                 self.read_bool()?;
                 Ok(())
@@ -143,6 +154,9 @@ impl<'data> OndemandParser<'data> {
                 U64 => {
                     self.read_u64()?;
                 }
+                I64 => {
+                    self.read_i64()?;
+                }
                 BOOL => {
                     self.read_bool()?;
                 }
@@ -397,6 +411,7 @@ where
             U32 => visitor.visit_u32(self.de.parser.read_u32()?),
             I32 => visitor.visit_i32(self.de.parser.read_i32()?),
             U64 => visitor.visit_u64(self.de.parser.read_u64()?),
+            I64 => visitor.visit_i64(self.de.parser.read_i64()?),
             BOOL => visitor.visit_bool(self.de.parser.read_bool()?),
             F32 => visitor.visit_f32(self.de.config.flavor.visit_f32(self.de.parser.read_f32()?)),
             F64 => visitor.visit_f64(self.de.config.flavor.visit_f64(self.de.parser.read_f64()?)),
@@ -440,7 +455,6 @@ impl<'a, 'de: 'a, 'res: 'de, RES: TokenResolver, F: BinaryFlavor> de::Deserializ
     deserialize_scalar!(deserialize_any);
     deserialize_scalar!(deserialize_i8);
     deserialize_scalar!(deserialize_i16);
-    deserialize_scalar!(deserialize_i64);
     deserialize_scalar!(deserialize_u8);
     deserialize_scalar!(deserialize_u16);
     deserialize_scalar!(deserialize_char);
@@ -492,6 +506,17 @@ impl<'a, 'de: 'a, 'res: 'de, RES: TokenResolver, F: BinaryFlavor> de::Deserializ
         }
     }
 
+    fn deserialize_i64<V>(self, visitor: V) -> Result<V::Value, Self::Error>
+    where
+        V: Visitor<'de>,
+    {
+        if self.token == I64 {
+            visitor.visit_i64(self.de.parser.read_i64()?)
+        } else {
+            Ok(self.deser(visitor)?)
+        }
+    }
+
     fn deserialize_f32<V>(self, visitor: V) -> Result<V::Value, Self::Error>
     where
         V: Visitor<'de>,
@@ -1104,6 +1129,7 @@ fn visit_key<'b, 'de: 'b, 'res: 'de, RES: TokenResolver, F: BinaryFlavor, V: Vis
         BinaryToken::Bool(x) => visitor.visit_bool(x),
         BinaryToken::U32(x) => visitor.visit_u32(x),
         BinaryToken::U64(x) => visitor.visit_u64(x),
+        BinaryToken::I64(x) => visitor.visit_i64(x),
         BinaryToken::I32(x) => visitor.visit_i32(x),
         BinaryToken::Quoted(x) | BinaryToken::Unquoted(x) => {
             match config.flavor.decode(x.as_bytes()) {
@@ -1643,6 +1669,24 @@ mod tests {
         assert_eq!(actual, MyStruct { field1: 128 });
     }
 
+    #[test]
+    fn test_i64_event() {
+        let data = [
+            0x6b, 0x32, 0x01, 0x00, 0x17, 0x03, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+        ];
+
+        #[derive(Deserialize, PartialEq, Eq, Debug)]
+        struct MyStruct {
+            field1: i64,
+        }
+
+        let mut map = HashMap::new();
+        map.insert(0x326b, String::from("field1"));
+
+        let actual: MyStruct = from_slice(&data[..], &map).unwrap();
+        assert_eq!(actual, MyStruct { field1: -1 });
+    }
+
     #[test]
     fn test_f32_event() {
         let data = [0x82, 0x2d, 0x01, 0x00, 0x0d, 0x00, 0x17, 0x00, 0x00, 0x00];

src/binary/tape.rs

@@ -36,6 +36,9 @@ pub enum BinaryToken<'a> {
     /// Represents a binary unsigned 64bit integer
     U64(u64),
 
+    /// Represents a binary signed 64bit integer
+    I64(i64),
+
     /// Represents a binary signed 32bit integer
     I32(i32),
 
@@ -173,6 +176,14 @@ impl<'a, 'b> ParserState<'a, 'b> {
         Ok(rest)
     }
 
+    #[inline]
+    fn parse_i64(&mut self, data: &'a [u8]) -> Result<&'a [u8], Error> {
+        let (head, rest) = get_split::<8>(data).ok_or_else(Error::eof)?;
+        let val = i64::from_le_bytes(head);
+        self.token_tape.alloc().init(BinaryToken::I64(val));
+        Ok(rest)
+    }
+
     #[inline]
     fn parse_i32(&mut self, data: &'a [u8]) -> Result<&'a [u8], Error> {
         let (head, rest) = get_split::<4>(data).ok_or_else(Error::eof)?;
@@ -613,6 +624,9 @@ impl<'a, 'b> ParserState<'a, 'b> {
                             state = ParseState::ObjectValue;
                         } else {
                             let len = self.token_tape.len();
+                            if len == 0 {
+                                panic!("EEEK");
+                            }
                             unsafe { ptr.add(len).write(BinaryToken::MixedContainer) };
                             unsafe { ptr.add(len + 1).write(last) };
                             unsafe { ptr.add(len + 2).write(BinaryToken::Equal) };
@@ -627,6 +641,10 @@ impl<'a, 'b> ParserState<'a, 'b> {
                     data = self.parse_rgb(d)?;
                     state = ParseState::Key;
                 }
+                I64 => {
+                    data = self.parse_i64(d)?;
+                    state = Self::next_state(state);
+                }
                 x => {
                     data = d;
                     self.token_tape.alloc().init(BinaryToken::Token(x));
@@ -1210,6 +1228,17 @@ mod tests {
         );
     }
 
+    #[test]
+    fn test_i64() {
+        let data = [
+            0x6b, 0x32, 0x01, 0x00, 0x17, 0x03, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+        ];
+        assert_eq!(
+            parse(&data).unwrap().token_tape,
+            vec![BinaryToken::Token(0x326b), BinaryToken::I64(-1),]
+        );
+    }
+
     #[test]
     fn test_nested_arrays() {
         let data = [

src/binary/tokens.rs

@@ -10,3 +10,4 @@ pub(crate) const UNQUOTED_STRING: u16 = 0x0017;
 pub(crate) const F32: u16 = 0x000d;
 pub(crate) const F64: u16 = 0x0167;
 pub(crate) const RGB: u16 = 0x0243;
+pub(crate) const I64: u16 = 0x0317;