runtime: add yaml parsing support to wasm runtime

Author: isum

Cargo.lock

@@ -83,3 +83,10 @@ pub const JSON_FROM_BYTES: GasOp = GasOp {
     base_cost: DEFAULT_BASE_COST,
     size_mult: DEFAULT_GAS_PER_BYTE * 100,
 };
+
+// Deeply nested YAML can take up more than 100 times the memory of the serialized format.
+// Multiplying the size cost by 100 accounts for this.
+pub const YAML_FROM_BYTES: GasOp = GasOp {
+    base_cost: DEFAULT_BASE_COST,
+    size_mult: DEFAULT_GAS_PER_BYTE * 100,
+};

graph/src/runtime/gas/costs.rs

@@ -371,7 +371,17 @@ pub enum IndexForAscTypeId {
     // Subgraph Data Source types
     AscEntityTrigger = 4500,
 
-    // Reserved discriminant space for a future blockchain type IDs: [4,500, 5,499]
+    // Reserved discriminant space for YAML type IDs: [5,500, 6,499]
+    YamlValue = 5500,
+    YamlTaggedValue = 5501,
+    YamlTypedMapEntryValueValue = 5502,
+    YamlTypedMapValueValue = 5503,
+    YamlArrayValue = 5504,
+    YamlArrayTypedMapEntryValueValue = 5505,
+    YamlWrappedValue = 5506,
+    YamlResultValueBool = 5507,
+
+    // Reserved discriminant space for a future blockchain type IDs: [6,500, 7,499]
     //
     // Generated with the following shell script:
     //

graph/src/runtime/mod.rs

@@ -20,3 +20,5 @@ wasm-instrument = { version = "0.2.0", features = ["std", "sign_ext"] }
 
 # AssemblyScript uses sign extensions
 parity-wasm = { version = "0.45", features = ["std", "sign_ext"] }
+
+serde_yaml = { workspace = true }

runtime/wasm/Cargo.toml

@@ -398,6 +398,17 @@ impl AscIndexId for Array<AscPtr<AscBigDecimal>> {
     const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::ArrayBigDecimal;
 }
 
+impl AscIndexId for Array<AscPtr<AscEnum<YamlValueKind>>> {
+    const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::YamlArrayValue;
+}
+
+impl AscIndexId
+    for Array<AscPtr<AscTypedMapEntry<AscEnum<YamlValueKind>, AscEnum<YamlValueKind>>>>
+{
+    const INDEX_ASC_TYPE_ID: IndexForAscTypeId =
+        IndexForAscTypeId::YamlArrayTypedMapEntryValueValue;
+}
+
 /// Represents any `AscValue` since they all fit in 64 bits.
 #[repr(C)]
 #[derive(Copy, Clone, Default)]
@@ -505,6 +516,10 @@ impl AscIndexId for AscEnum<JsonValueKind> {
     const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::JsonValue;
 }
 
+impl AscIndexId for AscEnum<YamlValueKind> {
+    const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::YamlValue;
+}
+
 pub type AscEnumArray<D> = AscPtr<Array<AscPtr<AscEnum<D>>>>;
 
 #[repr(u32)]
@@ -613,6 +628,10 @@ impl AscIndexId for AscTypedMapEntry<AscString, AscEnum<JsonValueKind>> {
     const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::TypedMapEntryStringJsonValue;
 }
 
+impl AscIndexId for AscTypedMapEntry<AscEnum<YamlValueKind>, AscEnum<YamlValueKind>> {
+    const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::YamlTypedMapEntryValueValue;
+}
+
 pub(crate) type AscTypedMapEntryArray<K, V> = Array<AscPtr<AscTypedMapEntry<K, V>>>;
 
 #[repr(C)]
@@ -638,6 +657,10 @@ impl AscIndexId for AscTypedMap<AscString, AscTypedMap<AscString, AscEnum<JsonVa
         IndexForAscTypeId::TypedMapStringTypedMapStringJsonValue;
 }
 
+impl AscIndexId for AscTypedMap<AscEnum<YamlValueKind>, AscEnum<YamlValueKind>> {
+    const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::YamlTypedMapValueValue;
+}
+
 pub type AscEntity = AscTypedMap<AscString, AscEnum<StoreValueKind>>;
 pub(crate) type AscJson = AscTypedMap<AscString, AscEnum<JsonValueKind>>;
 
@@ -725,6 +748,10 @@ impl AscIndexId for AscResult<AscPtr<AscEnum<JsonValueKind>>, bool> {
     const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::ResultJsonValueBool;
 }
 
+impl AscIndexId for AscResult<AscPtr<AscEnum<YamlValueKind>>, bool> {
+    const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::YamlResultValueBool;
+}
+
 #[repr(C)]
 #[derive(AscType, Copy, Clone)]
 pub struct AscWrapped<V: AscValue> {
@@ -742,3 +769,54 @@ impl AscIndexId for AscWrapped<bool> {
 impl AscIndexId for AscWrapped<AscPtr<AscEnum<JsonValueKind>>> {
     const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::WrappedJsonValue;
 }
+
+impl AscIndexId for AscWrapped<AscPtr<AscEnum<YamlValueKind>>> {
+    const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::YamlWrappedValue;
+}
+
+#[repr(u32)]
+#[derive(AscType, Clone, Copy)]
+pub enum YamlValueKind {
+    Null,
+    Bool,
+    Number,
+    String,
+    Array,
+    Object,
+    Tagged,
+}
+
+impl Default for YamlValueKind {
+    fn default() -> Self {
+        YamlValueKind::Null
+    }
+}
+
+impl AscValue for YamlValueKind {}
+
+impl YamlValueKind {
+    pub(crate) fn get_kind(value: &serde_yaml::Value) -> Self {
+        use serde_yaml::Value;
+
+        match value {
+            Value::Null => Self::Null,
+            Value::Bool(_) => Self::Bool,
+            Value::Number(_) => Self::Number,
+            Value::String(_) => Self::String,
+            Value::Sequence(_) => Self::Array,
+            Value::Mapping(_) => Self::Object,
+            Value::Tagged(_) => Self::Tagged,
+        }
+    }
+}
+
+#[repr(C)]
+#[derive(AscType)]
+pub struct AscYamlTaggedValue {
+    pub tag: AscPtr<AscString>,
+    pub value: AscPtr<AscEnum<YamlValueKind>>,
+}
+
+impl AscIndexId for AscYamlTaggedValue {
+    const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::YamlTaggedValue;
+}

runtime/wasm/src/asc_abi/class.rs

@@ -1236,6 +1236,36 @@ impl HostExports {
             .map(|mut tokens| tokens.pop().unwrap())
             .context("Failed to decode")
     }
+
+    pub(crate) fn yaml_from_bytes(
+        &self,
+        bytes: &[u8],
+        gas: &GasCounter,
+        state: &mut BlockState,
+    ) -> Result<serde_yaml::Value, DeterministicHostError> {
+        const YAML_MAX_SIZE_BYTES: usize = 10_000_000;
+
+        Self::track_gas_and_ops(
+            gas,
+            state,
+            gas::YAML_FROM_BYTES.with_args(complexity::Size, bytes),
+            "yaml_from_bytes",
+        )?;
+
+        if bytes.len() > YAML_MAX_SIZE_BYTES {
+            return Err(DeterministicHostError::Other(
+                anyhow!(
+                    "YAML size exceeds max size of {} bytes",
+                    YAML_MAX_SIZE_BYTES
+                )
+                .into(),
+            ));
+        }
+
+        serde_yaml::from_slice(bytes)
+            .context("failed to parse YAML from bytes")
+            .map_err(DeterministicHostError::from)
+    }
 }
 
 fn string_to_h160(string: &str) -> Result<H160, DeterministicHostError> {

runtime/wasm/src/host_exports.rs

@@ -1188,4 +1188,64 @@ impl WasmInstanceContext<'_> {
             "`box.profile` has been removed."
         )))
     }
+
+    /// function yaml.fromBytes(bytes: Bytes): YAMLValue
+    pub fn yaml_from_bytes(
+        &mut self,
+        gas: &GasCounter,
+        bytes_ptr: AscPtr<Uint8Array>,
+    ) -> Result<AscPtr<AscEnum<YamlValueKind>>, HostExportError> {
+        let bytes: Vec<u8> = asc_get(self, bytes_ptr, gas)?;
+        let host_exports = self.as_ref().ctx.host_exports.cheap_clone();
+        let ctx = &mut self.as_mut().ctx;
+
+        let yaml_value = host_exports
+            .yaml_from_bytes(&bytes, gas, &mut ctx.state)
+            .inspect_err(|_| {
+                debug!(
+                    &self.as_ref().ctx.logger,
+                    "Failed to parse YAML from byte array";
+                    "bytes" => truncate_yaml_bytes_for_logging(&bytes),
+                );
+            })?;
+
+        asc_new(self, &yaml_value, gas)
+    }
+
+    /// function yaml.try_fromBytes(bytes: Bytes): Result<YAMLValue, bool>
+    pub fn yaml_try_from_bytes(
+        &mut self,
+        gas: &GasCounter,
+        bytes_ptr: AscPtr<Uint8Array>,
+    ) -> Result<AscPtr<AscResult<AscPtr<AscEnum<YamlValueKind>>, bool>>, HostExportError> {
+        let bytes: Vec<u8> = asc_get(self, bytes_ptr, gas)?;
+        let host_exports = self.as_ref().ctx.host_exports.cheap_clone();
+        let ctx = &mut self.as_mut().ctx;
+
+        let result = host_exports
+            .yaml_from_bytes(&bytes, gas, &mut ctx.state)
+            .map_err(|err| {
+                warn!(
+                    &self.as_ref().ctx.logger,
+                    "Failed to parse YAML from byte array";
+                    "bytes" => truncate_yaml_bytes_for_logging(&bytes),
+                    "error" => format!("{:#}", err),
+                );
+
+                true
+            });
+
+        asc_new(self, &result, gas)
+    }
+}
+
+/// For debugging, it might be useful to know exactly which bytes could not be parsed as YAML, but
+/// since we can parse large YAML documents, even one bad mapping could produce terabytes of logs.
+/// To avoid this, we only log the first 1024 bytes of the failed YAML source.
+fn truncate_yaml_bytes_for_logging(bytes: &[u8]) -> String {
+    if bytes.len() > 1024 {
+        return format!("(truncated) 0x{}", hex::encode(&bytes[..1024]));
+    }
+
+    format!("0x{}", hex::encode(bytes))
 }

runtime/wasm/src/module/context.rs

@@ -468,6 +468,9 @@ impl WasmInstance {
         link!("json.toF64", json_to_f64, ptr);
         link!("json.toBigInt", json_to_big_int, ptr);
 
+        link!("yaml.fromBytes", yaml_from_bytes, ptr);
+        link!("yaml.try_fromBytes", yaml_try_from_bytes, ptr);
+
         link!("crypto.keccak256", crypto_keccak_256, ptr);
 
         link!("bigInt.plus", big_int_plus, x_ptr, y_ptr);

runtime/wasm/src/module/instance.rs

@@ -506,3 +506,54 @@ impl ToAscObj<AscEntityTrigger> for EntitySourceOperation {
 impl AscIndexId for AscEntityTrigger {
     const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::AscEntityTrigger;
 }
+
+impl ToAscObj<AscEnum<YamlValueKind>> for serde_yaml::Value {
+    fn to_asc_obj<H: AscHeap + ?Sized>(
+        &self,
+        heap: &mut H,
+        gas: &GasCounter,
+    ) -> Result<AscEnum<YamlValueKind>, HostExportError> {
+        use serde_yaml::Value;
+
+        let payload = match self {
+            Value::Null => EnumPayload(0),
+            Value::Bool(val) => EnumPayload::from(*val),
+            Value::Number(val) => asc_new(heap, &val.to_string(), gas)?.into(),
+            Value::String(val) => asc_new(heap, val, gas)?.into(),
+            Value::Sequence(val) => asc_new(heap, val.as_slice(), gas)?.into(),
+            Value::Mapping(val) => asc_new(heap, val, gas)?.into(),
+            Value::Tagged(val) => asc_new(heap, val.as_ref(), gas)?.into(),
+        };
+
+        Ok(AscEnum {
+            kind: YamlValueKind::get_kind(self),
+            _padding: 0,
+            payload,
+        })
+    }
+}
+
+impl ToAscObj<AscTypedMap<AscEnum<YamlValueKind>, AscEnum<YamlValueKind>>> for serde_yaml::Mapping {
+    fn to_asc_obj<H: AscHeap + ?Sized>(
+        &self,
+        heap: &mut H,
+        gas: &GasCounter,
+    ) -> Result<AscTypedMap<AscEnum<YamlValueKind>, AscEnum<YamlValueKind>>, HostExportError> {
+        Ok(AscTypedMap {
+            entries: asc_new(heap, &*self.iter().collect::<Vec<_>>(), gas)?,
+        })
+    }
+}
+
+impl ToAscObj<AscYamlTaggedValue> for serde_yaml::value::TaggedValue {
+    fn to_asc_obj<H: AscHeap + ?Sized>(
+        &self,
+        heap: &mut H,
+        gas: &GasCounter,
+    ) -> Result<AscYamlTaggedValue, HostExportError> {
+        Ok(AscYamlTaggedValue {
+            tag: asc_new(heap, &self.tag.to_string(), gas)?,
+            value: asc_new(heap, &self.value, gas)?,
+        })
+    }
+}