Refactor with futures

Cargo.toml

@@ -1,29 +1,10 @@
-[package]
-name = "roblox_rs"
-version = "0.1.0"
-authors = ["Firebolt <24422634+Fireboltofdeath@users.noreply.github.com>"]
-edition = "2018"
-build = "./build/main.rs"
-
-[lib]
-crate-type = ["cdylib", "rlib"]
-
-[dependencies]
-# Smaller allocator
-wee_alloc = { version = "0.4.5" }
-
-[build_dependencies]
-serde = { version = "1.0", features = ["derive"] }
-serde_json = "1.0"
-serde_with = "1.14.0"
-convert_case = "0.5"
-lazy_static = "1.4"
-chrono = "0.4"
-
-[features]
-default = ["multivalue"]
-multivalue = []
-deprecated_apis = []
+[workspace]
+members = [
+    "crates/roblox-rs",
+    "crates/roblox-dump",
+    "crates/codegen",
+    "crates/stream"
+]
 
 [profile.dev]
 # optimize for size even on dev to prevent excessive locals

crates/codegen/Cargo.toml

@@ -0,0 +1,16 @@
+[package]
+name = "codegen"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+stream = { path = "../stream" }
+roblox-dump = { path = "../roblox-dump" }
+lazy_static = "1.4"
+convert_case = "0.5"
+
+[features]
+multivalue = []
+deprecated_apis = []

build/main.rs → crates/codegen/src/lib.rs

@@ -1,34 +1,27 @@
-mod codegen;
 mod config;
-mod dump;
+pub mod luau;
 mod overrides;
-// mod macros;
+pub mod rust;
+mod structs;
+mod transform_dump;
 
-use codegen::{
-    luau, rust,
-    structs::{Namespace, NamespaceKind},
-};
 use config::DATATYPES;
-use dump::{transform_dump, types::Dump};
 use overrides::{get_datatypes, get_instances, internal_namespace};
-use std::{env, fs, path::Path};
-
-fn main() {
-    let output = env::var("OUT_DIR").unwrap();
-    let dump = &serde_json::from_slice::<Dump>(
-        fs::read("./dump.json")
-            .expect("Dump could not be read.")
-            .as_slice(),
-    )
-    .expect("could not deserialize dump");
+use roblox_dump::Dump;
+use structs::{Namespace, NamespaceKind};
+use transform_dump::transform_dump;
 
+pub fn get_namespaces(dump: &Dump) -> Vec<Namespace> {
     let mut namespaces = transform_dump(dump);
 
     let instance_overrides = get_instances();
     for partial in instance_overrides.into_iter() {
         if let Some(namespace) = namespaces.iter_mut().find(|v| v.name == partial.name) {
-            for member in partial.members.into_iter() {
+            for member in partial.members.iter() {
                 namespace.members.retain(|v| v.api_name != member.api_name);
+            }
+
+            for member in partial.members.into_iter() {
                 namespace.members.push(member);
             }
         }
@@ -68,10 +61,5 @@ fn main() {
         }
     }
 
-    fs::write("./roblox/abi.luau", luau::generate(&namespaces)).unwrap();
-    fs::write(
-        Path::new(&output).join("generated.rs"),
-        rust::generate(&namespaces, &dump.enums),
-    )
-    .unwrap();
+    namespaces
 }

build/codegen/luau/conversion.rs → crates/codegen/src/luau/conversion.rs

@@ -1,10 +1,6 @@
-use crate::codegen::{
-    stream::{note, pull, push, Stream},
-    structs::{CodegenKind, Parameter},
-    UniqueIds,
-};
-
 use super::{load_memory, store_memory};
+use crate::structs::{CodegenKind, Parameter};
+use stream::{note, pull, push, Stream, UniqueIds};
 
 fn convert_vec(
     identifier: &str,
@@ -42,7 +38,7 @@ pub fn convert_luau_to_rust(
     variadic: bool,
 ) -> Vec<String> {
     match &kind {
-        CodegenKind::Function(_, _) => panic!(),
+        CodegenKind::Function(_, _, _) => panic!(),
         CodegenKind::Optional(kind) => {
             let param_optional = ids.next_names(&[identifier, "optional"]);
             let width = kind.width();
@@ -138,7 +134,7 @@ pub fn convert_rust_to_luau(
     prereqs: &mut Vec<String>,
 ) -> String {
     match &parameter.kind {
-        CodegenKind::Function(parameters, output) => {
+        CodegenKind::Function(parameters, output, _) => {
             let layout = output.layout();
             let parameter_names: Vec<_> = parameters.iter().map(get_lua_parameter_name).collect();
 

build/codegen/luau/mod.rs → crates/codegen/src/luau/mod.rs

@@ -1,18 +1,12 @@
-pub mod constants;
+mod constants;
 mod conversion;
 
-use crate::codegen::{
-    luau::constants::{LUAU_CREATE_CONNECTION, LUAU_DISCONNECT_CONNECTION, LUA_VALUE_NUMBER_TYPES},
-    stream::{note, pull, push},
-    UniqueIds,
-};
-
 use self::conversion::{convert_luau_to_rust, convert_rust_to_luau};
-
-use super::{
-    stream::Stream,
-    structs::{CodegenKind, Member, Namespace, RenderData, Representation, TypeLayout},
+use crate::luau::constants::{
+    LUAU_CREATE_CONNECTION, LUAU_DISCONNECT_CONNECTION, LUA_VALUE_NUMBER_TYPES,
 };
+use crate::structs::{CodegenKind, Member, Namespace, RenderData, Representation, TypeLayout};
+use stream::{note, pull, push, Stream, UniqueIds};
 
 fn store_memory(layout: TypeLayout, address: &str, inputs: &[String]) -> String {
     Stream::expression(|stream| {
@@ -99,11 +93,12 @@ fn generate_member(out: &mut Vec<String>, namespace: &Namespace, member: &Member
         &mut effect_prereqs,
     );
 
+    let is_async = member.flags.yielding;
     let layout = TypeLayout::from_tuple(&member.outputs);
     let effect = if layout.width() == 0 {
         effect
     } else {
-        if layout.needs_spill() {
+        if layout.needs_spill() && !is_async {
             parameters.insert(0, "output".to_string());
         }
 
@@ -137,12 +132,32 @@ fn generate_member(out: &mut Vec<String>, namespace: &Namespace, member: &Member
             .flat_map(|(i, kind)| convert_luau_to_rust(&ids, &names[i], kind, &mut prereqs, false))
             .collect();
 
-        let stores = store_memory_or_return(layout, "output", &results);
+        if is_async {
+            let addr = ids.next_names(&["output_addr"]);
+            let future = ids.next_names(&["future"]);
+            let size = layout.size();
+
+            let stores = store_memory(layout, &addr, &results);
+
+            Stream::expression(|stream| {
+                note!(stream, "local {future} = createPointer(0);");
+                push!(stream, "task.spawn(function()");
+                note!(stream in prereqs);
+                note!(stream, "local {addr} = allocVec({size});");
+                note!(stream, "{stores}");
+                note!(stream, "updatePointer({future}, {addr});");
+                note!(stream, "wakeFuture({future});");
+                pull!(stream, "end)");
+                note!(stream, "return {future};");
+            })
+        } else {
+            let stores = store_memory_or_return(layout, "output", &results);
 
-        Stream::expression(|stream| {
-            note!(stream in prereqs);
-            note!(stream, "{stores}");
-        })
+            Stream::expression(|stream| {
+                note!(stream in prereqs);
+                note!(stream, "{stores}");
+            })
+        }
     };
 
     Stream::prereq(out, |stream| {
@@ -166,6 +181,7 @@ fn generate_abi_start() -> String {
     let paths = vec![
         ("getPointer", "util.getPointer"),
         ("createPointer", "util.createPointer"),
+        ("updatePointer", "util.updatePointer"),
         ("memory", "wasm.memory_list.memory"),
         ("storeU8", "rt.store.i32_n8"),
         ("storeU32", "rt.store.i32"),
@@ -179,6 +195,7 @@ fn generate_abi_start() -> String {
         ("functions", "wasm.table_list.__indirect_function_table"),
         ("invokeFunction", "util.invokeFunction"),
         ("dropFunctionRef", "util.dropFunctionRef"),
+        ("wakeFuture", "wasm.func_list.rors_wake_future"),
     ];
 
     Stream::expression(|stream| {

build/overrides.rs → crates/codegen/src/overrides.rs

@@ -1,6 +1,6 @@
 use convert_case::{Case, Casing};
 
-use crate::codegen::structs::{implementations::*, CodegenKind, Member, MemberFlags, Parameter};
+use crate::structs::{implementations::*, Async, CodegenKind, Member, MemberFlags, Parameter};
 
 pub struct PartialNamespace {
     /// Name of this namespace
@@ -189,6 +189,33 @@ pub fn internal_namespace() -> PartialNamespace {
                 new::instance_new(StaticFunction(Some("Instance"), None));
                 signature = (class_name: string) -> instance!(Instance);
             },
+            member! {
+                task_wait(StaticFunction(Some("task"), Some("wait")));
+                signature = (time: number) -> number;
+                flags = MemberFlags {
+                    yielding: true,
+                    ..MemberFlags::default()
+                };
+            },
+            member! {
+                task_spawn(StaticFunction(Some("task"), Some("spawn")));
+                signature = (
+                    closure: CodegenKind::Function(vec![], Box::new(CodegenKind::Void), Async::No)
+                ) -> CodegenKind::Void;
+            },
+            member! {
+                task_defer(StaticFunction(Some("task"), Some("defer")));
+                signature = (
+                    closure: CodegenKind::Function(vec![], Box::new(CodegenKind::Void), Async::No)
+                ) -> CodegenKind::Void;
+            },
+            member! {
+                task_delay(StaticFunction(Some("task"), Some("delay")));
+                signature = (
+                    delay: number,
+                    closure: CodegenKind::Function(vec![], Box::new(CodegenKind::Void), Async::No)
+                ) -> CodegenKind::Void;
+            },
         ],
     }
 }
@@ -276,7 +303,22 @@ macro_rules! parse_namespace_fields {
                 self: $kind,
                 callback: CodegenKind::Function(
                     vec![$(Parameter::new(stringify!($parameter), $type.clone())),*],
-                    Box::new(CodegenKind::Void)
+                    Box::new(CodegenKind::Void),
+                    Async::No
+                )
+            ) -> (datatype!(RbxScriptConnection));
+        });
+
+        $namespace.members.push(member! {
+            $api_name(Event);
+
+            name = format!("{}_async", stringify!($name).to_case(Case::Snake));
+            signature = (
+                self: $kind,
+                callback: CodegenKind::Function(
+                    vec![$(Parameter::new(stringify!($parameter), $type.clone())),*],
+                    Box::new(CodegenKind::Void),
+                    Async::Yes
                 )
             ) -> (datatype!(RbxScriptConnection));
         });
@@ -296,7 +338,22 @@ macro_rules! parse_namespace_fields {
                 self: $kind,
                 callback: CodegenKind::Function(
                     vec![$(Parameter::new(stringify!($parameter), $type.clone())),*],
-                    Box::new($result.clone())
+                    Box::new($result.clone()),
+                    Async::No
+                )
+            ) -> ();
+        });
+
+        $namespace.members.push(member! {
+            $api_name(Callback);
+
+            name = format!("{}_async", stringify!($name).to_case(Case::Snake));
+            signature = (
+                self: $kind,
+                callback: CodegenKind::Function(
+                    vec![$(Parameter::new(stringify!($parameter), $type.clone())),*],
+                    Box::new($result.clone()),
+                    Async::Yes
                 )
             ) -> ();
         });

build/codegen/rust/constants.rs → crates/codegen/src/rust/constants.rs

@@ -26,6 +26,14 @@ impl<T> From<RustVec<T>> for Vec<T> {
 }
 ";
 
+pub const RUST_CLOSURE: &str = "\
+#[repr(C)]
+pub struct RustClosure<F, O> {
+	closure: F,
+	poll: unsafe extern \"C\" fn(id: u32) -> TaskResult<O>,
+}
+";
+
 pub const RUST_OPTION: &str = "\
 #[repr(C)]
 pub enum RustOption<T> {