From 9d33cad3032b5895801c573bb92a981f36f8b085 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Cortier?= Date: Fri, 8 Sep 2023 17:16:27 -0400 Subject: [PATCH] feat(web): use softbuffer to draw into the canvas (#191) Instead of copying and sending image buffers to JavaScript, the WASM module now draws into the canvas by itself. This removes some overhead associated with the previous approach and open the door for further optimizations. In order to achieve good performance, the newest API of `softbuffer@0.3.0` is used: the "owned buffer" that can be written into by us with direct access and `present_with_damage` to apply partial updates. The presentation itself is currently not yet "no-copy" in the case of the web backend because the current API of `softbuffer` is expecting a pixel buffer in the BGRX format while the underlying canvas can only takes RGBA pixels. There is an open issue for this. There was a bug with the `present_with_damage` implementation for the web backend. I fixed the issue and opened a PR to upstream the patch. The cargo dependency patch will be removed once the fix is published on crates.io. Issue: ARC-164 --- Cargo.lock | 179 +++++++++++++++--- Cargo.toml | 7 +- crates/ironrdp-client/Cargo.toml | 4 +- crates/ironrdp-client/src/rdp.rs | 2 +- crates/ironrdp-graphics/src/rdp6/rle.rs | 2 +- crates/ironrdp-session/src/active_stage.rs | 2 +- crates/ironrdp-web/Cargo.toml | 28 +-- crates/ironrdp-web/src/canvas.rs | 86 +++++++++ crates/ironrdp-web/src/lib.rs | 3 +- crates/ironrdp-web/src/session.rs | 160 ++++++---------- web-client/iron-remote-gui/README.md | 1 - web-client/iron-remote-gui/index.html | 2 +- web-client/iron-remote-gui/public/test.html | 2 +- .../src/iron-remote-gui.svelte | 15 -- .../src/services/wasm-bridge.service.ts | 13 +- .../src/lib/login/login.svelte | 18 +- .../lib/remote-screen/remote-screen.svelte | 3 +- .../src/services/server-bridge.service.ts | 2 - .../src/services/services-injector.ts | 12 +- .../src/services/tauri-bridge.service.ts | 61 ------ .../src/services/wasm-bridge.service.ts | 3 - .../iron-svelte-client/svelte.config.js | 9 +- 22 files changed, 337 insertions(+), 277 deletions(-) create mode 100644 crates/ironrdp-web/src/canvas.rs delete mode 100644 web-client/iron-svelte-client/src/services/tauri-bridge.service.ts diff --git a/Cargo.lock b/Cargo.lock index 40a0b9d6a..559bc5e78 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -454,6 +454,20 @@ name = "bytemuck" version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "17febce684fd15d89027105661fec94afb475cb995fbc59d2865198446ba2eea" +dependencies = [ + "bytemuck_derive", +] + +[[package]] +name = "bytemuck_derive" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "965ab7eb5f8f97d2a083c799f3a1b994fc397b2fe2da5d1da1626ce15a39f2b1" +dependencies = [ + "proc-macro2 1.0.66", + "quote 1.0.33", + "syn 2.0.29", +] [[package]] name = "byteorder" @@ -587,16 +601,16 @@ checksum = "cd7cc57abe963c6d3b9d8be5b06ba7c8957a930305ca90304f24ef040aa6f961" [[package]] name = "cocoa" -version = "0.24.1" +version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f425db7937052c684daec3bd6375c8abe2d146dca4b8b143d6db777c39138f3a" +checksum = "f6140449f97a6e97f9511815c5632d84c8aacf8ac271ad77c559218161a1373c" dependencies = [ "bitflags 1.3.2", "block", "cocoa-foundation", "core-foundation", - "core-graphics", - "foreign-types", + "core-graphics 0.23.1", + "foreign-types 0.5.0", "libc", "objc", ] @@ -611,7 +625,7 @@ dependencies = [ "block", "core-foundation", "core-graphics-types", - "foreign-types", + "foreign-types 0.3.2", "libc", "objc", ] @@ -663,7 +677,20 @@ dependencies = [ "bitflags 1.3.2", "core-foundation", "core-graphics-types", - "foreign-types", + "foreign-types 0.3.2", + "libc", +] + +[[package]] +name = "core-graphics" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "970a29baf4110c26fedbc7f82107d42c23f7e88e404c4577ed73fe99ff85a212" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "core-graphics-types", + "foreign-types 0.5.0", "libc", ] @@ -772,6 +799,16 @@ dependencies = [ "subtle 2.4.1", ] +[[package]] +name = "ctor" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f34ba9a9bcb8645379e9de8cb3ecfcf4d1c85ba66d90deb3259206fa5aa193b" +dependencies = [ + "quote 1.0.33", + "syn 2.0.29", +] + [[package]] name = "curve25519-dalek" version = "4.0.0" @@ -956,7 +993,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "330c60081dcc4c72131f8eb70510f1ac07223e5d4163db481a04a0befcffa412" dependencies = [ - "libloading", + "libloading 0.7.4", ] [[package]] @@ -965,6 +1002,44 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ea835d29036a4087793836fa931b08837ad5e957da9e23886b29586fb9b6650" +[[package]] +name = "drm" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edf9159ef4bcecd0c5e4cbeb573b8d0037493403d542780dba5d840bbf9df56f" +dependencies = [ + "bitflags 1.3.2", + "bytemuck", + "drm-ffi", + "drm-fourcc", + "nix 0.26.2", +] + +[[package]] +name = "drm-ffi" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1352481b7b90e27a8a1bf8ef6b33cf18b98dba7c410e75c24bb3eef2f0d8d525" +dependencies = [ + "drm-sys", + "nix 0.26.2", +] + +[[package]] +name = "drm-fourcc" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0aafbcdb8afc29c1a7ee5fbe53b5d62f4565b35a042a662ca9fecd0b54dae6f4" + +[[package]] +name = "drm-sys" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1369f1679d6b706d234c4c1e0613c415c2c74b598a09ad28080ba2474b72e42d" +dependencies = [ + "libc", +] + [[package]] name = "dyn-clone" version = "1.0.13" @@ -1100,15 +1175,6 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" -[[package]] -name = "fastrand" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" -dependencies = [ - "instant", -] - [[package]] name = "fastrand" version = "2.0.0" @@ -1168,7 +1234,28 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" dependencies = [ - "foreign-types-shared", + "foreign-types-shared 0.1.1", +] + +[[package]] +name = "foreign-types" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d737d9aa519fb7b749cbc3b962edcf310a8dd1f4b67c91c4f83975dbdd17d965" +dependencies = [ + "foreign-types-macros", + "foreign-types-shared 0.3.1", +] + +[[package]] +name = "foreign-types-macros" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a5c6c585bc94aaf2c7b51dd4c2ba22680844aba4c687be581871a6f518c5742" +dependencies = [ + "proc-macro2 1.0.66", + "quote 1.0.33", + "syn 2.0.29", ] [[package]] @@ -1177,6 +1264,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" +[[package]] +name = "foreign-types-shared" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa9a19cbb55df58761df49b23516a86d432839add4af60fc256da840f66ed35b" + [[package]] name = "form_urlencoded" version = "1.2.0" @@ -2016,6 +2109,7 @@ dependencies = [ "js-sys", "semver", "smallvec", + "softbuffer", "tap", "time 0.3.27", "tracing", @@ -2024,6 +2118,7 @@ dependencies = [ "url", "wasm-bindgen", "wasm-bindgen-futures", + "web-sys", "x509-cert", ] @@ -2104,6 +2199,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "libloading" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d580318f95776505201b28cf98eb1fa5e4be3b689633ba6a3e6cd880ff22d8cb" +dependencies = [ + "cfg-if 1.0.0", + "windows-sys 0.48.0", +] + [[package]] name = "libm" version = "0.2.7" @@ -2248,9 +2353,9 @@ dependencies = [ [[package]] name = "memmap2" -version = "0.6.2" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d28bba84adfe6646737845bc5ebbfa2c08424eb1c37e94a1fd2a82adb56a872" +checksum = "f49388d20533534cd19360ad3d6a7dadc885944aa802ba3995040c5ec11288c6" dependencies = [ "libc", ] @@ -2679,7 +2784,7 @@ checksum = "729b745ad4a5575dd06a3e1af1414bd330ee561c01b3899eb584baeaa8def17e" dependencies = [ "bitflags 1.3.2", "cfg-if 1.0.0", - "foreign-types", + "foreign-types 0.3.2", "libc", "once_cell", "openssl-macros", @@ -3866,29 +3971,31 @@ dependencies = [ [[package]] name = "softbuffer" version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cc7e34c6782f6fe16e384f25d009ed7b30f70e022d77966769a0366232c4dc4" +source = "git+https://github.com/CBenoit/softbuffer.git?rev=8a9e7f95e054f9af5752682aa5f27890aeb1b094#8a9e7f95e054f9af5752682aa5f27890aeb1b094" dependencies = [ + "as-raw-xcb-connection", "bytemuck", "cfg_aliases", "cocoa", - "core-graphics", - "fastrand 1.9.0", - "foreign-types", + "core-graphics 0.23.1", + "drm", + "drm-sys", + "fastrand", + "foreign-types 0.5.0", "js-sys", "log", - "memmap2 0.6.2", + "memmap2 0.7.1", "nix 0.26.2", "objc", "raw-window-handle", "redox_syscall", + "tiny-xlib", "wasm-bindgen", "wayland-backend", "wayland-client 0.30.2", "wayland-sys 0.30.1", "web-sys", "windows-sys 0.48.0", - "x11-dl", "x11rb", ] @@ -4041,7 +4148,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef" dependencies = [ "cfg-if 1.0.0", - "fastrand 2.0.0", + "fastrand", "redox_syscall", "rustix", "windows-sys 0.48.0", @@ -4142,6 +4249,18 @@ dependencies = [ "strict-num", ] +[[package]] +name = "tiny-xlib" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4098d49269baa034a8d1eae9bd63e9fa532148d772121dace3bcd6a6c98eb6d" +dependencies = [ + "as-raw-xcb-connection", + "ctor", + "libloading 0.8.0", + "tracing", +] + [[package]] name = "tinyjson" version = "2.5.1" @@ -5009,7 +5128,7 @@ dependencies = [ "bitflags 1.3.2", "cfg_aliases", "core-foundation", - "core-graphics", + "core-graphics 0.22.3", "dispatch", "instant", "libc", @@ -5091,7 +5210,7 @@ dependencies = [ "as-raw-xcb-connection", "gethostname", "libc", - "libloading", + "libloading 0.7.4", "nix 0.26.2", "once_cell", "winapi", diff --git a/Cargo.toml b/Cargo.toml index 401d9c449..2d95bc9fe 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,10 +21,10 @@ keywords = ["rdp", "remote-desktop", "network", "client", "protocol"] categories = ["network-programming"] [workspace.dependencies] +ironrdp-acceptor = { version = "0.1", path = "crates/ironrdp-acceptor" } ironrdp-async = { version = "0.1", path = "crates/ironrdp-async" } ironrdp-cliprdr = { version = "0.1", path = "crates/ironrdp-cliprdr" } ironrdp-connector = { version = "0.1", path = "crates/ironrdp-connector" } -ironrdp-acceptor = { version = "0.1", path = "crates/ironrdp-acceptor" } ironrdp-dvc = { version = "0.1", path = "crates/ironrdp-dvc" } ironrdp-error = { version = "0.1", path = "crates/ironrdp-error" } ironrdp-futures = { version = "0.1", path = "crates/ironrdp-futures" } @@ -36,13 +36,13 @@ ironrdp-pdu = { version = "0.1", path = "crates/ironrdp-pdu" } ironrdp-rdcleanpath = { version = "0.1", path = "crates/ironrdp-rdcleanpath" } ironrdp-rdpdr = { version = "0.1", path = "crates/ironrdp-rdpdr" } ironrdp-rdpsnd = { version = "0.1", path = "crates/ironrdp-rdpsnd" } +ironrdp-server = { version = "0.1", path = "crates/ironrdp-server" } ironrdp-session-generators = { path = "crates/ironrdp-session-generators" } ironrdp-session = { version = "0.1", path = "crates/ironrdp-session" } ironrdp-svc = { version = "0.1", path = "crates/ironrdp-svc" } ironrdp-testsuite-core = { path = "crates/ironrdp-testsuite-core" } ironrdp-tls = { version = "0.1", path = "crates/ironrdp-tls" } ironrdp-tokio = { version = "0.1", path = "crates/ironrdp-tokio" } -ironrdp-server = { version = "0.1", path = "crates/ironrdp-server" } ironrdp = { version = "0.5", path = "crates/ironrdp" } expect-test = "1" @@ -63,3 +63,6 @@ lto = true inherits = "release" opt-level = "s" lto = true + +[patch.crates-io] +softbuffer = { git = "https://github.com/CBenoit/softbuffer.git", rev = "8a9e7f95e054f9af5752682aa5f27890aeb1b094" } diff --git a/crates/ironrdp-client/Cargo.toml b/crates/ironrdp-client/Cargo.toml index 3674da80d..90b55d3d1 100644 --- a/crates/ironrdp-client/Cargo.toml +++ b/crates/ironrdp-client/Cargo.toml @@ -33,9 +33,9 @@ ironrdp-tls.workspace = true ironrdp-tokio.workspace = true sspi = { workspace = true, features = ["network_client", "dns_resolver"] } # enable additional features -# GUI -softbuffer = "0.3" +# Windowing and rendering winit = "0.28" +softbuffer = "0.3" # CLI clap = { version = "4.2", features = ["derive", "cargo"] } diff --git a/crates/ironrdp-client/src/rdp.rs b/crates/ironrdp-client/src/rdp.rs index 2cf439a7f..d4cafb923 100644 --- a/crates/ironrdp-client/src/rdp.rs +++ b/crates/ironrdp-client/src/rdp.rs @@ -208,7 +208,6 @@ async fn active_session( }) .map_err(|e| session::custom_err!("event_loop_proxy", e))?; } - ActiveStageOutput::Terminate => break 'outer, ActiveStageOutput::PointerDefault => { event_loop_proxy .send_event(RdpOutputEvent::PointerDefault) @@ -224,6 +223,7 @@ async fn active_session( .send_event(RdpOutputEvent::PointerPosition { x, y }) .map_err(|e| session::custom_err!("event_loop_proxy", e))?; } + ActiveStageOutput::Terminate => break 'outer, } } } diff --git a/crates/ironrdp-graphics/src/rdp6/rle.rs b/crates/ironrdp-graphics/src/rdp6/rle.rs index cf6bed7e5..352f21f84 100644 --- a/crates/ironrdp-graphics/src/rdp6/rle.rs +++ b/crates/ironrdp-graphics/src/rdp6/rle.rs @@ -379,7 +379,7 @@ mod tests { decompress_8bpp_plane(src, dst.as_mut_slice(), width, height) } - pub fn compress(src: &[u8], dst: &mut Vec, width: usize, height: usize) -> Result { + pub fn compress(src: &[u8], dst: &mut [u8], width: usize, height: usize) -> Result { compress_8bpp_plane(src.iter().copied(), &mut WriteCursor::new(dst), width, height) } diff --git a/crates/ironrdp-session/src/active_stage.rs b/crates/ironrdp-session/src/active_stage.rs index 199a8094e..2bbd0d283 100644 --- a/crates/ironrdp-session/src/active_stage.rs +++ b/crates/ironrdp-session/src/active_stage.rs @@ -154,8 +154,8 @@ impl ActiveStage { pub enum ActiveStageOutput { ResponseFrame(Vec), GraphicsUpdate(InclusiveRectangle), - Terminate, PointerDefault, PointerHidden, PointerPosition { x: usize, y: usize }, + Terminate, } diff --git a/crates/ironrdp-web/Cargo.toml b/crates/ironrdp-web/Cargo.toml index 06975b4bf..cd03e1527 100644 --- a/crates/ironrdp-web/Cargo.toml +++ b/crates/ironrdp-web/Cargo.toml @@ -29,12 +29,16 @@ ironrdp-futures.workspace = true ironrdp-rdcleanpath.workspace = true # WASM -wasm-bindgen = "0.2.84" -wasm-bindgen-futures = "0.4.34" -js-sys = "0.3.61" -gloo-net = { version = "0.2.6", default-features = false, features = ["websocket", "http"] } -gloo-timers = { version = "0.2.6", default-features = false, features = ["futures"] } -tracing-web = "0.1.2" +wasm-bindgen = "0.2" +wasm-bindgen-futures = "0.4" +web-sys = { version = "0.3", features = ["HtmlCanvasElement"] } +js-sys = "0.3" +gloo-net = { version = "0.2", default-features = false, features = ["websocket", "http"] } +gloo-timers = { version = "0.2", default-features = false, features = ["futures"] } +tracing-web = "0.1" + +# Rendering +softbuffer = { version = "0.3", default-features = false } # Enable WebAssembly support for a few crates getrandom = { version = "0.2", features = ["js"] } @@ -45,7 +49,7 @@ time = { version = "0.3", features = ["wasm-bindgen"] } # logging them with `console.error`. This is great for development, but requires # all the `std::fmt` and `std::panicking` infrastructure, so isn't great for # code size when deploying. -console_error_panic_hook = { version = "0.1.7", optional = true } +console_error_panic_hook = { version = "0.1", optional = true } # Async futures-util = { version = "0.3", features = ["sink", "io"] } @@ -53,12 +57,12 @@ futures-channel = "0.3" # Logging tracing.workspace = true -tracing-subscriber = { version = "0.3.16", features = ["time"] } +tracing-subscriber = { version = "0.3", features = ["time"] } # Utils anyhow = "1" -smallvec = "1.10.0" -x509-cert = { version = "0.2.1", default-features = false, features = ["std"] } -tap = "1.0.1" +smallvec = "1.10" +x509-cert = { version = "0.2", default-features = false, features = ["std"] } +tap = "1" semver = "1" -url = "2.4.0" +url = "2" diff --git a/crates/ironrdp-web/src/canvas.rs b/crates/ironrdp-web/src/canvas.rs new file mode 100644 index 000000000..6fccdd731 --- /dev/null +++ b/crates/ironrdp-web/src/canvas.rs @@ -0,0 +1,86 @@ +use std::num::NonZeroU32; + +use ironrdp::pdu::geometry::{InclusiveRectangle, Rectangle as _}; +use web_sys::HtmlCanvasElement; + +pub struct Canvas { + width: u32, + surface: softbuffer::Surface, +} + +impl Canvas { + pub fn new(render_canvas: HtmlCanvasElement, width: u32, height: u32) -> anyhow::Result { + render_canvas.set_width(width); + render_canvas.set_height(height); + + #[cfg(target_arch = "wasm32")] + let mut surface = { + use softbuffer::SurfaceExtWeb as _; + softbuffer::Surface::from_canvas(render_canvas).expect("surface") + }; + + #[cfg(not(target_arch = "wasm32"))] + let mut surface = { + fn stub(_: HtmlCanvasElement) -> softbuffer::Surface { + unimplemented!() + } + + stub(render_canvas) + }; + + surface + .resize(NonZeroU32::new(width).unwrap(), NonZeroU32::new(height).unwrap()) + .expect("surface resize"); + + Ok(Self { width, surface }) + } + + pub fn draw(&mut self, buffer: &[u8], region: InclusiveRectangle) -> anyhow::Result<()> { + let region_width = region.width(); + let region_height = region.height(); + + let mut src = buffer.chunks_exact(4).map(|pixel| { + let r = pixel[0]; + let g = pixel[1]; + let b = pixel[2]; + u32::from_be_bytes([0, r, g, b]) + }); + + let mut dst = self.surface.buffer_mut().expect("surface buffer"); + + { + // Copy src into dst + + let region_top_usize = usize::from(region.top); + let region_height_usize = usize::from(region_height); + let region_left_usize = usize::from(region.left); + let region_width_usize = usize::from(region_width); + + for dst_row in dst + .chunks_exact_mut(self.width as usize) + .skip(region_top_usize) + .take(region_height_usize) + { + let src_row = src.by_ref().take(region_width_usize); + + dst_row + .iter_mut() + .skip(region_left_usize) + .take(region_width_usize) + .zip(src_row) + .for_each(|(dst, src)| *dst = src); + } + } + + let damage_rect = softbuffer::Rect { + x: u32::from(region.left), + y: u32::from(region.top), + width: NonZeroU32::new(u32::from(region_width)).unwrap(), + height: NonZeroU32::new(u32::from(region_height)).unwrap(), + }; + + dst.present_with_damage(&[damage_rect]).expect("buffer present"); + + Ok(()) + } +} diff --git a/crates/ironrdp-web/src/lib.rs b/crates/ironrdp-web/src/lib.rs index 2d5d6b617..cea3cc85a 100644 --- a/crates/ironrdp-web/src/lib.rs +++ b/crates/ironrdp-web/src/lib.rs @@ -3,6 +3,7 @@ #[macro_use] extern crate tracing; +mod canvas; mod error; mod image; mod input; @@ -27,7 +28,7 @@ pub fn ironrdp_init(log_level: &str) { // // For more details see // https://github.com/rustwasm/console_error_panic_hook#readme - #[cfg(feature = "console_error_panic_hook")] + #[cfg(feature = "panic_hook")] console_error_panic_hook::set_once(); if let Ok(level) = log_level.parse::() { diff --git a/crates/ironrdp-web/src/session.rs b/crates/ironrdp-web/src/session.rs index 94fa49080..98ee3bb27 100644 --- a/crates/ironrdp-web/src/session.rs +++ b/crates/ironrdp-web/src/session.rs @@ -5,12 +5,11 @@ use std::time::Duration; use anyhow::Context as _; use futures_channel::mpsc; use futures_util::io::{ReadHalf, WriteHalf}; -use futures_util::{select, AsyncReadExt as _, AsyncWriteExt as _, FutureExt, StreamExt as _}; +use futures_util::{select, AsyncReadExt as _, AsyncWriteExt as _, FutureExt as _, StreamExt as _}; use gloo_net::websocket; use gloo_net::websocket::futures::WebSocket; use ironrdp::connector::{self, ClientConnector}; use ironrdp::graphics::image_processing::PixelFormat; -use ironrdp::pdu::geometry::{InclusiveRectangle, Rectangle as _}; use ironrdp::pdu::input::fast_path::FastPathInputEvent; use ironrdp::pdu::write_buf::WriteBuf; use ironrdp::session::image::DecodedImage; @@ -18,9 +17,11 @@ use ironrdp::session::{ActiveStage, ActiveStageOutput}; use tap::prelude::*; use wasm_bindgen::prelude::*; use wasm_bindgen_futures::spawn_local; +use web_sys::HtmlCanvasElement; +use crate::canvas::Canvas; use crate::error::{IronRdpError, IronRdpErrorKind}; -use crate::image::{extract_partial_image, RectInfo}; +use crate::image::extract_partial_image; use crate::input::InputTransaction; use crate::network_client::WasmNetworkClientFactory; use crate::websocket::WebSocketCompat; @@ -44,8 +45,7 @@ struct SessionBuilderInner { client_name: String, desktop_size: DesktopSize, - update_callback: Option, - update_callback_context: Option, + render_canvas: Option, hide_pointer_callback: Option, hide_pointer_callback_context: Option, show_pointer_callback: Option, @@ -67,9 +67,8 @@ impl Default for SessionBuilderInner { width: DEFAULT_WIDTH, height: DEFAULT_HEIGHT, }, - update_callback: None, - update_callback_context: None, + render_canvas: None, hide_pointer_callback: None, hide_pointer_callback_context: None, show_pointer_callback: None, @@ -136,15 +135,9 @@ impl SessionBuilder { self.clone() } - /// Required - pub fn update_callback(&self, callback: js_sys::Function) -> SessionBuilder { - self.0.borrow_mut().update_callback = Some(callback); - self.clone() - } - - /// Required - pub fn update_callback_context(&self, context: JsValue) -> SessionBuilder { - self.0.borrow_mut().update_callback_context = Some(context); + /// Optional + pub fn render_canvas(&self, canvas: HtmlCanvasElement) -> SessionBuilder { + self.0.borrow_mut().render_canvas = Some(canvas); self.clone() } @@ -183,8 +176,7 @@ impl SessionBuilder { pcb, client_name, desktop_size, - update_callback, - update_callback_context, + render_canvas, hide_pointer_callback, hide_pointer_callback_context, show_pointer_callback, @@ -193,29 +185,34 @@ impl SessionBuilder { { let inner = self.0.borrow(); - username = inner.username.clone().expect("username"); - destination = inner.destination.clone().expect("destination"); + username = inner.username.clone().context("username missing")?; + destination = inner.destination.clone().context("destination missing")?; server_domain = inner.server_domain.clone(); - password = inner.password.clone().expect("password"); - proxy_address = inner.proxy_address.clone().expect("proxy_address"); - auth_token = inner.auth_token.clone().expect("auth_token"); + password = inner.password.clone().context("password missing")?; + proxy_address = inner.proxy_address.clone().context("proxy_address missing")?; + auth_token = inner.auth_token.clone().context("auth_token missing")?; pcb = inner.pcb.clone(); client_name = inner.client_name.clone(); desktop_size = inner.desktop_size.clone(); - update_callback = inner.update_callback.clone().expect("update_callback"); - update_callback_context = inner.update_callback_context.clone().expect("update_callback_context"); + render_canvas = inner.render_canvas.clone().context("render_canvas missing")?; - hide_pointer_callback = inner.hide_pointer_callback.clone().expect("hide_pointer_callback"); + hide_pointer_callback = inner + .hide_pointer_callback + .clone() + .context("hide_pointer_callback missing")?; hide_pointer_callback_context = inner .hide_pointer_callback_context .clone() - .expect("show_pointer_callback_context"); - show_pointer_callback = inner.show_pointer_callback.clone().expect("hide_pointer_callback"); + .context("show_pointer_callback_context missing")?; + show_pointer_callback = inner + .show_pointer_callback + .clone() + .context("hide_pointer_callback missing")?; show_pointer_callback_context = inner .show_pointer_callback_context .clone() - .expect("show_pointer_callback_context"); + .context("show_pointer_callback_context missing")?; } info!("Connect to RDP host"); @@ -268,8 +265,7 @@ impl SessionBuilder { writer_tx, input_events_tx, - update_callback, - update_callback_context, + render_canvas, hide_pointer_callback, hide_pointer_callback_context, show_pointer_callback, @@ -291,8 +287,7 @@ pub struct Session { writer_tx: mpsc::UnboundedSender>, input_events_tx: mpsc::UnboundedSender, - update_callback: js_sys::Function, - update_callback_context: JsValue, + render_canvas: HtmlCanvasElement, hide_pointer_callback: js_sys::Function, hide_pointer_callback_context: JsValue, show_pointer_callback: js_sys::Function, @@ -327,32 +322,55 @@ impl Session { let mut framed = ironrdp_futures::SingleThreadedFuturesFramed::new(rdp_reader); + debug!("Initialize canvas"); + + let mut gui = Canvas::new( + self.render_canvas.clone(), + u32::from(connection_result.desktop_size.width), + u32::from(connection_result.desktop_size.height), + ) + .context("canvas initialization")?; + + debug!("Canvas initialized"); + info!("Start RDP session"); - let mut image = DecodedImage::new(PixelFormat::RgbA32, self.desktop_size.width, self.desktop_size.height); + let mut image = DecodedImage::new( + PixelFormat::RgbA32, + connection_result.desktop_size.width, + connection_result.desktop_size.height, + ); let mut active_stage = ActiveStage::new(connection_result, None); - let mut frame_id = 0; 'outer: loop { let outputs = select! { - incoming_pdu = framed.read_pdu().fuse() => { - let (action, frame) = incoming_pdu.context("read next frame")?; + frame = framed.read_pdu().fuse() => { + let (action, payload) = frame.context("read frame")?; + trace!(?action, frame_length = payload.len(), "Frame received"); - active_stage - .process(&mut image, action, &frame) - .context("Active stage processing")? + active_stage.process(&mut image, action, &payload)? } input_events = fastpath_input_events.next() => { let events = input_events.context("read next fastpath input events")?; - active_stage.process_fastpath_input(&mut image, &events) - .context("Fast path input events processing")? + active_stage.process_fastpath_input(&mut image, &events).context("Fast path input events processing")? } }; for out in outputs { match out { + ActiveStageOutput::ResponseFrame(frame) => { + // PERF: unnecessary copy + self.writer_tx + .unbounded_send(frame.to_vec()) + .context("Send frame to writer task")?; + } + ActiveStageOutput::GraphicsUpdate(region) => { + // PERF: some copies and conversion could be optimized + let (region, buffer) = extract_partial_image(&image, region); + gui.draw(&buffer, region).context("draw updated region")?; + } ActiveStageOutput::PointerDefault => { let _ret = self .show_pointer_callback @@ -368,26 +386,6 @@ impl Session { ActiveStageOutput::PointerPosition { .. } => { // Not applicable for web } - ActiveStageOutput::ResponseFrame(frame) => { - // PERF: unnecessary copy - self.writer_tx - .unbounded_send(frame.to_vec()) - .context("Send frame to writer task")?; - } - ActiveStageOutput::GraphicsUpdate(updated_region) => { - let (partial_image_rectangle, partial_image) = extract_partial_image(&image, updated_region); - - send_update_rectangle( - &self.update_callback, - &self.update_callback_context, - frame_id, - partial_image_rectangle, - partial_image, - ) - .context("Failed to send update rectangle")?; - - frame_id += 1; - } ActiveStageOutput::Terminate => break 'outer, } } @@ -499,44 +497,6 @@ fn build_config( } } -fn send_update_rectangle( - update_callback: &js_sys::Function, - callback_context: &JsValue, - frame_id: usize, - region: InclusiveRectangle, - buffer: Vec, -) -> anyhow::Result<()> { - use js_sys::Uint8ClampedArray; - - let top = region.top; - let left = region.left; - let right = region.right; - let bottom = region.bottom; - let width = region.width(); - let height = region.height(); - - let update_rect = RectInfo { - frame_id, - top, - left, - right, - bottom, - width, - height, - }; - let update_rect = JsValue::from(update_rect); - - let js_array = Uint8ClampedArray::new_with_length(buffer.len() as u32); - js_array.copy_from(&buffer); - let js_array = JsValue::from(js_array); - - let _ret = update_callback - .call2(callback_context, &update_rect, &js_array) - .map_err(|e| anyhow::Error::msg(format!("update callback failed: {e:?}")))?; - - Ok(()) -} - async fn writer_task(rx: mpsc::UnboundedReceiver>, rdp_writer: WriteHalf) { debug!("writer task started"); diff --git a/web-client/iron-remote-gui/README.md b/web-client/iron-remote-gui/README.md index 92594bc5e..01c1af3fa 100644 --- a/web-client/iron-remote-gui/README.md +++ b/web-client/iron-remote-gui/README.md @@ -46,7 +46,6 @@ You can add some parameters for default initialization on the component ` Note that due to a limitation of the framework all parameters need to be lower-cased. - `scale`: The scaling behavior of the distant screen. Can be `fit`, `real` or `full`. Default is `real`; -- `targetplatform`: Can be `web` or `native`. Default is `web`. - `verbose`: Show logs from `iron-remote-gui`. `true` or `false`. Default is `false`. - `debugwasm`: Show debug info from web assembly. Can be `"OFF"`, `"ERROR"`, `"WARN"`, `"INFO"`, `"DEBUG"`, `"TRACE"`. Default is `"OFF"`. - `flexcentre`: Helper to force `iron-remote-gui` a flex and centering the content automatically. Otherwise, you need to manage manually. Default is `true`. diff --git a/web-client/iron-remote-gui/index.html b/web-client/iron-remote-gui/index.html index e2223d5f0..6d50f0dc2 100644 --- a/web-client/iron-remote-gui/index.html +++ b/web-client/iron-remote-gui/index.html @@ -7,7 +7,7 @@ - + - +