From 65c0987fe3be1cdc90c54e6b3e0e133607720653 Mon Sep 17 00:00:00 2001 From: Brendan Allan Date: Sat, 9 Nov 2024 17:01:40 +0800 Subject: [PATCH 1/6] av-asset-writer --- Cargo.lock | 250 ++++++++++++++++----- apps/desktop/src-tauri/Cargo.toml | 3 - apps/desktop/src-tauri/src/recording.rs | 16 +- crates/media/Cargo.toml | 1 + crates/media/src/encoders/h264.rs | 64 ++++-- crates/media/src/sources/screen_capture.rs | 173 +++++++++----- 6 files changed, 372 insertions(+), 135 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f025fdecf..da1011848 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -139,6 +139,55 @@ dependencies = [ "winapi", ] +[[package]] +name = "anstream" +version = "0.6.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" + +[[package]] +name = "anstyle-parse" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" +dependencies = [ + "windows-sys 0.59.0", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2109dbce0e72be3ec00bed26e6a7479ca384ad226efdd66db8fa2e3a38c83125" +dependencies = [ + "anstyle", + "windows-sys 0.59.0", +] + [[package]] name = "anyhow" version = "1.0.86" @@ -162,7 +211,7 @@ checksum = "0ae92a5119aa49cdbcf6b9f893fe4e1d98b04ccbf82ee0584ad948a44a734dea" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -307,7 +356,7 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -342,7 +391,7 @@ checksum = "6e0c28dcc82d7c8ead5cb13beb15405b57b8546e93215673ff8ca0349a028107" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -508,7 +557,7 @@ dependencies = [ "bitflags 1.3.2", "cexpr 0.4.0", "clang-sys", - "clap", + "clap 2.34.0", "env_logger", "lazy_static", "lazycell", @@ -540,7 +589,7 @@ dependencies = [ "regex", "rustc-hash 1.1.0", "shlex 1.3.0", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -558,7 +607,7 @@ dependencies = [ "regex", "rustc-hash 1.1.0", "shlex 1.3.0", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -690,7 +739,7 @@ checksum = "0cc8b54b395f2fcfbb3d90c47b01c7f444d94d05bdeb775811dec868ac3bbc26" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -799,6 +848,7 @@ dependencies = [ "cap-flags", "cap-gpu-converters", "cap-project", + "cidre", "cocoa 0.26.0", "core-foundation 0.10.0", "core-graphics 0.24.0", @@ -993,6 +1043,21 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "cidre" +version = "0.4.0" +source = "git+https://github.com/yury/cidre?rev=1e008bec49a0f97aeaaea6130a0ba20fe00aa03b#1e008bec49a0f97aeaaea6130a0ba20fe00aa03b" +dependencies = [ + "cidre-macros", + "parking_lot", + "tokio", +] + +[[package]] +name = "cidre-macros" +version = "0.1.0" +source = "git+https://github.com/yury/cidre?rev=1e008bec49a0f97aeaaea6130a0ba20fe00aa03b#1e008bec49a0f97aeaaea6130a0ba20fe00aa03b" + [[package]] name = "clang-sys" version = "1.8.1" @@ -1019,6 +1084,46 @@ dependencies = [ "vec_map", ] +[[package]] +name = "clap" +version = "4.5.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97f376d85a664d5837dbae44bf546e6477a679ff6610010f17276f686d867e8" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.5.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19bc80abd44e4bed93ca373a0704ccbd1b710dc5749406201bb018272808dc54" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim 0.11.1", +] + +[[package]] +name = "clap_derive" +version = "4.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ac6a0c7b1a9e9a5186361f67dfa1b88213572f427fb9ab038efb2bd8c582dab" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "syn 2.0.87", +] + +[[package]] +name = "clap_lex" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97" + [[package]] name = "claxon" version = "0.4.3" @@ -1116,6 +1221,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" +[[package]] +name = "colorchoice" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" + [[package]] name = "com" version = "0.6.0" @@ -1493,7 +1604,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13b588ba4ac1a99f7f2964d24b3d896ddc6bf847ee3855dbd4366f058cfcd331" dependencies = [ "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -1503,7 +1614,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "edb49164822f3ee45b17acd4a208cfc1251410cf0cad9a833234c9890774dd9f" dependencies = [ "quote", - "syn 2.0.85", + "syn 2.0.87", +] + +[[package]] +name = "ctrlc" +version = "3.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90eeab0aa92f3f9b4e87f258c72b139c207d251f9cbc1080a0086b86a8870dd3" +dependencies = [ + "nix 0.29.0", + "windows-sys 0.59.0", ] [[package]] @@ -1538,7 +1659,7 @@ dependencies = [ "proc-macro2", "quote", "strsim 0.11.1", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -1549,7 +1670,7 @@ checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" dependencies = [ "darling_core", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -1610,7 +1731,7 @@ checksum = "67e77553c4162a157adbf834ebae5b415acbecbeafc7a74b0e886657506a7611" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -1623,7 +1744,7 @@ dependencies = [ "proc-macro2", "quote", "rustc_version", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -1693,7 +1814,6 @@ dependencies = [ "uuid", "wgpu", "windows 0.52.0", - "windows-capture", ] [[package]] @@ -1777,7 +1897,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -1809,7 +1929,7 @@ checksum = "f2b99bf03862d7f545ebc28ddd33a665b50865f4dfd84031a393823879bd4c54" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -1962,7 +2082,7 @@ checksum = "de0d48a183585823424a4ce1aa132d174a6a81bd540895822eb4c8373a8e49e8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -2225,7 +2345,7 @@ checksum = "1a5c6c585bc94aaf2c7b51dd4c2ba22680844aba4c687be581871a6f518c5742" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -2349,7 +2469,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -2627,7 +2747,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -2796,7 +2916,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -3224,7 +3344,7 @@ checksum = "c34819042dc3d3971c46c2190835914dfbe0c3c13f61449b2997f4e9722dfa60" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -3252,6 +3372,12 @@ dependencies = [ "once_cell", ] +[[package]] +name = "is_terminal_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" + [[package]] name = "itertools" version = "0.12.1" @@ -4169,7 +4295,7 @@ checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -4230,7 +4356,7 @@ dependencies = [ "proc-macro-crate 2.0.2", "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -4572,7 +4698,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -4822,7 +4948,7 @@ dependencies = [ "phf_shared 0.11.2", "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -4869,7 +4995,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -5072,7 +5198,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8021cf59c8ec9c432cfc2526ac6b8aa508ecaf29cd415f271b8406c1b851c3fd" dependencies = [ "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -5697,7 +5823,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -5815,7 +5941,7 @@ checksum = "de523f781f095e28fa605cdce0f8307e451cc0fd14e2eb4cd2e98a355b147766" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -5826,7 +5952,7 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -5859,7 +5985,7 @@ checksum = "6c64451ba24fc7a6a2d60fc75dd9c83c90903b19028d4eff35e88fc1e86564e9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -5910,7 +6036,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -6122,7 +6248,7 @@ dependencies = [ "Inflector", "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -6293,9 +6419,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.85" +version = "2.0.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5023162dfcd14ef8f32034d8bcd4cc5ddc61ef7a247c024a33e24e1f24d21b56" +checksum = "25aa4ce346d03a6dcd68dd8b4010bcb74e54e62c90c573f394c46eae99aba32d" dependencies = [ "proc-macro2", "quote", @@ -6434,7 +6560,7 @@ checksum = "f4e16beb8b2ac17db28eab8bca40e62dbfbb34c0fcdc6d9826b11b7b5d047dfd" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -6547,7 +6673,7 @@ dependencies = [ "serde", "serde_json", "sha2", - "syn 2.0.85", + "syn 2.0.87", "tauri-utils", "thiserror", "time", @@ -6565,7 +6691,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", "tauri-codegen", "tauri-utils", ] @@ -6894,7 +7020,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -7004,22 +7130,22 @@ checksum = "8eaa81235c7058867fa8c0e7314f33dcce9c215f535d1913822a2b3f5e289f3c" [[package]] name = "thiserror" -version = "1.0.63" +version = "1.0.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0342370b38b6a11b6cc11d6a805569958d54cfa061a29969c3b5ce2ea405724" +checksum = "02dd99dc800bbb97186339685293e1cc5d9df1f8fae2d0aecd9ff1c77efea892" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.63" +version = "1.0.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261" +checksum = "a7c61ec9a6f64d2793d8a45faba21efbe3ced62a886d44c36a009b2b519b4c7e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -7105,7 +7231,7 @@ checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -7276,7 +7402,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -7494,6 +7620,12 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + [[package]] name = "uuid" version = "1.10.0" @@ -7632,7 +7764,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", "wasm-bindgen-shared", ] @@ -7666,7 +7798,7 @@ checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -7835,7 +7967,7 @@ checksum = "1d228f15bba3b9d56dde8bddbee66fa24545bd17b48d5128ccf4a8742b18e431" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -8096,14 +8228,16 @@ dependencies = [ [[package]] name = "windows-capture" -version = "1.2.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15e2f94b43842205eb84814f505badade792e7b8cd132e436636ccbf7baa08fc" +checksum = "308113a004a94ea5fb83dc7d19e2c66050879b6767f1eb10231af2e77b66475d" dependencies = [ + "clap 4.5.20", + "ctrlc", "parking_lot", "rayon", "thiserror", - "windows 0.56.0", + "windows 0.58.0", ] [[package]] @@ -8167,7 +8301,7 @@ checksum = "f6fc35f58ecd95a9b71c4f2329b911016e6bec66b3f2e6a4aad86bd2e99e2f9b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -8178,7 +8312,7 @@ checksum = "2bbd5b46c938e506ecbce286b6628a02171d56153ba733b6c741fc627ec9579b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -8189,7 +8323,7 @@ checksum = "08990546bf4edef8f431fa6326e032865f27138718c587dc21bc0265bbcb57cc" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -8200,7 +8334,7 @@ checksum = "053c4c462dc91d3b1504c6fe5a726dd15e216ba718e84a0e46a88fbe5ded3515" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -8665,7 +8799,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] diff --git a/apps/desktop/src-tauri/Cargo.toml b/apps/desktop/src-tauri/Cargo.toml index f2f1273dd..afde32334 100644 --- a/apps/desktop/src-tauri/Cargo.toml +++ b/apps/desktop/src-tauri/Cargo.toml @@ -97,9 +97,6 @@ windows = { version = "0.52.0", features = [ "Win32_UI_WindowsAndMessaging", "Win32_Graphics_Gdi", ] } -# Lock the version of windows-capture used by scap@691bd88798d3 -# TODO: Remove this once scap uses the latest version of `windows` and `windows-capture` -windows-capture = "=1.2.0" [target.'cfg(unix)'.dependencies] nix = { version = "0.29.0", features = ["fs"] } diff --git a/apps/desktop/src-tauri/src/recording.rs b/apps/desktop/src-tauri/src/recording.rs index 8d0021a1b..df85cbd1e 100644 --- a/apps/desktop/src-tauri/src/recording.rs +++ b/apps/desktop/src-tauri/src/recording.rs @@ -5,10 +5,10 @@ use serde::Serialize; use specta::Type; use std::collections::HashMap; use std::fs::File; +use std::path::PathBuf; use std::sync::atomic::AtomicBool; use std::sync::Arc; use std::time::{SystemTime, UNIX_EPOCH}; -use std::path::PathBuf; use tokio::sync::oneshot; use crate::cursor::spawn_cursor_recorder; @@ -206,7 +206,7 @@ pub async fn start( let screen_config = screen_source.info(); let screen_bounds = screen_source.bounds; - let output_config = screen_config.scaled(1920, 30); + let output_config = screen_config; // .scaled(1920, 30); let screen_filter = VideoFilter::init("screen", screen_config, output_config)?; let screen_encoder = H264Encoder::init( "screen", @@ -215,7 +215,7 @@ pub async fn start( )?; pipeline_builder = pipeline_builder .source("screen_capture", screen_source) - .pipe("screen_capture_filter", screen_filter) + // .pipe("screen_capture_filter", screen_filter) .sink("screen_capture_encoder", screen_encoder); if let Some(mic_source) = recording_options @@ -241,7 +241,7 @@ pub async fn start( if let Some(camera_source) = CameraSource::init(camera_feed) { let camera_config = camera_source.info(); - let output_config = camera_config.scaled(1920, 30); + let output_config = camera_config; //.scaled(1920, 30); camera_output_path = Some(content_dir.join("camera.mp4")); let camera_filter = VideoFilter::init("camera", camera_config, output_config)?; @@ -263,9 +263,11 @@ pub async fn start( let stop_signal = Arc::new(AtomicBool::new(false)); // Initialize default values for cursor channels - let (mouse_moves, mouse_clicks) = if FLAGS.record_mouse { - spawn_cursor_recorder(stop_signal.clone(), screen_bounds, content_dir, cursors_dir) - } else { + let (mouse_moves, mouse_clicks) = + // if FLAGS.record_mouse + { + // spawn_cursor_recorder(stop_signal.clone(), screen_bounds, content_dir, cursors_dir) + // } else { // Create dummy channels that will never receive data let (move_tx, move_rx) = oneshot::channel(); let (click_tx, click_rx) = oneshot::channel(); diff --git a/crates/media/Cargo.toml b/crates/media/Cargo.toml index 4e3182d83..eafc0ba22 100644 --- a/crates/media/Cargo.toml +++ b/crates/media/Cargo.toml @@ -37,6 +37,7 @@ objc = "0.2.7" objc-foundation = "0.1.1" objc2-foundation = { version = "0.2.2", features = ["NSValue"] } nokhwa-bindings-macos.workspace = true +cidre = { git = "https://github.com/yury/cidre", rev = "1e008bec49a0f97aeaaea6130a0ba20fe00aa03b" } [target.'cfg(target_os = "windows")'.dependencies] windows = { version = "0.52.0", features = [ diff --git a/crates/media/src/encoders/h264.rs b/crates/media/src/encoders/h264.rs index 819c8efba..15082961a 100644 --- a/crates/media/src/encoders/h264.rs +++ b/crates/media/src/encoders/h264.rs @@ -6,6 +6,7 @@ use crate::{ use ffmpeg::{ codec::{codec::Codec, context, encoder}, format::{self}, + software, threading::Config, Dictionary, }; @@ -17,11 +18,13 @@ pub struct H264Encoder { encoder: encoder::Video, output_ctx: format::context::Output, last_pts: Option, + config: VideoInfo, } impl H264Encoder { pub fn init(tag: &'static str, config: VideoInfo, output: Output) -> Result { let Output::File(destination) = output; + let mut output_ctx = format::output(&destination)?; let (codec, options) = get_codec_and_options(&config)?; @@ -37,7 +40,14 @@ impl H264Encoder { encoder.set_format(config.pixel_format); encoder.set_time_base(config.frame_rate.invert()); encoder.set_frame_rate(Some(config.frame_rate)); - encoder.set_bit_rate(5_000_000); + + if codec.name() == "h264_videotoolbox" { + encoder.set_bit_rate(1_200_000); + encoder.set_max_bit_rate(120_000); + } else { + encoder.set_bit_rate(8_000_000); + encoder.set_max_bit_rate(8_000_000); + } let video_encoder = encoder.open_with(options)?; @@ -52,10 +62,30 @@ impl H264Encoder { encoder: video_encoder, output_ctx, last_pts: None, + config, }) } fn queue_frame(&mut self, frame: FFVideo) { + // let out_frame = if frame.width() > self.config.width || frame.height() > self.config.height + // { + // let mut conv = software::scaler( + // frame.format(), + // software::scaling::Flags::FAST_BILINEAR, + // (frame.width(), frame.height()), + // (self.config.width, self.config.height), + // ) + // .unwrap(); + + // let mut out_frame = FFVideo::empty(); + // conv.run(&frame, &mut out_frame).unwrap(); + + // out_frame.set_pts(frame.pts()); + // out_frame + // } else { + // frame + // }; + self.encoder.send_frame(&frame).unwrap(); } @@ -66,7 +96,8 @@ impl H264Encoder { while self.encoder.receive_packet(&mut encoded_packet).is_ok() { encoded_packet.set_stream(0); encoded_packet.rescale_ts( - self.encoder.time_base(), + ffmpeg::Rational::new(1, 1_000_000), + // self.encoder.time_base(), self.output_ctx.stream(0).unwrap().time_base(), ); // TODO: Possibly move writing to disk to its own file, to increase encoding throughput? @@ -109,6 +140,8 @@ impl PipelineSinkTask for H264Encoder { fn get_codec_and_options(config: &VideoInfo) -> Result<(Codec, Dictionary), MediaError> { let encoder_name = { if cfg!(target_os = "macos") { + // "libx264" + // looks terrible rn :( "h264_videotoolbox" } else { "libx264" @@ -117,17 +150,22 @@ fn get_codec_and_options(config: &VideoInfo) -> Result<(Codec, Dictionary), Medi if let Some(codec) = encoder::find_by_name(encoder_name) { let mut options = Dictionary::new(); - let keyframe_interval_secs = 2; - let keyframe_interval = keyframe_interval_secs * config.frame_rate.numerator(); - let keyframe_interval_str = keyframe_interval.to_string(); - - options.set("preset", "ultrafast"); - options.set("tune", "zerolatency"); - options.set("vsync", "1"); - options.set("g", &keyframe_interval_str); - options.set("keyint_min", &keyframe_interval_str); - // TODO: Is it worth limiting quality? Maybe make this configurable - // options.set("crf", "23"); + if encoder_name == "h264_videotoolbox" { + // options.set("constant_bit_rate", "true"); + options.set("realtime", "true"); + } else { + let keyframe_interval_secs = 2; + let keyframe_interval = keyframe_interval_secs * config.frame_rate.numerator(); + let keyframe_interval_str = keyframe_interval.to_string(); + + options.set("preset", "ultrafast"); + options.set("tune", "zerolatency"); + options.set("vsync", "1"); + options.set("g", &keyframe_interval_str); + options.set("keyint_min", &keyframe_interval_str); + // // TODO: Is it worth limiting quality? Maybe make this configurable + // options.set("crf", "14"); + } return Ok((codec, options)); } diff --git a/crates/media/src/sources/screen_capture.rs b/crates/media/src/sources/screen_capture.rs index 0282de5ec..91026d955 100644 --- a/crates/media/src/sources/screen_capture.rs +++ b/crates/media/src/sources/screen_capture.rs @@ -7,15 +7,12 @@ use scap::{ }; use serde::{Deserialize, Serialize}; use specta::Type; -use std::collections::HashMap; +use std::{collections::HashMap, path::PathBuf}; use crate::{ data::{FFVideo, RawVideoFormat, VideoInfo}, - platform::{Bounds, Window}, -}; -use crate::{ pipeline::{clock::*, control::Control, task::PipelineSourceTask}, - platform, + platform::{self, Bounds, Window}, }; static EXCLUDED_WINDOWS: [&str; 4] = [ @@ -81,13 +78,9 @@ impl ScreenCaptureSource { let excluded_targets: Vec = targets .iter() - .filter(|target| match target { - Target::Window(scap_window) - if EXCLUDED_WINDOWS.contains(&scap_window.title.as_str()) => - { - true - } - _ => false, + .filter(|target| { + matches!(target, Target::Window(scap_window) + if EXCLUDED_WINDOWS.contains(&scap_window.title.as_str())) }) .cloned() .collect(); @@ -229,6 +222,46 @@ impl PipelineSourceTask for ScreenCaptureSource { let mut capturing = false; ready_signal.send(Ok(())).unwrap(); + use cidre::{objc::Obj, *}; + + let mut asset_writer = av::AssetWriter::with_url_and_file_type( + cf::Url::with_path(&PathBuf::from("../bruh.mp4").as_path(), false) + .unwrap() + .as_ns(), + av::FileType::mp4(), + ) + .unwrap(); + + let assistant = + av::OutputSettingsAssistant::with_preset(av::OutputSettingsPreset::h264_3840x2160()) + .unwrap(); + + let mut output_settings = assistant.video_settings().unwrap().copy_mut(); + + output_settings.insert( + av::video_settings_keys::width(), + ns::Number::with_u32(self.video_info.width).as_id_ref(), + ); + + output_settings.insert( + av::video_settings_keys::height(), + ns::Number::with_u32(self.video_info.height).as_id_ref(), + ); + + let mut video_input = av::AssetWriterInput::with_media_type_and_output_settings( + av::MediaType::video(), + Some(output_settings.as_ref()), + ) + .unwrap(); + video_input.set_expects_media_data_in_real_time(true); + + asset_writer.add_input(&video_input).unwrap(); + + asset_writer.start_writing(); + + let mut first_timestamp = None; + let mut last_timestamp = None; + loop { match control_signal.last() { Some(Control::Play) => { @@ -248,48 +281,75 @@ impl PipelineSourceTask for ScreenCaptureSource { continue; } - let raw_timestamp = RawNanoseconds(pixel_buffer.display_time()); - match clock.timestamp_for(raw_timestamp) { - None => { - eprintln!("Clock is currently stopped. Dropping frames."); - } - Some(timestamp) => { - let mut frame = FFVideo::new( - self.video_info.pixel_format, - self.video_info.width, - self.video_info.height, - ); - frame.set_pts(Some(timestamp)); - - let planes = pixel_buffer.planes(); - - for (i, plane) in planes.into_iter().enumerate() { - let data = plane.data(); - - for y in 0..plane.height() { - let buffer_y_offset = y * plane.bytes_per_row(); - let frame_y_offset = y * frame.stride(i); - - let num_bytes = - frame.stride(i).min(plane.bytes_per_row()); - - frame.data_mut(i) - [frame_y_offset..frame_y_offset + num_bytes] - .copy_from_slice( - &data[buffer_y_offset - ..buffer_y_offset + num_bytes], - ); - } - } - - if let Err(_) = output.send(frame) { - eprintln!( - "Pipeline is unreachable. Shutting down recording." - ); - break; - } - } - }; + let timestamp = + pixel_buffer.buffer().sys_ref.get_presentation_timestamp(); + let time = cm::Time::with_epoch( + timestamp.value, + timestamp.timescale, + timestamp.epoch, + ); + + if first_timestamp.is_none() { + asset_writer.start_session_at_src_time(time); + first_timestamp = Some(time); + } + last_timestamp = Some(time); + + video_input + .append_sample_buf(unsafe { + let ptr = &*pixel_buffer.buffer().sys_ref as *const _ + as *const cm::SampleBuf; + + // let ret = std::mem::transmute(ptr); + + // std::mem::forget(pixel_buffer); + + &*ptr + }) + .unwrap(); + + // let raw_timestamp = RawNanoseconds(pixel_buffer.display_time()); + // match clock.timestamp_for(raw_timestamp) { + // None => { + // eprintln!("Clock is currently stopped. Dropping frames."); + // } + // Some(timestamp) => { + // // let mut frame = FFVideo::new( + // // self.video_info.pixel_format, + // // self.video_info.width, + // // self.video_info.height, + // // ); + // // frame.set_pts(Some(timestamp)); + + // // let planes = pixel_buffer.planes(); + + // // for (i, plane) in planes.into_iter().enumerate() { + // // let data = plane.data(); + + // // for y in 0..plane.height() { + // // let buffer_y_offset = y * plane.bytes_per_row(); + // // let frame_y_offset = y * frame.stride(i); + + // // let num_bytes = + // // frame.stride(i).min(plane.bytes_per_row()); + + // // frame.data_mut(i) + // // [frame_y_offset..frame_y_offset + num_bytes] + // // .copy_from_slice( + // // &data[buffer_y_offset + // // ..buffer_y_offset + num_bytes], + // // ); + // // } + // // } + + // if let Err(_) = output.send(frame) { + // eprintln!( + // "Pipeline is unreachable. Shutting down recording." + // ); + // break; + // } + // } + // }; } Err(error) => { eprintln!("Capture error: {error}"); @@ -308,6 +368,11 @@ impl PipelineSourceTask for ScreenCaptureSource { println!("Received shutdown signal"); if capturing { capturer.stop_capture(); + asset_writer.end_session_at_src_time( + last_timestamp.take().unwrap_or(cm::Time::zero()), + ); + video_input.mark_as_finished(); + asset_writer.finish_writing(); } break; } From db978af806532a74d8e1936bac47c8dba5b728b7 Mon Sep 17 00:00:00 2001 From: Brendan Allan Date: Sun, 10 Nov 2024 19:06:06 +0800 Subject: [PATCH 2/6] integrate apple encoding stack into pipeline + handle missing and duplicate frames in decoder --- Cargo.lock | 7 +- Cargo.toml | 2 +- apps/desktop/src-tauri/src/recording.rs | 63 +++-- crates/media/Cargo.toml | 1 + crates/media/src/encoders/h264.rs | 26 +- .../media/src/encoders/h264_avassetwriter.rs | 130 ++++++++++ crates/media/src/encoders/mod.rs | 4 + crates/media/src/sources/screen_capture.rs | 231 ++++++++++-------- crates/rendering/src/decoder.rs | 119 +++++---- crates/rendering/src/lib.rs | 4 +- 10 files changed, 387 insertions(+), 200 deletions(-) create mode 100644 crates/media/src/encoders/h264_avassetwriter.rs diff --git a/Cargo.lock b/Cargo.lock index da1011848..7085ec299 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -865,6 +865,7 @@ dependencies = [ "objc-foundation", "objc2-foundation", "scap", + "screencapturekit", "serde", "specta", "tempfile", @@ -5773,8 +5774,8 @@ dependencies = [ [[package]] name = "scap" -version = "0.0.6" -source = "git+https://github.com/CapSoftware/scap?rev=4d6be030ba2b0cea565ccaee28b1999f25b8dd5d#4d6be030ba2b0cea565ccaee28b1999f25b8dd5d" +version = "0.0.7" +source = "git+https://github.com/CapSoftware/scap?rev=b1e140a3fe90#b1e140a3fe905c19b845dfea66b3b1aea02f0472" dependencies = [ "cocoa 0.25.0", "core-graphics-helmer-fork", @@ -5786,7 +5787,7 @@ dependencies = [ "screencapturekit-sys", "sysinfo", "tao-core-video-sys", - "windows 0.52.0", + "windows 0.58.0", "windows-capture", ] diff --git a/Cargo.toml b/Cargo.toml index 29b82b2b9..721327c49 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,7 +17,7 @@ tokio = { version = "1.39.3", features = [ ] } tauri = { version = "2.0.0" } specta = { version = "=2.0.0-rc.20" } -scap = { git = "https://github.com/CapSoftware/scap", rev = "4d6be030ba2b0cea565ccaee28b1999f25b8dd5d" } +scap = { git = "https://github.com/CapSoftware/scap", rev = "b1e140a3fe90" } nokhwa = { git = "https://github.com/CapSoftware/nokhwa", rev = "c5c7e2298764", features = [ "input-native", "serialize", diff --git a/apps/desktop/src-tauri/src/recording.rs b/apps/desktop/src-tauri/src/recording.rs index df85cbd1e..c1e45eb99 100644 --- a/apps/desktop/src-tauri/src/recording.rs +++ b/apps/desktop/src-tauri/src/recording.rs @@ -20,13 +20,13 @@ pub const FPS: u32 = 30; #[tauri::command(async)] #[specta::specta] pub fn list_capture_screens() -> Vec { - ScreenCaptureSource::list_screens() + ScreenCaptureSource::::list_screens() } #[tauri::command(async)] #[specta::specta] pub fn list_capture_windows() -> Vec { - ScreenCaptureSource::list_targets() + ScreenCaptureSource::::list_targets() } #[tauri::command(async)] @@ -201,22 +201,47 @@ pub async fn start( let mut audio_output_path = None; let mut camera_output_path = None; - let screen_source = - ScreenCaptureSource::init(dbg!(&recording_options.capture_target), None, None); - let screen_config = screen_source.info(); - let screen_bounds = screen_source.bounds; - - let output_config = screen_config; // .scaled(1920, 30); - let screen_filter = VideoFilter::init("screen", screen_config, output_config)?; - let screen_encoder = H264Encoder::init( - "screen", - output_config, - Output::File(display_output_path.clone()), - )?; - pipeline_builder = pipeline_builder - .source("screen_capture", screen_source) - // .pipe("screen_capture_filter", screen_filter) - .sink("screen_capture_encoder", screen_encoder); + #[cfg(target_os = "macos")] + { + let screen_source = ScreenCaptureSource::::init( + dbg!(&recording_options.capture_target), + None, + None, + ); + let screen_config = screen_source.info(); + + let output_config = screen_config.scaled(1920, 30); + let screen_encoder = cap_media::encoders::H264AVAssetWriterEncoder::init( + "screen", + output_config, + Output::File(display_output_path.clone()), + )?; + pipeline_builder = pipeline_builder + .source("screen_capture", screen_source) + .sink("screen_capture_encoder", screen_encoder); + } + #[cfg(not(target_os = "macos"))] + { + let screen_source = ScreenCaptureSource::::init( + dbg!(&recording_options.capture_target), + None, + None, + ); + let screen_config = screen_source.info(); + let screen_bounds = screen_source.bounds; + + let output_config = screen_config.scaled(1920, 30); + let screen_filter = VideoFilter::init("screen", screen_config, output_config)?; + let screen_encoder = H264Encoder::init( + "screen", + output_config, + Output::File(display_output_path.clone()), + )?; + pipeline_builder = pipeline_builder + .source("screen_capture", screen_source) + // .pipe("screen_capture_filter", screen_filter) + .sink("screen_capture_encoder", screen_encoder); + } if let Some(mic_source) = recording_options .audio_input_name @@ -241,7 +266,7 @@ pub async fn start( if let Some(camera_source) = CameraSource::init(camera_feed) { let camera_config = camera_source.info(); - let output_config = camera_config; //.scaled(1920, 30); + let output_config = camera_config.scaled(1920, 30); camera_output_path = Some(content_dir.join("camera.mp4")); let camera_filter = VideoFilter::init("camera", camera_config, output_config)?; diff --git a/crates/media/Cargo.toml b/crates/media/Cargo.toml index eafc0ba22..c18b6199e 100644 --- a/crates/media/Cargo.toml +++ b/crates/media/Cargo.toml @@ -38,6 +38,7 @@ objc-foundation = "0.1.1" objc2-foundation = { version = "0.2.2", features = ["NSValue"] } nokhwa-bindings-macos.workspace = true cidre = { git = "https://github.com/yury/cidre", rev = "1e008bec49a0f97aeaaea6130a0ba20fe00aa03b" } +screencapturekit = "0.2.8" [target.'cfg(target_os = "windows")'.dependencies] windows = { version = "0.52.0", features = [ diff --git a/crates/media/src/encoders/h264.rs b/crates/media/src/encoders/h264.rs index 15082961a..fc4c355cf 100644 --- a/crates/media/src/encoders/h264.rs +++ b/crates/media/src/encoders/h264.rs @@ -67,25 +67,6 @@ impl H264Encoder { } fn queue_frame(&mut self, frame: FFVideo) { - // let out_frame = if frame.width() > self.config.width || frame.height() > self.config.height - // { - // let mut conv = software::scaler( - // frame.format(), - // software::scaling::Flags::FAST_BILINEAR, - // (frame.width(), frame.height()), - // (self.config.width, self.config.height), - // ) - // .unwrap(); - - // let mut out_frame = FFVideo::empty(); - // conv.run(&frame, &mut out_frame).unwrap(); - - // out_frame.set_pts(frame.pts()); - // out_frame - // } else { - // frame - // }; - self.encoder.send_frame(&frame).unwrap(); } @@ -96,8 +77,7 @@ impl H264Encoder { while self.encoder.receive_packet(&mut encoded_packet).is_ok() { encoded_packet.set_stream(0); encoded_packet.rescale_ts( - ffmpeg::Rational::new(1, 1_000_000), - // self.encoder.time_base(), + self.encoder.time_base(), self.output_ctx.stream(0).unwrap().time_base(), ); // TODO: Possibly move writing to disk to its own file, to increase encoding throughput? @@ -140,9 +120,9 @@ impl PipelineSinkTask for H264Encoder { fn get_codec_and_options(config: &VideoInfo) -> Result<(Codec, Dictionary), MediaError> { let encoder_name = { if cfg!(target_os = "macos") { - // "libx264" + "libx264" // looks terrible rn :( - "h264_videotoolbox" + // "h264_videotoolbox" } else { "libx264" } diff --git a/crates/media/src/encoders/h264_avassetwriter.rs b/crates/media/src/encoders/h264_avassetwriter.rs new file mode 100644 index 000000000..c9f91b394 --- /dev/null +++ b/crates/media/src/encoders/h264_avassetwriter.rs @@ -0,0 +1,130 @@ +use crate::{data::VideoInfo, pipeline::task::PipelineSinkTask, MediaError}; + +use super::Output; +use arc::Retained; +use cidre::{objc::Obj, *}; + +pub struct H264AVAssetWriterEncoder { + tag: &'static str, + last_pts: Option, + config: VideoInfo, + asset_writer: Retained, + video_input: Retained, + first_timestamp: Option, + last_timestamp: Option, +} + +impl H264AVAssetWriterEncoder { + pub fn init(tag: &'static str, config: VideoInfo, output: Output) -> Result { + let Output::File(destination) = output; + + let mut asset_writer = av::AssetWriter::with_url_and_file_type( + cf::Url::with_path(&destination.as_path(), false) + .unwrap() + .as_ns(), + av::FileType::mp4(), + ) + .unwrap(); + + let assistant = + av::OutputSettingsAssistant::with_preset(av::OutputSettingsPreset::h264_3840x2160()) + .unwrap(); + + let mut output_settings = assistant.video_settings().unwrap().copy_mut(); + + output_settings.insert( + av::video_settings_keys::width(), + ns::Number::with_u32(config.width).as_id_ref(), + ); + + output_settings.insert( + av::video_settings_keys::height(), + ns::Number::with_u32(config.height).as_id_ref(), + ); + + output_settings.insert( + av::video_settings_keys::compression_props(), + ns::Dictionary::with_keys_values( + &[unsafe { AVVideoAverageBitRateKey }], + &[ns::Number::with_u32(10_000_000).as_id_ref()], + ) + .as_id_ref(), + ); + + let mut video_input = av::AssetWriterInput::with_media_type_and_output_settings( + av::MediaType::video(), + Some(output_settings.as_ref()), + ) + .unwrap(); + video_input.set_expects_media_data_in_real_time(true); + + asset_writer.add_input(&video_input).unwrap(); + + asset_writer.start_writing(); + + Ok(Self { + tag, + last_pts: None, + config, + asset_writer, + video_input, + first_timestamp: None, + last_timestamp: None, + }) + } + + fn queue_frame(&mut self, frame: screencapturekit::cm_sample_buffer::CMSampleBuffer) { + let sample_buf = unsafe { + let ptr = &*frame.sys_ref as *const _ as *const cm::SampleBuf; + &*ptr + }; + + let time = sample_buf.pts(); + + if self.first_timestamp.is_none() { + self.asset_writer.start_session_at_src_time(time); + self.first_timestamp = Some(time); + } + + self.last_timestamp = Some(time); + + self.video_input.append_sample_buf(sample_buf).unwrap(); + } + + fn process_frame(&mut self) {} + + fn finish(&mut self) { + self.asset_writer + .end_session_at_src_time(self.last_timestamp.take().unwrap_or(cm::Time::zero())); + self.video_input.mark_as_finished(); + self.asset_writer.finish_writing(); + } +} + +impl PipelineSinkTask for H264AVAssetWriterEncoder { + type Input = screencapturekit::cm_sample_buffer::CMSampleBuffer; + + fn run( + &mut self, + ready_signal: crate::pipeline::task::PipelineReadySignal, + input: flume::Receiver, + ) { + println!("Starting {} video encoding thread", self.tag); + ready_signal.send(Ok(())).unwrap(); + + while let Ok(frame) = input.recv() { + self.queue_frame(frame); + self.process_frame(); + } + + println!("Received last {} frame. Finishing up encoding.", self.tag); + self.finish(); + + println!("Shutting down {} video encoding thread", self.tag); + } +} + +#[link(name = "AVFoundation", kind = "framework")] +extern "C" { + static AVVideoAverageBitRateKey: &'static cidre::ns::String; +} diff --git a/crates/media/src/encoders/mod.rs b/crates/media/src/encoders/mod.rs index bc184da53..27d9ba0cd 100644 --- a/crates/media/src/encoders/mod.rs +++ b/crates/media/src/encoders/mod.rs @@ -1,9 +1,13 @@ use std::path::PathBuf; mod h264; +#[cfg(target_os = "macos")] +mod h264_avassetwriter; mod mp3; pub use h264::*; +#[cfg(target_os = "macos")] +pub use h264_avassetwriter::*; pub use mp3::*; pub enum Output { diff --git a/crates/media/src/sources/screen_capture.rs b/crates/media/src/sources/screen_capture.rs index 91026d955..8d1e36a44 100644 --- a/crates/media/src/sources/screen_capture.rs +++ b/crates/media/src/sources/screen_capture.rs @@ -1,4 +1,6 @@ use cap_flags::FLAGS; +use cidre::cm; +use core_foundation::base::{kCFAllocatorDefault, CFAllocatorRef}; use flume::Sender; use scap::{ capturer::{get_output_frame_size, Area, Capturer, Options, Point, Resolution, Size}, @@ -7,7 +9,12 @@ use scap::{ }; use serde::{Deserialize, Serialize}; use specta::Type; -use std::{collections::HashMap, path::PathBuf}; +use std::{ + collections::HashMap, + ffi::c_void, + path::PathBuf, + ptr::{null, null_mut}, +}; use crate::{ data::{FFVideo, RawVideoFormat, VideoInfo}, @@ -57,14 +64,15 @@ impl PartialEq for ScreenCaptureTarget { } } -pub struct ScreenCaptureSource { +pub struct ScreenCaptureSource { options: Options, video_info: VideoInfo, target: ScreenCaptureTarget, pub bounds: Bounds, + phantom: std::marker::PhantomData, } -impl ScreenCaptureSource { +impl ScreenCaptureSource { pub const DEFAULT_FPS: u32 = 30; pub fn init( @@ -134,6 +142,7 @@ impl ScreenCaptureSource { target: capture_target.clone(), bounds, video_info: VideoInfo::from_raw(RawVideoFormat::Nv12, frame_width, frame_height, fps), + phantom: Default::default(), } } @@ -201,7 +210,9 @@ impl ScreenCaptureSource { } } -impl PipelineSourceTask for ScreenCaptureSource { +pub struct AVFrameCapture; + +impl PipelineSourceTask for ScreenCaptureSource { type Clock = SynchronisedClock; type Output = FFVideo; @@ -222,45 +233,121 @@ impl PipelineSourceTask for ScreenCaptureSource { let mut capturing = false; ready_signal.send(Ok(())).unwrap(); - use cidre::{objc::Obj, *}; + loop { + match control_signal.last() { + Some(Control::Play) => { + if !capturing { + if let Some(window_id) = maybe_capture_window_id { + crate::platform::bring_window_to_focus(window_id); + } + capturer.start_capture(); + capturing = true; - let mut asset_writer = av::AssetWriter::with_url_and_file_type( - cf::Url::with_path(&PathBuf::from("../bruh.mp4").as_path(), false) - .unwrap() - .as_ns(), - av::FileType::mp4(), - ) - .unwrap(); + println!("Screen recording started."); + } - let assistant = - av::OutputSettingsAssistant::with_preset(av::OutputSettingsPreset::h264_3840x2160()) - .unwrap(); + match capturer.raw().get_next_pixel_buffer() { + Ok(pixel_buffer) => { + if pixel_buffer.height() == 0 || pixel_buffer.width() == 0 { + continue; + } - let mut output_settings = assistant.video_settings().unwrap().copy_mut(); + let raw_timestamp = RawNanoseconds(pixel_buffer.display_time()); + match clock.timestamp_for(raw_timestamp) { + None => { + eprintln!("Clock is currently stopped. Dropping frames."); + } + Some(timestamp) => { + let mut frame = FFVideo::new( + self.video_info.pixel_format, + self.video_info.width, + self.video_info.height, + ); + frame.set_pts(Some(timestamp)); + + let planes = pixel_buffer.planes(); + + for (i, plane) in planes.into_iter().enumerate() { + let data = plane.data(); + + for y in 0..plane.height() { + let buffer_y_offset = y * plane.bytes_per_row(); + let frame_y_offset = y * frame.stride(i); + + let num_bytes = + frame.stride(i).min(plane.bytes_per_row()); + + frame.data_mut(i) + [frame_y_offset..frame_y_offset + num_bytes] + .copy_from_slice( + &data[buffer_y_offset + ..buffer_y_offset + num_bytes], + ); + } + } + + if let Err(_) = output.send(frame) { + eprintln!( + "Pipeline is unreachable. Shutting down recording." + ); + break; + } + } + }; + } + Err(error) => { + eprintln!("Capture error: {error}"); + break; + } + } + } + Some(Control::Pause) => { + println!("Received pause signal"); + if capturing { + capturer.stop_capture(); + capturing = false; + } + } + Some(Control::Shutdown) | None => { + println!("Received shutdown signal"); + if capturing { + capturer.stop_capture(); + } + break; + } + } + } - output_settings.insert( - av::video_settings_keys::width(), - ns::Number::with_u32(self.video_info.width).as_id_ref(), - ); + println!("Shutting down screen capture source thread."); + } +} - output_settings.insert( - av::video_settings_keys::height(), - ns::Number::with_u32(self.video_info.height).as_id_ref(), - ); +#[cfg(target_os = "macos")] +pub struct CMSampleBufferCapture; - let mut video_input = av::AssetWriterInput::with_media_type_and_output_settings( - av::MediaType::video(), - Some(output_settings.as_ref()), - ) - .unwrap(); - video_input.set_expects_media_data_in_real_time(true); +#[cfg(target_os = "macos")] +impl PipelineSourceTask for ScreenCaptureSource { + type Clock = SynchronisedClock; + type Output = screencapturekit::cm_sample_buffer::CMSampleBuffer; - asset_writer.add_input(&video_input).unwrap(); + fn run( + &mut self, + _: Self::Clock, + ready_signal: crate::pipeline::task::PipelineReadySignal, + mut control_signal: crate::pipeline::control::PipelineControlSignal, + output: Sender, + ) { + use cidre::*; - asset_writer.start_writing(); + println!("Preparing screen capture source thread..."); - let mut first_timestamp = None; - let mut last_timestamp = None; + let maybe_capture_window_id = match &self.target { + ScreenCaptureTarget::Window(window) => Some(window.id), + _ => None, + }; + let mut capturer = Capturer::new(dbg!(self.options.clone())); + let mut capturing = false; + ready_signal.send(Ok(())).unwrap(); loop { match control_signal.last() { @@ -281,75 +368,10 @@ impl PipelineSourceTask for ScreenCaptureSource { continue; } - let timestamp = - pixel_buffer.buffer().sys_ref.get_presentation_timestamp(); - let time = cm::Time::with_epoch( - timestamp.value, - timestamp.timescale, - timestamp.epoch, - ); - - if first_timestamp.is_none() { - asset_writer.start_session_at_src_time(time); - first_timestamp = Some(time); + if let Err(_) = output.send(pixel_buffer.into()) { + eprintln!("Pipeline is unreachable. Shutting down recording."); + break; } - last_timestamp = Some(time); - - video_input - .append_sample_buf(unsafe { - let ptr = &*pixel_buffer.buffer().sys_ref as *const _ - as *const cm::SampleBuf; - - // let ret = std::mem::transmute(ptr); - - // std::mem::forget(pixel_buffer); - - &*ptr - }) - .unwrap(); - - // let raw_timestamp = RawNanoseconds(pixel_buffer.display_time()); - // match clock.timestamp_for(raw_timestamp) { - // None => { - // eprintln!("Clock is currently stopped. Dropping frames."); - // } - // Some(timestamp) => { - // // let mut frame = FFVideo::new( - // // self.video_info.pixel_format, - // // self.video_info.width, - // // self.video_info.height, - // // ); - // // frame.set_pts(Some(timestamp)); - - // // let planes = pixel_buffer.planes(); - - // // for (i, plane) in planes.into_iter().enumerate() { - // // let data = plane.data(); - - // // for y in 0..plane.height() { - // // let buffer_y_offset = y * plane.bytes_per_row(); - // // let frame_y_offset = y * frame.stride(i); - - // // let num_bytes = - // // frame.stride(i).min(plane.bytes_per_row()); - - // // frame.data_mut(i) - // // [frame_y_offset..frame_y_offset + num_bytes] - // // .copy_from_slice( - // // &data[buffer_y_offset - // // ..buffer_y_offset + num_bytes], - // // ); - // // } - // // } - - // if let Err(_) = output.send(frame) { - // eprintln!( - // "Pipeline is unreachable. Shutting down recording." - // ); - // break; - // } - // } - // }; } Err(error) => { eprintln!("Capture error: {error}"); @@ -368,11 +390,6 @@ impl PipelineSourceTask for ScreenCaptureSource { println!("Received shutdown signal"); if capturing { capturer.stop_capture(); - asset_writer.end_session_at_src_time( - last_timestamp.take().unwrap_or(cm::Time::zero()), - ); - video_input.mark_as_finished(); - asset_writer.finish_writing(); } break; } diff --git a/crates/rendering/src/decoder.rs b/crates/rendering/src/decoder.rs index 996e1065f..13bae9e23 100644 --- a/crates/rendering/src/decoder.rs +++ b/crates/rendering/src/decoder.rs @@ -6,8 +6,8 @@ use std::{ use ffmpeg::{ codec, - format::{self, context::input::PacketIter}, - frame, rescale, Codec, Packet, Rational, Rescale, Stream, + format::{self}, + frame, rescale, Codec, Rational, Rescale, }; use ffmpeg_hw_device::{CodecContextExt, HwDevice}; use ffmpeg_sys_next::{avcodec_find_decoder, AVHWDeviceType}; @@ -18,13 +18,16 @@ enum VideoDecoderMessage { GetFrame(u32, tokio::sync::oneshot::Sender>>>), } -fn ts_to_frame(ts: i64, time_base: Rational, frame_rate: Rational) -> u32 { - // dbg!((ts, time_base, frame_rate)); - ((ts * time_base.numerator() as i64 * frame_rate.numerator() as i64) - / (time_base.denominator() as i64 * frame_rate.denominator() as i64)) as u32 +fn pts_to_frame(pts: i64, time_base: Rational) -> u32 { + (FPS as f64 * ((pts as f64 * time_base.numerator() as f64) / (time_base.denominator() as f64))) + .round() as u32 } const FRAME_CACHE_SIZE: usize = 50; +// TODO: Allow dynamic FPS values by either passing it into `spawn` +// or changing `get_frame` to take the requested time instead of frame number, +// so that the lookup can be done by PTS instead of frame number. +const FPS: u32 = 30; pub struct AsyncVideoDecoder; @@ -90,97 +93,114 @@ impl AsyncVideoDecoder { let mut temp_frame = ffmpeg::frame::Video::empty(); - let render_more_margin = (FRAME_CACHE_SIZE / 4) as u32; - let mut cache = BTreeMap::>>::new(); // active frame is a frame that triggered decode. // frames that are within render_more_margin of this frame won't trigger decode. let mut last_active_frame = None::; let mut last_decoded_frame = None::; - - struct PacketStuff<'a> { - packets: PacketIter<'a>, - skipped_packet: Option<(Stream<'a>, Packet)>, - } + let mut last_sent_frame = None::<(u32, DecodedFrame)>; let mut peekable_requests = PeekableReceiver { rx, peeked: None }; let mut packets = input.packets(); - // let mut packet_stuff = PacketStuff { - // packets: input.packets(), - // skipped_packet: None, - // }; while let Ok(r) = peekable_requests.recv() { match r { - VideoDecoderMessage::GetFrame(frame_number, sender) => { - // println!("retrieving frame {frame_number}"); - - let mut sender = if let Some(cached) = cache.get(&frame_number) { - // println!("sending frame {frame_number} from cache"); + VideoDecoderMessage::GetFrame(requested_frame, sender) => { + let mut sender = if let Some(cached) = cache.get(&requested_frame) { sender.send(Some(cached.clone())).ok(); + last_sent_frame = Some((requested_frame, cached.clone())); continue; } else { Some(sender) }; - let cache_min = frame_number.saturating_sub(FRAME_CACHE_SIZE as u32 / 2); - let cache_max = frame_number + FRAME_CACHE_SIZE as u32 / 2; + let cache_min = requested_frame.saturating_sub(FRAME_CACHE_SIZE as u32 / 2); + let cache_max = requested_frame + FRAME_CACHE_SIZE as u32 / 2; - if frame_number <= 0 - || last_decoded_frame - .map(|f| { - frame_number < f || + if requested_frame <= 0 + || last_sent_frame + .as_ref() + .map(|last| { + requested_frame < last.0 || // seek forward for big jumps. this threshold is arbitrary but should be derived from i-frames in future - frame_number - f > FRAME_CACHE_SIZE as u32 + requested_frame - last.0 > FRAME_CACHE_SIZE as u32 }) .unwrap_or(true) { + dbg!((requested_frame, last_decoded_frame)); let timestamp_us = - ((frame_number as f32 / frame_rate.numerator() as f32) + ((requested_frame as f32 / frame_rate.numerator() as f32) * 1_000_000.0) as i64; let position = timestamp_us.rescale((1, 1_000_000), rescale::TIME_BASE); - println!("seeking to {position} for frame {frame_number}"); + println!("seeking to {position} for frame {requested_frame}"); decoder.flush(); input.seek(position, ..position).unwrap(); cache.clear(); last_decoded_frame = None; + last_sent_frame = None; packets = input.packets(); } - last_active_frame = Some(frame_number); + // handle when requested_frame == last_decoded_frame or last_decoded_frame > requested_frame. + // the latter can occur when there are skips in frame numbers. + // in future we should alleviate this by using time + pts values instead of frame numbers. + if let Some((_, last_sent_frame)) = last_decoded_frame + .zip(last_sent_frame.as_ref()) + .filter(|(last_decoded_frame, last_sent_frame)| { + last_sent_frame.0 < requested_frame + && requested_frame < *last_decoded_frame + }) + { + if let Some(sender) = sender.take() { + sender.send(Some(last_sent_frame.1.clone())).ok(); + continue; + } + } + + last_active_frame = Some(requested_frame); loop { if peekable_requests.peek().is_some() { break; } let Some((stream, packet)) = packets.next() else { + sender.take().map(|s| s.send(None)); break; }; if stream.index() == input_stream_index { let start_offset = stream.start_time(); - let packet_frame = - ts_to_frame(packet.pts().unwrap(), time_base, frame_rate); - // println!("sending frame {packet_frame} packet"); decoder.send_packet(&packet).ok(); // decode failures are ok, we just fail to return a frame let mut exit = false; while decoder.receive_frame(&mut temp_frame).is_ok() { - let current_frame = ts_to_frame( + let current_frame = pts_to_frame( temp_frame.pts().unwrap() - start_offset, time_base, - frame_rate, ); - // println!("processing frame {current_frame}"); + last_decoded_frame = Some(current_frame); + // we repeat the similar section as above to do the check per-frame instead of just per-request + if let Some((_, last_sent_frame)) = last_decoded_frame + .zip(last_sent_frame.as_ref()) + .filter(|(last_decoded_frame, last_sent_frame)| { + last_sent_frame.0 <= requested_frame + && requested_frame < *last_decoded_frame + }) + { + if let Some(sender) = sender.take() { + sender.send(Some(last_sent_frame.1.clone())).ok(); + } + } + let exceeds_cache_bounds = current_frame > cache_max; let too_small_for_cache_bounds = current_frame < cache_min; @@ -223,18 +243,22 @@ impl AsyncVideoDecoder { let frame = Arc::new(frame_buffer); - if current_frame == frame_number { + if current_frame == requested_frame { if let Some(sender) = sender.take() { + last_sent_frame = Some((current_frame, frame.clone())); sender.send(Some(frame.clone())).ok(); + + break; } } if !too_small_for_cache_bounds { if cache.len() >= FRAME_CACHE_SIZE { if let Some(last_active_frame) = &last_active_frame { - let frame = if frame_number > *last_active_frame { + let frame = if requested_frame > *last_active_frame + { *cache.keys().next().unwrap() - } else if frame_number < *last_active_frame { + } else if requested_frame < *last_active_frame { *cache.keys().next_back().unwrap() } else { let min = *cache.keys().min().unwrap(); @@ -265,8 +289,13 @@ impl AsyncVideoDecoder { } } - if sender.is_some() { - println!("failed to send frame {frame_number}"); + if let Some(s) = sender.take() { + println!("sending None for {requested_frame}"); + dbg!(last_sent_frame.as_ref().map(|f| f.0)); + dbg!(&last_decoded_frame); + s.send(None); + } else { + println!("sent frame {requested_frame}"); } } } @@ -283,10 +312,10 @@ pub struct AsyncVideoDecoderHandle { } impl AsyncVideoDecoderHandle { - pub async fn get_frame(&self, frame_number: u32) -> Option>> { + pub async fn get_frame(&self, time: u32) -> Option>> { let (tx, rx) = tokio::sync::oneshot::channel(); self.sender - .send(VideoDecoderMessage::GetFrame(frame_number, tx)) + .send(VideoDecoderMessage::GetFrame(time, tx)) .unwrap(); rx.await.ok().flatten() } diff --git a/crates/rendering/src/lib.rs b/crates/rendering/src/lib.rs index 4b024b0d7..8b8561b0d 100644 --- a/crates/rendering/src/lib.rs +++ b/crates/rendering/src/lib.rs @@ -12,8 +12,8 @@ use wgpu::util::DeviceExt; use wgpu::{CommandEncoder, COPY_BYTES_PER_ROW_ALIGNMENT}; use cap_project::{ - AspectRatio, BackgroundSource, CameraXPosition, CameraYPosition, Crop, - CursorData, CursorMoveEvent, ProjectConfiguration, XY, + AspectRatio, BackgroundSource, CameraXPosition, CameraYPosition, Crop, CursorData, + CursorMoveEvent, ProjectConfiguration, XY, }; use image::GenericImageView; From c8210e6f33a23b809d19249c9f2f0e6cab7f99a1 Mon Sep 17 00:00:00 2001 From: Brendan Allan Date: Sun, 10 Nov 2024 19:14:41 +0800 Subject: [PATCH 3/6] remove logs --- crates/rendering/src/decoder.rs | 6 ------ 1 file changed, 6 deletions(-) diff --git a/crates/rendering/src/decoder.rs b/crates/rendering/src/decoder.rs index 13bae9e23..0c5ad774e 100644 --- a/crates/rendering/src/decoder.rs +++ b/crates/rendering/src/decoder.rs @@ -129,7 +129,6 @@ impl AsyncVideoDecoder { }) .unwrap_or(true) { - dbg!((requested_frame, last_decoded_frame)); let timestamp_us = ((requested_frame as f32 / frame_rate.numerator() as f32) * 1_000_000.0) as i64; @@ -290,12 +289,7 @@ impl AsyncVideoDecoder { } if let Some(s) = sender.take() { - println!("sending None for {requested_frame}"); - dbg!(last_sent_frame.as_ref().map(|f| f.0)); - dbg!(&last_decoded_frame); s.send(None); - } else { - println!("sent frame {requested_frame}"); } } } From 29ff6e71a26c8ca99550255f056f08139781c521 Mon Sep 17 00:00:00 2001 From: PJ Date: Sun, 10 Nov 2024 16:04:39 +0100 Subject: [PATCH 4/6] implement recording settings UI --- apps/desktop/src-tauri/src/lib.rs | 3 + apps/desktop/src-tauri/src/recording.rs | 58 +++++- apps/desktop/src/components/SwitchButton.tsx | 34 ++++ .../src/routes/(window-chrome)/settings.tsx | 1 + .../(window-chrome)/settings/general.tsx | 60 ++----- .../(window-chrome)/settings/recording.tsx | 166 ++++++++++++++++++ apps/desktop/src/store.ts | 10 ++ apps/desktop/src/utils/tauri.ts | 11 ++ crates/project/src/configuration.rs | 16 ++ 9 files changed, 309 insertions(+), 50 deletions(-) create mode 100644 apps/desktop/src/components/SwitchButton.tsx create mode 100644 apps/desktop/src/routes/(window-chrome)/settings/recording.tsx diff --git a/apps/desktop/src-tauri/src/lib.rs b/apps/desktop/src-tauri/src/lib.rs index 24c81747d..4fac021cd 100644 --- a/apps/desktop/src-tauri/src/lib.rs +++ b/apps/desktop/src-tauri/src/lib.rs @@ -2459,6 +2459,7 @@ pub async fn run() { check_upgraded_and_update, open_external_link, hotkeys::set_hotkey, + recording::set_recording_settings, set_general_settings, delete_auth_open_signin, reset_camera_permissions, @@ -2489,6 +2490,7 @@ pub async fn run() { .typ::() .typ::() .typ::() + .typ::() .typ::(); #[cfg(debug_assertions)] @@ -2528,6 +2530,7 @@ pub async fn run() { specta_builder.mount_events(app); hotkeys::init(app.handle()); general_settings::init(app.handle()); + recording::init_settings(app.handle()); let app_handle = app.handle().clone(); diff --git a/apps/desktop/src-tauri/src/recording.rs b/apps/desktop/src-tauri/src/recording.rs index b613e1f21..158a65449 100644 --- a/apps/desktop/src-tauri/src/recording.rs +++ b/apps/desktop/src-tauri/src/recording.rs @@ -1,14 +1,16 @@ use cap_flags::FLAGS; use cap_media::{encoders::*, feeds::*, filters::*, pipeline::*, sources::*, MediaError}; -use cap_project::{CursorClickEvent, CursorMoveEvent, RecordingMeta}; -use serde::Serialize; +use cap_project::{CursorClickEvent, CursorMoveEvent, RecordingMeta, TargetFPS, TargetResolution}; +use serde::{Deserialize, Serialize}; use specta::Type; use std::collections::HashMap; use std::fs::File; use std::path::PathBuf; use std::sync::atomic::AtomicBool; -use std::sync::Arc; +use std::sync::{Arc, Mutex}; use std::time::{SystemTime, UNIX_EPOCH}; +use tauri::{AppHandle, Manager, Wry}; +use tauri_plugin_store::StoreExt; use tokio::sync::oneshot; use crate::cursor::spawn_cursor_recorder; @@ -17,6 +19,52 @@ use crate::RecordingOptions; // TODO: Hacky, please fix pub const FPS: u32 = 30; +#[derive(Serialize, Deserialize, Type, Default, Debug)] +pub struct RecordingSettingsStore { + pub use_hardware_acceleration: bool, + pub recording_resolution: TargetResolution, + pub recording_fps: TargetFPS, +} + +pub type RecordingSettingsState = Mutex; + +pub fn init_settings(app: &AppHandle) { + println!("Initializing RecordingSettingsStore"); + let store = RecordingSettingsStore::get(app) + .unwrap() + .unwrap_or_default(); + app.manage(RecordingSettingsState::new(store)); + println!("RecordingSettingsState managed"); +} + +impl RecordingSettingsStore { + pub fn get(app: &AppHandle) -> Result, String> { + let Some(Some(store)) = app.get_store("store").map(|s| s.get("recording_settings")) else { + return Ok(None); + }; + + serde_json::from_value(store).map_err(|e| e.to_string()) + } + + pub fn set(app: &AppHandle, settings: Self) -> Result<(), String> { + let Some(store) = app.get_store("store") else { + return Err("Store not found".to_string()); + }; + + store.set("recording_settings", serde_json::json!(settings)); + store.save().map_err(|e| e.to_string()) + } +} + +#[tauri::command(async)] +#[specta::specta] +pub fn set_recording_settings( + app: AppHandle, + settings: RecordingSettingsStore, +) -> Result<(), String> { + RecordingSettingsStore::set(&app, settings) +} + #[tauri::command(async)] #[specta::specta] pub fn list_capture_screens() -> Vec { @@ -239,7 +287,7 @@ pub async fn start( )?; pipeline_builder = pipeline_builder .source("screen_capture", screen_source) - // .pipe("screen_capture_filter", screen_filter) + .pipe("screen_capture_filter", screen_filter) .sink("screen_capture_encoder", screen_encoder); } @@ -251,7 +299,6 @@ pub async fn start( let mic_config = mic_source.info(); audio_output_path = Some(content_dir.join("audio-input.mp3")); - // let mic_filter = AudioFilter::init("microphone", mic_config, "aresample=async=1:min_hard_comp=0.100000:first_pts=0")?; let mic_encoder = MP3Encoder::init( "microphone", mic_config, @@ -260,7 +307,6 @@ pub async fn start( pipeline_builder = pipeline_builder .source("microphone_capture", mic_source) - // .pipe("microphone_filter", mic_filter) .sink("microphone_encoder", mic_encoder); } diff --git a/apps/desktop/src/components/SwitchButton.tsx b/apps/desktop/src/components/SwitchButton.tsx new file mode 100644 index 000000000..39898f684 --- /dev/null +++ b/apps/desktop/src/components/SwitchButton.tsx @@ -0,0 +1,34 @@ +export type SwitchButtonProps = { + name: T, + value: boolean, + disabled?: boolean, + onChange: (name: T, value: boolean) => void, +}; + +export function SwitchButton(props: SwitchButtonProps) { + return ( + + ); +} diff --git a/apps/desktop/src/routes/(window-chrome)/settings.tsx b/apps/desktop/src/routes/(window-chrome)/settings.tsx index 410070708..374a79076 100644 --- a/apps/desktop/src/routes/(window-chrome)/settings.tsx +++ b/apps/desktop/src/routes/(window-chrome)/settings.tsx @@ -24,6 +24,7 @@ export default function (props: RouteSectionProps) { each={[ { href: "general", name: "General", icon: IconCapSettings }, { href: "hotkeys", name: "Shortcuts", icon: IconCapHotkeys }, + { href: "recording", name: "Recording", icon: IconLucideVideo }, { href: "recordings", name: "Previous Recordings", diff --git a/apps/desktop/src/routes/(window-chrome)/settings/general.tsx b/apps/desktop/src/routes/(window-chrome)/settings/general.tsx index 2b5767a83..8e60ca265 100644 --- a/apps/desktop/src/routes/(window-chrome)/settings/general.tsx +++ b/apps/desktop/src/routes/(window-chrome)/settings/general.tsx @@ -6,8 +6,16 @@ import { isPermissionGranted, requestPermission, } from "@tauri-apps/plugin-notification"; +import { SwitchButton } from "~/components/SwitchButton"; -const settingsList = [ +type Setting = { + key: keyof GeneralSettingsStore, + label: string, + description: string, + requiresPermission?: boolean, +}; + +const settingsList: Setting[] = [ { key: "upload_individual_files", label: "Upload individual recording files when creating shareable link", @@ -62,7 +70,7 @@ function Inner(props: { initialStore: GeneralSettingsStore | null }) { } ); - const handleChange = async (key: string, value: boolean) => { + const handleChange = async (key: keyof GeneralSettingsStore, value: boolean) => { console.log(`Handling settings change for ${key}: ${value}`); // Special handling for notifications permission if (key === "enable_notifications") { @@ -86,7 +94,7 @@ function Inner(props: { initialStore: GeneralSettingsStore | null }) { } } - setSettings(key as keyof GeneralSettingsStore, value); + setSettings(key, value); await commands.setGeneralSettings({ ...settings, [key]: value, @@ -102,47 +110,11 @@ function Inner(props: { initialStore: GeneralSettingsStore | null }) {

{setting.label}

- +
{setting.description && (

{setting.description}

diff --git a/apps/desktop/src/routes/(window-chrome)/settings/recording.tsx b/apps/desktop/src/routes/(window-chrome)/settings/recording.tsx new file mode 100644 index 000000000..43b900750 --- /dev/null +++ b/apps/desktop/src/routes/(window-chrome)/settings/recording.tsx @@ -0,0 +1,166 @@ +import { createResource, Show, For } from "solid-js"; +import { createStore } from "solid-js/store"; +import { Select as KSelect } from "@kobalte/core/select"; +import { recordingSettingsStore } from "~/store"; +import { createCurrentRecordingQuery } from "~/utils/queries"; +import { commands, type RecordingSettingsStore, TargetResolution, TargetFPS } from "~/utils/tauri"; +import { + MenuItem, + MenuItemList, + PopperContent, + topLeftAnimateClasses, +} from "../../editor/ui"; +import { SwitchButton } from "~/components/SwitchButton"; + +export default function RecordingSettings() { + const [store] = createResource(() => recordingSettingsStore.get()); + + return ( + + {(store) => } + + ); +} + +type SettingOption = { + label: string, + value: T, +} + +type RecordingSetting = { + label: string +} & ({ + key: "recording_resolution", + options: SettingOption[], +} | { + key: "recording_fps", + options: SettingOption[], +}); + +const recordingSettings: RecordingSetting[] = [ + { + key: "recording_resolution", + label: "Target resolution for screen capturing", + options: [ + { + label: "720p", + value: "_720p", + }, + { + label: "1080p", + value: "_1080p", + }, + { + label: "Native resolution", + value: "Native", + }, + ], + }, + { + key: "recording_fps", + label: "Target FPS for screen capturing", + options: [ + { + label: "30 fps", + value: "_30", + }, + { + label: "60 fps", + value: "_60", + }, + { + label: "Native refresh rate", + value: "Native", + }, + ], + }, +]; + +function Inner(props: { initialStore: RecordingSettingsStore | null }) { + const currentRecording = createCurrentRecordingQuery(); + const [settings, setSettings] = createStore( + props.initialStore ?? { + use_hardware_acceleration: false, + recording_resolution: "_1080p", + recording_fps: "_30", + } + ); + + const handleChange = async (key: K, value: RecordingSettingsStore[K]) => { + console.log(`Handling settings change for ${key}: ${value}`); + + setSettings(key, value); + const result = await commands.setRecordingSettings({ + ...settings, + [key]: value, + }); + if (result.status === "error") console.error(result.error); + } + + const settingOption = (setting: RecordingSetting): typeof setting.options[0] => { + return setting.options.find((option) => option.value === settings[setting.key])!; + } + + const recordingInProgress = !!currentRecording.data; + + return ( +
+
+
+
+
+

Use hardware acceleration

+ +
+ + {(setting) => ( +
+

{setting.label}

+ + options={setting.options} + optionValue="value" + optionTextValue="value" + value={settingOption(setting)} + disabled={recordingInProgress} + onChange={(option) => { + handleChange(setting.key, option!.value) + }} + itemComponent={(props) => ( + as={KSelect.Item} item={props.item}> + + {props.item.rawValue.label} + + + )} + > + + class=""> + {(state) => {state.selectedOption().label}} + + + + + as={KSelect.Content} + class={topLeftAnimateClasses} + > + + class="max-h-36 overflow-y-auto" + as={KSelect.Listbox} + /> + + + +
+ )} + +
+
+
+
+ ); +} diff --git a/apps/desktop/src/store.ts b/apps/desktop/src/store.ts index ce3c2d40d..ddc9776fe 100644 --- a/apps/desktop/src/store.ts +++ b/apps/desktop/src/store.ts @@ -5,6 +5,7 @@ import type { ProjectConfiguration, HotkeysStore, GeneralSettingsStore, + RecordingSettingsStore, } from "~/utils/tauri"; let _store: Promise | undefined; @@ -64,3 +65,12 @@ export const generalSettingsStore = { await s.save(); }, }; + +export const recordingSettingsStore = { + get: () => store().then((s) => s.get("recording_settings")), + set: async (value: RecordingSettingsStore) => { + const s = await store(); + await s.set("recording_settings", value); + await s.save(); + } +}; diff --git a/apps/desktop/src/utils/tauri.ts b/apps/desktop/src/utils/tauri.ts index a16f2614f..18ababaff 100644 --- a/apps/desktop/src/utils/tauri.ts +++ b/apps/desktop/src/utils/tauri.ts @@ -279,6 +279,14 @@ async setHotkey(action: HotkeyAction, hotkey: Hotkey | null) : Promise> { + try { + return { status: "ok", data: await TAURI_INVOKE("set_recording_settings", { settings }) }; +} catch (e) { + if(e instanceof Error) throw e; + else return { status: "error", error: e as any }; +} +}, async setGeneralSettings(settings: GeneralSettingsStore) : Promise> { try { return { status: "ok", data: await TAURI_INVOKE("set_general_settings", { settings }) }; @@ -411,6 +419,7 @@ export type RecordingMetaChanged = { id: string } export type RecordingOptions = { captureTarget: ScreenCaptureTarget; cameraLabel: string | null; audioInputName: string | null } export type RecordingOptionsChanged = null export type RecordingSegment = { start: number; end: number } +export type RecordingSettingsStore = { use_hardware_acceleration: boolean; recording_resolution: TargetResolution; recording_fps: TargetFPS } export type RecordingStarted = null export type RecordingStopped = { path: string } export type RenderFrameEvent = { frame_number: number } @@ -425,6 +434,8 @@ export type ScreenCaptureTarget = ({ variant: "window" } & CaptureWindow) | ({ v export type SerializedEditorInstance = { framesSocketUrl: string; recordingDuration: number; savedProjectConfig: ProjectConfiguration; recordings: ProjectRecordings; path: string; prettyName: string } export type SharingMeta = { id: string; link: string } export type ShowCapturesPanel = null +export type TargetFPS = "_30" | "_60" | "Native" +export type TargetResolution = "_720p" | "_1080p" | "Native" export type TimelineConfiguration = { segments: TimelineSegment[]; zoomSegments?: ZoomSegment[] } export type TimelineSegment = { timescale: number; start: number; end: number } export type UploadResult = { Success: string } | "NotAuthenticated" | "PlanCheckFailed" | "UpgradeRequired" diff --git a/crates/project/src/configuration.rs b/crates/project/src/configuration.rs index c3df56a83..6019d4498 100644 --- a/crates/project/src/configuration.rs +++ b/crates/project/src/configuration.rs @@ -51,6 +51,22 @@ impl Default for BackgroundSource { } } +#[derive(Type, Serialize, Deserialize, Clone, Debug, Default)] +pub enum TargetResolution { + _720p, + #[default] + _1080p, + Native, +} + +#[derive(Type, Serialize, Deserialize, Clone, Debug, Default)] +pub enum TargetFPS { + #[default] + _30, + _60, + Native, +} + #[derive(Type, Serialize, Deserialize, Clone, Copy, Debug, Default)] #[serde(rename_all = "camelCase")] pub struct XY { From f2383f172537d117075ea7d21e4bc82b314037b1 Mon Sep 17 00:00:00 2001 From: PJ Date: Tue, 12 Nov 2024 16:38:22 +0100 Subject: [PATCH 5/6] remove all hardcoded FPS references --- apps/desktop/src-tauri/src/lib.rs | 20 +++--- apps/desktop/src-tauri/src/recording.rs | 64 +++++++++++++------ .../(window-chrome)/settings/recording.tsx | 53 ++++++++++----- apps/desktop/src/utils/tauri.ts | 8 +-- crates/editor/src/editor.rs | 4 ++ crates/editor/src/editor_instance.rs | 18 ++++-- crates/editor/src/playback.rs | 27 ++++---- crates/editor/src/project_recordings.rs | 4 +- crates/media/src/data.rs | 36 ++++++++--- .../media/src/encoders/h264_avassetwriter.rs | 28 ++++---- crates/media/src/feeds/camera.rs | 2 +- crates/media/src/sources/screen_capture.rs | 17 +++-- crates/project/src/configuration.rs | 42 ++++++++++-- crates/rendering/src/decoder.rs | 13 ++-- crates/rendering/src/lib.rs | 22 ++++--- 15 files changed, 245 insertions(+), 113 deletions(-) diff --git a/apps/desktop/src-tauri/src/lib.rs b/apps/desktop/src-tauri/src/lib.rs index 4fac021cd..9076fa790 100644 --- a/apps/desktop/src-tauri/src/lib.rs +++ b/apps/desktop/src-tauri/src/lib.rs @@ -21,7 +21,7 @@ use cap_editor::{EditorInstance, FRAMES_WS_PATH}; use cap_editor::{EditorState, ProjectRecordings}; use cap_media::sources::CaptureScreen; use cap_media::{ - feeds::{AudioData, AudioFrameBuffer, CameraFeed, CameraFrameSender}, + feeds::{AudioFrameBuffer, CameraFeed, CameraFrameSender}, platform::Bounds, sources::{AudioInputSource, ScreenCaptureTarget}, }; @@ -34,10 +34,10 @@ use cap_rendering::{ProjectUniforms, ZOOM_DURATION}; use general_settings::GeneralSettingsStore; use image::{ImageBuffer, Rgba}; use mp4::Mp4Reader; -use num_traits::ToBytes; use png::{ColorType, Encoder}; use recording::{ - list_cameras, list_capture_screens, list_capture_windows, InProgressRecording, FPS, + list_cameras, list_capture_screens, list_capture_windows, InProgressRecording, + RecordingSettingsStore, }; use scap::capturer::Capturer; use scap::frame::Frame; @@ -338,10 +338,13 @@ async fn start_recording(app: AppHandle, state: MutableState<'_, App>) -> Result } } + let recording_settings = RecordingSettingsStore::get(&app)?; + match recording::start( id, recording_dir, &state.start_recording_options, + recording_settings, state.camera_feed.as_ref(), ) .await @@ -969,6 +972,7 @@ async fn render_to_file_impl( let audio = editor_instance.audio.clone(); let decoders = editor_instance.decoders.clone(); let options = editor_instance.render_constants.options.clone(); + let screen_fps = editor_instance.recordings.display.fps; let (tx_image_data, mut rx_image_data) = tokio::sync::mpsc::unbounded_channel::>(); @@ -997,7 +1001,7 @@ async fn render_to_file_impl( ffmpeg.add_input(cap_ffmpeg_cli::FFmpegRawVideoInput { width: output_size.0, height: output_size.1, - fps: 30, + fps: screen_fps, pix_fmt: "rgba", input: pipe_path.clone().into_os_string(), }); @@ -1082,7 +1086,7 @@ async fn render_to_file_impl( let audio_info = audio.buffer.info(); let estimated_samples_per_frame = - f64::from(audio_info.sample_rate) / f64::from(FPS); + f64::from(audio_info.sample_rate) / f64::from(screen_fps); let samples = estimated_samples_per_frame.ceil() as usize; if let Some((_, frame_data)) = @@ -1158,6 +1162,7 @@ async fn render_to_file_impl( decoders, editor_instance.cursor.clone(), editor_instance.project_path.clone(), + screen_fps, ) .await?; @@ -1629,10 +1634,9 @@ async fn render_to_file( .await .unwrap(); - // 30 FPS (calculated for output video) - let total_frames = (duration * 30.0).round() as u32; - let editor_instance = upsert_editor_instance(&app, video_id.clone()).await; + let screen_fps = editor_instance.recordings.display.fps as f64; + let total_frames = (duration * screen_fps).round() as u32; render_to_file_impl( &editor_instance, diff --git a/apps/desktop/src-tauri/src/recording.rs b/apps/desktop/src-tauri/src/recording.rs index 158a65449..ade2e1727 100644 --- a/apps/desktop/src-tauri/src/recording.rs +++ b/apps/desktop/src-tauri/src/recording.rs @@ -16,16 +16,31 @@ use tokio::sync::oneshot; use crate::cursor::spawn_cursor_recorder; use crate::RecordingOptions; -// TODO: Hacky, please fix -pub const FPS: u32 = 30; - -#[derive(Serialize, Deserialize, Type, Default, Debug)] +#[derive(Serialize, Deserialize, Type, Debug)] pub struct RecordingSettingsStore { pub use_hardware_acceleration: bool, - pub recording_resolution: TargetResolution, + #[serde(default = "default_recording_resolution")] + pub capture_resolution: Option, + #[serde(default = "default_recording_resolution")] + pub output_resolution: Option, pub recording_fps: TargetFPS, } +fn default_recording_resolution() -> Option { + Some(TargetResolution::_1080p) +} + +impl Default for RecordingSettingsStore { + fn default() -> Self { + Self { + use_hardware_acceleration: false, + capture_resolution: Some(TargetResolution::_1080p), + output_resolution: None, + recording_fps: TargetFPS::_30, + } + } +} + pub type RecordingSettingsState = Mutex; pub fn init_settings(app: &AppHandle) { @@ -234,6 +249,7 @@ pub async fn start( id: String, recording_dir: PathBuf, recording_options: &RecordingOptions, + recording_settings: Option, camera_feed: Option<&CameraFeed>, ) -> Result { let content_dir = recording_dir.join("content"); @@ -249,16 +265,21 @@ pub async fn start( let mut audio_output_path = None; let mut camera_output_path = None; - #[cfg(target_os = "macos")] - { - let screen_source = ScreenCaptureSource::::init( + let settings = recording_settings.unwrap_or_default(); + + if settings.use_hardware_acceleration && cfg!(target_os = "macos") { + let screen_source = ScreenCaptureSource::::init( dbg!(&recording_options.capture_target), - None, - None, + settings.recording_fps, + settings.capture_resolution, ); let screen_config = screen_source.info(); + let output_config = settings.output_resolution.map(|output| { + screen_config + .with_resolution(output.to_width()) + .with_hardware_format() + }); - let output_config = screen_config.scaled(1920, 30); let screen_encoder = cap_media::encoders::H264AVAssetWriterEncoder::init( "screen", output_config, @@ -267,19 +288,20 @@ pub async fn start( pipeline_builder = pipeline_builder .source("screen_capture", screen_source) .sink("screen_capture_encoder", screen_encoder); - } - #[cfg(not(target_os = "macos"))] - { + } else { let screen_source = ScreenCaptureSource::::init( dbg!(&recording_options.capture_target), - None, + settings.recording_fps, None, ); let screen_config = screen_source.info(); - let screen_bounds = screen_source.bounds; - let output_config = screen_config.scaled(1920, 30); + let mut output_config = screen_config.with_software_format(); + if let Some(output_resolution) = settings.output_resolution { + output_config = output_config.with_resolution(output_resolution.to_width()); + } let screen_filter = VideoFilter::init("screen", screen_config, output_config)?; + let screen_encoder = H264Encoder::init( "screen", output_config, @@ -312,7 +334,13 @@ pub async fn start( if let Some(camera_source) = CameraSource::init(camera_feed) { let camera_config = camera_source.info(); - let output_config = camera_config.scaled(1920, 30); + // TODO: I'm not sure if there's a point to scaling the camera capture to the same resolution + // as the display capture (since it will be scaled down anyway, but matching at least the frame + // rate here is easier than trying to sync different frame rates while editing/rendering). + // Also, use hardware filters maybe? + let output_config = camera_config + .with_software_format() + .with_fps(settings.recording_fps.to_raw()); camera_output_path = Some(content_dir.join("camera.mp4")); let camera_filter = VideoFilter::init("camera", camera_config, output_config)?; diff --git a/apps/desktop/src/routes/(window-chrome)/settings/recording.tsx b/apps/desktop/src/routes/(window-chrome)/settings/recording.tsx index 43b900750..afe6d9ab9 100644 --- a/apps/desktop/src/routes/(window-chrome)/settings/recording.tsx +++ b/apps/desktop/src/routes/(window-chrome)/settings/recording.tsx @@ -30,30 +30,52 @@ type SettingOption = { type RecordingSetting = { label: string } & ({ - key: "recording_resolution", - options: SettingOption[], + key: "capture_resolution", + options: SettingOption[], +} | { + key: "output_resolution", + options: SettingOption[], } | { key: "recording_fps", options: SettingOption[], }); +const resolutionOptions: SettingOption[] = [ + { + label: "720p (1280x720)", + value: "_720p", + }, + { + label: "1080p (1920x1080)", + value: "_1080p", + }, + { + label: "4K (3840x2160)", + value: "_4K", + } +] + const recordingSettings: RecordingSetting[] = [ { - key: "recording_resolution", - label: "Target resolution for screen capturing", + key: "capture_resolution", + label: "Screen capture resolution", options: [ { - label: "720p", - value: "_720p", - }, - { - label: "1080p", - value: "_1080p", + label: "Same as display", + value: null, }, + ...resolutionOptions, + ], + }, + { + key: "output_resolution", + label: "Output (scaled) resolution", + options: [ { - label: "Native resolution", - value: "Native", + label: "Same as captured", + value: null, }, + ...resolutionOptions, ], }, { @@ -68,10 +90,6 @@ const recordingSettings: RecordingSetting[] = [ label: "60 fps", value: "_60", }, - { - label: "Native refresh rate", - value: "Native", - }, ], }, ]; @@ -81,7 +99,8 @@ function Inner(props: { initialStore: RecordingSettingsStore | null }) { const [settings, setSettings] = createStore( props.initialStore ?? { use_hardware_acceleration: false, - recording_resolution: "_1080p", + capture_resolution: "_1080p", + output_resolution: null, recording_fps: "_30", } ); diff --git a/apps/desktop/src/utils/tauri.ts b/apps/desktop/src/utils/tauri.ts index 18ababaff..d63ec5cd1 100644 --- a/apps/desktop/src/utils/tauri.ts +++ b/apps/desktop/src/utils/tauri.ts @@ -419,7 +419,7 @@ export type RecordingMetaChanged = { id: string } export type RecordingOptions = { captureTarget: ScreenCaptureTarget; cameraLabel: string | null; audioInputName: string | null } export type RecordingOptionsChanged = null export type RecordingSegment = { start: number; end: number } -export type RecordingSettingsStore = { use_hardware_acceleration: boolean; recording_resolution: TargetResolution; recording_fps: TargetFPS } +export type RecordingSettingsStore = { use_hardware_acceleration: boolean; capture_resolution?: TargetResolution | null; output_resolution?: TargetResolution | null; recording_fps: TargetFPS } export type RecordingStarted = null export type RecordingStopped = { path: string } export type RenderFrameEvent = { frame_number: number } @@ -434,12 +434,12 @@ export type ScreenCaptureTarget = ({ variant: "window" } & CaptureWindow) | ({ v export type SerializedEditorInstance = { framesSocketUrl: string; recordingDuration: number; savedProjectConfig: ProjectConfiguration; recordings: ProjectRecordings; path: string; prettyName: string } export type SharingMeta = { id: string; link: string } export type ShowCapturesPanel = null -export type TargetFPS = "_30" | "_60" | "Native" -export type TargetResolution = "_720p" | "_1080p" | "Native" +export type TargetFPS = "_30" | "_60" +export type TargetResolution = "_720p" | "_1080p" | "_4K" export type TimelineConfiguration = { segments: TimelineSegment[]; zoomSegments?: ZoomSegment[] } export type TimelineSegment = { timescale: number; start: number; end: number } export type UploadResult = { Success: string } | "NotAuthenticated" | "PlanCheckFailed" | "UpgradeRequired" -export type Video = { duration: number; width: number; height: number } +export type Video = { duration: number; width: number; height: number; fps: number } export type VideoType = "screen" | "output" export type XY = { x: T; y: T } export type ZoomSegment = { start: number; end: number; amount: number } diff --git a/crates/editor/src/editor.rs b/crates/editor/src/editor.rs index f0b8a414c..d669f252f 100644 --- a/crates/editor/src/editor.rs +++ b/crates/editor/src/editor.rs @@ -115,6 +115,7 @@ pub struct Renderer { rx: mpsc::Receiver, frame_tx: mpsc::UnboundedSender, render_constants: Arc, + fps: u32, } pub struct RendererHandle { @@ -125,6 +126,7 @@ impl Renderer { pub fn spawn( render_constants: Arc, frame_tx: mpsc::UnboundedSender, + fps: u32, ) -> RendererHandle { let (tx, rx) = mpsc::channel(4); @@ -132,6 +134,7 @@ impl Renderer { rx, frame_tx, render_constants, + fps, }; tokio::spawn(this.run()); @@ -173,6 +176,7 @@ impl Renderer { cap_rendering::Background::from(background), &uniforms, time, // Pass the actual time value + self.fps, ) .await .unwrap(); diff --git a/crates/editor/src/editor_instance.rs b/crates/editor/src/editor_instance.rs index 16bb6987b..b16fe31ea 100644 --- a/crates/editor/src/editor_instance.rs +++ b/crates/editor/src/editor_instance.rs @@ -10,8 +10,6 @@ use std::sync::Mutex as StdMutex; use std::{path::PathBuf, sync::Arc}; use tokio::sync::{mpsc, watch, Mutex}; -const FPS: u32 = 30; - pub struct EditorInstance { pub project_path: PathBuf, pub id: String, @@ -95,7 +93,11 @@ impl EditorInstance { .unwrap(), ); - let renderer = Arc::new(editor::Renderer::spawn(render_constants.clone(), frame_tx)); + let renderer = Arc::new(editor::Renderer::spawn( + render_constants.clone(), + frame_tx, + recordings.display.fps, + )); let (preview_tx, preview_rx) = watch::channel(None); @@ -236,6 +238,8 @@ impl EditorInstance { self: Arc, mut preview_rx: watch::Receiver>, ) -> tokio::task::JoinHandle<()> { + let fps = self.recordings.display.fps; + tokio::spawn(async move { loop { preview_rx.changed().await.unwrap(); @@ -248,14 +252,14 @@ impl EditorInstance { let Some(time) = project .timeline .as_ref() - .map(|timeline| timeline.get_recording_time(frame_number as f64 / FPS as f64)) - .unwrap_or(Some(frame_number as f64 / FPS as f64)) + .map(|timeline| timeline.get_recording_time(frame_number as f64 / fps as f64)) + .unwrap_or(Some(frame_number as f64 / fps as f64)) else { continue; }; let Some((screen_frame, camera_frame)) = - self.decoders.get_frames((time * FPS as f64) as u32).await + self.decoders.get_frames((time * fps as f64) as u32).await else { continue; }; @@ -265,7 +269,7 @@ impl EditorInstance { screen_frame, camera_frame, project.background.source.clone(), - ProjectUniforms::new(&self.render_constants, &project, time as f32), + ProjectUniforms::new(&self.render_constants, &project, time as f32, fps), time as f32, // Add the time parameter ) .await; diff --git a/crates/editor/src/playback.rs b/crates/editor/src/playback.rs index 0298acd26..f47cd12d4 100644 --- a/crates/editor/src/playback.rs +++ b/crates/editor/src/playback.rs @@ -22,8 +22,6 @@ pub struct Playback { pub recordings: ProjectRecordings, } -const FPS: u32 = 30; - #[derive(Clone, Copy)] pub enum PlaybackEvent { Start, @@ -53,6 +51,7 @@ impl Playback { tokio::spawn(async move { let start = Instant::now(); + let fps = self.recordings.display.fps; let mut frame_number = self.start_frame_number + 1; let duration = self @@ -68,35 +67,36 @@ impl Playback { audio: audio_data.clone(), stop_rx: stop_rx.clone(), start_frame_number: self.start_frame_number, - duration, + // duration, + fps, project: self.project.clone(), } .spawn(); }; loop { - if frame_number as f64 > FPS as f64 * duration { + if frame_number as f64 > fps as f64 * duration { break; }; let project = self.project.borrow().clone(); let time = if let Some(timeline) = project.timeline() { - match timeline.get_recording_time(frame_number as f64 / FPS as f64) { + match timeline.get_recording_time(frame_number as f64 / fps as f64) { Some(time) => time, None => break, } } else { - frame_number as f64 / FPS as f64 + frame_number as f64 / fps as f64 }; tokio::select! { _ = stop_rx.changed() => { break; }, - Some((screen_frame, camera_frame)) = self.decoders.get_frames((time * FPS as f64) as u32) => { + Some((screen_frame, camera_frame)) = self.decoders.get_frames((time * fps as f64) as u32) => { // println!("decoded frame in {:?}", debug.elapsed()); - let uniforms = ProjectUniforms::new(&self.render_constants, &project, time as f32); + let uniforms = ProjectUniforms::new(&self.render_constants, &project, time as f32, fps); self .renderer @@ -109,7 +109,7 @@ impl Playback { ) .await; - tokio::time::sleep_until(start + (frame_number - self.start_frame_number) * Duration::from_secs_f32(1.0 / FPS as f32)).await; + tokio::time::sleep_until(start + (frame_number - self.start_frame_number) * Duration::from_secs_f64(1.0 / fps as f64)).await; event_tx.send(PlaybackEvent::Frame(frame_number)).ok(); @@ -145,7 +145,8 @@ struct AudioPlayback { audio: AudioData, stop_rx: watch::Receiver, start_frame_number: u32, - duration: f64, + // duration: f64, + fps: u32, project: watch::Receiver, } @@ -194,17 +195,15 @@ impl AudioPlayback { stop_rx, start_frame_number, project, + fps, .. } = self; let mut output_info = AudioInfo::from_stream_config(&supported_config); output_info.sample_format = output_info.sample_format.packed(); - // TODO: Get fps and duration from video (once we start supporting other frame rates) - // Also, it's a bit weird that self.duration can ever be infinity to begin with, since - // pre-recorded videos are obviously a fixed size let mut audio_renderer = AudioPlaybackBuffer::new(audio, output_info); - let playhead = f64::from(start_frame_number) / f64::from(FPS); + let playhead = f64::from(start_frame_number) / fps as f64; audio_renderer.set_playhead(playhead, project.borrow().timeline()); // Prerender enough for smooth playback diff --git a/crates/editor/src/project_recordings.rs b/crates/editor/src/project_recordings.rs index 04906f6c9..2fcb26894 100644 --- a/crates/editor/src/project_recordings.rs +++ b/crates/editor/src/project_recordings.rs @@ -1,6 +1,6 @@ use std::path::PathBuf; -use cap_project::RecordingMeta; +use cap_project::{RecordingMeta, TargetFPS}; use serde::Serialize; use specta::Type; @@ -9,6 +9,7 @@ pub struct Video { pub duration: f64, pub width: u32, pub height: u32, + pub fps: u32, } impl Video { @@ -26,6 +27,7 @@ impl Video { width: video_decoder.width(), height: video_decoder.height(), duration: input.duration() as f64 / 1_000_000.0, + fps: TargetFPS::round(stream.rate().into()), } } } diff --git a/crates/media/src/data.rs b/crates/media/src/data.rs index 131ce1190..26d24c5c5 100644 --- a/crates/media/src/data.rs +++ b/crates/media/src/data.rs @@ -210,24 +210,42 @@ impl VideoInfo { } } - pub fn scaled(&self, width: u32, fps: u32) -> Self { - let (width, height) = match self.width <= width { - true => (self.width, self.height), + pub fn with_resolution(&self, width: u32) -> Self { + match self.width <= width { + true => self.clone(), false => { let new_width = width & !1; let new_height = (((new_width as f32) * (self.height as f32) / (self.width as f32)) .round() as u32) & !1; - (new_width, new_height) + + Self { + width: new_width, + height: new_height, + ..self.clone() + } } - }; + } + } + pub fn with_fps(&self, fps: u32) -> Self { Self { - pixel_format: Pixel::NV12, - width, - height, - time_base: self.time_base, frame_rate: FFRational(fps.try_into().unwrap(), 1), + ..self.clone() + } + } + + pub fn with_software_format(&self) -> Self { + Self { + pixel_format: Pixel::YUV420P, + ..self.clone() + } + } + + pub fn with_hardware_format(&self) -> Self { + Self { + pixel_format: Pixel::NV12, + ..self.clone() } } diff --git a/crates/media/src/encoders/h264_avassetwriter.rs b/crates/media/src/encoders/h264_avassetwriter.rs index c9f91b394..49143d351 100644 --- a/crates/media/src/encoders/h264_avassetwriter.rs +++ b/crates/media/src/encoders/h264_avassetwriter.rs @@ -7,7 +7,7 @@ use cidre::{objc::Obj, *}; pub struct H264AVAssetWriterEncoder { tag: &'static str, last_pts: Option, - config: VideoInfo, + config: Option, asset_writer: Retained, video_input: Retained, first_timestamp: Option, @@ -15,7 +15,11 @@ pub struct H264AVAssetWriterEncoder { } impl H264AVAssetWriterEncoder { - pub fn init(tag: &'static str, config: VideoInfo, output: Output) -> Result { + pub fn init( + tag: &'static str, + maybe_config: Option, + output: Output, + ) -> Result { let Output::File(destination) = output; let mut asset_writer = av::AssetWriter::with_url_and_file_type( @@ -32,15 +36,17 @@ impl H264AVAssetWriterEncoder { let mut output_settings = assistant.video_settings().unwrap().copy_mut(); - output_settings.insert( - av::video_settings_keys::width(), - ns::Number::with_u32(config.width).as_id_ref(), - ); + if let Some(config) = maybe_config.as_ref() { + output_settings.insert( + av::video_settings_keys::width(), + ns::Number::with_u32(config.width).as_id_ref(), + ); - output_settings.insert( - av::video_settings_keys::height(), - ns::Number::with_u32(config.height).as_id_ref(), - ); + output_settings.insert( + av::video_settings_keys::height(), + ns::Number::with_u32(config.height).as_id_ref(), + ); + } output_settings.insert( av::video_settings_keys::compression_props(), @@ -65,7 +71,7 @@ impl H264AVAssetWriterEncoder { Ok(Self { tag, last_pts: None, - config, + config: maybe_config, asset_writer, video_input, first_timestamp: None, diff --git a/crates/media/src/feeds/camera.rs b/crates/media/src/feeds/camera.rs index 10122efe6..757cde9f3 100644 --- a/crates/media/src/feeds/camera.rs +++ b/crates/media/src/feeds/camera.rs @@ -177,7 +177,7 @@ fn find_camera(selected_camera: &String) -> Result { fn create_camera(info: &CameraInfo) -> Result { dbg!(info); - // TODO: Make selected format more flexible + // TODO: Make selected format more flexible (also record at higher FPS maybe? Leaving it at 30 is fine for now) // let format = RequestedFormat::new::(RequestedFormatType::AbsoluteHighestResolution); let format = RequestedFormat::with_formats( RequestedFormatType::ClosestIgnoringFormat { diff --git a/crates/media/src/sources/screen_capture.rs b/crates/media/src/sources/screen_capture.rs index 03034291c..e15711494 100644 --- a/crates/media/src/sources/screen_capture.rs +++ b/crates/media/src/sources/screen_capture.rs @@ -1,4 +1,5 @@ use cap_flags::FLAGS; +use cap_project::{TargetFPS, TargetResolution}; use cidre::cm; use core_foundation::base::{kCFAllocatorDefault, CFAllocatorRef}; use flume::Sender; @@ -73,15 +74,19 @@ pub struct ScreenCaptureSource { } impl ScreenCaptureSource { - pub const DEFAULT_FPS: u32 = 30; - pub fn init( capture_target: &ScreenCaptureTarget, - fps: Option, - resolution: Option, + fps: TargetFPS, + resolution: Option, ) -> Self { - let fps = fps.unwrap_or(Self::DEFAULT_FPS); - let output_resolution = resolution.unwrap_or(Resolution::Captured); + let fps = fps.to_raw(); + let output_resolution = resolution + .map(|target_resolution| match target_resolution { + TargetResolution::_720p => Resolution::_720p, + TargetResolution::_1080p => Resolution::_1080p, + TargetResolution::_4K => Resolution::_2160p, + }) + .unwrap_or(Resolution::Captured); let targets = dbg!(scap::get_all_targets()); let excluded_targets: Vec = targets diff --git a/crates/project/src/configuration.rs b/crates/project/src/configuration.rs index 6019d4498..5ab42726c 100644 --- a/crates/project/src/configuration.rs +++ b/crates/project/src/configuration.rs @@ -51,20 +51,54 @@ impl Default for BackgroundSource { } } -#[derive(Type, Serialize, Deserialize, Clone, Debug, Default)] +#[derive(Type, Serialize, Deserialize, Clone, Copy, Debug, Default)] pub enum TargetResolution { _720p, #[default] _1080p, - Native, + _4K, } -#[derive(Type, Serialize, Deserialize, Clone, Debug, Default)] +impl TargetResolution { + pub fn to_width(&self) -> u32 { + match self { + TargetResolution::_720p => 1280, + TargetResolution::_1080p => 1920, + TargetResolution::_4K => 3840, + } + } +} + +#[derive(Type, Serialize, Deserialize, Clone, Copy, Debug, Default)] pub enum TargetFPS { #[default] _30, _60, - Native, +} + +impl TargetFPS { + pub fn to_raw(&self) -> u32 { + match self { + TargetFPS::_30 => 30, + TargetFPS::_60 => 60, + } + } + + pub fn round(fps: f64) -> u32 { + // Common monitor refresh rates + match fps.round() { + 29.0..31.0 => 30, + 59.0..61.0 => 60, + 74.0..76.0 => 75, + 89.0..91.0 => 90, + 119.0..121.0 => 120, + 143.0..145.0 => 144, + 164.0..166.0 => 165, + 239.0..241.0 => 240, + 359.0..361.0 => 360, + _ => unimplemented!("Unknown refresh rate"), + } + } } #[derive(Type, Serialize, Deserialize, Clone, Copy, Debug, Default)] diff --git a/crates/rendering/src/decoder.rs b/crates/rendering/src/decoder.rs index 0c5ad774e..3fcd3ecbc 100644 --- a/crates/rendering/src/decoder.rs +++ b/crates/rendering/src/decoder.rs @@ -4,6 +4,7 @@ use std::{ sync::{mpsc, Arc}, }; +use cap_project::TargetFPS; use ffmpeg::{ codec, format::{self}, @@ -18,16 +19,16 @@ enum VideoDecoderMessage { GetFrame(u32, tokio::sync::oneshot::Sender>>>), } -fn pts_to_frame(pts: i64, time_base: Rational) -> u32 { - (FPS as f64 * ((pts as f64 * time_base.numerator() as f64) / (time_base.denominator() as f64))) - .round() as u32 +fn pts_to_frame(fps: f64, pts: i64, time_base: Rational) -> u32 { + (fps * ((pts as f64 * time_base.numerator() as f64) / (time_base.denominator() as f64))).round() + as u32 } const FRAME_CACHE_SIZE: usize = 50; // TODO: Allow dynamic FPS values by either passing it into `spawn` // or changing `get_frame` to take the requested time instead of frame number, // so that the lookup can be done by PTS instead of frame number. -const FPS: u32 = 30; +// const FPS: u32 = 30; pub struct AsyncVideoDecoder; @@ -53,6 +54,7 @@ impl AsyncVideoDecoder { let input_stream_index = input_stream.index(); let time_base = input_stream.time_base(); let frame_rate = input_stream.rate(); + let fps = TargetFPS::round(frame_rate.into()); // Create a decoder for the video stream let mut decoder = context.decoder().video().unwrap(); @@ -181,6 +183,7 @@ impl AsyncVideoDecoder { while decoder.receive_frame(&mut temp_frame).is_ok() { let current_frame = pts_to_frame( + fps as f64, temp_frame.pts().unwrap() - start_offset, time_base, ); @@ -289,7 +292,7 @@ impl AsyncVideoDecoder { } if let Some(s) = sender.take() { - s.send(None); + let _ = s.send(None); } } } diff --git a/crates/rendering/src/lib.rs b/crates/rendering/src/lib.rs index 8b8561b0d..5b6339d36 100644 --- a/crates/rendering/src/lib.rs +++ b/crates/rendering/src/lib.rs @@ -119,6 +119,7 @@ pub async fn render_video_to_channel( decoders: RecordingDecoders, cursor: Arc, project_path: PathBuf, // Add project_path parameter + fps: u32, ) -> Result<(), String> { let constants = RenderVideoConstants::new(options, cursor, project_path).await?; @@ -136,23 +137,23 @@ pub async fn render_video_to_channel( let background = Background::from(project.background.source.clone()); loop { - if frame_number as f64 > 30_f64 * duration { + if frame_number as f64 > fps as f64 * duration { break; }; let time = if let Some(timeline) = project.timeline() { - match timeline.get_recording_time(frame_number as f64 / 30_f64) { + match timeline.get_recording_time(frame_number as f64 / fps as f64) { Some(time) => time, None => break, } } else { - frame_number as f64 / 30_f64 + frame_number as f64 / fps as f64 }; - let uniforms = ProjectUniforms::new(&constants, &project, time as f32); + let uniforms = ProjectUniforms::new(&constants, &project, time as f32, fps); let Some((screen_frame, camera_frame)) = - decoders.get_frames((time * 30.0) as u32).await + decoders.get_frames((time * fps as f64) as u32).await else { break; }; @@ -164,6 +165,7 @@ pub async fn render_video_to_channel( background, &uniforms, time as f32, + fps, ) .await { @@ -472,6 +474,7 @@ impl ProjectUniforms { constants: &RenderVideoConstants, project: &ProjectConfiguration, time: f32, + fps: u32, ) -> Self { let options = &constants.options; let output_size = Self::get_output_size(options, project); @@ -480,7 +483,7 @@ impl ProjectUniforms { let zoom_keyframes = ZoomKeyframes::new(project); let current_zoom = zoom_keyframes.get_amount(time as f64); - let prev_zoom = zoom_keyframes.get_amount((time - 1.0 / 30.0) as f64); + let prev_zoom = zoom_keyframes.get_amount(time as f64 - 1.0 / (fps as f64)); let velocity = if current_zoom != prev_zoom { let scale_change = (current_zoom - prev_zoom) as f32; @@ -633,7 +636,7 @@ impl ProjectUniforms { let zoom_delta = (current_zoom - prev_zoom).abs() as f32; // Calculate a smooth transition factor - let transition_speed = 30.0f32; // Frames per second + let transition_speed = fps as f32; // Frames per second let transition_factor = (zoom_delta * transition_speed).min(1.0); // Reduce multiplier from 3.0 to 2.0 for weaker blur @@ -757,6 +760,7 @@ pub async fn produce_frame( background: Background, uniforms: &ProjectUniforms, time: f32, + fps: u32, ) -> Result, String> { let mut encoder = constants.device.create_command_encoder( &(wgpu::CommandEncoderDescriptor { @@ -886,6 +890,7 @@ pub async fn produce_frame( constants, uniforms, time, + fps, &mut encoder, get_either(texture_views, !output_is_left), ); @@ -1043,6 +1048,7 @@ fn draw_cursor( constants: &RenderVideoConstants, uniforms: &ProjectUniforms, time: f32, + fps: u32, encoder: &mut CommandEncoder, view: &wgpu::TextureView, ) { @@ -1051,7 +1057,7 @@ fn draw_cursor( }; // Calculate previous position for velocity - let prev_position = interpolate_cursor_position(&constants.cursor, time - 1.0 / 30.0); + let prev_position = interpolate_cursor_position(&constants.cursor, time - 1.0 / (fps as f32)); // Calculate velocity in screen space let velocity = if let Some(prev_pos) = prev_position { From 9180db880a2f21702fc4597172e54d312d62a623 Mon Sep 17 00:00:00 2001 From: Brendan Allan Date: Wed, 13 Nov 2024 15:13:00 +0800 Subject: [PATCH 6/6] handle null properly with KSelect --- .../src/routes/(window-chrome)/settings.tsx | 32 ++++---- .../(window-chrome)/settings/recording.tsx | 73 ++++++++++++------- 2 files changed, 60 insertions(+), 45 deletions(-) diff --git a/apps/desktop/src/routes/(window-chrome)/settings.tsx b/apps/desktop/src/routes/(window-chrome)/settings.tsx index 374a79076..f993b95b4 100644 --- a/apps/desktop/src/routes/(window-chrome)/settings.tsx +++ b/apps/desktop/src/routes/(window-chrome)/settings.tsx @@ -37,25 +37,19 @@ export default function (props: RouteSectionProps) { }, ]} > - {(item) => { - const isActive = () => location.pathname.includes(item.href); - return ( -
  • - - - {item.name} - -
    • - ); - }} + {(item) => ( +
  • + + + {item.name} + +
    • + )}
    diff --git a/apps/desktop/src/routes/(window-chrome)/settings/recording.tsx b/apps/desktop/src/routes/(window-chrome)/settings/recording.tsx index afe6d9ab9..72a966d72 100644 --- a/apps/desktop/src/routes/(window-chrome)/settings/recording.tsx +++ b/apps/desktop/src/routes/(window-chrome)/settings/recording.tsx @@ -3,7 +3,12 @@ import { createStore } from "solid-js/store"; import { Select as KSelect } from "@kobalte/core/select"; import { recordingSettingsStore } from "~/store"; import { createCurrentRecordingQuery } from "~/utils/queries"; -import { commands, type RecordingSettingsStore, TargetResolution, TargetFPS } from "~/utils/tauri"; +import { + commands, + type RecordingSettingsStore, + TargetResolution, + TargetFPS, +} from "~/utils/tauri"; import { MenuItem, MenuItemList, @@ -23,22 +28,26 @@ export default function RecordingSettings() { } type SettingOption = { - label: string, - value: T, -} + label: string; + value: T; +}; type RecordingSetting = { - label: string -} & ({ - key: "capture_resolution", - options: SettingOption[], -} | { - key: "output_resolution", - options: SettingOption[], -} | { - key: "recording_fps", - options: SettingOption[], -}); + label: string; +} & ( + | { + key: "capture_resolution"; + options: SettingOption[]; + } + | { + key: "output_resolution"; + options: SettingOption[]; + } + | { + key: "recording_fps"; + options: SettingOption[]; + } +); const resolutionOptions: SettingOption[] = [ { @@ -52,8 +61,8 @@ const resolutionOptions: SettingOption[] = [ { label: "4K (3840x2160)", value: "_4K", - } -] + }, +]; const recordingSettings: RecordingSetting[] = [ { @@ -105,7 +114,10 @@ function Inner(props: { initialStore: RecordingSettingsStore | null }) { } ); - const handleChange = async (key: K, value: RecordingSettingsStore[K]) => { + const handleChange = async ( + key: K, + value: RecordingSettingsStore[K] + ) => { console.log(`Handling settings change for ${key}: ${value}`); setSettings(key, value); @@ -114,11 +126,15 @@ function Inner(props: { initialStore: RecordingSettingsStore | null }) { [key]: value, }); if (result.status === "error") console.error(result.error); - } + }; - const settingOption = (setting: RecordingSetting): typeof setting.options[0] => { - return setting.options.find((option) => option.value === settings[setting.key])!; - } + const settingOption = (setting: RecordingSetting) => { + return ( + setting.options.find( + (option) => option.value === settings[setting.key] + ) ?? null + ); + }; const recordingInProgress = !!currentRecording.data; @@ -140,25 +156,30 @@ function Inner(props: { initialStore: RecordingSettingsStore | null }) { {(setting) => (

    {setting.label}

    - + options={setting.options} optionValue="value" optionTextValue="value" value={settingOption(setting)} disabled={recordingInProgress} onChange={(option) => { - handleChange(setting.key, option!.value) + handleChange(setting.key, option?.value ?? null); }} + allowDuplicateSelectionEvents={false} itemComponent={(props) => ( - as={KSelect.Item} item={props.item}> + + as={KSelect.Item} + item={props.item} + > {props.item.rawValue.label} )} + placeholder={setting.options[0].label} > - class=""> + class=""> {(state) => {state.selectedOption().label}}