Skip to content

Commit

Permalink
feat(video/surfaces): add surface mapping node (#358) [WIP]
Browse files Browse the repository at this point in the history
Most of what the node requires to function is already implemented.

* Renaming functionality of the surface and section is not implemented
* The transform doesn't update in realtime
* Currently there is an issue with the transform, as you can see the seem between the triangles rendering the video when the transform is non rectangular
* The input fields in the settings pane don't actually change the transform
* The surface is not deleted when deleting the node
* The Open surface button doesn't navigate to the surfaces view
  • Loading branch information
maxjoehnk committed Oct 14, 2023
1 parent 2a24543 commit 2382a06
Show file tree
Hide file tree
Showing 63 changed files with 2,430 additions and 17 deletions.
43 changes: 43 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 3 additions & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ members = [
"crates/components/media",
"crates/components/sequencer",
"crates/components/sequencer/commands",
"crates/components/surfaces",
"crates/components/timecode",
"crates/projects",
"crates/runtime",
Expand Down Expand Up @@ -74,6 +75,7 @@ members = [
"crates/runtime/pipeline/nodes/transport",
"crates/runtime/pipeline/nodes/video",
"crates/runtime/pipeline/nodes/video/screen-capture",
"crates/runtime/pipeline/nodes/video/surfaces",
"crates/runtime/pipeline/nodes/video/text",
"crates/runtime/pipeline/nodes/video/webcams",
"crates/runtime/pipeline/ports",
Expand Down Expand Up @@ -148,6 +150,7 @@ mizer-protocol-dmx = { path = "crates/components/connections/protocols/dmx" }
mizer-protocol-midi = { path = "crates/components/connections/protocols/midi" }
mizer-protocol-mqtt = { path = "crates/components/connections/protocols/mqtt" }
mizer-protocol-osc = { path = "crates/components/connections/protocols/osc" }
mizer-surfaces = { path = "crates/components/surfaces" }
mizer-processing = { path = "crates/runtime/processing" }
mizer-module = { path = "crates/runtime/module" }
mizer-ui = { path = "crates/ui", optional = true }
Expand Down
1 change: 1 addition & 0 deletions crates/api/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ mizer-media = { path = "../components/media" }
mizer-connections = { path = "../components/connections" }
mizer-devices = { path = "../components/connections/devices" }
mizer-gamepads = { path = "../components/connections/devices/gamepads" }
mizer-surfaces = { path = "../components/surfaces" }
mizer-node = { path = "../runtime/pipeline/node" }
mizer-nodes = { path = "../runtime/pipeline/nodes" }
mizer-runtime = { path = "../runtime" }
Expand Down
1 change: 1 addition & 0 deletions crates/api/build.rs
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ fn main() {
"protos/plans.proto",
"protos/mappings.proto",
"protos/timecode.proto",
"protos/surfaces.proto",
],
&["protos/"],
)
Expand Down
1 change: 1 addition & 0 deletions crates/api/protos/nodes.proto
Original file line number Diff line number Diff line change
Expand Up @@ -161,6 +161,7 @@ message Node {
IMAGE_FILE = 91;
TEXTURE_MASK = 92;
TEXTURE_OPACITY = 93;
SURFACE_MAPPING = 94;
CONTAINER = 100;
}

Expand Down
38 changes: 38 additions & 0 deletions crates/api/protos/surfaces.proto
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
syntax = "proto3";

package mizer.surfaces;

message Surface {
string id = 1;
string name = 2;
repeated SurfaceSection sections = 3;
}

message Surfaces {
repeated Surface surfaces = 1;
}

message SurfaceSection {
uint32 id = 1;
string name = 2;
SurfaceTransform input = 3;
SurfaceTransform output = 4;
}

message SurfaceTransform {
SurfaceTransformPoint top_left = 1;
SurfaceTransformPoint top_right = 2;
SurfaceTransformPoint bottom_left = 3;
SurfaceTransformPoint bottom_right = 4;
}

message SurfaceTransformPoint {
double x = 1;
double y = 2;
}

message UpdateSectionTransform {
string surface_id = 1;
uint32 section_id = 2;
SurfaceTransform transform = 3;
}
8 changes: 7 additions & 1 deletion crates/api/src/handlers/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ use mizer_fixtures::manager::FixtureManager;
use mizer_media::MediaServer;
use mizer_sequencer::{EffectEngine, Sequencer};
use mizer_status_bus::StatusBus;
use mizer_surfaces::SurfaceRegistryApi;
use mizer_timecode::TimecodeManager;

use crate::RuntimeApi;
Expand All @@ -20,6 +21,7 @@ pub use self::sequencer::*;
pub use self::session::*;
pub use self::settings::*;
pub use self::status::*;
pub use self::surfaces::*;
pub use self::timecode::*;
pub use self::transport::*;

Expand All @@ -36,6 +38,7 @@ mod sequencer;
mod session;
mod settings;
mod status;
mod surfaces;
mod timecode;
mod transport;

Expand All @@ -56,6 +59,7 @@ pub struct Handlers<R: RuntimeApi> {
pub mappings: MappingsHandler<R>,
pub timecode: TimecodeHandler<R>,
pub status: StatusHandler,
pub surfaces: SurfacesHandler<R>,
}

impl<R: RuntimeApi> Handlers<R> {
Expand All @@ -68,6 +72,7 @@ impl<R: RuntimeApi> Handlers<R> {
effect_engine: EffectEngine,
timecode_manager: TimecodeManager,
status_bus: StatusBus,
surface_registry_api: SurfaceRegistryApi,
) -> Self {

Check warning on line 76 in crates/api/src/handlers/mod.rs

View workflow job for this annotation

GitHub Actions / clippy

this function has too many arguments (9/7)

warning: this function has too many arguments (9/7) --> crates/api/src/handlers/mod.rs:66:5 | 66 | / pub fn new( 67 | | runtime: R, 68 | | fixture_manager: FixtureManager, 69 | | fixture_library: FixtureLibrary, ... | 75 | | surface_registry_api: SurfaceRegistryApi, 76 | | ) -> Self { | |_____________^ | = help: for further information visit https://rust-lang.github.io/rust-clippy/master/index.html#too_many_arguments = note: `#[warn(clippy::too_many_arguments)]` on by default
Handlers {
connections: ConnectionsHandler::new(runtime.clone()),
Expand All @@ -87,8 +92,9 @@ impl<R: RuntimeApi> Handlers<R> {
settings: SettingsHandler::new(runtime.clone()),
plans: PlansHandler::new(fixture_manager, runtime.clone()),
mappings: MappingsHandler::new(runtime.clone()),
timecode: TimecodeHandler::new(timecode_manager, runtime),
timecode: TimecodeHandler::new(timecode_manager, runtime.clone()),
status: StatusHandler::new(status_bus),
surfaces: SurfacesHandler::new(runtime, surface_registry_api),
}
}
}
74 changes: 74 additions & 0 deletions crates/api/src/handlers/surfaces.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
use futures::{Stream, StreamExt};

use mizer_command_executor::UpdateSurfaceSectionCommand;
use mizer_surfaces::{SurfaceRegistryApi, SurfaceSectionId};

use crate::proto::surfaces::*;
use crate::RuntimeApi;

#[derive(Clone)]
pub struct SurfacesHandler<R: RuntimeApi> {
api: SurfaceRegistryApi,
runtime: R,
}

impl<R: RuntimeApi> SurfacesHandler<R> {
pub fn new(runtime: R, api: SurfaceRegistryApi) -> Self {
Self { runtime, api }
}

#[tracing::instrument(skip(self))]
#[profiling::function]
pub fn update_input_mapping(
&self,
surface_id: String,
section_id: usize,
mapping: SurfaceTransform,
) -> anyhow::Result<()> {
let id = SurfaceSectionId {
surface_id: surface_id.parse()?,
index: section_id,
};
self.runtime.run_command(UpdateSurfaceSectionCommand {
id,
input: Some(mapping.into()),
output: None,
})?;
Ok(())
}

#[tracing::instrument(skip(self))]
#[profiling::function]
pub fn update_output_mapping(
&self,
surface_id: String,
section_id: usize,
mapping: SurfaceTransform,
) -> anyhow::Result<()> {
let id = SurfaceSectionId {
surface_id: surface_id.parse()?,
index: section_id,
};
self.runtime.run_command(UpdateSurfaceSectionCommand {
id,
input: None,
output: Some(mapping.into()),
})?;
Ok(())
}

#[tracing::instrument(skip(self))]
#[profiling::function]
pub fn observe_surfaces(&self) -> impl Stream<Item = Surfaces> {
log::debug!("Observing surfaces");
self.api
.bus
.subscribe()
.into_stream()
.map(|surfaces| surfaces.into_iter().map(|surface| surface.into()).collect())
.map(|surfaces| {
log::debug!("Emitting new surfaces");
Surfaces { surfaces }
})
}
}
1 change: 1 addition & 0 deletions crates/api/src/mappings/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,5 +8,6 @@ mod plans;
mod programmer;
mod sequencer;
mod settings;
mod surfaces;
mod timecode;
mod transport;
4 changes: 3 additions & 1 deletion crates/api/src/mappings/nodes.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@ use mizer_nodes::{MidiInputConfig, NodeDowncast};
use mizer_runtime::commands::StaticNodeDescriptor;
use mizer_runtime::NodeDescriptor;

use crate::proto::nodes::node_setting::{spline_value, SplineValue};
use crate::proto::nodes::*;
use crate::proto::nodes::node_setting::{spline_value, SplineValue};

impl TryFrom<mizer_nodes::Node> for node_config::Type {
type Error = ();
Expand Down Expand Up @@ -222,6 +222,7 @@ impl From<NodeType> for node::NodeType {
NodeType::ColorizeTexture => node::NodeType::ColorizeTexture,
NodeType::TextureMask => node::NodeType::TextureMask,
NodeType::TextureOpacity => node::NodeType::TextureOpacity,
NodeType::SurfaceMapping => node::NodeType::SurfaceMapping,
NodeType::TestSink => unimplemented!("only for test"),
}
}
Expand Down Expand Up @@ -309,6 +310,7 @@ impl From<node::NodeType> for NodeType {
node::NodeType::ColorizeTexture => NodeType::ColorizeTexture,
node::NodeType::TextureMask => NodeType::TextureMask,
node::NodeType::TextureOpacity => NodeType::TextureOpacity,
node::NodeType::SurfaceMapping => NodeType::SurfaceMapping,
}
}
}
Expand Down
62 changes: 62 additions & 0 deletions crates/api/src/mappings/surfaces.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
use crate::proto::surfaces::*;

impl From<mizer_surfaces::Surface> for Surface {
fn from(surface: mizer_surfaces::Surface) -> Self {
Self {
id: surface.id.to_string(),
name: surface.name,
sections: surface.sections.into_iter().map(Into::into).collect(),
}
}
}

impl From<mizer_surfaces::SurfaceSection> for SurfaceSection {
fn from(value: mizer_surfaces::SurfaceSection) -> Self {
Self {
id: value.id.index as u32,
name: value.name,
input: Some(value.input.into()),
output: Some(value.output.into()),
}
}
}

impl From<mizer_surfaces::SurfaceTransform> for SurfaceTransform {
fn from(value: mizer_surfaces::SurfaceTransform) -> Self {
Self {
top_left: Some(value.top_left.into()),
top_right: Some(value.top_right.into()),
bottom_left: Some(value.bottom_left.into()),
bottom_right: Some(value.bottom_right.into()),
}
}
}

impl From<mizer_surfaces::SurfaceTransformPoint> for SurfaceTransformPoint {
fn from(value: mizer_surfaces::SurfaceTransformPoint) -> Self {
Self {
x: value.x,
y: value.y,
}
}
}

impl From<SurfaceTransform> for mizer_surfaces::SurfaceTransform {
fn from(value: SurfaceTransform) -> Self {
Self {
top_left: value.top_left.unwrap().into(),
top_right: value.top_right.unwrap().into(),
bottom_left: value.bottom_left.unwrap().into(),
bottom_right: value.bottom_right.unwrap().into(),
}
}
}

impl From<SurfaceTransformPoint> for mizer_surfaces::SurfaceTransformPoint {
fn from(value: SurfaceTransformPoint) -> Self {
Self {
x: value.x,
y: value.y,
}
}
}
Loading

0 comments on commit 2382a06

Please sign in to comment.