backend: forward keyboard and mouse input from client

Cargo.lock

@@ -537,6 +537,16 @@ version = "2.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
 
+[[package]]
+name = "flatbuffers"
+version = "25.2.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1045398c1bfd89168b5fd3f1fc11f6e70b34f6f66300c87d44d3de849463abf1"
+dependencies = [
+ "bitflags",
+ "rustc_version",
+]
+
 [[package]]
 name = "fnv"
 version = "1.0.7"
@@ -668,6 +678,7 @@ version = "0.1.0"
 dependencies = [
  "anyhow",
  "directories",
+ "flatbuffers",
  "getrandom 0.3.3",
  "hex",
  "hmac-sha256",
@@ -2038,6 +2049,15 @@ version = "2.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d"
 
+[[package]]
+name = "rustc_version"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92"
+dependencies = [
+ "semver",
+]
+
 [[package]]
 name = "rustix"
 version = "1.0.7"
@@ -2443,6 +2463,12 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "semver"
+version = "1.0.26"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "56e6fa9c48d24d85fb3de5ad847117517440f6beceb7798af16b4a87d616b8d0"
+
 [[package]]
 name = "serde"
 version = "1.0.219"

gamestream-webtransport-proxy/Cargo.toml

@@ -6,6 +6,7 @@ edition = "2024"
 [dependencies]
 anyhow = "1.0.98"
 directories = "6.0.0"
+flatbuffers = "25.2.10"
 getrandom = { version = "0.3.3", features = ["std"] }
 hex = "0.4.3"
 hmac-sha256 = "1.1.12"

gamestream-webtransport-proxy/src/gamestream/decoder.rs

@@ -256,7 +256,6 @@ extern "C" fn start_cb() {
 }
 
 extern "C" fn submit_decode_unit_cb(decode_unit: PDECODE_UNIT) -> std::os::raw::c_int {
-    debug!("SUBMIT DECODE UNIT CB");
     if decode_unit.is_null() {
         error!("Decode unit pointer was null");
         return -1;

gamestream-webtransport-proxy/src/gamestream/mod.rs

@@ -20,7 +20,7 @@ pub fn start_connection(
         address,
         stream.server_codec_mode_support,
     )?;
-    let mut stream_config = config::stream_config(&stream);
+    let mut stream_config = config::stream_config(stream);
     let mut listener_callbacks = config::listener_callbacks();
     let (mut decoder_callbacks, decoder_rx) = decoder::decoder_callbacks()?;
     let ret;
@@ -47,3 +47,31 @@ pub fn start_connection(
 pub fn stop_connection() {
     unsafe { moonlight_common_c_sys::LiStopConnection() };
 }
+
+pub fn send_keyboard_event(keycode: i16, keyaction: i8, modifiers: u8) -> Result<()> {
+    let ret =
+        unsafe { moonlight_common_c_sys::LiSendKeyboardEvent(keycode, keyaction, modifiers as i8) };
+
+    match ret {
+        0 => Ok(()),
+        _ => Err(anyhow!("Could not send keyboard event: {ret}")),
+    }
+}
+
+pub fn send_mouse_move_event(movement_x: i16, movement_y: i16) -> Result<()> {
+    let ret = unsafe { moonlight_common_c_sys::LiSendMouseMoveEvent(movement_x, movement_y) };
+
+    match ret {
+        0 => Ok(()),
+        _ => Err(anyhow!("Could not send mouse movement event: {ret}")),
+    }
+}
+
+pub fn send_mouse_input_event(action: i8, button: i32) -> Result<()> {
+    let ret = unsafe { moonlight_common_c_sys::LiSendMouseButtonEvent(action, button) };
+
+    match ret {
+        0 => Ok(()),
+        _ => Err(anyhow!("Could not send mouse input event: {ret}")),
+    }
+}

gamestream-webtransport-proxy/src/proxy/input/mod.rs

@@ -0,0 +1 @@
+pub mod input_generated;

gamestream-webtransport-proxy/src/proxy/mod.rs

@@ -1,11 +1,12 @@
 use anyhow::{Context, Result};
-use salvo::{conn::http1::Connection, hyper::body::Bytes, proto::WebTransportSession};
 use tokio::{io::AsyncReadExt, io::AsyncWriteExt, select, sync::RwLock};
 use tracing::{debug, error, info};
 
 use crate::{backend, gamestream};
 
 pub mod handler;
+mod input;
+mod packet_parser;
 
 pub struct Proxy {
     pub cert_hash: [u8; 32],
@@ -40,14 +41,16 @@ async fn proxy_main(
 
     let mut channels = spawn_gamestream(stream).await?;
 
-    let mut buffer = vec![0; 65536].into_boxed_slice();
+    let mut packet_buffer = packet_parser::PacketBuffer::new();
+    //let mut buffer = vec![0; 65536].into_boxed_slice();
+    let mut buffer = [0u8; 65536];
+    //let mut buffer = vec![0; 65536].into_boxed_slice();
 
     loop {
         select! {
             gamestream_packet = channels.gamestream_channels.decoder_rx.recv() => {
                 match gamestream_packet {
                     Some(frame) => {
-                        info!("Got decoder packet");
                         let frame_json = serde_json::to_vec(&frame)?;
                         let frame_json_len: u32 = <u32>::try_from(frame_json.len())?;
 
@@ -60,8 +63,11 @@ async fn proxy_main(
                     }
                 }
             },
-            webtransport_packet = wt_recv.read(&mut buffer) => {
+            ret = wt_recv.read(&mut buffer) => {
-                info!("Got packet from client");
+                let bytes_read = ret?;
+
+                packet_parser::handle_client_packet(&mut packet_buffer, &buffer[..bytes_read])?;
+
             }
         }
     }

gamestream-webtransport-proxy/src/proxy/packet_parser.rs

@@ -0,0 +1,158 @@
+use anyhow::{Result, anyhow};
+use tracing::{debug, error, info};
+
+use super::input::input_generated::input_event;
+use crate::gamestream;
+
+pub struct PacketBuffer {
+    buffer: Vec<u8>,
+    expected_length: Option<usize>,
+    length_bytes_needed: usize,
+}
+
+impl PacketBuffer {
+    pub fn new() -> Self {
+        Self {
+            buffer: Vec::new(),
+            expected_length: None,
+            length_bytes_needed: 4, // Assuming u32 length (4 bytes)
+        }
+    }
+
+    /// Process incoming data and return complete packets
+    pub fn process_data(&mut self, data: &[u8]) -> Vec<Vec<u8>> {
+        let mut complete_packets = Vec::new();
+        self.buffer.extend_from_slice(data);
+
+        loop {
+            // If we don't know the expected length yet, try to read it
+            if self.expected_length.is_none() {
+                if self.buffer.len() >= self.length_bytes_needed {
+                    // Read length from first 4 bytes (adjust as needed)
+                    let length = u32::from_be_bytes([
+                        self.buffer[0],
+                        self.buffer[1],
+                        self.buffer[2],
+                        self.buffer[3],
+                    ]) as usize;
+
+                    self.expected_length = Some(length);
+                } else {
+                    // Not enough data to read length yet
+                    break;
+                }
+            }
+
+            // We know the expected length, check if we have a complete packet
+            if let Some(expected_len) = self.expected_length {
+                let total_packet_size = self.length_bytes_needed + expected_len;
+
+                if self.buffer.len() >= total_packet_size {
+                    // We have a complete packet - extract just the data portion
+                    let packet_data =
+                        self.buffer[self.length_bytes_needed..total_packet_size].to_vec();
+                    complete_packets.push(packet_data);
+
+                    // Remove the processed packet from buffer
+                    self.buffer.drain(0..total_packet_size);
+                    self.expected_length = None;
+                } else {
+                    // Packet is incomplete
+                    break;
+                }
+            }
+        }
+
+        complete_packets
+    }
+}
+
+pub fn process_keyboard_event(input_event: input_event::InputEvent) -> Result<()> {
+    let Some(table) = input_event.input() else {
+        debug!("Keyboard event table was empty, ignoring");
+        return Err(anyhow!("Keyboard event table was empty"));
+    };
+    let keyboard_event = unsafe { input_event::KeyboardInput::init_from_table(table) };
+
+    let key_action_i8 = match keyboard_event.key_action() {
+        input_event::KeyAction::DOWN => moonlight_common_c_sys::KEY_ACTION_DOWN,
+        input_event::KeyAction::UP => moonlight_common_c_sys::KEY_ACTION_UP,
+        _ => {
+            debug!("Invalid KeyAction value, ignoring");
+            return Err(anyhow!("Invalid KeyAction value"));
+        }
+    };
+
+    gamestream::send_keyboard_event(keyboard_event.key_code(), key_action_i8, 0)
+}
+
+pub fn process_mouse_move_event(input_event: input_event::InputEvent) -> Result<()> {
+    let Some(table) = input_event.input() else {
+        debug!("MouseMovement event table was empty, ignoring");
+        return Err(anyhow!("MouseMovement event table was empty"));
+    };
+    let mouse_event = unsafe { input_event::MouseMovement::init_from_table(table) };
+
+    gamestream::send_mouse_move_event(mouse_event.movement_x(), mouse_event.movement_y())
+}
+
+pub fn process_mouse_input_event(input_event: input_event::InputEvent) -> Result<()> {
+    let Some(table) = input_event.input() else {
+        debug!("MouseInput event table was empty, ignoring");
+        return Err(anyhow!("MouseInput event table was empty"));
+    };
+    let mouse_event = unsafe { input_event::MouseInput::init_from_table(table) };
+
+    let button_action_i8 = match mouse_event.button_action() {
+        input_event::KeyAction::DOWN => moonlight_common_c_sys::BUTTON_ACTION_PRESS,
+        input_event::KeyAction::UP => moonlight_common_c_sys::BUTTON_ACTION_RELEASE,
+        _ => {
+            error!("Invalid KeyAction value, ignoring",);
+            return Err(anyhow!("Invalid KeyAction value"));
+        }
+    };
+
+    let mouse_button_i32 = match mouse_event.button() {
+        input_event::MouseButton::LEFT => moonlight_common_c_sys::BUTTON_LEFT,
+        input_event::MouseButton::MIDDLE => moonlight_common_c_sys::BUTTON_MIDDLE,
+        input_event::MouseButton::RIGHT => moonlight_common_c_sys::BUTTON_RIGHT,
+        input_event::MouseButton::X1 => moonlight_common_c_sys::BUTTON_X1,
+        input_event::MouseButton::X2 => moonlight_common_c_sys::BUTTON_X2,
+        _ => {
+            error!("Invalid MouseButton value, ignoring",);
+            return Err(anyhow!("Invalid MouseButton value"));
+        }
+    };
+
+    gamestream::send_mouse_input_event(button_action_i8, mouse_button_i32)
+}
+
+pub fn handle_client_packet(packet_buffer: &mut PacketBuffer, buffer: &[u8]) -> anyhow::Result<()> {
+    let complete_packets = packet_buffer.process_data(buffer);
+
+    // TODO: only supports input packets. this should use a union wrapper or something
+    for packet_data in complete_packets {
+        let input_event = input_event::root_as_input_event(&packet_data)?;
+        match input_event.input_type() {
+            input_event::Input::Keyboard => {
+                process_keyboard_event(input_event)?;
+            }
+            input_event::Input::MouseMovement => {
+                process_mouse_move_event(input_event)?;
+            }
+            input_event::Input::MouseInput => {
+                process_mouse_input_event(input_event)?;
+            }
+
+            input_event::Input::NONE => {
+                debug!("Input event was empty, ignoring.")
+            }
+            _ => {
+                error!("Unknown InputEvent type");
+                return Err(anyhow!("Unknown InputEvent type"));
+            }
+        }
+    }
+
+    Ok(())
+}

moonlight-common-c-sys/build.rs

@@ -12,9 +12,13 @@ struct CustomCallbacks;
 impl ParseCallbacks for CustomCallbacks {
     fn int_macro(&self, name: &str, _value: i64) -> Option<IntKind> {
         match name {
-            "STREAM_CFG_LOCAL" => Some(IntKind::I32),
+            //"STREAM_CFG_LOCAL" => Some(IntKind::I32),
-            "STREAM_CFG_REMOTE" => Some(IntKind::I32),
+            //"STREAM_CFG_REMOTE" => Some(IntKind::I32),
-            "STREAM_CFG_AUTO" => Some(IntKind::I32),
+            //"STREAM_CFG_AUTO" => Some(IntKind::I32),
+            "KEY_ACTION_DOWN" => Some(IntKind::I8),
+            "KEY_ACTION_UP" => Some(IntKind::I8),
+            "BUTTON_ACTION_PRESS" => Some(IntKind::I8),
+            "BUTTON_ACTION_RELEASE" => Some(IntKind::I8),
             _ => None, // Default behavior for all others
         }
     }
@@ -34,7 +38,7 @@ fn main() {
     let bindings = bindgen::Builder::default()
         .header("moonlight-common-c/src/Limelight.h")
         .clang_arg(format!("-I{}/src", dst.display())) // Include built headers
-        //.parse_callbacks(Box::new(CustomCallbacks))
+        .parse_callbacks(Box::new(CustomCallbacks))
         .default_macro_constant_type(bindgen::MacroTypeVariation::Signed)
         .generate()
         .expect("Failed to generate bindings");

schema/input.fbs

@@ -0,0 +1,53 @@
+namespace InputEvent;
+
+
+enum KeyAction: byte {
+  DOWN,
+  UP,
+}
+
+
+table MouseMovement {
+  movement_x: int16;
+  movement_y: int16;
+}
+
+enum MouseButton: byte {
+  LEFT,
+  MIDDLE,
+  RIGHT,
+  X1,
+  X2
+}
+
+table MouseInput {
+  button: MouseButton;
+  button_action: KeyAction;
+}
+
+
+struct ModifierState {
+  shift: bool;
+  ctrl: bool;
+  alt: bool;
+  meta: bool;
+}
+
+table KeyboardInput {
+  key_code: int16;
+  key_action: KeyAction;
+  modifiers: ModifierState;
+}
+
+union Input {
+  Keyboard:KeyboardInput,
+  MouseMovement:MouseMovement,
+  MouseInput:MouseInput,
+}
+
+table InputEvent {
+  input: Input;
+}
+
+root_type InputEvent;
+