frontend: refactor input functions to their own file

Cargo.lock

@@ -537,6 +537,16 @@ version = "2.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
 
+[[package]]
+name = "flatbuffers"
+version = "25.2.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1045398c1bfd89168b5fd3f1fc11f6e70b34f6f66300c87d44d3de849463abf1"
+dependencies = [
+ "bitflags",
+ "rustc_version",
+]
+
 [[package]]
 name = "fnv"
 version = "1.0.7"
@@ -668,6 +678,7 @@ version = "0.1.0"
 dependencies = [
  "anyhow",
  "directories",
+ "flatbuffers",
  "getrandom 0.3.3",
  "hex",
  "hmac-sha256",
@@ -2038,6 +2049,15 @@ version = "2.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d"
 
+[[package]]
+name = "rustc_version"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92"
+dependencies = [
+ "semver",
+]
+
 [[package]]
 name = "rustix"
 version = "1.0.7"
@@ -2443,6 +2463,12 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "semver"
+version = "1.0.26"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "56e6fa9c48d24d85fb3de5ad847117517440f6beceb7798af16b4a87d616b8d0"
+
 [[package]]
 name = "serde"
 version = "1.0.219"

frontend/package.json

@@ -42,6 +42,7 @@
 		]
 	},
 	"dependencies": {
+		"flatbuffers": "^25.2.10",
 		"lucide-svelte": "^0.539.0",
 		"svelte-loading-spinners": "^0.3.6"
 	}

frontend/pnpm-lock.yaml

@@ -8,6 +8,9 @@ importers:
 
   .:
     dependencies:
+      flatbuffers:
+        specifier: ^25.2.10
+        version: 25.2.10
       lucide-svelte:
         specifier: ^0.539.0
         version: 0.539.0(svelte@5.36.12)
@@ -864,6 +867,9 @@ packages:
     resolution: {integrity: sha512-f7ccFPK3SXFHpx15UIGyRJ/FJQctuKZ0zVuN3frBo4HnK3cay9VEW0R6yPYFHC0AgqhukPzKjq22t5DmAyqGyw==}
     engines: {node: '>=16'}
 
+  flatbuffers@25.2.10:
+    resolution: {integrity: sha512-7JlN9ZvLDG1McO3kbX0k4v+SUAg48L1rIwEvN6ZQl/eCtgJz9UylTMzE9wrmYrcorgxm3CX/3T/w5VAub99UUw==}
+
   flatted@3.3.3:
     resolution: {integrity: sha512-GX+ysw4PBCz0PzosHDepZGANEuFCMLrnRTiEy9McGjmkCQYwRq4A/X786G/fjM/+OjsWSU1ZrY5qyARZmO/uwg==}
 
@@ -2167,6 +2173,8 @@ snapshots:
       flatted: 3.3.3
       keyv: 4.5.4
 
+  flatbuffers@25.2.10: {}
+
   flatted@3.3.3: {}
 
   fsevents@2.3.3:

frontend/src/lib/input.ts

@@ -0,0 +1,152 @@
+import { Input } from "$lib/proto/input-event"
+import { InputEvent } from "$lib/proto/input"
+import { Builder } from "flatbuffers";
+
+export enum KeyAction {
+  DOWN,
+  UP,
+}
+
+export async function sendKeyboardEvent(
+  writer: WritableStreamDefaultWriter,
+  event: KeyboardEvent,
+  keyAction: KeyAction,
+) {
+  const builder = new Builder(1024);
+
+  InputEvent.KeyboardInput.startKeyboardInput(builder);
+  // Have to use keyCode even though it's deprecated beacuse .code is a unhelpful string :sigh:
+  InputEvent.KeyboardInput.addKeyCode(builder, event.keyCode);
+  InputEvent.KeyboardInput.addKeyAction(builder, keyAction);
+  const modifierState = InputEvent.ModifierState.createModifierState(
+    builder,
+    event.shiftKey,
+    event.ctrlKey,
+    event.altKey,
+    event.metaKey
+  );
+
+  InputEvent.KeyboardInput.addModifiers(builder, modifierState);
+  const keyboardInput = InputEvent.KeyboardInput.endKeyboardInput(builder);
+
+  // Create InputEvent table with union
+  InputEvent.InputEvent.startInputEvent(builder);
+  InputEvent.InputEvent.addInputType(builder, Input.Keyboard);
+  InputEvent.InputEvent.addInput(builder, keyboardInput);
+  const inputEvent = InputEvent.InputEvent.endInputEvent(builder);
+
+  // Finish the buffer
+  builder.finish(inputEvent);
+
+  // Get the buffer data
+  const bufferData = builder.asUint8Array();
+
+  // Create size prefix as big-endian u32
+  const sizePrefix = new Uint8Array(4);
+  const size = bufferData.length;
+  sizePrefix[0] = (size >> 24) & 0xFF;  // Most significant byte
+  sizePrefix[1] = (size >> 16) & 0xFF;
+  sizePrefix[2] = (size >> 8) & 0xFF;
+  sizePrefix[3] = size & 0xFF;          // Least significant byte
+
+  // Create combined buffer with size prefix + flatbuffer data
+  const combined = new Uint8Array(sizePrefix.length + bufferData.length);
+  combined.set(sizePrefix, 0);
+  combined.set(bufferData, sizePrefix.length);
+
+  // Write to stream
+  await writer.write(combined);
+}
+
+export async function sendMouseMoveEvent(
+  writer: WritableStreamDefaultWriter,
+  event: MouseEvent,
+) {
+  const builder = new Builder(1024);
+
+  const mouseMovement = InputEvent.MouseMovement.createMouseMovement(builder, event.movementX, event.movementY);
+
+  // Create InputEvent table with union
+  InputEvent.InputEvent.startInputEvent(builder);
+  InputEvent.InputEvent.addInputType(builder, Input.MouseMovement);
+  InputEvent.InputEvent.addInput(builder, mouseMovement);
+  const inputEvent = InputEvent.InputEvent.endInputEvent(builder);
+
+  // Finish the buffer
+  builder.finish(inputEvent);
+
+  // Get the buffer data
+  const bufferData = builder.asUint8Array();
+
+  // Create size prefix as big-endian u32
+  const sizePrefix = new Uint8Array(4);
+  const size = bufferData.length;
+  sizePrefix[0] = (size >> 24) & 0xFF;  // Most significant byte
+  sizePrefix[1] = (size >> 16) & 0xFF;
+  sizePrefix[2] = (size >> 8) & 0xFF;
+  sizePrefix[3] = size & 0xFF;          // Least significant byte
+
+  // Create combined buffer with size prefix + flatbuffer data
+  const combined = new Uint8Array(sizePrefix.length + bufferData.length);
+  combined.set(sizePrefix, 0);
+  combined.set(bufferData, sizePrefix.length);
+
+  // Write to stream
+  await writer.write(combined);
+}
+
+export async function sendMouseInputEvent(
+  writer: WritableStreamDefaultWriter,
+  event: MouseEvent,
+  keyAction: KeyAction,
+) {
+  const builder = new Builder(1024);
+
+  var mouseButton: InputEvent.MouseButton;
+
+  switch (event.button) {
+    case 0:
+      mouseButton = InputEvent.MouseButton.LEFT;
+      break;
+    case 1:
+      mouseButton = InputEvent.MouseButton.MIDDLE;
+      break;
+    case 2:
+      mouseButton = InputEvent.MouseButton.RIGHT;
+      break;
+    default:
+      console.log(`Unknown mouse button ${event.button}, defaulting to LEFT`);
+      mouseButton = InputEvent.MouseButton.LEFT;
+      break;
+  }
+
+  const mouseInput = InputEvent.MouseInput.createMouseInput(builder, mouseButton, keyAction);
+
+  // Create InputEvent table with union
+  InputEvent.InputEvent.startInputEvent(builder);
+  InputEvent.InputEvent.addInputType(builder, Input.MouseInput);
+  InputEvent.InputEvent.addInput(builder, mouseInput);
+  const inputEvent = InputEvent.InputEvent.endInputEvent(builder);
+
+  // Finish the buffer
+  builder.finish(inputEvent);
+
+  // Get the buffer data
+  const bufferData = builder.asUint8Array();
+
+  // Create size prefix as big-endian u32
+  const sizePrefix = new Uint8Array(4);
+  const size = bufferData.length;
+  sizePrefix[0] = (size >> 24) & 0xFF;  // Most significant byte
+  sizePrefix[1] = (size >> 16) & 0xFF;
+  sizePrefix[2] = (size >> 8) & 0xFF;
+  sizePrefix[3] = size & 0xFF;          // Least significant byte
+
+  // Create combined buffer with size prefix + flatbuffer data
+  const combined = new Uint8Array(sizePrefix.length + bufferData.length);
+  combined.set(sizePrefix, 0);
+  combined.set(bufferData, sizePrefix.length);
+
+  // Write to stream
+  await writer.write(combined);
+}

frontend/src/lib/proto/input-event.ts

@@ -0,0 +1,12 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */
+
+export { Input } from './input-event/input.js';
+export { InputEvent } from './input-event/input-event.js';
+export { KeyAction } from './input-event/key-action.js';
+export { KeyboardInput } from './input-event/keyboard-input.js';
+export { ModifierState } from './input-event/modifier-state.js';
+export { MouseButton } from './input-event/mouse-button.js';
+export { MouseInput } from './input-event/mouse-input.js';
+export { MouseMovement } from './input-event/mouse-movement.js';

frontend/src/lib/proto/input-event/input-event.ts

@@ -0,0 +1,69 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */
+
+import * as flatbuffers from 'flatbuffers';
+
+import { Input, unionToInput, unionListToInput } from '../input-event/input.js';
+
+
+export class InputEvent {
+  bb: flatbuffers.ByteBuffer|null = null;
+  bb_pos = 0;
+  __init(i:number, bb:flatbuffers.ByteBuffer):InputEvent {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+}
+
+static getRootAsInputEvent(bb:flatbuffers.ByteBuffer, obj?:InputEvent):InputEvent {
+  return (obj || new InputEvent()).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+}
+
+static getSizePrefixedRootAsInputEvent(bb:flatbuffers.ByteBuffer, obj?:InputEvent):InputEvent {
+  bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH);
+  return (obj || new InputEvent()).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+}
+
+inputType():Input {
+  const offset = this.bb!.__offset(this.bb_pos, 4);
+  return offset ? this.bb!.readUint8(this.bb_pos + offset) : Input.NONE;
+}
+
+input<T extends flatbuffers.Table>(obj:any):any|null {
+  const offset = this.bb!.__offset(this.bb_pos, 6);
+  return offset ? this.bb!.__union(obj, this.bb_pos + offset) : null;
+}
+
+static startInputEvent(builder:flatbuffers.Builder) {
+  builder.startObject(2);
+}
+
+static addInputType(builder:flatbuffers.Builder, inputType:Input) {
+  builder.addFieldInt8(0, inputType, Input.NONE);
+}
+
+static addInput(builder:flatbuffers.Builder, inputOffset:flatbuffers.Offset) {
+  builder.addFieldOffset(1, inputOffset, 0);
+}
+
+static endInputEvent(builder:flatbuffers.Builder):flatbuffers.Offset {
+  const offset = builder.endObject();
+  return offset;
+}
+
+static finishInputEventBuffer(builder:flatbuffers.Builder, offset:flatbuffers.Offset) {
+  builder.finish(offset);
+}
+
+static finishSizePrefixedInputEventBuffer(builder:flatbuffers.Builder, offset:flatbuffers.Offset) {
+  builder.finish(offset, undefined, true);
+}
+
+static createInputEvent(builder:flatbuffers.Builder, inputType:Input, inputOffset:flatbuffers.Offset):flatbuffers.Offset {
+  InputEvent.startInputEvent(builder);
+  InputEvent.addInputType(builder, inputType);
+  InputEvent.addInput(builder, inputOffset);
+  return InputEvent.endInputEvent(builder);
+}
+}

frontend/src/lib/proto/input-event/input.ts

@@ -0,0 +1,42 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */
+
+import { KeyboardInput } from '../input-event/keyboard-input.js';
+import { MouseInput } from '../input-event/mouse-input.js';
+import { MouseMovement } from '../input-event/mouse-movement.js';
+
+
+export enum Input {
+  NONE = 0,
+  Keyboard = 1,
+  MouseMovement = 2,
+  MouseInput = 3
+}
+
+export function unionToInput(
+  type: Input,
+  accessor: (obj:KeyboardInput|MouseInput|MouseMovement) => KeyboardInput|MouseInput|MouseMovement|null
+): KeyboardInput|MouseInput|MouseMovement|null {
+  switch(Input[type]) {
+    case 'NONE': return null; 
+    case 'Keyboard': return accessor(new KeyboardInput())! as KeyboardInput;
+    case 'MouseMovement': return accessor(new MouseMovement())! as MouseMovement;
+    case 'MouseInput': return accessor(new MouseInput())! as MouseInput;
+    default: return null;
+  }
+}
+
+export function unionListToInput(
+  type: Input, 
+  accessor: (index: number, obj:KeyboardInput|MouseInput|MouseMovement) => KeyboardInput|MouseInput|MouseMovement|null, 
+  index: number
+): KeyboardInput|MouseInput|MouseMovement|null {
+  switch(Input[type]) {
+    case 'NONE': return null; 
+    case 'Keyboard': return accessor(index, new KeyboardInput())! as KeyboardInput;
+    case 'MouseMovement': return accessor(index, new MouseMovement())! as MouseMovement;
+    case 'MouseInput': return accessor(index, new MouseInput())! as MouseInput;
+    default: return null;
+  }
+}

frontend/src/lib/proto/input-event/key-action.ts

@@ -0,0 +1,8 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */
+
+export enum KeyAction {
+  DOWN = 0,
+  UP = 1
+}

frontend/src/lib/proto/input-event/keyboard-input.ts

@@ -0,0 +1,65 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */
+
+import * as flatbuffers from 'flatbuffers';
+
+import { KeyAction } from '../input-event/key-action.js';
+import { ModifierState } from '../input-event/modifier-state.js';
+
+
+export class KeyboardInput {
+  bb: flatbuffers.ByteBuffer|null = null;
+  bb_pos = 0;
+  __init(i:number, bb:flatbuffers.ByteBuffer):KeyboardInput {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+}
+
+static getRootAsKeyboardInput(bb:flatbuffers.ByteBuffer, obj?:KeyboardInput):KeyboardInput {
+  return (obj || new KeyboardInput()).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+}
+
+static getSizePrefixedRootAsKeyboardInput(bb:flatbuffers.ByteBuffer, obj?:KeyboardInput):KeyboardInput {
+  bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH);
+  return (obj || new KeyboardInput()).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+}
+
+keyCode():number {
+  const offset = this.bb!.__offset(this.bb_pos, 4);
+  return offset ? this.bb!.readInt16(this.bb_pos + offset) : 0;
+}
+
+keyAction():KeyAction {
+  const offset = this.bb!.__offset(this.bb_pos, 6);
+  return offset ? this.bb!.readInt8(this.bb_pos + offset) : KeyAction.DOWN;
+}
+
+modifiers(obj?:ModifierState):ModifierState|null {
+  const offset = this.bb!.__offset(this.bb_pos, 8);
+  return offset ? (obj || new ModifierState()).__init(this.bb_pos + offset, this.bb!) : null;
+}
+
+static startKeyboardInput(builder:flatbuffers.Builder) {
+  builder.startObject(3);
+}
+
+static addKeyCode(builder:flatbuffers.Builder, keyCode:number) {
+  builder.addFieldInt16(0, keyCode, 0);
+}
+
+static addKeyAction(builder:flatbuffers.Builder, keyAction:KeyAction) {
+  builder.addFieldInt8(1, keyAction, KeyAction.DOWN);
+}
+
+static addModifiers(builder:flatbuffers.Builder, modifiersOffset:flatbuffers.Offset) {
+  builder.addFieldStruct(2, modifiersOffset, 0);
+}
+
+static endKeyboardInput(builder:flatbuffers.Builder):flatbuffers.Offset {
+  const offset = builder.endObject();
+  return offset;
+}
+
+}

frontend/src/lib/proto/input-event/modifier-state.ts

@@ -0,0 +1,45 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */
+
+import * as flatbuffers from 'flatbuffers';
+
+export class ModifierState {
+  bb: flatbuffers.ByteBuffer|null = null;
+  bb_pos = 0;
+  __init(i:number, bb:flatbuffers.ByteBuffer):ModifierState {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+}
+
+shift():boolean {
+  return !!this.bb!.readInt8(this.bb_pos);
+}
+
+ctrl():boolean {
+  return !!this.bb!.readInt8(this.bb_pos + 1);
+}
+
+alt():boolean {
+  return !!this.bb!.readInt8(this.bb_pos + 2);
+}
+
+meta():boolean {
+  return !!this.bb!.readInt8(this.bb_pos + 3);
+}
+
+static sizeOf():number {
+  return 4;
+}
+
+static createModifierState(builder:flatbuffers.Builder, shift: boolean, ctrl: boolean, alt: boolean, meta: boolean):flatbuffers.Offset {
+  builder.prep(1, 4);
+  builder.writeInt8(Number(Boolean(meta)));
+  builder.writeInt8(Number(Boolean(alt)));
+  builder.writeInt8(Number(Boolean(ctrl)));
+  builder.writeInt8(Number(Boolean(shift)));
+  return builder.offset();
+}
+
+}

frontend/src/lib/proto/input-event/mouse-button.ts

@@ -0,0 +1,11 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */
+
+export enum MouseButton {
+  LEFT = 0,
+  MIDDLE = 1,
+  RIGHT = 2,
+  X1 = 3,
+  X2 = 4
+}

frontend/src/lib/proto/input-event/mouse-input.ts

@@ -0,0 +1,62 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */
+
+import * as flatbuffers from 'flatbuffers';
+
+import { KeyAction } from '../input-event/key-action.js';
+import { MouseButton } from '../input-event/mouse-button.js';
+
+
+export class MouseInput {
+  bb: flatbuffers.ByteBuffer|null = null;
+  bb_pos = 0;
+  __init(i:number, bb:flatbuffers.ByteBuffer):MouseInput {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+}
+
+static getRootAsMouseInput(bb:flatbuffers.ByteBuffer, obj?:MouseInput):MouseInput {
+  return (obj || new MouseInput()).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+}
+
+static getSizePrefixedRootAsMouseInput(bb:flatbuffers.ByteBuffer, obj?:MouseInput):MouseInput {
+  bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH);
+  return (obj || new MouseInput()).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+}
+
+button():MouseButton {
+  const offset = this.bb!.__offset(this.bb_pos, 4);
+  return offset ? this.bb!.readInt8(this.bb_pos + offset) : MouseButton.LEFT;
+}
+
+buttonAction():KeyAction {
+  const offset = this.bb!.__offset(this.bb_pos, 6);
+  return offset ? this.bb!.readInt8(this.bb_pos + offset) : KeyAction.DOWN;
+}
+
+static startMouseInput(builder:flatbuffers.Builder) {
+  builder.startObject(2);
+}
+
+static addButton(builder:flatbuffers.Builder, button:MouseButton) {
+  builder.addFieldInt8(0, button, MouseButton.LEFT);
+}
+
+static addButtonAction(builder:flatbuffers.Builder, buttonAction:KeyAction) {
+  builder.addFieldInt8(1, buttonAction, KeyAction.DOWN);
+}
+
+static endMouseInput(builder:flatbuffers.Builder):flatbuffers.Offset {
+  const offset = builder.endObject();
+  return offset;
+}
+
+static createMouseInput(builder:flatbuffers.Builder, button:MouseButton, buttonAction:KeyAction):flatbuffers.Offset {
+  MouseInput.startMouseInput(builder);
+  MouseInput.addButton(builder, button);
+  MouseInput.addButtonAction(builder, buttonAction);
+  return MouseInput.endMouseInput(builder);
+}
+}

frontend/src/lib/proto/input-event/mouse-movement.ts

@@ -0,0 +1,58 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */
+
+import * as flatbuffers from 'flatbuffers';
+
+export class MouseMovement {
+  bb: flatbuffers.ByteBuffer|null = null;
+  bb_pos = 0;
+  __init(i:number, bb:flatbuffers.ByteBuffer):MouseMovement {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+}
+
+static getRootAsMouseMovement(bb:flatbuffers.ByteBuffer, obj?:MouseMovement):MouseMovement {
+  return (obj || new MouseMovement()).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+}
+
+static getSizePrefixedRootAsMouseMovement(bb:flatbuffers.ByteBuffer, obj?:MouseMovement):MouseMovement {
+  bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH);
+  return (obj || new MouseMovement()).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+}
+
+movementX():number {
+  const offset = this.bb!.__offset(this.bb_pos, 4);
+  return offset ? this.bb!.readInt16(this.bb_pos + offset) : 0;
+}
+
+movementY():number {
+  const offset = this.bb!.__offset(this.bb_pos, 6);
+  return offset ? this.bb!.readInt16(this.bb_pos + offset) : 0;
+}
+
+static startMouseMovement(builder:flatbuffers.Builder) {
+  builder.startObject(2);
+}
+
+static addMovementX(builder:flatbuffers.Builder, movementX:number) {
+  builder.addFieldInt16(0, movementX, 0);
+}
+
+static addMovementY(builder:flatbuffers.Builder, movementY:number) {
+  builder.addFieldInt16(1, movementY, 0);
+}
+
+static endMouseMovement(builder:flatbuffers.Builder):flatbuffers.Offset {
+  const offset = builder.endObject();
+  return offset;
+}
+
+static createMouseMovement(builder:flatbuffers.Builder, movementX:number, movementY:number):flatbuffers.Offset {
+  MouseMovement.startMouseMovement(builder);
+  MouseMovement.addMovementX(builder, movementX);
+  MouseMovement.addMovementY(builder, movementY);
+  return MouseMovement.endMouseMovement(builder);
+}
+}

frontend/src/lib/proto/input.ts

@@ -0,0 +1,5 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */
+
+export * as InputEvent from './input-event.js';

frontend/src/routes/getStreamData.ts

@@ -37,15 +37,15 @@ export async function getStreamData(appId: number, server_name: string): Promise
       server: server_name,
       server_mode: {
         fps: 60,
-        width: 1280,
-        height: 720,
+        width: 1920,
+        height: 1080,
       },
       stream_config: {
-        bitrate_kbps: 5120,
+        bitrate_kbps: 10240,
         mode: {
           fps: 60,
-          width: 1280,
-          height: 720,
+          width: 1920,
+          height: 1080,
         }
       }
     };

frontend/src/routes/stream/Stream.svelte

@@ -1,6 +1,6 @@
 <script lang="ts">
 	import { onMount } from 'svelte';
-	import { streamUrl } from './stream';
+	import { startWebtransportStream } from './stream';
 	import StreamUi from './StreamUi.svelte';
 
 	interface Props {
@@ -15,8 +15,7 @@
 	let gameplayCanvas: HTMLCanvasElement;
 
 	async function startStream() {
-		console.log(`Connecting to stream at ${url} with cert_hash ${certHash}`);
-		await streamUrl(url, certHash, gameplayCanvas);
+		await startWebtransportStream(url, certHash, gameplayCanvas, gameplayCanvas, gameplayCanvas);
 	}
 
 	async function requestFullscreen() {
@@ -29,10 +28,8 @@
 			await document.exitFullscreen();
 			fullscreen = false;
 		} else {
-			if (gameplayView.requestFullscreen) {
-				await gameplayView.requestFullscreen();
-				fullscreen = true;
-			}
+			await gameplayCanvas.requestFullscreen();
+			fullscreen = true;
 		}
 	}
 
@@ -42,7 +39,8 @@
 </script>
 
 <div id="gameplay-view" class="gameplay-view" bind:this={gameplayView}>
-	<canvas id="gamestream-canvas" class="gamestream-canvas" bind:this={gameplayCanvas}></canvas>
+	<canvas id="gamestream-canvas" class="gamestream-canvas" bind:this={gameplayCanvas} tabindex="0"
+	></canvas>
 	<StreamUi fullscreenFunc={requestFullscreen}></StreamUi>
 </div>
 
@@ -56,6 +54,9 @@
 		background-color: black;
 
 		position: relative;
+
+		display: flex;
+		flex-direction: column;
 	}
 
 	.gamestream-canvas {
@@ -65,6 +66,6 @@
 		position: relative;
 		margin: auto;
 
-		z-index: 1;
+		/*z-index: 1;*/
 	}
 </style>

frontend/src/routes/stream/StreamUi.svelte

@@ -10,11 +10,11 @@
 
 <div class="stream-controls">
 	<div style="flex-grow: 1"></div>
-	<div class="bar-button" role="button" tabindex="0">
+	<div class="bar-button" role="button">
 		<Settings size="100%" />
 	</div>
 	<div style="width: 0.5%"></div>
-	<div class="bar-button" onclick={fullscreenFunc} role="button" tabindex="0">
+	<div class="bar-button" onclick={fullscreenFunc} role="button">
 		<Maximize size="100%" />
 	</div>
 	<div style="width: 0.5%"></div>
@@ -41,12 +41,12 @@
 		height: 5%;
 		width: 100%;
 
-		position: absolute;
+		/*position: absolute;*/
 
 		bottom: 0;
 		left: 0;
 
-		z-index: 2;
+		/*z-index: 2;*/
 
 		border-top-color: oklch(0.454 0 360);
 		border-top-width: 2px;

frontend/src/routes/stream/stream.ts

@@ -1,3 +1,5 @@
+import { sendKeyboardEvent, sendMouseInputEvent, sendMouseMoveEvent, KeyAction } from "$lib/input"
+
 type Setup = {
   video_format: string,
   width: number,
@@ -26,9 +28,8 @@ type DecodeUnitPacket = {
   DecodeUnit: DecodeUnit
 }
 
-async function connectToUrl(url: string, certHash: Uint8Array): Promise<WebTransport> {
-  console.log('Hash: ', certHash);
-  console.log(`Connecting to stream`);
+export async function getStreamTransport(url: string, certHash: Array<number>): Promise<WebTransport> {
+  let certHashArray = new Uint8Array(certHash);
 
   // Check if WebTransport is supported
   if (!window.WebTransport) {
@@ -39,7 +40,7 @@ async function connectToUrl(url: string, certHash: Uint8Array): Promise<WebTrans
     serverCertificateHashes: [
       {
         algorithm: "sha-256",
-        value: certHash,
+        value: certHashArray,
       }
     ]
   });
@@ -84,20 +85,12 @@ function parseData(newBuffer: Uint8Array, oldBuffer: Uint8Array): [Array<Object>
   return [packets, unparsedData];
 }
 
-export async function streamUrl(url: string, certHash: Array<number>, canvasElement: HTMLCanvasElement) {
-  const certHashArray = new Uint8Array(certHash);
-
-  const transport = await connectToUrl(url, certHashArray);
-  console.log('Creating WebTransport bidirectional stream');
-  const stream = await transport.createBidirectionalStream();
-  console.log('Bidirectional stream created');
-
+export async function streamVideoFromReader(reader: ReadableStreamDefaultReader, canvasElement: HTMLCanvasElement) {
   const canvasCtx: CanvasRenderingContext2D | null = canvasElement.getContext('2d');
   if (canvasCtx == null) {
     throw new Error(`Could not get 2d canvas context`);
   }
 
-  const reader = stream.readable.getReader();
   try {
     let unparsedData = new Uint8Array();
 
@@ -171,16 +164,6 @@ export async function streamUrl(url: string, certHash: Array<number>, canvasElem
 
     }
 
-    // Handle connection close
-    transport.closed.then(() => {
-      console.log('WebTransport connection closed');
-    }).catch((error) => {
-      console.error('WebTransport connection closed with error:', error);
-    });
-
-    // You can add more WebTransport handling logic here
-    // For example, handling incoming streams, sending data, etc.
-
   } catch (e) {
     var error = <Error>e;
     console.error('Error connecting to stream:', error);
@@ -188,3 +171,55 @@ export async function streamUrl(url: string, certHash: Array<number>, canvasElem
   }
 }
 
+
+
+
+
+export async function startWebtransportStream(
+  url: string,
+  certHash: Array<number>,
+  gameplayCanvas: HTMLCanvasElement,
+  keyEventElement: HTMLElement,
+  mouseElement: HTMLElement,
+) {
+  console.log(`Connecting to stream at ${url} with cert_hash ${certHash}`);
+  const transport = await getStreamTransport(url, certHash);
+
+  const stream = await transport.createBidirectionalStream();
+
+  const reader = stream.readable.getReader();
+  const writer = stream.writable.getWriter();
+
+  keyEventElement.addEventListener("keydown", (event: KeyboardEvent) => { sendKeyboardEvent(writer, event, KeyAction.DOWN) });
+  keyEventElement.addEventListener("keyup", (event: KeyboardEvent) => { sendKeyboardEvent(writer, event, KeyAction.UP) });
+
+  mouseElement.addEventListener("mousedown", (event: MouseEvent) => { sendMouseInputEvent(writer, event, KeyAction.DOWN) });
+  mouseElement.addEventListener("mouseup", (event: MouseEvent) => { sendMouseInputEvent(writer, event, KeyAction.UP) });
+
+
+  mouseElement.addEventListener("click", async () => {
+    console.log("Requesting pointer lock");
+    await mouseElement.requestPointerLock({
+      unadjustedMovement: true,
+    });
+    console.log("Pointer lock aquired");
+  })
+
+  mouseElement.addEventListener("mousemove", (event: MouseEvent) => {
+    if (document.pointerLockElement === mouseElement) {
+      sendMouseMoveEvent(writer, event);
+    }
+  });
+
+
+  await streamVideoFromReader(reader, gameplayCanvas);
+
+  // Handle connection close
+  transport.closed
+    .then(() => {
+      console.log('WebTransport connection closed');
+    })
+    .catch((error) => {
+      console.error('WebTransport connection closed with error:', error);
+    });
+}

gamestream-webtransport-proxy/Cargo.toml

@@ -6,6 +6,7 @@ edition = "2024"
 [dependencies]
 anyhow = "1.0.98"
 directories = "6.0.0"
+flatbuffers = "25.2.10"
 getrandom = { version = "0.3.3", features = ["std"] }
 hex = "0.4.3"
 hmac-sha256 = "1.1.12"

gamestream-webtransport-proxy/src/gamestream/decoder.rs

@@ -256,7 +256,6 @@ extern "C" fn start_cb() {
 }
 
 extern "C" fn submit_decode_unit_cb(decode_unit: PDECODE_UNIT) -> std::os::raw::c_int {
-    debug!("SUBMIT DECODE UNIT CB");
     if decode_unit.is_null() {
         error!("Decode unit pointer was null");
         return -1;

gamestream-webtransport-proxy/src/gamestream/mod.rs

@@ -20,7 +20,7 @@ pub fn start_connection(
         address,
         stream.server_codec_mode_support,
     )?;
-    let mut stream_config = config::stream_config(&stream);
+    let mut stream_config = config::stream_config(stream);
     let mut listener_callbacks = config::listener_callbacks();
     let (mut decoder_callbacks, decoder_rx) = decoder::decoder_callbacks()?;
     let ret;
@@ -47,3 +47,31 @@ pub fn start_connection(
 pub fn stop_connection() {
     unsafe { moonlight_common_c_sys::LiStopConnection() };
 }
+
+pub fn send_keyboard_event(keycode: i16, keyaction: i8, modifiers: u8) -> Result<()> {
+    let ret =
+        unsafe { moonlight_common_c_sys::LiSendKeyboardEvent(keycode, keyaction, modifiers as i8) };
+
+    match ret {
+        0 => Ok(()),
+        _ => Err(anyhow!("Could not send keyboard event: {ret}")),
+    }
+}
+
+pub fn send_mouse_move_event(movement_x: i16, movement_y: i16) -> Result<()> {
+    let ret = unsafe { moonlight_common_c_sys::LiSendMouseMoveEvent(movement_x, movement_y) };
+
+    match ret {
+        0 => Ok(()),
+        _ => Err(anyhow!("Could not send mouse movement event: {ret}")),
+    }
+}
+
+pub fn send_mouse_input_event(action: i8, button: i32) -> Result<()> {
+    let ret = unsafe { moonlight_common_c_sys::LiSendMouseButtonEvent(action, button) };
+
+    match ret {
+        0 => Ok(()),
+        _ => Err(anyhow!("Could not send mouse input event: {ret}")),
+    }
+}

gamestream-webtransport-proxy/src/proxy/input/mod.rs

@@ -0,0 +1 @@
+pub mod input_generated;

gamestream-webtransport-proxy/src/proxy/mod.rs

@@ -1,11 +1,12 @@
 use anyhow::{Context, Result};
-use salvo::{conn::http1::Connection, hyper::body::Bytes, proto::WebTransportSession};
 use tokio::{io::AsyncReadExt, io::AsyncWriteExt, select, sync::RwLock};
 use tracing::{debug, error, info};
 
 use crate::{backend, gamestream};
 
 pub mod handler;
+mod input;
+mod packet_parser;
 
 pub struct Proxy {
     pub cert_hash: [u8; 32],
@@ -40,14 +41,16 @@ async fn proxy_main(
 
     let mut channels = spawn_gamestream(stream).await?;
 
-    let mut buffer = vec![0; 65536].into_boxed_slice();
+    let mut packet_buffer = packet_parser::PacketBuffer::new();
+    //let mut buffer = vec![0; 65536].into_boxed_slice();
+    let mut buffer = [0u8; 65536];
+    //let mut buffer = vec![0; 65536].into_boxed_slice();
 
     loop {
         select! {
             gamestream_packet = channels.gamestream_channels.decoder_rx.recv() => {
                 match gamestream_packet {
                     Some(frame) => {
-                        info!("Got decoder packet");
                         let frame_json = serde_json::to_vec(&frame)?;
                         let frame_json_len: u32 = <u32>::try_from(frame_json.len())?;
 
@@ -60,8 +63,11 @@ async fn proxy_main(
                     }
                 }
             },
-            webtransport_packet = wt_recv.read(&mut buffer) => {
-                info!("Got packet from client");
+            ret = wt_recv.read(&mut buffer) => {
+                let bytes_read = ret?;
+
+                packet_parser::handle_client_packet(&mut packet_buffer, &buffer[..bytes_read])?;
+
             }
         }
     }

gamestream-webtransport-proxy/src/proxy/packet_parser.rs

@@ -0,0 +1,158 @@
+use anyhow::{Result, anyhow};
+use tracing::{debug, error, info};
+
+use super::input::input_generated::input_event;
+use crate::gamestream;
+
+pub struct PacketBuffer {
+    buffer: Vec<u8>,
+    expected_length: Option<usize>,
+    length_bytes_needed: usize,
+}
+
+impl PacketBuffer {
+    pub fn new() -> Self {
+        Self {
+            buffer: Vec::new(),
+            expected_length: None,
+            length_bytes_needed: 4, // Assuming u32 length (4 bytes)
+        }
+    }
+
+    /// Process incoming data and return complete packets
+    pub fn process_data(&mut self, data: &[u8]) -> Vec<Vec<u8>> {
+        let mut complete_packets = Vec::new();
+        self.buffer.extend_from_slice(data);
+
+        loop {
+            // If we don't know the expected length yet, try to read it
+            if self.expected_length.is_none() {
+                if self.buffer.len() >= self.length_bytes_needed {
+                    // Read length from first 4 bytes (adjust as needed)
+                    let length = u32::from_be_bytes([
+                        self.buffer[0],
+                        self.buffer[1],
+                        self.buffer[2],
+                        self.buffer[3],
+                    ]) as usize;
+
+                    self.expected_length = Some(length);
+                } else {
+                    // Not enough data to read length yet
+                    break;
+                }
+            }
+
+            // We know the expected length, check if we have a complete packet
+            if let Some(expected_len) = self.expected_length {
+                let total_packet_size = self.length_bytes_needed + expected_len;
+
+                if self.buffer.len() >= total_packet_size {
+                    // We have a complete packet - extract just the data portion
+                    let packet_data =
+                        self.buffer[self.length_bytes_needed..total_packet_size].to_vec();
+                    complete_packets.push(packet_data);
+
+                    // Remove the processed packet from buffer
+                    self.buffer.drain(0..total_packet_size);
+                    self.expected_length = None;
+                } else {
+                    // Packet is incomplete
+                    break;
+                }
+            }
+        }
+
+        complete_packets
+    }
+}
+
+pub fn process_keyboard_event(input_event: input_event::InputEvent) -> Result<()> {
+    let Some(table) = input_event.input() else {
+        debug!("Keyboard event table was empty, ignoring");
+        return Err(anyhow!("Keyboard event table was empty"));
+    };
+    let keyboard_event = unsafe { input_event::KeyboardInput::init_from_table(table) };
+
+    let key_action_i8 = match keyboard_event.key_action() {
+        input_event::KeyAction::DOWN => moonlight_common_c_sys::KEY_ACTION_DOWN,
+        input_event::KeyAction::UP => moonlight_common_c_sys::KEY_ACTION_UP,
+        _ => {
+            debug!("Invalid KeyAction value, ignoring");
+            return Err(anyhow!("Invalid KeyAction value"));
+        }
+    };
+
+    gamestream::send_keyboard_event(keyboard_event.key_code(), key_action_i8, 0)
+}
+
+pub fn process_mouse_move_event(input_event: input_event::InputEvent) -> Result<()> {
+    let Some(table) = input_event.input() else {
+        debug!("MouseMovement event table was empty, ignoring");
+        return Err(anyhow!("MouseMovement event table was empty"));
+    };
+    let mouse_event = unsafe { input_event::MouseMovement::init_from_table(table) };
+
+    gamestream::send_mouse_move_event(mouse_event.movement_x(), mouse_event.movement_y())
+}
+
+pub fn process_mouse_input_event(input_event: input_event::InputEvent) -> Result<()> {
+    let Some(table) = input_event.input() else {
+        debug!("MouseInput event table was empty, ignoring");
+        return Err(anyhow!("MouseInput event table was empty"));
+    };
+    let mouse_event = unsafe { input_event::MouseInput::init_from_table(table) };
+
+    let button_action_i8 = match mouse_event.button_action() {
+        input_event::KeyAction::DOWN => moonlight_common_c_sys::BUTTON_ACTION_PRESS,
+        input_event::KeyAction::UP => moonlight_common_c_sys::BUTTON_ACTION_RELEASE,
+        _ => {
+            error!("Invalid KeyAction value, ignoring",);
+            return Err(anyhow!("Invalid KeyAction value"));
+        }
+    };
+
+    let mouse_button_i32 = match mouse_event.button() {
+        input_event::MouseButton::LEFT => moonlight_common_c_sys::BUTTON_LEFT,
+        input_event::MouseButton::MIDDLE => moonlight_common_c_sys::BUTTON_MIDDLE,
+        input_event::MouseButton::RIGHT => moonlight_common_c_sys::BUTTON_RIGHT,
+        input_event::MouseButton::X1 => moonlight_common_c_sys::BUTTON_X1,
+        input_event::MouseButton::X2 => moonlight_common_c_sys::BUTTON_X2,
+        _ => {
+            error!("Invalid MouseButton value, ignoring",);
+            return Err(anyhow!("Invalid MouseButton value"));
+        }
+    };
+
+    gamestream::send_mouse_input_event(button_action_i8, mouse_button_i32)
+}
+
+pub fn handle_client_packet(packet_buffer: &mut PacketBuffer, buffer: &[u8]) -> anyhow::Result<()> {
+    let complete_packets = packet_buffer.process_data(buffer);
+
+    // TODO: only supports input packets. this should use a union wrapper or something
+    for packet_data in complete_packets {
+        let input_event = input_event::root_as_input_event(&packet_data)?;
+        match input_event.input_type() {
+            input_event::Input::Keyboard => {
+                process_keyboard_event(input_event)?;
+            }
+            input_event::Input::MouseMovement => {
+                process_mouse_move_event(input_event)?;
+            }
+            input_event::Input::MouseInput => {
+                process_mouse_input_event(input_event)?;
+            }
+
+            input_event::Input::NONE => {
+                debug!("Input event was empty, ignoring.")
+            }
+            _ => {
+                error!("Unknown InputEvent type");
+                return Err(anyhow!("Unknown InputEvent type"));
+            }
+        }
+    }
+
+    Ok(())
+}

moonlight-common-c-sys/build.rs

@@ -12,9 +12,13 @@ struct CustomCallbacks;
 impl ParseCallbacks for CustomCallbacks {
     fn int_macro(&self, name: &str, _value: i64) -> Option<IntKind> {
         match name {
-            "STREAM_CFG_LOCAL" => Some(IntKind::I32),
-            "STREAM_CFG_REMOTE" => Some(IntKind::I32),
-            "STREAM_CFG_AUTO" => Some(IntKind::I32),
+            //"STREAM_CFG_LOCAL" => Some(IntKind::I32),
+            //"STREAM_CFG_REMOTE" => Some(IntKind::I32),
+            //"STREAM_CFG_AUTO" => Some(IntKind::I32),
+            "KEY_ACTION_DOWN" => Some(IntKind::I8),
+            "KEY_ACTION_UP" => Some(IntKind::I8),
+            "BUTTON_ACTION_PRESS" => Some(IntKind::I8),
+            "BUTTON_ACTION_RELEASE" => Some(IntKind::I8),
             _ => None, // Default behavior for all others
         }
     }
@@ -34,7 +38,7 @@ fn main() {
     let bindings = bindgen::Builder::default()
         .header("moonlight-common-c/src/Limelight.h")
         .clang_arg(format!("-I{}/src", dst.display())) // Include built headers
-        //.parse_callbacks(Box::new(CustomCallbacks))
+        .parse_callbacks(Box::new(CustomCallbacks))
         .default_macro_constant_type(bindgen::MacroTypeVariation::Signed)
         .generate()
         .expect("Failed to generate bindings");

schema/input.fbs

@@ -0,0 +1,53 @@
+namespace InputEvent;
+
+
+enum KeyAction: byte {
+  DOWN,
+  UP,
+}
+
+
+table MouseMovement {
+  movement_x: int16;
+  movement_y: int16;
+}
+
+enum MouseButton: byte {
+  LEFT,
+  MIDDLE,
+  RIGHT,
+  X1,
+  X2
+}
+
+table MouseInput {
+  button: MouseButton;
+  button_action: KeyAction;
+}
+
+
+struct ModifierState {
+  shift: bool;
+  ctrl: bool;
+  alt: bool;
+  meta: bool;
+}
+
+table KeyboardInput {
+  key_code: int16;
+  key_action: KeyAction;
+  modifiers: ModifierState;
+}
+
+union Input {
+  Keyboard:KeyboardInput,
+  MouseMovement:MouseMovement,
+  MouseInput:MouseInput,
+}
+
+table InputEvent {
+  input: Input;
+}
+
+root_type InputEvent;
+