backend and frontend: support out of order chunks + now it's performant on chrome

- changed the backend to append the buffers together and dispatch them
  as such (this is what moonlight-embedded does)
- fixed the frontend trying to playback an empty buffer because types
  are hard (this is why I should have used protobuf...)

Cargo.lock

@@ -537,6 +537,16 @@ version = "2.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
 
+[[package]]
+name = "flatbuffers"
+version = "25.2.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1045398c1bfd89168b5fd3f1fc11f6e70b34f6f66300c87d44d3de849463abf1"
+dependencies = [
+ "bitflags",
+ "rustc_version",
+]
+
 [[package]]
 name = "fnv"
 version = "1.0.7"
@@ -668,9 +678,13 @@ version = "0.1.0"
 dependencies = [
  "anyhow",
  "directories",
+ "flatbuffers",
  "getrandom 0.3.3",
+ "h3-datagram",
+ "h3-quinn",
  "hex",
  "hmac-sha256",
+ "http",
  "libc",
  "moonlight-common-c-sys",
  "openssl",
@@ -2037,6 +2051,15 @@ version = "2.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d"
 
+[[package]]
+name = "rustc_version"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92"
+dependencies = [
+ "semver",
+]
+
 [[package]]
 name = "rustix"
 version = "1.0.7"
@@ -2442,6 +2465,12 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "semver"
+version = "1.0.26"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "56e6fa9c48d24d85fb3de5ad847117517440f6beceb7798af16b4a87d616b8d0"
+
 [[package]]
 name = "serde"
 version = "1.0.219"

frontend/package.json

@@ -40,5 +40,10 @@
 		"onlyBuiltDependencies": [
 			"esbuild"
 		]
+	},
+	"dependencies": {
+		"flatbuffers": "^25.2.10",
+		"lucide-svelte": "^0.539.0",
+		"svelte-loading-spinners": "^0.3.6"
 	}
 }

frontend/pnpm-lock.yaml

@@ -7,6 +7,16 @@ settings:
 importers:
 
   .:
+    dependencies:
+      flatbuffers:
+        specifier: ^25.2.10
+        version: 25.2.10
+      lucide-svelte:
+        specifier: ^0.539.0
+        version: 0.539.0(svelte@5.36.12)
+      svelte-loading-spinners:
+        specifier: ^0.3.6
+        version: 0.3.6
     devDependencies:
       '@eslint/compat':
         specifier: ^1.2.5
@@ -857,6 +867,9 @@ packages:
     resolution: {integrity: sha512-f7ccFPK3SXFHpx15UIGyRJ/FJQctuKZ0zVuN3frBo4HnK3cay9VEW0R6yPYFHC0AgqhukPzKjq22t5DmAyqGyw==}
     engines: {node: '>=16'}
 
+  flatbuffers@25.2.10:
+    resolution: {integrity: sha512-7JlN9ZvLDG1McO3kbX0k4v+SUAg48L1rIwEvN6ZQl/eCtgJz9UylTMzE9wrmYrcorgxm3CX/3T/w5VAub99UUw==}
+
   flatted@3.3.3:
     resolution: {integrity: sha512-GX+ysw4PBCz0PzosHDepZGANEuFCMLrnRTiEy9McGjmkCQYwRq4A/X786G/fjM/+OjsWSU1ZrY5qyARZmO/uwg==}
 
@@ -1034,6 +1047,11 @@ packages:
   lodash.merge@4.6.2:
     resolution: {integrity: sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==}
 
+  lucide-svelte@0.539.0:
+    resolution: {integrity: sha512-p4k3GOje/9Si1eIkg1W1OQUhozeja5Ka5shjVpfyP5X2ye+B7sfyMnX3d5D2et+MYJwUFGrMna5MIYgq6bLfqw==}
+    peerDependencies:
+      svelte: ^3 || ^4 || ^5.0.0-next.42
+
   magic-string@0.30.17:
     resolution: {integrity: sha512-sNPKHvyjVf7gyjwS4xGTaW/mCnF8wnjtifKBEhxfZ7E/S8tQ0rssrwGNn6q8JH/ohItJfSQp9mBtQYuTlH5QnA==}
 
@@ -1307,6 +1325,9 @@ packages:
       svelte:
         optional: true
 
+  svelte-loading-spinners@0.3.6:
+    resolution: {integrity: sha512-mthHQ2TwiwzTWzbFry3CBnVEfzqPOD9WkVw84OfSYzHRq6N9wgQ+yv37u81uPeuLU/ZOIPqhujpXquB1aol5ZQ==}
+
   svelte@5.36.12:
     resolution: {integrity: sha512-c3mWT+b0yBLl3gPGSHiy4pdSQCsPNTjLC0tVoOhrGJ6PPfCzD/RQpAmAfJtQZ304CAae2ph+L3C4aqds3R3seQ==}
     engines: {node: '>=18'}
@@ -2152,6 +2173,8 @@ snapshots:
       flatted: 3.3.3
       keyv: 4.5.4
 
+  flatbuffers@25.2.10: {}
+
   flatted@3.3.3: {}
 
   fsevents@2.3.3:
@@ -2280,6 +2303,10 @@ snapshots:
 
   lodash.merge@4.6.2: {}
 
+  lucide-svelte@0.539.0(svelte@5.36.12):
+    dependencies:
+      svelte: 5.36.12
+
   magic-string@0.30.17:
     dependencies:
       '@jridgewell/sourcemap-codec': 1.5.4
@@ -2480,6 +2507,8 @@ snapshots:
     optionalDependencies:
       svelte: 5.36.12
 
+  svelte-loading-spinners@0.3.6: {}
+
   svelte@5.36.12:
     dependencies:
       '@ampproject/remapping': 2.3.0

frontend/src/lib/canvas.worker.ts

@@ -0,0 +1,24 @@
+import { streamVideoFromReader } from "./video";
+
+let canvas: OffscreenCanvas | undefined = undefined;
+let reader: ReadableStream | undefined = undefined;
+let resolver: any = undefined;
+const messagePromise = new Promise((resolve) => resolver = resolve);
+
+
+self.onmessage = async ({ data: payload }: { data: any }) => {
+  canvas = payload["canvas"];
+  reader = payload["reader"];
+  self.onmessage = null;
+  resolver?.();
+};
+
+console.log(`Worker spawned successfully`)
+await messagePromise;
+console.log("Worker received required objects")
+console.log("Starting video streaming")
+
+await streamVideoFromReader(reader!.getReader(), canvas!);
+
+
+export { }; // This makes TypeScript happy

frontend/src/lib/input.ts

@@ -0,0 +1,152 @@
+import { Input } from "$lib/proto/input-event"
+import { InputEvent } from "$lib/proto/input"
+import { Builder } from "flatbuffers";
+
+export enum KeyAction {
+  DOWN,
+  UP,
+}
+
+export async function sendKeyboardEvent(
+  writer: WritableStreamDefaultWriter,
+  event: KeyboardEvent,
+  keyAction: KeyAction,
+) {
+  const builder = new Builder(1024);
+
+  InputEvent.KeyboardInput.startKeyboardInput(builder);
+  // Have to use keyCode even though it's deprecated beacuse .code is a unhelpful string :sigh:
+  InputEvent.KeyboardInput.addKeyCode(builder, event.keyCode);
+  InputEvent.KeyboardInput.addKeyAction(builder, keyAction);
+  const modifierState = InputEvent.ModifierState.createModifierState(
+    builder,
+    event.shiftKey,
+    event.ctrlKey,
+    event.altKey,
+    event.metaKey
+  );
+
+  InputEvent.KeyboardInput.addModifiers(builder, modifierState);
+  const keyboardInput = InputEvent.KeyboardInput.endKeyboardInput(builder);
+
+  // Create InputEvent table with union
+  InputEvent.InputEvent.startInputEvent(builder);
+  InputEvent.InputEvent.addInputType(builder, Input.Keyboard);
+  InputEvent.InputEvent.addInput(builder, keyboardInput);
+  const inputEvent = InputEvent.InputEvent.endInputEvent(builder);
+
+  // Finish the buffer
+  builder.finish(inputEvent);
+
+  // Get the buffer data
+  const bufferData = builder.asUint8Array();
+
+  // Create size prefix as big-endian u32
+  const sizePrefix = new Uint8Array(4);
+  const size = bufferData.length;
+  sizePrefix[0] = (size >> 24) & 0xFF;  // Most significant byte
+  sizePrefix[1] = (size >> 16) & 0xFF;
+  sizePrefix[2] = (size >> 8) & 0xFF;
+  sizePrefix[3] = size & 0xFF;          // Least significant byte
+
+  // Create combined buffer with size prefix + flatbuffer data
+  const combined = new Uint8Array(sizePrefix.length + bufferData.length);
+  combined.set(sizePrefix, 0);
+  combined.set(bufferData, sizePrefix.length);
+
+  // Write to stream
+  await writer.write(combined);
+}
+
+export async function sendMouseMoveEvent(
+  writer: WritableStreamDefaultWriter,
+  event: MouseEvent,
+) {
+  const builder = new Builder(1024);
+
+  const mouseMovement = InputEvent.MouseMovement.createMouseMovement(builder, event.movementX, event.movementY);
+
+  // Create InputEvent table with union
+  InputEvent.InputEvent.startInputEvent(builder);
+  InputEvent.InputEvent.addInputType(builder, Input.MouseMovement);
+  InputEvent.InputEvent.addInput(builder, mouseMovement);
+  const inputEvent = InputEvent.InputEvent.endInputEvent(builder);
+
+  // Finish the buffer
+  builder.finish(inputEvent);
+
+  // Get the buffer data
+  const bufferData = builder.asUint8Array();
+
+  // Create size prefix as big-endian u32
+  const sizePrefix = new Uint8Array(4);
+  const size = bufferData.length;
+  sizePrefix[0] = (size >> 24) & 0xFF;  // Most significant byte
+  sizePrefix[1] = (size >> 16) & 0xFF;
+  sizePrefix[2] = (size >> 8) & 0xFF;
+  sizePrefix[3] = size & 0xFF;          // Least significant byte
+
+  // Create combined buffer with size prefix + flatbuffer data
+  const combined = new Uint8Array(sizePrefix.length + bufferData.length);
+  combined.set(sizePrefix, 0);
+  combined.set(bufferData, sizePrefix.length);
+
+  // Write to stream
+  await writer.write(combined);
+}
+
+export async function sendMouseInputEvent(
+  writer: WritableStreamDefaultWriter,
+  event: MouseEvent,
+  keyAction: KeyAction,
+) {
+  const builder = new Builder(1024);
+
+  var mouseButton: InputEvent.MouseButton;
+
+  switch (event.button) {
+    case 0:
+      mouseButton = InputEvent.MouseButton.LEFT;
+      break;
+    case 1:
+      mouseButton = InputEvent.MouseButton.MIDDLE;
+      break;
+    case 2:
+      mouseButton = InputEvent.MouseButton.RIGHT;
+      break;
+    default:
+      console.log(`Unknown mouse button ${event.button}, defaulting to LEFT`);
+      mouseButton = InputEvent.MouseButton.LEFT;
+      break;
+  }
+
+  const mouseInput = InputEvent.MouseInput.createMouseInput(builder, mouseButton, keyAction);
+
+  // Create InputEvent table with union
+  InputEvent.InputEvent.startInputEvent(builder);
+  InputEvent.InputEvent.addInputType(builder, Input.MouseInput);
+  InputEvent.InputEvent.addInput(builder, mouseInput);
+  const inputEvent = InputEvent.InputEvent.endInputEvent(builder);
+
+  // Finish the buffer
+  builder.finish(inputEvent);
+
+  // Get the buffer data
+  const bufferData = builder.asUint8Array();
+
+  // Create size prefix as big-endian u32
+  const sizePrefix = new Uint8Array(4);
+  const size = bufferData.length;
+  sizePrefix[0] = (size >> 24) & 0xFF;  // Most significant byte
+  sizePrefix[1] = (size >> 16) & 0xFF;
+  sizePrefix[2] = (size >> 8) & 0xFF;
+  sizePrefix[3] = size & 0xFF;          // Least significant byte
+
+  // Create combined buffer with size prefix + flatbuffer data
+  const combined = new Uint8Array(sizePrefix.length + bufferData.length);
+  combined.set(sizePrefix, 0);
+  combined.set(bufferData, sizePrefix.length);
+
+  // Write to stream
+  await writer.write(combined);
+}

frontend/src/lib/proto/input-event.ts

@@ -0,0 +1,12 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */
+
+export { Input } from './input-event/input.js';
+export { InputEvent } from './input-event/input-event.js';
+export { KeyAction } from './input-event/key-action.js';
+export { KeyboardInput } from './input-event/keyboard-input.js';
+export { ModifierState } from './input-event/modifier-state.js';
+export { MouseButton } from './input-event/mouse-button.js';
+export { MouseInput } from './input-event/mouse-input.js';
+export { MouseMovement } from './input-event/mouse-movement.js';

frontend/src/lib/proto/input-event/input-event.ts

@@ -0,0 +1,69 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */
+
+import * as flatbuffers from 'flatbuffers';
+
+import { Input, unionToInput, unionListToInput } from '../input-event/input.js';
+
+
+export class InputEvent {
+  bb: flatbuffers.ByteBuffer|null = null;
+  bb_pos = 0;
+  __init(i:number, bb:flatbuffers.ByteBuffer):InputEvent {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+}
+
+static getRootAsInputEvent(bb:flatbuffers.ByteBuffer, obj?:InputEvent):InputEvent {
+  return (obj || new InputEvent()).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+}
+
+static getSizePrefixedRootAsInputEvent(bb:flatbuffers.ByteBuffer, obj?:InputEvent):InputEvent {
+  bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH);
+  return (obj || new InputEvent()).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+}
+
+inputType():Input {
+  const offset = this.bb!.__offset(this.bb_pos, 4);
+  return offset ? this.bb!.readUint8(this.bb_pos + offset) : Input.NONE;
+}
+
+input<T extends flatbuffers.Table>(obj:any):any|null {
+  const offset = this.bb!.__offset(this.bb_pos, 6);
+  return offset ? this.bb!.__union(obj, this.bb_pos + offset) : null;
+}
+
+static startInputEvent(builder:flatbuffers.Builder) {
+  builder.startObject(2);
+}
+
+static addInputType(builder:flatbuffers.Builder, inputType:Input) {
+  builder.addFieldInt8(0, inputType, Input.NONE);
+}
+
+static addInput(builder:flatbuffers.Builder, inputOffset:flatbuffers.Offset) {
+  builder.addFieldOffset(1, inputOffset, 0);
+}
+
+static endInputEvent(builder:flatbuffers.Builder):flatbuffers.Offset {
+  const offset = builder.endObject();
+  return offset;
+}
+
+static finishInputEventBuffer(builder:flatbuffers.Builder, offset:flatbuffers.Offset) {
+  builder.finish(offset);
+}
+
+static finishSizePrefixedInputEventBuffer(builder:flatbuffers.Builder, offset:flatbuffers.Offset) {
+  builder.finish(offset, undefined, true);
+}
+
+static createInputEvent(builder:flatbuffers.Builder, inputType:Input, inputOffset:flatbuffers.Offset):flatbuffers.Offset {
+  InputEvent.startInputEvent(builder);
+  InputEvent.addInputType(builder, inputType);
+  InputEvent.addInput(builder, inputOffset);
+  return InputEvent.endInputEvent(builder);
+}
+}

frontend/src/lib/proto/input-event/input.ts

@@ -0,0 +1,42 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */
+
+import { KeyboardInput } from '../input-event/keyboard-input.js';
+import { MouseInput } from '../input-event/mouse-input.js';
+import { MouseMovement } from '../input-event/mouse-movement.js';
+
+
+export enum Input {
+  NONE = 0,
+  Keyboard = 1,
+  MouseMovement = 2,
+  MouseInput = 3
+}
+
+export function unionToInput(
+  type: Input,
+  accessor: (obj:KeyboardInput|MouseInput|MouseMovement) => KeyboardInput|MouseInput|MouseMovement|null
+): KeyboardInput|MouseInput|MouseMovement|null {
+  switch(Input[type]) {
+    case 'NONE': return null; 
+    case 'Keyboard': return accessor(new KeyboardInput())! as KeyboardInput;
+    case 'MouseMovement': return accessor(new MouseMovement())! as MouseMovement;
+    case 'MouseInput': return accessor(new MouseInput())! as MouseInput;
+    default: return null;
+  }
+}
+
+export function unionListToInput(
+  type: Input, 
+  accessor: (index: number, obj:KeyboardInput|MouseInput|MouseMovement) => KeyboardInput|MouseInput|MouseMovement|null, 
+  index: number
+): KeyboardInput|MouseInput|MouseMovement|null {
+  switch(Input[type]) {
+    case 'NONE': return null; 
+    case 'Keyboard': return accessor(index, new KeyboardInput())! as KeyboardInput;
+    case 'MouseMovement': return accessor(index, new MouseMovement())! as MouseMovement;
+    case 'MouseInput': return accessor(index, new MouseInput())! as MouseInput;
+    default: return null;
+  }
+}

frontend/src/lib/proto/input-event/key-action.ts

@@ -0,0 +1,8 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */
+
+export enum KeyAction {
+  DOWN = 0,
+  UP = 1
+}

frontend/src/lib/proto/input-event/keyboard-input.ts

@@ -0,0 +1,65 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */
+
+import * as flatbuffers from 'flatbuffers';
+
+import { KeyAction } from '../input-event/key-action.js';
+import { ModifierState } from '../input-event/modifier-state.js';
+
+
+export class KeyboardInput {
+  bb: flatbuffers.ByteBuffer|null = null;
+  bb_pos = 0;
+  __init(i:number, bb:flatbuffers.ByteBuffer):KeyboardInput {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+}
+
+static getRootAsKeyboardInput(bb:flatbuffers.ByteBuffer, obj?:KeyboardInput):KeyboardInput {
+  return (obj || new KeyboardInput()).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+}
+
+static getSizePrefixedRootAsKeyboardInput(bb:flatbuffers.ByteBuffer, obj?:KeyboardInput):KeyboardInput {
+  bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH);
+  return (obj || new KeyboardInput()).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+}
+
+keyCode():number {
+  const offset = this.bb!.__offset(this.bb_pos, 4);
+  return offset ? this.bb!.readInt16(this.bb_pos + offset) : 0;
+}
+
+keyAction():KeyAction {
+  const offset = this.bb!.__offset(this.bb_pos, 6);
+  return offset ? this.bb!.readInt8(this.bb_pos + offset) : KeyAction.DOWN;
+}
+
+modifiers(obj?:ModifierState):ModifierState|null {
+  const offset = this.bb!.__offset(this.bb_pos, 8);
+  return offset ? (obj || new ModifierState()).__init(this.bb_pos + offset, this.bb!) : null;
+}
+
+static startKeyboardInput(builder:flatbuffers.Builder) {
+  builder.startObject(3);
+}
+
+static addKeyCode(builder:flatbuffers.Builder, keyCode:number) {
+  builder.addFieldInt16(0, keyCode, 0);
+}
+
+static addKeyAction(builder:flatbuffers.Builder, keyAction:KeyAction) {
+  builder.addFieldInt8(1, keyAction, KeyAction.DOWN);
+}
+
+static addModifiers(builder:flatbuffers.Builder, modifiersOffset:flatbuffers.Offset) {
+  builder.addFieldStruct(2, modifiersOffset, 0);
+}
+
+static endKeyboardInput(builder:flatbuffers.Builder):flatbuffers.Offset {
+  const offset = builder.endObject();
+  return offset;
+}
+
+}

frontend/src/lib/proto/input-event/modifier-state.ts

@@ -0,0 +1,45 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */
+
+import * as flatbuffers from 'flatbuffers';
+
+export class ModifierState {
+  bb: flatbuffers.ByteBuffer|null = null;
+  bb_pos = 0;
+  __init(i:number, bb:flatbuffers.ByteBuffer):ModifierState {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+}
+
+shift():boolean {
+  return !!this.bb!.readInt8(this.bb_pos);
+}
+
+ctrl():boolean {
+  return !!this.bb!.readInt8(this.bb_pos + 1);
+}
+
+alt():boolean {
+  return !!this.bb!.readInt8(this.bb_pos + 2);
+}
+
+meta():boolean {
+  return !!this.bb!.readInt8(this.bb_pos + 3);
+}
+
+static sizeOf():number {
+  return 4;
+}
+
+static createModifierState(builder:flatbuffers.Builder, shift: boolean, ctrl: boolean, alt: boolean, meta: boolean):flatbuffers.Offset {
+  builder.prep(1, 4);
+  builder.writeInt8(Number(Boolean(meta)));
+  builder.writeInt8(Number(Boolean(alt)));
+  builder.writeInt8(Number(Boolean(ctrl)));
+  builder.writeInt8(Number(Boolean(shift)));
+  return builder.offset();
+}
+
+}

frontend/src/lib/proto/input-event/mouse-button.ts

@@ -0,0 +1,11 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */
+
+export enum MouseButton {
+  LEFT = 0,
+  MIDDLE = 1,
+  RIGHT = 2,
+  X1 = 3,
+  X2 = 4
+}

frontend/src/lib/proto/input-event/mouse-input.ts

@@ -0,0 +1,62 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */
+
+import * as flatbuffers from 'flatbuffers';
+
+import { KeyAction } from '../input-event/key-action.js';
+import { MouseButton } from '../input-event/mouse-button.js';
+
+
+export class MouseInput {
+  bb: flatbuffers.ByteBuffer|null = null;
+  bb_pos = 0;
+  __init(i:number, bb:flatbuffers.ByteBuffer):MouseInput {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+}
+
+static getRootAsMouseInput(bb:flatbuffers.ByteBuffer, obj?:MouseInput):MouseInput {
+  return (obj || new MouseInput()).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+}
+
+static getSizePrefixedRootAsMouseInput(bb:flatbuffers.ByteBuffer, obj?:MouseInput):MouseInput {
+  bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH);
+  return (obj || new MouseInput()).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+}
+
+button():MouseButton {
+  const offset = this.bb!.__offset(this.bb_pos, 4);
+  return offset ? this.bb!.readInt8(this.bb_pos + offset) : MouseButton.LEFT;
+}
+
+buttonAction():KeyAction {
+  const offset = this.bb!.__offset(this.bb_pos, 6);
+  return offset ? this.bb!.readInt8(this.bb_pos + offset) : KeyAction.DOWN;
+}
+
+static startMouseInput(builder:flatbuffers.Builder) {
+  builder.startObject(2);
+}
+
+static addButton(builder:flatbuffers.Builder, button:MouseButton) {
+  builder.addFieldInt8(0, button, MouseButton.LEFT);
+}
+
+static addButtonAction(builder:flatbuffers.Builder, buttonAction:KeyAction) {
+  builder.addFieldInt8(1, buttonAction, KeyAction.DOWN);
+}
+
+static endMouseInput(builder:flatbuffers.Builder):flatbuffers.Offset {
+  const offset = builder.endObject();
+  return offset;
+}
+
+static createMouseInput(builder:flatbuffers.Builder, button:MouseButton, buttonAction:KeyAction):flatbuffers.Offset {
+  MouseInput.startMouseInput(builder);
+  MouseInput.addButton(builder, button);
+  MouseInput.addButtonAction(builder, buttonAction);
+  return MouseInput.endMouseInput(builder);
+}
+}

frontend/src/lib/proto/input-event/mouse-movement.ts

@@ -0,0 +1,58 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */
+
+import * as flatbuffers from 'flatbuffers';
+
+export class MouseMovement {
+  bb: flatbuffers.ByteBuffer|null = null;
+  bb_pos = 0;
+  __init(i:number, bb:flatbuffers.ByteBuffer):MouseMovement {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+}
+
+static getRootAsMouseMovement(bb:flatbuffers.ByteBuffer, obj?:MouseMovement):MouseMovement {
+  return (obj || new MouseMovement()).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+}
+
+static getSizePrefixedRootAsMouseMovement(bb:flatbuffers.ByteBuffer, obj?:MouseMovement):MouseMovement {
+  bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH);
+  return (obj || new MouseMovement()).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+}
+
+movementX():number {
+  const offset = this.bb!.__offset(this.bb_pos, 4);
+  return offset ? this.bb!.readInt16(this.bb_pos + offset) : 0;
+}
+
+movementY():number {
+  const offset = this.bb!.__offset(this.bb_pos, 6);
+  return offset ? this.bb!.readInt16(this.bb_pos + offset) : 0;
+}
+
+static startMouseMovement(builder:flatbuffers.Builder) {
+  builder.startObject(2);
+}
+
+static addMovementX(builder:flatbuffers.Builder, movementX:number) {
+  builder.addFieldInt16(0, movementX, 0);
+}
+
+static addMovementY(builder:flatbuffers.Builder, movementY:number) {
+  builder.addFieldInt16(1, movementY, 0);
+}
+
+static endMouseMovement(builder:flatbuffers.Builder):flatbuffers.Offset {
+  const offset = builder.endObject();
+  return offset;
+}
+
+static createMouseMovement(builder:flatbuffers.Builder, movementX:number, movementY:number):flatbuffers.Offset {
+  MouseMovement.startMouseMovement(builder);
+  MouseMovement.addMovementX(builder, movementX);
+  MouseMovement.addMovementY(builder, movementY);
+  return MouseMovement.endMouseMovement(builder);
+}
+}

frontend/src/lib/proto/input.ts

@@ -0,0 +1,5 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */
+
+export * as InputEvent from './input-event.js';

frontend/src/lib/proto/video-update.ts

@@ -0,0 +1,10 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */
+
+export { DecodeUnitBuffer } from './video-update/decode-unit-buffer.js';
+export { DecodeUnitStart } from './video-update/decode-unit-start.js';
+export { FrameType } from './video-update/frame-type.js';
+export { Setup } from './video-update/setup.js';
+export { Update } from './video-update/update.js';
+export { VideoUpdate } from './video-update/video-update.js';

frontend/src/lib/proto/video-update/decode-unit-buffer.ts

@@ -0,0 +1,100 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */
+
+import * as flatbuffers from 'flatbuffers';
+
+export class DecodeUnitBuffer {
+  bb: flatbuffers.ByteBuffer|null = null;
+  bb_pos = 0;
+  __init(i:number, bb:flatbuffers.ByteBuffer):DecodeUnitBuffer {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+}
+
+static getRootAsDecodeUnitBuffer(bb:flatbuffers.ByteBuffer, obj?:DecodeUnitBuffer):DecodeUnitBuffer {
+  return (obj || new DecodeUnitBuffer()).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+}
+
+static getSizePrefixedRootAsDecodeUnitBuffer(bb:flatbuffers.ByteBuffer, obj?:DecodeUnitBuffer):DecodeUnitBuffer {
+  bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH);
+  return (obj || new DecodeUnitBuffer()).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+}
+
+frameNumber():bigint {
+  const offset = this.bb!.__offset(this.bb_pos, 4);
+  return offset ? this.bb!.readUint64(this.bb_pos + offset) : BigInt('0');
+}
+
+bufferIndex():bigint {
+  const offset = this.bb!.__offset(this.bb_pos, 6);
+  return offset ? this.bb!.readUint64(this.bb_pos + offset) : BigInt('0');
+}
+
+bufferOffset():bigint {
+  const offset = this.bb!.__offset(this.bb_pos, 8);
+  return offset ? this.bb!.readUint64(this.bb_pos + offset) : BigInt('0');
+}
+
+data(index: number):number|null {
+  const offset = this.bb!.__offset(this.bb_pos, 10);
+  return offset ? this.bb!.readUint8(this.bb!.__vector(this.bb_pos + offset) + index) : 0;
+}
+
+dataLength():number {
+  const offset = this.bb!.__offset(this.bb_pos, 10);
+  return offset ? this.bb!.__vector_len(this.bb_pos + offset) : 0;
+}
+
+dataArray():Uint8Array|null {
+  const offset = this.bb!.__offset(this.bb_pos, 10);
+  return offset ? new Uint8Array(this.bb!.bytes().buffer, this.bb!.bytes().byteOffset + this.bb!.__vector(this.bb_pos + offset), this.bb!.__vector_len(this.bb_pos + offset)) : null;
+}
+
+static startDecodeUnitBuffer(builder:flatbuffers.Builder) {
+  builder.startObject(4);
+}
+
+static addFrameNumber(builder:flatbuffers.Builder, frameNumber:bigint) {
+  builder.addFieldInt64(0, frameNumber, BigInt('0'));
+}
+
+static addBufferIndex(builder:flatbuffers.Builder, bufferIndex:bigint) {
+  builder.addFieldInt64(1, bufferIndex, BigInt('0'));
+}
+
+static addBufferOffset(builder:flatbuffers.Builder, bufferOffset:bigint) {
+  builder.addFieldInt64(2, bufferOffset, BigInt('0'));
+}
+
+static addData(builder:flatbuffers.Builder, dataOffset:flatbuffers.Offset) {
+  builder.addFieldOffset(3, dataOffset, 0);
+}
+
+static createDataVector(builder:flatbuffers.Builder, data:number[]|Uint8Array):flatbuffers.Offset {
+  builder.startVector(1, data.length, 1);
+  for (let i = data.length - 1; i >= 0; i--) {
+    builder.addInt8(data[i]!);
+  }
+  return builder.endVector();
+}
+
+static startDataVector(builder:flatbuffers.Builder, numElems:number) {
+  builder.startVector(1, numElems, 1);
+}
+
+static endDecodeUnitBuffer(builder:flatbuffers.Builder):flatbuffers.Offset {
+  const offset = builder.endObject();
+  return offset;
+}
+
+static createDecodeUnitBuffer(builder:flatbuffers.Builder, frameNumber:bigint, bufferIndex:bigint, bufferOffset:bigint, dataOffset:flatbuffers.Offset):flatbuffers.Offset {
+  DecodeUnitBuffer.startDecodeUnitBuffer(builder);
+  DecodeUnitBuffer.addFrameNumber(builder, frameNumber);
+  DecodeUnitBuffer.addBufferIndex(builder, bufferIndex);
+  DecodeUnitBuffer.addBufferOffset(builder, bufferOffset);
+  DecodeUnitBuffer.addData(builder, dataOffset);
+  return DecodeUnitBuffer.endDecodeUnitBuffer(builder);
+}
+}

frontend/src/lib/proto/video-update/decode-unit-start.ts

@@ -0,0 +1,91 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */
+
+import * as flatbuffers from 'flatbuffers';
+
+import { FrameType } from '../video-update/frame-type.js';
+
+
+export class DecodeUnitStart {
+  bb: flatbuffers.ByteBuffer|null = null;
+  bb_pos = 0;
+  __init(i:number, bb:flatbuffers.ByteBuffer):DecodeUnitStart {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+}
+
+static getRootAsDecodeUnitStart(bb:flatbuffers.ByteBuffer, obj?:DecodeUnitStart):DecodeUnitStart {
+  return (obj || new DecodeUnitStart()).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+}
+
+static getSizePrefixedRootAsDecodeUnitStart(bb:flatbuffers.ByteBuffer, obj?:DecodeUnitStart):DecodeUnitStart {
+  bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH);
+  return (obj || new DecodeUnitStart()).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+}
+
+frameNumber():bigint {
+  const offset = this.bb!.__offset(this.bb_pos, 4);
+  return offset ? this.bb!.readUint64(this.bb_pos + offset) : BigInt('0');
+}
+
+frameType():FrameType {
+  const offset = this.bb!.__offset(this.bb_pos, 6);
+  return offset ? this.bb!.readInt8(this.bb_pos + offset) : FrameType.PFRAME;
+}
+
+numBuffers():bigint {
+  const offset = this.bb!.__offset(this.bb_pos, 8);
+  return offset ? this.bb!.readUint64(this.bb_pos + offset) : BigInt('0');
+}
+
+receiveTimeMs():number {
+  const offset = this.bb!.__offset(this.bb_pos, 10);
+  return offset ? this.bb!.readUint16(this.bb_pos + offset) : 0;
+}
+
+fullLength():bigint {
+  const offset = this.bb!.__offset(this.bb_pos, 12);
+  return offset ? this.bb!.readUint64(this.bb_pos + offset) : BigInt('0');
+}
+
+static startDecodeUnitStart(builder:flatbuffers.Builder) {
+  builder.startObject(5);
+}
+
+static addFrameNumber(builder:flatbuffers.Builder, frameNumber:bigint) {
+  builder.addFieldInt64(0, frameNumber, BigInt('0'));
+}
+
+static addFrameType(builder:flatbuffers.Builder, frameType:FrameType) {
+  builder.addFieldInt8(1, frameType, FrameType.PFRAME);
+}
+
+static addNumBuffers(builder:flatbuffers.Builder, numBuffers:bigint) {
+  builder.addFieldInt64(2, numBuffers, BigInt('0'));
+}
+
+static addReceiveTimeMs(builder:flatbuffers.Builder, receiveTimeMs:number) {
+  builder.addFieldInt16(3, receiveTimeMs, 0);
+}
+
+static addFullLength(builder:flatbuffers.Builder, fullLength:bigint) {
+  builder.addFieldInt64(4, fullLength, BigInt('0'));
+}
+
+static endDecodeUnitStart(builder:flatbuffers.Builder):flatbuffers.Offset {
+  const offset = builder.endObject();
+  return offset;
+}
+
+static createDecodeUnitStart(builder:flatbuffers.Builder, frameNumber:bigint, frameType:FrameType, numBuffers:bigint, receiveTimeMs:number, fullLength:bigint):flatbuffers.Offset {
+  DecodeUnitStart.startDecodeUnitStart(builder);
+  DecodeUnitStart.addFrameNumber(builder, frameNumber);
+  DecodeUnitStart.addFrameType(builder, frameType);
+  DecodeUnitStart.addNumBuffers(builder, numBuffers);
+  DecodeUnitStart.addReceiveTimeMs(builder, receiveTimeMs);
+  DecodeUnitStart.addFullLength(builder, fullLength);
+  return DecodeUnitStart.endDecodeUnitStart(builder);
+}
+}

frontend/src/lib/proto/video-update/decode-unit.ts

@@ -0,0 +1,103 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */
+
+import * as flatbuffers from 'flatbuffers';
+
+import { FrameType } from '../video-update/frame-type.js';
+
+
+export class DecodeUnit {
+  bb: flatbuffers.ByteBuffer|null = null;
+  bb_pos = 0;
+  __init(i:number, bb:flatbuffers.ByteBuffer):DecodeUnit {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+}
+
+static getRootAsDecodeUnit(bb:flatbuffers.ByteBuffer, obj?:DecodeUnit):DecodeUnit {
+  return (obj || new DecodeUnit()).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+}
+
+static getSizePrefixedRootAsDecodeUnit(bb:flatbuffers.ByteBuffer, obj?:DecodeUnit):DecodeUnit {
+  bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH);
+  return (obj || new DecodeUnit()).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+}
+
+frameNumber():number {
+  const offset = this.bb!.__offset(this.bb_pos, 4);
+  return offset ? this.bb!.readUint16(this.bb_pos + offset) : 0;
+}
+
+frameType():FrameType {
+  const offset = this.bb!.__offset(this.bb_pos, 6);
+  return offset ? this.bb!.readInt8(this.bb_pos + offset) : FrameType.PFRAME;
+}
+
+receiveTimeMs():number {
+  const offset = this.bb!.__offset(this.bb_pos, 8);
+  return offset ? this.bb!.readUint16(this.bb_pos + offset) : 0;
+}
+
+data(index: number):number|null {
+  const offset = this.bb!.__offset(this.bb_pos, 10);
+  return offset ? this.bb!.readUint8(this.bb!.__vector(this.bb_pos + offset) + index) : 0;
+}
+
+dataLength():number {
+  const offset = this.bb!.__offset(this.bb_pos, 10);
+  return offset ? this.bb!.__vector_len(this.bb_pos + offset) : 0;
+}
+
+dataArray():Uint8Array|null {
+  const offset = this.bb!.__offset(this.bb_pos, 10);
+  return offset ? new Uint8Array(this.bb!.bytes().buffer, this.bb!.bytes().byteOffset + this.bb!.__vector(this.bb_pos + offset), this.bb!.__vector_len(this.bb_pos + offset)) : null;
+}
+
+static startDecodeUnit(builder:flatbuffers.Builder) {
+  builder.startObject(4);
+}
+
+static addFrameNumber(builder:flatbuffers.Builder, frameNumber:number) {
+  builder.addFieldInt16(0, frameNumber, 0);
+}
+
+static addFrameType(builder:flatbuffers.Builder, frameType:FrameType) {
+  builder.addFieldInt8(1, frameType, FrameType.PFRAME);
+}
+
+static addReceiveTimeMs(builder:flatbuffers.Builder, receiveTimeMs:number) {
+  builder.addFieldInt16(2, receiveTimeMs, 0);
+}
+
+static addData(builder:flatbuffers.Builder, dataOffset:flatbuffers.Offset) {
+  builder.addFieldOffset(3, dataOffset, 0);
+}
+
+static createDataVector(builder:flatbuffers.Builder, data:number[]|Uint8Array):flatbuffers.Offset {
+  builder.startVector(1, data.length, 1);
+  for (let i = data.length - 1; i >= 0; i--) {
+    builder.addInt8(data[i]!);
+  }
+  return builder.endVector();
+}
+
+static startDataVector(builder:flatbuffers.Builder, numElems:number) {
+  builder.startVector(1, numElems, 1);
+}
+
+static endDecodeUnit(builder:flatbuffers.Builder):flatbuffers.Offset {
+  const offset = builder.endObject();
+  return offset;
+}
+
+static createDecodeUnit(builder:flatbuffers.Builder, frameNumber:number, frameType:FrameType, receiveTimeMs:number, dataOffset:flatbuffers.Offset):flatbuffers.Offset {
+  DecodeUnit.startDecodeUnit(builder);
+  DecodeUnit.addFrameNumber(builder, frameNumber);
+  DecodeUnit.addFrameType(builder, frameType);
+  DecodeUnit.addReceiveTimeMs(builder, receiveTimeMs);
+  DecodeUnit.addData(builder, dataOffset);
+  return DecodeUnit.endDecodeUnit(builder);
+}
+}

frontend/src/lib/proto/video-update/frame-type.ts

@@ -0,0 +1,8 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */
+
+export enum FrameType {
+  PFRAME = 0,
+  IDR = 1
+}

frontend/src/lib/proto/video-update/setup.ts

@@ -0,0 +1,80 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */
+
+import * as flatbuffers from 'flatbuffers';
+
+export class Setup {
+  bb: flatbuffers.ByteBuffer|null = null;
+  bb_pos = 0;
+  __init(i:number, bb:flatbuffers.ByteBuffer):Setup {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+}
+
+static getRootAsSetup(bb:flatbuffers.ByteBuffer, obj?:Setup):Setup {
+  return (obj || new Setup()).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+}
+
+static getSizePrefixedRootAsSetup(bb:flatbuffers.ByteBuffer, obj?:Setup):Setup {
+  bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH);
+  return (obj || new Setup()).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+}
+
+videoFormat():string|null
+videoFormat(optionalEncoding:flatbuffers.Encoding):string|Uint8Array|null
+videoFormat(optionalEncoding?:any):string|Uint8Array|null {
+  const offset = this.bb!.__offset(this.bb_pos, 4);
+  return offset ? this.bb!.__string(this.bb_pos + offset, optionalEncoding) : null;
+}
+
+width():number {
+  const offset = this.bb!.__offset(this.bb_pos, 6);
+  return offset ? this.bb!.readUint16(this.bb_pos + offset) : 0;
+}
+
+height():number {
+  const offset = this.bb!.__offset(this.bb_pos, 8);
+  return offset ? this.bb!.readUint16(this.bb_pos + offset) : 0;
+}
+
+redrawRate():number {
+  const offset = this.bb!.__offset(this.bb_pos, 10);
+  return offset ? this.bb!.readUint16(this.bb_pos + offset) : 0;
+}
+
+static startSetup(builder:flatbuffers.Builder) {
+  builder.startObject(4);
+}
+
+static addVideoFormat(builder:flatbuffers.Builder, videoFormatOffset:flatbuffers.Offset) {
+  builder.addFieldOffset(0, videoFormatOffset, 0);
+}
+
+static addWidth(builder:flatbuffers.Builder, width:number) {
+  builder.addFieldInt16(1, width, 0);
+}
+
+static addHeight(builder:flatbuffers.Builder, height:number) {
+  builder.addFieldInt16(2, height, 0);
+}
+
+static addRedrawRate(builder:flatbuffers.Builder, redrawRate:number) {
+  builder.addFieldInt16(3, redrawRate, 0);
+}
+
+static endSetup(builder:flatbuffers.Builder):flatbuffers.Offset {
+  const offset = builder.endObject();
+  return offset;
+}
+
+static createSetup(builder:flatbuffers.Builder, videoFormatOffset:flatbuffers.Offset, width:number, height:number, redrawRate:number):flatbuffers.Offset {
+  Setup.startSetup(builder);
+  Setup.addVideoFormat(builder, videoFormatOffset);
+  Setup.addWidth(builder, width);
+  Setup.addHeight(builder, height);
+  Setup.addRedrawRate(builder, redrawRate);
+  return Setup.endSetup(builder);
+}
+}

frontend/src/lib/proto/video-update/update.ts

@@ -0,0 +1,42 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */
+
+import { DecodeUnitBuffer } from '../video-update/decode-unit-buffer.js';
+import { DecodeUnitStart } from '../video-update/decode-unit-start.js';
+import { Setup } from '../video-update/setup.js';
+
+
+export enum Update {
+  NONE = 0,
+  Setup = 1,
+  DecodeUnitStart = 2,
+  DecodeUnitBuffer = 3
+}
+
+export function unionToUpdate(
+  type: Update,
+  accessor: (obj:DecodeUnitBuffer|DecodeUnitStart|Setup) => DecodeUnitBuffer|DecodeUnitStart|Setup|null
+): DecodeUnitBuffer|DecodeUnitStart|Setup|null {
+  switch(Update[type]) {
+    case 'NONE': return null; 
+    case 'Setup': return accessor(new Setup())! as Setup;
+    case 'DecodeUnitStart': return accessor(new DecodeUnitStart())! as DecodeUnitStart;
+    case 'DecodeUnitBuffer': return accessor(new DecodeUnitBuffer())! as DecodeUnitBuffer;
+    default: return null;
+  }
+}
+
+export function unionListToUpdate(
+  type: Update, 
+  accessor: (index: number, obj:DecodeUnitBuffer|DecodeUnitStart|Setup) => DecodeUnitBuffer|DecodeUnitStart|Setup|null, 
+  index: number
+): DecodeUnitBuffer|DecodeUnitStart|Setup|null {
+  switch(Update[type]) {
+    case 'NONE': return null; 
+    case 'Setup': return accessor(index, new Setup())! as Setup;
+    case 'DecodeUnitStart': return accessor(index, new DecodeUnitStart())! as DecodeUnitStart;
+    case 'DecodeUnitBuffer': return accessor(index, new DecodeUnitBuffer())! as DecodeUnitBuffer;
+    default: return null;
+  }
+}

frontend/src/lib/proto/video-update/video-update.ts

@@ -0,0 +1,69 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */
+
+import * as flatbuffers from 'flatbuffers';
+
+import { Update, unionToUpdate, unionListToUpdate } from '../video-update/update.js';
+
+
+export class VideoUpdate {
+  bb: flatbuffers.ByteBuffer|null = null;
+  bb_pos = 0;
+  __init(i:number, bb:flatbuffers.ByteBuffer):VideoUpdate {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+}
+
+static getRootAsVideoUpdate(bb:flatbuffers.ByteBuffer, obj?:VideoUpdate):VideoUpdate {
+  return (obj || new VideoUpdate()).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+}
+
+static getSizePrefixedRootAsVideoUpdate(bb:flatbuffers.ByteBuffer, obj?:VideoUpdate):VideoUpdate {
+  bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH);
+  return (obj || new VideoUpdate()).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+}
+
+updateType():Update {
+  const offset = this.bb!.__offset(this.bb_pos, 4);
+  return offset ? this.bb!.readUint8(this.bb_pos + offset) : Update.NONE;
+}
+
+update<T extends flatbuffers.Table>(obj:any):any|null {
+  const offset = this.bb!.__offset(this.bb_pos, 6);
+  return offset ? this.bb!.__union(obj, this.bb_pos + offset) : null;
+}
+
+static startVideoUpdate(builder:flatbuffers.Builder) {
+  builder.startObject(2);
+}
+
+static addUpdateType(builder:flatbuffers.Builder, updateType:Update) {
+  builder.addFieldInt8(0, updateType, Update.NONE);
+}
+
+static addUpdate(builder:flatbuffers.Builder, updateOffset:flatbuffers.Offset) {
+  builder.addFieldOffset(1, updateOffset, 0);
+}
+
+static endVideoUpdate(builder:flatbuffers.Builder):flatbuffers.Offset {
+  const offset = builder.endObject();
+  return offset;
+}
+
+static finishVideoUpdateBuffer(builder:flatbuffers.Builder, offset:flatbuffers.Offset) {
+  builder.finish(offset);
+}
+
+static finishSizePrefixedVideoUpdateBuffer(builder:flatbuffers.Builder, offset:flatbuffers.Offset) {
+  builder.finish(offset, undefined, true);
+}
+
+static createVideoUpdate(builder:flatbuffers.Builder, updateType:Update, updateOffset:flatbuffers.Offset):flatbuffers.Offset {
+  VideoUpdate.startVideoUpdate(builder);
+  VideoUpdate.addUpdateType(builder, updateType);
+  VideoUpdate.addUpdate(builder, updateOffset);
+  return VideoUpdate.endVideoUpdate(builder);
+}
+}

frontend/src/lib/proto/video.ts

@@ -0,0 +1,5 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */
+
+export * as VideoUpdate from './video-update.js';

frontend/src/lib/video.ts

@@ -0,0 +1,189 @@
+import { VideoUpdate } from "$lib/proto/video";
+import { ByteBuffer } from "flatbuffers";
+import { DecodeUnitBuffer } from "./proto/video-update";
+import { DoorClosed, Video } from "lucide-svelte";
+
+function getVideoDecoder(canvasElement: OffscreenCanvas): VideoDecoder {
+  const canvasCtx: OffscreenCanvasRenderingContext2D | null = canvasElement.getContext('2d');
+  if (canvasCtx == null) {
+    throw new Error(`Could not get 2d canvas context`);
+  }
+
+  const videoDecoder = new VideoDecoder({
+    output: (frame) => {
+      //console.log(`rendering frame start: ${performance.now()}`);
+      //canvasElement.width = frame.displayWidth;
+      //canvasElement.height = frame.displayHeight;
+
+      //console.log(`rendering frame drawImage: ${performance.now()}`);
+      canvasCtx.drawImage(frame, 0, 0);
+
+      //console.log(`rendering frame end: ${performance.now()}`);
+      frame.close();
+      //console.log(`rendering frame close: ${performance.now()}`);
+    },
+    error: (e) => {
+      console.error('Decode error:', e);
+    }
+  });
+  return videoDecoder;
+}
+
+async function configureDecoder(videoDecoder: VideoDecoder, videoFormat: string, width: number, height: number) {
+  let config: VideoDecoderConfig = {
+    codec: videoFormat,
+    codedWidth: width,
+    codedHeight: height,
+    optimizeForLatency: true,
+    //hardwareAcceleration: "prefer-hardware",
+  };
+
+  const codecSupport = await VideoDecoder.isConfigSupported(config);
+  console.log(codecSupport);
+  if (codecSupport.supported) {
+    videoDecoder.configure(config);
+  } else {
+    throw new Error(`Could not configure decoder`);
+  }
+}
+
+class Decoder {
+  videoDecoder: VideoDecoder;
+
+  frameNumber: bigint | undefined;
+  frameType: EncodedAudioChunkType = "delta";
+  fullLength: bigint = 0n;
+  receiveTimeMs: number = 0;
+
+  //frameBroken: boolean = false;
+  //lastBufferIndex: bigint = 0n;
+  frameCollector: boolean[] = new Array<boolean>();
+  dataOffset: number = 0;
+  data: Uint8Array = new Uint8Array();
+
+  constructor(videoDecoder: VideoDecoder) {
+    this.videoDecoder = videoDecoder;
+  }
+
+  print() {
+    console.log(
+      `
+    frameNumber: ${this.frameNumber}
+    frameType: ${this.frameType}
+    fullLength: ${this.fullLength}
+    receiveTimeMs: ${this.receiveTimeMs}
+    frameCollector: ${this.frameCollector}
+    ts: ${performance.now()}
+    `
+    );
+  }
+
+  processStart(decodeUnitStart: VideoUpdate.DecodeUnitStart) {
+    //this.print();
+    const frameCompleted = !this.frameCollector.includes(false);
+    if (!frameCompleted) {
+      console.log(`Got setup packet for frame ${decodeUnitStart.frameNumber()} but the last frame has not been completed`);
+    }
+
+    this.frameNumber = decodeUnitStart.frameNumber();
+    this.frameType = "delta";
+    if (decodeUnitStart.frameType() == VideoUpdate.FrameType.IDR) {
+      this.frameType = "key";
+    }
+    this.fullLength = decodeUnitStart.fullLength();
+    this.receiveTimeMs = decodeUnitStart.receiveTimeMs();
+
+    //this.frameBroken = false;
+    //this.lastBufferIndex = -1n;
+    //this.dataOffset = 0;
+    this.frameCollector = new Array(Number(decodeUnitStart.numBuffers())).fill(false);
+
+    this.data = new Uint8Array(Number(this.fullLength));
+
+
+    //this.print();
+    //console.log(`start: `, this);
+    //console.log(performance.now());
+  }
+
+  processBuffer(decodeUnitBuffer: VideoUpdate.DecodeUnitBuffer) {
+    //console.log(`buffer: `, this);
+    //console.log(performance.now());
+    //this.print();
+
+
+    const frameNumber = decodeUnitBuffer.frameNumber();
+    if (this.frameNumber === undefined) {
+      console.log("frameNumber is undefined but we got a buffer, ignoring...");
+      return;
+    }
+    if (this.frameNumber != frameNumber) {
+      console.log(`Got buffer for frame ${frameNumber} but we are processing frame ${this.frameNumber}, ignoring...`);
+      return;
+    }
+
+    let offset = decodeUnitBuffer.bufferOffset();
+
+    for (var i = 0; i < decodeUnitBuffer.dataLength(); i++) {
+      this.data[Number(offset) + i] = decodeUnitBuffer.data(i)!;
+    }
+
+    this.frameCollector[Number(decodeUnitBuffer.bufferIndex())] = true;
+
+
+    const gotAllframes = !this.frameCollector.includes(false);
+
+    if (gotAllframes) {
+      const chunk = new EncodedVideoChunk({
+        //timestamp: this.receiveTimeMs,
+        timestamp: 0,
+        type: this.frameType,
+        data: this.data,
+      });
+
+      //console.log(`${ performance.now() }: Enqueing a new decode request, current queue size ${ this.videoDecoder.decodeQueueSize } `);
+      this.videoDecoder.decode(chunk);
+    }
+    //this.print();
+  }
+}
+
+export async function streamVideoFromReader(reader: ReadableStreamDefaultReader, canvasElement: OffscreenCanvas) {
+  const videoDecoder = getVideoDecoder(canvasElement);
+  try {
+    let decodeUnitBuffer: VideoUpdate.DecodeUnitBuffer = new VideoUpdate.DecodeUnitBuffer();
+    let decoder = new Decoder(videoDecoder);
+
+    while (true) {
+      const { value, done } = await reader.read();
+      if (done) break;
+
+      const dataToParse = new ByteBuffer(value);
+      const videoUpdate = VideoUpdate.VideoUpdate.getRootAsVideoUpdate(dataToParse);
+
+      if (videoUpdate.updateType() == VideoUpdate.Update.Setup) {
+        let setup = videoUpdate.update(new VideoUpdate.Setup());
+        await configureDecoder(videoDecoder, setup.videoFormat(), setup.width(), setup.height());
+
+      } else if (videoUpdate.updateType() == VideoUpdate.Update.DecodeUnitStart) {
+        let decodeUnitStart: VideoUpdate.DecodeUnitStart = new VideoUpdate.DecodeUnitStart();
+        videoUpdate.update(decodeUnitStart);
+        decoder.processStart(decodeUnitStart);
+
+      } else if (videoUpdate.updateType() == VideoUpdate.Update.DecodeUnitBuffer) {
+        videoUpdate.update(decodeUnitBuffer);
+        decoder.processBuffer(decodeUnitBuffer);
+
+      } else {
+        throw new Error(`Got packet of unknown type`);
+      }
+    }
+
+  } catch (e) {
+    var error = <Error>e;
+    console.error('Error connecting to stream:', error);
+    alert('Failed to connect to stream: ' + error.message);
+  }
+}
+
+

frontend/src/routes/+layout.svelte

@@ -27,12 +27,12 @@
 	}
 
 	main {
-		flex: 1;
+		/*flex: 1;*/
-		display: flex;
+		/*display: flex;*/
-		flex-direction: column;
+		/*flex-direction: column;*/
-		padding: 1rem;
+		/*padding: 1rem;*/
 		width: 100%;
-		max-width: 64rem;
+		/*max-width: 64rem;*/
 		margin: 0 auto;
 		box-sizing: border-box;
 	}

frontend/src/routes/+page.svelte

@@ -1,35 +1,16 @@
 <script lang="ts">
 	import Covers from './Covers.svelte';
-	import Stream from './Stream.svelte';
 	import welcome from '$lib/images/svelte-welcome.webp';
 	import welcomeFallback from '$lib/images/svelte-welcome.png';
 </script>
 
-<!--
 <svelte:head>
 	<title>Home</title>
 	<meta name="description" content="Svelte demo app" />
 </svelte:head>
--->
 
 <section>
-	<!--<h1>
-		<span class="welcome">
-			<picture>
-				<source srcset={welcome} type="image/webp" />
-				<img src={welcomeFallback} alt="Welcome" />
-			</picture>
-		</span>
-
-		to your new<br />SvelteKit app
-	</h1>
-
-	<h2>
-		try editing <strong>src/routes/+page.svelte</strong>
-	</h2>-->
-
 	<Covers />
-	<Stream />
 </section>
 
 <style>
@@ -40,24 +21,4 @@
 		align-items: center;
 		flex: 0.6;
 	}
-
-	h1 {
-		width: 100%;
-	}
-
-	.welcome {
-		display: block;
-		position: relative;
-		width: 100%;
-		height: 0;
-		padding: 0 0 calc(100% * 495 / 2048) 0;
-	}
-
-	.welcome img {
-		position: absolute;
-		width: 100%;
-		height: 100%;
-		top: 0;
-		display: block;
-	}
 </style>

frontend/src/routes/Cover.svelte

@@ -1,43 +1,70 @@
 <script lang="ts">
 	import type { HTMLButtonAttributes } from 'svelte/elements';
-	import { startStream } from './connect';
+	import { getStreamData } from './getStreamData';
 	import { type App } from './apps';
+	import { Circle } from 'svelte-loading-spinners';
+	import { goto } from '$app/navigation';
+	import { streamStore } from './stores/streamStore.svelte';
 
 	interface Props {
 		app: App;
 		server_name: string;
+		tab_index: number;
 	}
 
-	let { app, server_name }: Props = $props();
+	let { app, server_name, tab_index }: Props = $props();
+
+	let loading = $state(false);
 
 	async function streamApp() {
-		await startStream(app, server_name);
+		loading = true;
+
+		console.log(`Getting stream data for ${app.title} on ${server_name}`);
+		let streamData = await getStreamData(app.id, server_name);
+		streamStore.Url = streamData.Url;
+		streamStore.CertHash = streamData.CertHash;
+		streamStore.Width = streamData.Width;
+		streamStore.Height = streamData.Height;
+
+		console.log(`Stream data retrieved. Navigating to /stream.`);
+		await goto('/stream');
 	}
 </script>
 
-<div class="app-box" on:click={streamApp}>
+{#if loading}
+	<div class="app-box">
 		<div class="app-artwork">
-		<div class="play-button"></div>
+			<div class="app-loading-box">
+				<Circle size="60" color="#FF3E00" unit="px" duration="1s" />
+			</div>
+			{#if app.active}
+				<div class="running-pill">Running</div>
+			{/if}
 		</div>
 		<div class="app-title">{app.title}</div>
 		<div class="app-server">{server_name}</div>
-</div>
+	</div>
+{:else}
+	<div class="app-box" onclick={streamApp} onkeydown={streamApp} role="button" tabindex={tab_index}>
+		<div class="app-artwork">
+			{#if app.active}
+				<div class="app-control-box">
+					<div class="resume-button"></div>
+					<div class="stop-button"></div>
+				</div>
+				<div class="running-pill">Running</div>
+			{:else}
+				<div class="app-control-box">
+					<div class="resume-button"></div>
+				</div>
+			{/if}
+		</div>
+		<div class="app-title">{app.title}</div>
+		<div class="app-server">{server_name}</div>
+	</div>
+{/if}
 
 <style>
-	body {
-		font-family: Arial, sans-serif;
-		margin: 20px;
-		background-color: #1a1a1a;
-		color: white;
-	}
-
-	.apps-container {
-		display: flex;
-		flex-wrap: wrap;
-		gap: 20px;
-		padding: 20px 0;
-	}
-
 	.app-box {
 		position: relative;
 		width: 200px;
@@ -53,7 +80,7 @@
 
 	.app-artwork {
 		width: 200px;
-		height: 200px;
+		height: 280px;
 		background-color: #333;
 		border: 2px solid #555;
 		border-radius: 8px;
@@ -66,6 +93,73 @@
 		border-color: #00aaff;
 	}
 
+	.app-loading-box,
+	.app-control-box {
+		width: 200px;
+		height: 60px;
+		position: relative;
+		top: 50%;
+		transform: translate(0%, -50%);
+		opacity: 0;
+
+		display: flex;
+		justify-content: space-evenly;
+	}
+
+	.app-loading-box {
+		opacity: 1;
+	}
+
+	.app-box:hover .app-control-box {
+		opacity: 1;
+	}
+
+	.stop-button,
+	.resume-button {
+		width: 60px;
+		height: 60px;
+		border-radius: 50%;
+
+		background-color: rgba(0, 170, 255, 0.9);
+		display: flex;
+		align-items: center;
+		justify-content: center;
+		transition: opacity 0.3s ease;
+		pointer-events: none;
+	}
+
+	.resume-button::after {
+		content: '';
+		width: 0;
+		height: 0;
+		border-left: 20px solid white;
+		border-top: 12px solid transparent;
+		border-bottom: 12px solid transparent;
+		margin-left: 4px;
+	}
+
+	.stop-button::after {
+		content: '';
+		width: 24px;
+		height: 24px;
+		background-color: white;
+	}
+
+	.running-pill {
+		position: absolute;
+		bottom: 12px;
+		left: 50%;
+		transform: translateX(-50%);
+		background-color: #22c55e;
+		color: white;
+		padding: 6px 16px;
+		border-radius: 20px;
+		font-size: 20px;
+		font-weight: 500;
+		text-align: center;
+		box-shadow: 0 2px 4px rgba(0, 0, 0, 0.2);
+	}
+
 	.app-title {
 		text-align: center;
 		margin: 10px 0 5px 0;
@@ -82,52 +176,4 @@
 		color: #aaa;
 		margin: 0;
 	}
-
-	.play-button {
-		position: absolute;
-		top: 50%;
-		left: 50%;
-		transform: translate(-50%, -50%);
-		width: 60px;
-		height: 60px;
-		background-color: rgba(0, 170, 255, 0.9);
-		border-radius: 50%;
-		display: flex;
-		align-items: center;
-		justify-content: center;
-		opacity: 0;
-		transition: opacity 0.3s ease;
-		pointer-events: none;
-	}
-
-	.play-button::after {
-		content: '';
-		width: 0;
-		height: 0;
-		border-left: 20px solid white;
-		border-top: 12px solid transparent;
-		border-bottom: 12px solid transparent;
-		margin-left: 4px;
-	}
-
-	.app-box:hover .play-button {
-		opacity: 1;
-	}
-
-	.app-box.clicked {
-		transform: scale(0.95);
-	}
-
-	.loading {
-		text-align: center;
-		font-size: 18px;
-		margin-top: 50px;
-	}
-
-	.error {
-		color: #ff4444;
-		text-align: center;
-		font-size: 18px;
-		margin-top: 50px;
-	}
 </style>

frontend/src/routes/Covers.svelte

@@ -1,12 +1,7 @@
 <script lang="ts">
 	import Cover from './Cover.svelte';
 	import { fetchApps } from './apps';
-
+	import type { App } from './apps';
-	interface App {
-		title: string;
-		id: number;
-		hdr_supported: boolean;
-	}
 
 	interface AppsResponse {
 		apps: Record<string, App[]>;
@@ -14,15 +9,6 @@
 
 	let appsPromise: Promise<AppsResponse>;
 
-	//async function fetchApps() {
-	//	console.log('apps');
-	//	const response = await fetch('/api/apps');
-	//	console.log(response);
-	//	const data = (await response.json()) as AppsResponse;
-	//	console.log(data);
-	//	return data;
-	//}
-
 	appsPromise = fetchApps();
 </script>
 
@@ -31,8 +17,8 @@
 {:then resp}
 	<div class="apps-container">
 		{#each Object.entries(resp.apps) as [server_name, apps]}
-			{#each apps as app}
+			{#each apps as app, tab_index}
-				<Cover {app} {server_name}></Cover>
+				<Cover {app} {server_name} {tab_index}></Cover>
 			{/each}
 		{/each}
 	</div>

frontend/src/routes/Stream.svelte

@@ -1,35 +0,0 @@
-<script lang="ts">
-	import Cover from './Cover.svelte';
-	import { fetchApps } from './apps';
-
-	interface Props {
-		id: string;
-	}
-
-	let { id }: Props = $props();
-</script>
-
-<canvas id="canvas"></canvas>
-
-<!--{#await appsPromise}
-	<p>Loading...</p>
-{:then resp}
-	<div class="apps-container">
-		{#each Object.entries(resp.apps) as [server_name, apps]}
-			{#each apps as app}
-				<Cover {app} {server_name}></Cover>
-			{/each}
-		{/each}
-	</div>
-{:catch error}
-	<p>Error: {error.message}</p>
-{/await}-->
-
-<style>
-	.apps-container {
-		display: flex;
-		flex-wrap: wrap;
-		gap: 20px;
-		padding: 20px 0;
-	}
-</style>

frontend/src/routes/apps.ts

@@ -2,6 +2,7 @@ export interface App {
   title: string;
   id: number;
   hdr_supported: boolean;
+  active: boolean;
 }
 
 export interface AppsResponse {

frontend/src/routes/connect.ts

@@ -1,268 +0,0 @@
-import { type App } from './apps'
-
-
-
-//Setup {
-//    video_format: VideoFormat,
-//    width: u64,
-//    height: u64,
-//    redraw_rate: u64,
-//    dr_flags: i32,
-//},
-//DecodeUnit {
-//    frame_number: u64,
-//    frame_type: FrameType,
-
-//    host_processing_latency: u16,
-//    receieve_time_ms: u64,
-//    enqueue_time_ms: u64,
-//    presentation_time: u64,
-
-//    full_length: usize,
-//    //buffers: Vec<Buffer>,
-//    buffer: Buffer,
-//    index: u64,
-
-//    hdr_active: bool,
-//    colorspace: u8,
-//},
-
-type Setup = {
-  video_format: string,
-  width: number,
-  height: number,
-  redraw_rate: number,
-  dr_flags: number,
-}
-
-type SetupPacket = {
-  Setup: Setup
-}
-
-type DecodeUnit = {
-  frame_number: number,
-  frame_type: string,
-  buffer: Array<number>,
-}
-
-type DecodeUnitPacket = {
-  DecodeUnit: DecodeUnit
-}
-
-export async function startStream(app: App, server_name: string) {
-  try {
-    console.log(`Starting stream for ${app.title} on ${server_name}`);
-
-    // Create the POST request payload
-    const payload = {
-      id: app.id,
-      server: server_name,
-      server_mode: {
-        fps: 60,
-        height: 1280,
-        width: 720
-      },
-      stream_config: {
-        bitrate_kbps: 5120,
-        mode: {
-          fps: 60,
-          height: 1280,
-          width: 720
-        }
-      }
-    };
-
-    // Make POST request to start stream
-    const response = await fetch('/api/stream/start', {
-      method: 'POST',
-      headers: {
-        'Content-Type': 'application/json',
-      },
-      body: JSON.stringify(payload)
-    });
-
-    if (!response.ok) {
-      throw new Error(`HTTP error! status: ${response.status}: ${await response.text()}`);
-    }
-
-    const streamData = await response.json();
-    console.log('Stream started:', streamData);
-
-    if (streamData.url && streamData.cert_hash) {
-      await connectToStream(streamData.url, streamData.cert_hash);
-    } else {
-      throw new Error('Response was missing required parameters');
-    }
-
-  } catch (error) {
-    console.error('Error starting stream:', error);
-    alert('Failed to start stream: ' + error.message);
-  }
-}
-
-
-export async function connectToStream(url: string, cert_hash: Array<number>) {
-  const buffer = new Uint8Array(cert_hash);
-  console.log('Hash: ', buffer);
-  try {
-    console.log(`Connecting to stream`);
-
-    // Check if WebTransport is supported
-    if (!window.WebTransport) {
-      throw new Error('WebTransport is not supported in this browser');
-    }
-
-    //const url = new URL();
-    const transport = new WebTransport(url, {
-      serverCertificateHashes: [
-        {
-          algorithm: "sha-256",
-          value: buffer,
-        }
-      ]
-    });
-
-    console.log('Connecting to WebTransport at ', url);
-    // Wait for the connection to be ready
-    await transport.ready;
-    console.log('WebTransport connection established');
-
-
-    console.log('Creating WebTransport bidirectional stream');
-    const stream = await transport.createBidirectionalStream();
-    console.log('Bidirectional stream created');
-
-    const reader = stream.readable.getReader();
-
-    function parseData(newBuffer: Uint8Array, oldBuffer: Uint8Array): [Array<Object>, Uint8Array<ArrayBuffer>] {
-      let packets = new Array<Object>();
-      let unparsedData = new Uint8Array();
-
-      let data = new Uint8Array([...oldBuffer, ...newBuffer]);
-      let index = 0;
-      while (true) {
-        if (index >= data.length) {
-          break
-        }
-        const view = new DataView(data.buffer.slice(index, index + 4));
-        const dataLength = view.getUint32(0, true);
-
-        const slice_start_index = index + 4;
-        const slice_end_index = index + 4 + dataLength;
-
-        if (data.length < slice_end_index) {
-          unparsedData = new Uint8Array(data.buffer.slice(index, data.length));
-          break;
-        }
-
-        const dataToParse = data.buffer.slice(slice_start_index, slice_end_index);
-        const decoder = new TextDecoder('utf-8');
-        const jsonString = decoder.decode(dataToParse);
-
-        packets.push(JSON.parse(jsonString));
-
-        index += 4 + dataLength;
-      }
-      return [packets, unparsedData];
-    }
-
-    let unparsedData = new Uint8Array();
-
-    const canvas: HTMLCanvasElement | null = <HTMLCanvasElement>document.getElementById('canvas');
-    if (canvas == null) {
-      throw new Error(`Could not find canvas`);
-    }
-    const ctx: CanvasRenderingContext2D | null = canvas.getContext('2d');
-    if (ctx == null) {
-      throw new Error(`Could not get 2d canvas context`);
-    }
-
-    const videoDecoder = new VideoDecoder({
-      output: (frame) => {
-        // Set canvas dimensions to match the frame
-        canvas.width = frame.displayWidth;
-        canvas.height = frame.displayHeight;
-
-        // Draw the decoded frame to canvas
-        ctx.drawImage(frame, 0, 0);
-
-        // Important: close the frame to free memory
-        frame.close();
-      },
-      error: (e) => {
-        console.error('Decode error:', e);
-      }
-    });
-
-    while (true) {
-      const { value, done } = await reader.read();
-      if (done) break;
-
-      let [packets, remainingData] = parseData(value, unparsedData);
-      unparsedData = remainingData;
-
-      for (let i = 0; i < packets.length; i++) {
-        console.log(packets[i]);
-        if (Object.hasOwn(packets[i], "Setup")) {
-          let packet = packets[i] as SetupPacket;
-
-          let config: VideoDecoderConfig | undefined = undefined;
-          if (packet.Setup.video_format == "H264") {
-            config = {
-              codec: 'avc1.42E01E', // H.264 codec
-              codedWidth: packet.Setup.width,
-              codedHeight: packet.Setup.height,
-            };
-          } else {
-            throw new Error(`Unsupported video codec ${packet.Setup.video_format}`);
-          }
-
-          const codecSupport = await VideoDecoder.isConfigSupported(config);
-          if (codecSupport.supported) {
-            videoDecoder.configure(config);
-          }
-
-        } else if (Object.hasOwn(packets[i], "DecodeUnit")) {
-          let packet = packets[i] as DecodeUnitPacket;
-
-
-          let frame_type: EncodedAudioChunkType = "delta";
-          if (packet.DecodeUnit.frame_type == "IDR") {
-            frame_type = "key";
-          }
-          const chunk = new EncodedVideoChunk({
-            timestamp: 0,
-            type: frame_type,
-            data: new Uint8Array(packet.DecodeUnit.buffer),
-          });
-
-          videoDecoder.decode(chunk);
-
-        } else {
-          throw new Error(`Got packet of unknown type`);
-        }
-      }
-
-    }
-
-    //console.log(`Received: ${new TextDecoder().decode(value)}`);
-
-
-    //readData();
-
-    // Handle connection close
-    transport.closed.then(() => {
-      console.log('WebTransport connection closed');
-    }).catch((error) => {
-      console.error('WebTransport connection closed with error:', error);
-    });
-
-    // You can add more WebTransport handling logic here
-    // For example, handling incoming streams, sending data, etc.
-
-  } catch (error) {
-    console.error('Error connecting to stream:', error);
-    alert('Failed to connect to stream: ' + error.message);
-  }
-}
-

frontend/src/routes/getStreamData.ts

@@ -0,0 +1,59 @@
+type StreamData = {
+  Url: string,
+  CertHash: Array<number>,
+  Width: number,
+  Height: number,
+}
+
+export async function getStreamData(appId: number, server_name: string): Promise<StreamData> {
+  try {
+    // Create the POST request payload
+    const width = 1920;
+    const height = 1080;
+
+    const payload = {
+      id: appId,
+      server: server_name,
+      server_mode: {
+        fps: 60,
+        width: width,
+        height: height,
+      },
+      stream_config: {
+        bitrate_kbps: 1024 * 10 * 5,
+        mode: {
+          fps: 60,
+          width: width,
+          height: height,
+        }
+      }
+    };
+
+    // Make POST request to start stream
+    const response = await fetch('/api/stream/start', {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+      },
+      body: JSON.stringify(payload)
+    });
+
+    if (!response.ok) {
+      throw new Error(`HTTP error! status: ${response.status}: ${await response.text()}`);
+    }
+
+    const streamDataResp = await response.json();
+    console.log('Stream started:', streamDataResp);
+
+    let streamData: StreamData = { Url: streamDataResp.url, CertHash: streamDataResp.cert_hash, Width: width, Height: height };
+    return streamData;
+
+
+  } catch (error) {
+    console.error('Error getting stream data: ', error);
+    throw new Error('Failed to start stream: ' + error);
+  }
+}
+
+
+

frontend/src/routes/stores/streamStore.svelte.ts

@@ -0,0 +1,6 @@
+export const streamStore = $state({
+  Url: '',
+  CertHash: [0],
+  Width: 0,
+  Height: 0,
+});

frontend/src/routes/stream/+page.svelte

@@ -0,0 +1,32 @@
+<script lang="ts">
+	import Stream from './Stream.svelte';
+	import { streamStore } from '../stores/streamStore.svelte';
+
+	$: url = streamStore.Url;
+	$: certHash = streamStore.CertHash;
+	$: width = streamStore.Width;
+	$: height = streamStore.Height;
+</script>
+
+<svelte:head>
+	<title>Stream</title>
+	<meta name="description" content="Streaming game" />
+</svelte:head>
+
+<!--<section>
+</section>-->
+<Stream {url} {certHash} {width} {height} />
+
+<style>
+	section {
+		min-height: 100vh;
+		min-width: 100vw;
+	}
+	/*section {
+		display: flex;
+		flex-direction: column;
+		justify-content: center;
+		align-items: center;
+		flex: 0.6;
+	}*/
+</style>

frontend/src/routes/stream/Stream.svelte

@@ -0,0 +1,81 @@
+<script lang="ts">
+	import { onMount } from 'svelte';
+	import { startWebtransportStream } from './stream';
+	import StreamUi from './StreamUi.svelte';
+
+	interface Props {
+		url: string;
+		certHash: Array<number>;
+		width: number;
+		height: number;
+	}
+
+	let { url, certHash, width, height }: Props = $props();
+	let loading = $state(true);
+	let fullscreen = $state(false);
+	let gameplayView: HTMLDivElement;
+	let gameplayCanvas: HTMLCanvasElement;
+
+	async function startStream() {
+		await startWebtransportStream(
+			url,
+			certHash,
+			width,
+			height,
+			gameplayCanvas,
+			gameplayCanvas,
+			gameplayCanvas
+		);
+	}
+
+	async function requestFullscreen() {
+		// Update fullscreen var if fullscreen was exited outside our control
+		if (document.fullscreenElement == null) {
+			fullscreen = false;
+		}
+
+		if (fullscreen) {
+			await document.exitFullscreen();
+			fullscreen = false;
+		} else {
+			await gameplayCanvas.requestFullscreen();
+			fullscreen = true;
+		}
+	}
+
+	onMount(async () => {
+		await startStream();
+	});
+</script>
+
+<div id="gameplay-view" class="gameplay-view" bind:this={gameplayView}>
+	<canvas id="gamestream-canvas" class="gamestream-canvas" bind:this={gameplayCanvas} tabindex="0"
+	></canvas>
+	<StreamUi fullscreenFunc={requestFullscreen}></StreamUi>
+</div>
+
+<style>
+	.gameplay-view {
+		height: 100vh;
+		aspect-ratio: 16 / 9;
+
+		margin-left: auto;
+		margin-right: auto;
+		background-color: black;
+
+		position: relative;
+
+		display: flex;
+		flex-direction: column;
+	}
+
+	.gamestream-canvas {
+		height: 100%;
+		aspect-ratio: 16 / 9;
+
+		position: relative;
+		margin: auto;
+
+		/*z-index: 1;*/
+	}
+</style>

frontend/src/routes/stream/StreamUi.svelte

@@ -0,0 +1,57 @@
+<script lang="ts">
+	import { Maximize, Settings } from 'lucide-svelte';
+
+	interface Props {
+		fullscreenFunc: () => void;
+	}
+
+	let { fullscreenFunc }: Props = $props();
+</script>
+
+<div class="stream-controls">
+	<div style="flex-grow: 1"></div>
+	<div class="bar-button" role="button">
+		<Settings size="100%" />
+	</div>
+	<div style="width: 0.5%"></div>
+	<div class="bar-button" onclick={fullscreenFunc} role="button">
+		<Maximize size="100%" />
+	</div>
+	<div style="width: 0.5%"></div>
+</div>
+
+<style>
+	.bar-button {
+		height: 85%;
+		aspect-ratio: 1 / 1;
+		color: oklch(0.454 0 360);
+
+		transition:
+			transform 0.3s ease,
+			filter 0.3s ease;
+		/* Set transform origin to center so it grows from the middle */
+		transform-origin: center;
+	}
+	.bar-button:hover {
+		transform: scale(1.05);
+		filter: brightness(1.2);
+	}
+
+	.stream-controls {
+		height: 5%;
+		width: 100%;
+
+		/*position: absolute;*/
+
+		bottom: 0;
+		left: 0;
+
+		/*z-index: 2;*/
+
+		border-top-color: oklch(0.454 0 360);
+		border-top-width: 2px;
+
+		display: flex;
+		align-items: center;
+	}
+</style>

frontend/src/routes/stream/stream.ts

@@ -0,0 +1,99 @@
+import { sendKeyboardEvent, sendMouseInputEvent, sendMouseMoveEvent, KeyAction } from "$lib/input"
+import CanvasWorker from "$lib/canvas.worker?worker";
+
+export async function getStreamTransport(url: string, certHash: Array<number>): Promise<WebTransport> {
+  let certHashArray = new Uint8Array(certHash);
+
+  if (!window.WebTransport) {
+    throw new Error('WebTransport is not supported in this browser');
+  }
+
+  const transport = new WebTransport(url, {
+    //congestionControl: "low-latency",
+    serverCertificateHashes: [
+      {
+        algorithm: "sha-256",
+        value: certHashArray,
+      }
+    ]
+  });
+
+  console.log('Connecting to WebTransport at ', url);
+  await transport.ready;
+  console.log(`WebTransport connection established`);
+
+  return transport;
+}
+
+
+export async function spawnWorker(gameplayCanvas: HTMLCanvasElement, reader: ReadableStream) {
+  const offscreenCanvas = gameplayCanvas.transferControlToOffscreen();
+
+  const worker = new CanvasWorker();
+
+  worker.postMessage({
+    canvas: offscreenCanvas,
+    reader: reader,
+  }, [offscreenCanvas, reader]);
+
+}
+
+
+export async function startWebtransportStream(
+  url: string,
+  certHash: Array<number>,
+  width: number,
+  height: number,
+  gameplayCanvas: HTMLCanvasElement,
+  keyEventElement: HTMLElement,
+  mouseElement: HTMLElement,
+) {
+  console.log(width, height);
+  gameplayCanvas.width = width;
+  gameplayCanvas.height = height;
+
+
+  console.log(`Connecting to stream at ${url} with cert_hash ${certHash}`);
+  const transport = await getStreamTransport(url, certHash);
+
+  const datagrams = transport.datagrams;
+  datagrams.incomingHighWaterMark = 20000;
+  const stream = await transport.createBidirectionalStream();
+
+  //const reader = stream.readable
+  const datagramReader = datagrams.readable
+  const writer = stream.writable.getWriter();
+
+  spawnWorker(gameplayCanvas, datagramReader);
+
+  keyEventElement.addEventListener("keydown", (event: KeyboardEvent) => { sendKeyboardEvent(writer, event, KeyAction.DOWN) });
+  keyEventElement.addEventListener("keyup", (event: KeyboardEvent) => { sendKeyboardEvent(writer, event, KeyAction.UP) });
+
+  mouseElement.addEventListener("mousedown", (event: MouseEvent) => { sendMouseInputEvent(writer, event, KeyAction.DOWN) });
+  mouseElement.addEventListener("mouseup", (event: MouseEvent) => { sendMouseInputEvent(writer, event, KeyAction.UP) });
+
+
+  mouseElement.addEventListener("click", async () => {
+    console.log("Requesting pointer lock");
+    await mouseElement.requestPointerLock({
+      //unadjustedMovement: true,
+    });
+    console.log("Pointer lock aquired");
+  })
+
+  mouseElement.addEventListener("mousemove", (event: MouseEvent) => {
+    if (document.pointerLockElement === mouseElement) {
+      sendMouseMoveEvent(writer, event);
+    }
+  });
+
+
+  // Handle connection close
+  transport.closed
+    .then(() => {
+      console.log('WebTransport connection closed');
+    })
+    .catch((error) => {
+      console.error('WebTransport connection closed with error:', error);
+    });
+}

frontend/src/routes/sverdle/+page.server.ts

@@ -1,69 +0,0 @@
-import { fail } from '@sveltejs/kit';
-import { Game } from './game';
-import type { PageServerLoad, Actions } from './$types';
-
-export const load = (({ cookies }) => {
-	const game = new Game(cookies.get('sverdle'));
-
-	return {
-		/**
-		 * The player's guessed words so far
-		 */
-		guesses: game.guesses,
-
-		/**
-		 * An array of strings like '__x_c' corresponding to the guesses, where 'x' means
-		 * an exact match, and 'c' means a close match (right letter, wrong place)
-		 */
-		answers: game.answers,
-
-		/**
-		 * The correct answer, revealed if the game is over
-		 */
-		answer: game.answers.length >= 6 ? game.answer : null
-	};
-}) satisfies PageServerLoad;
-
-export const actions = {
-	/**
-	 * Modify game state in reaction to a keypress. If client-side JavaScript
-	 * is available, this will happen in the browser instead of here
-	 */
-	update: async ({ request, cookies }) => {
-		const game = new Game(cookies.get('sverdle'));
-
-		const data = await request.formData();
-		const key = data.get('key');
-
-		const i = game.answers.length;
-
-		if (key === 'backspace') {
-			game.guesses[i] = game.guesses[i].slice(0, -1);
-		} else {
-			game.guesses[i] += key;
-		}
-
-		cookies.set('sverdle', game.toString(), { path: '/' });
-	},
-
-	/**
-	 * Modify game state in reaction to a guessed word. This logic always runs on
-	 * the server, so that people can't cheat by peeking at the JavaScript
-	 */
-	enter: async ({ request, cookies }) => {
-		const game = new Game(cookies.get('sverdle'));
-
-		const data = await request.formData();
-		const guess = data.getAll('guess') as string[];
-
-		if (!game.enter(guess)) {
-			return fail(400, { badGuess: true });
-		}
-
-		cookies.set('sverdle', game.toString(), { path: '/' });
-	},
-
-	restart: async ({ cookies }) => {
-		cookies.delete('sverdle', { path: '/' });
-	}
-} satisfies Actions;

frontend/src/routes/sverdle/+page.svelte

@@ -1,411 +0,0 @@
-<script lang="ts">
-	import { enhance } from '$app/forms';
-	import { confetti } from '@neoconfetti/svelte';
-	import type { ActionData, PageData } from './$types';
-	import { MediaQuery } from 'svelte/reactivity';
-
-	interface Props {
-		data: PageData;
-		form: ActionData;
-	}
-	let { data, form = $bindable() }: Props = $props();
-
-	/** Whether the user prefers reduced motion */
-	const reducedMotion = new MediaQuery('(prefers-reduced-motion: reduce)');
-
-	/** Whether or not the user has won */
-	let won = $derived(data.answers.at(-1) === 'xxxxx');
-
-	/** The index of the current guess */
-	let i = $derived(won ? -1 : data.answers.length);
-
-	/** The current guess */
-	let currentGuess = $derived(data.guesses[i] || '');
-
-	/** Whether the current guess can be submitted */
-	let submittable = $derived(currentGuess.length === 5);
-
-	const { classnames, description } = $derived.by(() => {
-		/**
-		 * A map of classnames for all letters that have been guessed,
-		 * used for styling the keyboard
-		 */
-		let classnames: Record<string, 'exact' | 'close' | 'missing'> = {};
-		/**
-		 * A map of descriptions for all letters that have been guessed,
-		 * used for adding text for assistive technology (e.g. screen readers)
-		 */
-		let description: Record<string, string> = {};
-		data.answers.forEach((answer, i) => {
-			const guess = data.guesses[i];
-			for (let i = 0; i < 5; i += 1) {
-				const letter = guess[i];
-				if (answer[i] === 'x') {
-					classnames[letter] = 'exact';
-					description[letter] = 'correct';
-				} else if (!classnames[letter]) {
-					classnames[letter] = answer[i] === 'c' ? 'close' : 'missing';
-					description[letter] = answer[i] === 'c' ? 'present' : 'absent';
-				}
-			}
-		});
-		return { classnames, description };
-	});
-
-	/**
-	 * Modify the game state without making a trip to the server,
-	 * if client-side JavaScript is enabled
-	 */
-	function update(event: MouseEvent) {
-		event.preventDefault();
-		const key = (event.target as HTMLButtonElement).getAttribute(
-			'data-key'
-		);
-
-		if (key === 'backspace') {
-			currentGuess = currentGuess.slice(0, -1);
-			if (form?.badGuess) form.badGuess = false;
-		} else if (currentGuess.length < 5) {
-			currentGuess += key;
-		}
-	}
-
-	/**
-	 * Trigger form logic in response to a keydown event, so that
-	 * desktop users can use the keyboard to play the game
-	 */
-	function keydown(event: KeyboardEvent) {
-		if (event.metaKey) return;
-
-		if (event.key === 'Enter' && !submittable) return;
-
-		document
-			.querySelector(`[data-key="${event.key}" i]`)
-			?.dispatchEvent(new MouseEvent('click', { cancelable: true, bubbles: true }));
-	}
-</script>
-
-<svelte:window onkeydown={keydown} />
-
-<svelte:head>
-	<title>Sverdle</title>
-	<meta name="description" content="A Wordle clone written in SvelteKit" />
-</svelte:head>
-
-<h1 class="visually-hidden">Sverdle</h1>
-
-<form
-	method="post"
-	action="?/enter"
-	use:enhance={() => {
-		// prevent default callback from resetting the form
-		return ({ update }) => {
-			update({ reset: false });
-		};
-	}}
->
-	<a class="how-to-play" href="/sverdle/how-to-play">How to play</a>
-
-	<div class="grid" class:playing={!won} class:bad-guess={form?.badGuess}>
-		{#each Array.from(Array(6).keys()) as row (row)}
-			{@const current = row === i}
-			<h2 class="visually-hidden">Row {row + 1}</h2>
-			<div class="row" class:current>
-				{#each Array.from(Array(5).keys()) as column (column)}
-					{@const guess = current ? currentGuess : data.guesses[row]}
-					{@const answer = data.answers[row]?.[column]}
-					{@const value = guess?.[column] ?? ''}
-					{@const selected = current && column === guess.length}
-					{@const exact = answer === 'x'}
-					{@const close = answer === 'c'}
-					{@const missing = answer === '_'}
-					<div class="letter" class:exact class:close class:missing class:selected>
-						{value}
-						<span class="visually-hidden">
-							{#if exact}
-								(correct)
-							{:else if close}
-								(present)
-							{:else if missing}
-								(absent)
-							{:else}
-								empty
-							{/if}
-						</span>
-						<input name="guess" disabled={!current} type="hidden" {value} />
-					</div>
-				{/each}
-			</div>
-		{/each}
-	</div>
-
-	<div class="controls">
-		{#if won || data.answers.length >= 6}
-			{#if !won && data.answer}
-				<p>the answer was "{data.answer}"</p>
-			{/if}
-			<button data-key="enter" class="restart selected" formaction="?/restart">
-				{won ? 'you won :)' : `game over :(`} play again?
-			</button>
-		{:else}
-			<div class="keyboard">
-				<button data-key="enter" class:selected={submittable} disabled={!submittable}>enter</button>
-
-				<button
-					onclick={update}
-					data-key="backspace"
-					formaction="?/update"
-					name="key"
-					value="backspace"
-				>
-					back
-				</button>
-
-				{#each ['qwertyuiop', 'asdfghjkl', 'zxcvbnm'] as row (row)}
-					<div class="row">
-						{#each row as letter, index (index)}
-							<button
-								onclick={update}
-								data-key={letter}
-								class={classnames[letter]}
-								disabled={submittable}
-								formaction="?/update"
-								name="key"
-								value={letter}
-								aria-label="{letter} {description[letter] || ''}"
-							>
-								{letter}
-							</button>
-						{/each}
-					</div>
-				{/each}
-			</div>
-		{/if}
-	</div>
-</form>
-
-{#if won}
-	<div
-		style="position: absolute; left: 50%; top: 30%"
-		use:confetti={{
-			particleCount: reducedMotion.current ? 0 : undefined,
-			force: 0.7,
-			stageWidth: window.innerWidth,
-			stageHeight: window.innerHeight,
-			colors: ['#ff3e00', '#40b3ff', '#676778']
-		}}
-	></div>
-{/if}
-
-<style>
-	form {
-		width: 100%;
-		height: 100%;
-		display: flex;
-		flex-direction: column;
-		align-items: center;
-		justify-content: center;
-		gap: 1rem;
-		flex: 1;
-	}
-
-	.how-to-play {
-		color: var(--color-text);
-	}
-
-	.how-to-play::before {
-		content: 'i';
-		display: inline-block;
-		font-size: 0.8em;
-		font-weight: 900;
-		width: 1em;
-		height: 1em;
-		padding: 0.2em;
-		line-height: 1;
-		border: 1.5px solid var(--color-text);
-		border-radius: 50%;
-		text-align: center;
-		margin: 0 0.5em 0 0;
-		position: relative;
-		top: -0.05em;
-	}
-
-	.grid {
-		--width: min(100vw, 40vh, 380px);
-		max-width: var(--width);
-		align-self: center;
-		justify-self: center;
-		width: 100%;
-		height: 100%;
-		display: flex;
-		flex-direction: column;
-		justify-content: flex-start;
-	}
-
-	.grid .row {
-		display: grid;
-		grid-template-columns: repeat(5, 1fr);
-		grid-gap: 0.2rem;
-		margin: 0 0 0.2rem 0;
-	}
-
-	@media (prefers-reduced-motion: no-preference) {
-		.grid.bad-guess .row.current {
-			animation: wiggle 0.5s;
-		}
-	}
-
-	.grid.playing .row.current {
-		filter: drop-shadow(3px 3px 10px var(--color-bg-0));
-	}
-
-	.letter {
-		aspect-ratio: 1;
-		width: 100%;
-		display: flex;
-		align-items: center;
-		justify-content: center;
-		text-align: center;
-		box-sizing: border-box;
-		text-transform: lowercase;
-		border: none;
-		font-size: calc(0.08 * var(--width));
-		border-radius: 2px;
-		background: white;
-		margin: 0;
-		color: rgba(0, 0, 0, 0.7);
-	}
-
-	.letter.missing {
-		background: rgba(255, 255, 255, 0.5);
-		color: rgba(0, 0, 0, 0.5);
-	}
-
-	.letter.exact {
-		background: var(--color-theme-2);
-		color: white;
-	}
-
-	.letter.close {
-		border: 2px solid var(--color-theme-2);
-	}
-
-	.selected {
-		outline: 2px solid var(--color-theme-1);
-	}
-
-	.controls {
-		text-align: center;
-		justify-content: center;
-		height: min(18vh, 10rem);
-	}
-
-	.keyboard {
-		--gap: 0.2rem;
-		position: relative;
-		display: flex;
-		flex-direction: column;
-		gap: var(--gap);
-		height: 100%;
-	}
-
-	.keyboard .row {
-		display: flex;
-		justify-content: center;
-		gap: 0.2rem;
-		flex: 1;
-	}
-
-	.keyboard button,
-	.keyboard button:disabled {
-		--size: min(8vw, 4vh, 40px);
-		background-color: white;
-		color: black;
-		width: var(--size);
-		border: none;
-		border-radius: 2px;
-		font-size: calc(var(--size) * 0.5);
-		margin: 0;
-	}
-
-	.keyboard button.exact {
-		background: var(--color-theme-2);
-		color: white;
-	}
-
-	.keyboard button.missing {
-		opacity: 0.5;
-	}
-
-	.keyboard button.close {
-		border: 2px solid var(--color-theme-2);
-	}
-
-	.keyboard button:focus {
-		background: var(--color-theme-1);
-		color: white;
-		outline: none;
-	}
-
-	.keyboard button[data-key='enter'],
-	.keyboard button[data-key='backspace'] {
-		position: absolute;
-		bottom: 0;
-		width: calc(1.5 * var(--size));
-		height: calc(1 / 3 * (100% - 2 * var(--gap)));
-		text-transform: uppercase;
-		font-size: calc(0.3 * var(--size));
-		padding-top: calc(0.15 * var(--size));
-	}
-
-	.keyboard button[data-key='enter'] {
-		right: calc(50% + 3.5 * var(--size) + 0.8rem);
-	}
-
-	.keyboard button[data-key='backspace'] {
-		left: calc(50% + 3.5 * var(--size) + 0.8rem);
-	}
-
-	.keyboard button[data-key='enter']:disabled {
-		opacity: 0.5;
-	}
-
-	.restart {
-		width: 100%;
-		padding: 1rem;
-		background: rgba(255, 255, 255, 0.5);
-		border-radius: 2px;
-		border: none;
-	}
-
-	.restart:focus,
-	.restart:hover {
-		background: var(--color-theme-1);
-		color: white;
-		outline: none;
-	}
-
-	@keyframes wiggle {
-		0% {
-			transform: translateX(0);
-		}
-		10% {
-			transform: translateX(-2px);
-		}
-		30% {
-			transform: translateX(4px);
-		}
-		50% {
-			transform: translateX(-6px);
-		}
-		70% {
-			transform: translateX(+4px);
-		}
-		90% {
-			transform: translateX(-2px);
-		}
-		100% {
-			transform: translateX(0);
-		}
-	}
-</style>

frontend/src/routes/sverdle/game.ts

@@ -1,75 +0,0 @@
-import { words, allowed } from './words.server';
-
-export class Game {
-	index: number;
-	guesses: string[];
-	answers: string[];
-	answer: string;
-
-	/**
-	 * Create a game object from the player's cookie, or initialise a new game
-	 */
-	constructor(serialized: string | undefined = undefined) {
-		if (serialized) {
-			const [index, guesses, answers] = serialized.split('-');
-
-			this.index = +index;
-			this.guesses = guesses ? guesses.split(' ') : [];
-			this.answers = answers ? answers.split(' ') : [];
-		} else {
-			this.index = Math.floor(Math.random() * words.length);
-			this.guesses = ['', '', '', '', '', ''];
-			this.answers = [];
-		}
-
-		this.answer = words[this.index];
-	}
-
-	/**
-	 * Update game state based on a guess of a five-letter word. Returns
-	 * true if the guess was valid, false otherwise
-	 */
-	enter(letters: string[]) {
-		const word = letters.join('');
-		const valid = allowed.has(word);
-
-		if (!valid) return false;
-
-		this.guesses[this.answers.length] = word;
-
-		const available = Array.from(this.answer);
-		const answer = Array(5).fill('_');
-
-		// first, find exact matches
-		for (let i = 0; i < 5; i += 1) {
-			if (letters[i] === available[i]) {
-				answer[i] = 'x';
-				available[i] = ' ';
-			}
-		}
-
-		// then find close matches (this has to happen
-		// in a second step, otherwise an early close
-		// match can prevent a later exact match)
-		for (let i = 0; i < 5; i += 1) {
-			if (answer[i] === '_') {
-				const index = available.indexOf(letters[i]);
-				if (index !== -1) {
-					answer[i] = 'c';
-					available[index] = ' ';
-				}
-			}
-		}
-
-		this.answers.push(answer.join(''));
-
-		return true;
-	}
-
-	/**
-	 * Serialize game state so it can be set as a cookie
-	 */
-	toString() {
-		return `${this.index}-${this.guesses.join(' ')}-${this.answers.join(' ')}`;
-	}
-}

frontend/src/routes/sverdle/how-to-play/+page.svelte

@@ -1,95 +0,0 @@
-<svelte:head>
-	<title>How to play Sverdle</title>
-	<meta name="description" content="How to play Sverdle" />
-</svelte:head>
-
-<div class="text-column">
-	<h1>How to play Sverdle</h1>
-
-	<p>
-		Sverdle is a clone of <a href="https://www.nytimes.com/games/wordle/index.html">Wordle</a>, the
-		word guessing game. To play, enter a five-letter English word. For example:
-	</p>
-
-	<div class="example">
-		<span class="close">r</span>
-		<span class="missing">i</span>
-		<span class="close">t</span>
-		<span class="missing">z</span>
-		<span class="exact">y</span>
-	</div>
-
-	<p>
-		The <span class="exact">y</span> is in the right place. <span class="close">r</span> and
-		<span class="close">t</span>
-		are the right letters, but in the wrong place. The other letters are wrong, and can be discarded.
-		Let's make another guess:
-	</p>
-
-	<div class="example">
-		<span class="exact">p</span>
-		<span class="exact">a</span>
-		<span class="exact">r</span>
-		<span class="exact">t</span>
-		<span class="exact">y</span>
-	</div>
-
-	<p>This time we guessed right! You have <strong>six</strong> guesses to get the word.</p>
-
-	<p>
-		Unlike the original Wordle, Sverdle runs on the server instead of in the browser, making it
-		impossible to cheat. It uses <code>&lt;form&gt;</code> and cookies to submit data, meaning you can
-		even play with JavaScript disabled!
-	</p>
-</div>
-
-<style>
-	span {
-		display: inline-flex;
-		justify-content: center;
-		align-items: center;
-		font-size: 0.8em;
-		width: 2.4em;
-		height: 2.4em;
-		background-color: white;
-		box-sizing: border-box;
-		border-radius: 2px;
-		border-width: 2px;
-		color: rgba(0, 0, 0, 0.7);
-	}
-
-	.missing {
-		background: rgba(255, 255, 255, 0.5);
-		color: rgba(0, 0, 0, 0.5);
-	}
-
-	.close {
-		border-style: solid;
-		border-color: var(--color-theme-2);
-	}
-
-	.exact {
-		background: var(--color-theme-2);
-		color: white;
-	}
-
-	.example {
-		display: flex;
-		justify-content: flex-start;
-		margin: 1rem 0;
-		gap: 0.2rem;
-	}
-
-	.example span {
-		font-size: 1.4rem;
-	}
-
-	p span {
-		position: relative;
-		border-width: 1px;
-		border-radius: 1px;
-		font-size: 0.4em;
-		transform: scale(2) translate(0, -10%);
-		margin: 0 1em;
-	}
-</style>

frontend/src/routes/sverdle/how-to-play/+page.ts

@@ -1,9 +0,0 @@
-import { dev } from '$app/environment';
-
-// we don't need any JS on this page, though we'll load
-// it in dev so that we get hot module replacement
-export const csr = dev;
-
-// since there's no dynamic data here, we can prerender
-// it so that it gets served as a static asset in production
-export const prerender = true;

gamestream-webtransport-proxy/Cargo.toml

@@ -6,9 +6,13 @@ edition = "2024"
 [dependencies]
 anyhow = "1.0.98"
 directories = "6.0.0"
+flatbuffers = "25.2.10"
 getrandom = { version = "0.3.3", features = ["std"] }
+h3-datagram = "0.0.2"
+h3-quinn = "0.0.10"
 hex = "0.4.3"
 hmac-sha256 = "1.1.12"
+http = "1.3.1"
 libc = "0.2.174"
 moonlight-common-c-sys = { path = "../moonlight-common-c-sys" }
 openssl = "0.10.73"

gamestream-webtransport-proxy/src/apps.rs

@@ -4,7 +4,12 @@ use salvo::prelude::*;
 use serde::{Deserialize, Serialize};
 use tracing::{debug, error};
 
-use crate::{common::AppResult, state::StateReader};
+use crate::{
+    common,
+    common::{AppError, AppResult},
+    responses,
+    state::StateReader,
+};
 
 #[derive(Deserialize)]
 struct AppListRespApp {
@@ -29,6 +34,7 @@ struct App {
     title: String,
     id: u64,
     hdr_supported: bool,
+    active: bool,
 }
 
 #[derive(Debug, Serialize, ToSchema)]
@@ -40,7 +46,7 @@ struct GetAppsResponse {
 impl crate::backend::Backend {
     #[craft(endpoint(status_codes(StatusCode::OK, StatusCode::INTERNAL_SERVER_ERROR)))]
     pub async fn get_apps(self: ::std::sync::Arc<Self>) -> AppResult<Json<GetAppsResponse>> {
-        let standard_error = Err(crate::common::AppError {
+        let standard_error = Err(AppError {
             status_code: StatusCode::INTERNAL_SERVER_ERROR,
             description: "failed to get available apps".to_string(),
         });
@@ -67,7 +73,33 @@ impl crate::backend::Backend {
         };
 
         for (_, server) in servers.into_iter() {
-            let mut base_url = crate::common::base_url(
+            let mut server_info_base_url = common::base_url(
+                "https",
+                &server.host,
+                server.https_port(),
+                &unique_id,
+                "serverinfo",
+                None,
+            );
+
+            let server_info = match common::get_url(&mut server_info_base_url, true).await {
+                Ok(s) => s,
+                Err(e) => {
+                    error!("Could not get server_info: {}", e);
+                    return standard_error;
+                }
+            };
+            debug!("server_info: {server_info}");
+            let server_info: responses::ServerInfoResponse =
+                match serde_xml_rs::from_str(&server_info) {
+                    Ok(s) => s,
+                    Err(e) => {
+                        error!("Could not parse serverinfo response: {e}");
+                        return standard_error;
+                    }
+                };
+
+            let mut base_url = common::base_url(
                 "https",
                 &server.host,
                 server.https_port(),
@@ -76,7 +108,7 @@ impl crate::backend::Backend {
                 None,
             );
 
-            let resp = match crate::common::get_url(&mut base_url, true).await {
+            let resp = match common::get_url(&mut base_url, true).await {
                 Ok(r) => r,
                 Err(e) => {
                     error!("could not get applist from server {}: {}", server.name, e);
@@ -103,6 +135,7 @@ impl crate::backend::Backend {
                     title: a.app_title,
                     hdr_supported: a.is_hdr_supported,
                     id: a.id,
+                    active: a.id == server_info.currentgame,
                 })
                 .rev()
                 .collect();

gamestream-webtransport-proxy/src/certs.rs

@@ -120,15 +120,41 @@ fn generate_http_cert_and_key(
 
     cert_builder.set_version(2)?;
 
+    let serial = openssl::bn::BigNum::from_u32(1)?;
+    let asn_serial = openssl::asn1::Asn1Integer::from_bn(&serial)?;
+    cert_builder.set_serial_number(&asn_serial)?;
+
     // Set subject (Distinguished Name)
     let mut name_builder = X509NameBuilder::new()?;
-    name_builder.append_entry_by_text("CN", "mumble-web self-signed")?;
+    name_builder.append_entry_by_text("CN", "localhost")?;
     let subject_name = name_builder.build();
     cert_builder.set_subject_name(&subject_name)?;
-
-    // Set issuer (same as subject for self-signed)
     cert_builder.set_issuer_name(&subject_name)?;
 
+    let context = cert_builder.x509v3_context(None, None);
+
+    let mut san = openssl::x509::extension::SubjectAlternativeName::new();
+    san.dns("localhost");
+
+    let san_extension = san.build(&context)?;
+
+    let key_usage = openssl::x509::extension::KeyUsage::new()
+        .digital_signature()
+        .key_encipherment()
+        .build()?;
+
+    let ext_key_usage = openssl::x509::extension::ExtendedKeyUsage::new()
+        .server_auth()
+        .build()?;
+
+    // Add Subject Key Identifier
+    let subject_key_id = openssl::x509::extension::SubjectKeyIdentifier::new().build(&context)?;
+
+    cert_builder.append_extension(san_extension)?;
+    cert_builder.append_extension(key_usage)?;
+    cert_builder.append_extension(ext_key_usage)?;
+    cert_builder.append_extension(subject_key_id)?;
+
     cert_builder.set_not_before(&now)?;
     cert_builder.set_not_after(&expiration_time)?;
     cert_builder.set_pubkey(&key)?;

gamestream-webtransport-proxy/src/gamestream/config.rs

@@ -40,7 +40,7 @@ pub fn stream_config(stream: &crate::backend::Stream) -> _STREAM_CONFIGURATION {
         height: stream.stream_config.mode.height,
         fps: stream.stream_config.mode.fps,
         bitrate: stream.stream_config.bitrate_kbps,
-        packetSize: 512,
+        packetSize: 1024,
         streamingRemotely: STREAM_CFG_AUTO,
         audioConfiguration: (0x3 << 16) | (2 << 8) | 0xCA,
         supportedVideoFormats: VIDEO_FORMAT_H264,

gamestream-webtransport-proxy/src/gamestream/decoder.rs

@@ -12,7 +12,7 @@ use tokio::sync::mpsc;
 use tracing::{debug, error};
 
 #[derive(Serialize)]
-enum FrameType {
+pub enum FrameType {
     PFRAME,
     IDR,
 }
@@ -32,7 +32,7 @@ impl TryFrom<i32> for FrameType {
 }
 
 #[derive(Serialize)]
-enum VideoFormat {
+pub enum VideoFormat {
     H264,
     H264_HIGH8_444,
     H265,
@@ -92,8 +92,8 @@ impl TryFrom<i32> for BufferType {
 }
 
 #[derive(Serialize)]
-struct Buffer {
+pub struct Buffer {
-    data: Vec<u8>,
+    pub data: Vec<u8>,
     buffer_type: BufferType,
 }
 
@@ -119,23 +119,27 @@ pub enum RendererMessage {
         redraw_rate: u64,
         dr_flags: i32,
     },
-    DecodeUnit {
+    DecodeUnitStart {
         frame_number: u64,
         frame_type: FrameType,
+        num_buffers: u64,
 
         host_processing_latency: u16,
-        receieve_time_ms: u64,
+        receive_time_ms: u64,
         enqueue_time_ms: u64,
         presentation_time: u64,
 
-        full_length: usize,
+        full_length: u64,
-        //buffers: Vec<Buffer>,
-        buffer: Buffer,
-        index: u64,
 
         hdr_active: bool,
         colorspace: u8,
     },
+    DecodeUnitBuffer {
+        frame_number: u64,
+        buffer_index: u64,
+        buffer_offset: u64,
+        buffer: Buffer,
+    },
 }
 
 impl RendererMessage {
@@ -156,41 +160,59 @@ impl RendererMessage {
     }
 
     fn from_decode_unit(decode_unit: _DECODE_UNIT) -> Result<Vec<Self>> {
-        let mut buffers = Vec::new();
+        let mut messages = Vec::new();
 
         if decode_unit.bufferList.is_null() {
             return Err(anyhow!("DecodeUnit bufferList is null"));
         }
+        let frame_number = <u64>::try_from(decode_unit.frameNumber)?;
+
+        messages.push(RendererMessage::DecodeUnitStart {
+            frame_number,
+            frame_type: FrameType::try_from(decode_unit.frameType)?,
+            num_buffers: 0,
+            host_processing_latency: decode_unit.frameHostProcessingLatency,
+            receive_time_ms: decode_unit.receiveTimeMs,
+            enqueue_time_ms: decode_unit.enqueueTimeMs,
+            presentation_time: decode_unit.presentationTimeMs as u64,
+            full_length: <u64>::try_from(decode_unit.fullLength)?,
+            hdr_active: decode_unit.hdrActive,
+            colorspace: decode_unit.colorspace,
+        });
+
         let mut next = unsafe { *decode_unit.bufferList };
 
         let mut index = 0;
+        let mut offset = 0;
         loop {
+            let b = Buffer::try_from(next)?;
+            let buffer_len = b.data.len() as u64;
+
+            messages.push(RendererMessage::DecodeUnitBuffer {
+                frame_number,
+                buffer_index: index,
+                buffer_offset: offset,
+                buffer: b,
+            });
+
+            offset = offset + buffer_len;
+            index = index + 1;
             if next.next.is_null() {
                 break;
             }
-            let buffer = Buffer::try_from(next)?;
 
-            let msg = RendererMessage::DecodeUnit {
-                frame_number: <u64>::try_from(decode_unit.frameNumber)?,
-                frame_type: FrameType::try_from(decode_unit.frameType)?,
-                host_processing_latency: decode_unit.frameHostProcessingLatency,
-                receieve_time_ms: decode_unit.receiveTimeMs,
-                enqueue_time_ms: decode_unit.enqueueTimeMs,
-                presentation_time: decode_unit.presentationTimeMs as u64,
-                full_length: <usize>::try_from(decode_unit.fullLength)?,
-                buffer,
-                index,
-                hdr_active: decode_unit.hdrActive,
-                colorspace: decode_unit.colorspace,
-            };
-
-            buffers.push(msg);
-
-            index = index + 1;
             next = unsafe { *next.next };
         }
 
-        Ok(buffers)
+        if let RendererMessage::DecodeUnitStart {
+            ref mut num_buffers,
+            ..
+        } = messages[0]
+        {
+            *num_buffers = index;
+        }
+
+        Ok(messages)
     }
 }
 
@@ -251,12 +273,12 @@ extern "C" fn start_cb() {
 }
 
 extern "C" fn submit_decode_unit_cb(decode_unit: PDECODE_UNIT) -> std::os::raw::c_int {
-    debug!("SUBMIT DECODE UNIT CB");
     if decode_unit.is_null() {
         error!("Decode unit pointer was null");
         return -1;
     }
     let decode_unit = unsafe { *decode_unit };
+    //debug!("decode unit bytes: {}", decode_unit.fullLength);
 
     let messages = match RendererMessage::from_decode_unit(decode_unit) {
         Ok(m) => m,
@@ -265,6 +287,11 @@ extern "C" fn submit_decode_unit_cb(decode_unit: PDECODE_UNIT) -> std::os::raw::
             return -1;
         }
     };
+    debug!(
+        "got decode unit with {} buffers: {:?}",
+        messages.len() - 1,
+        std::time::Instant::now()
+    );
 
     for msg in messages {
         let ret = send_message(msg);
@@ -272,6 +299,7 @@ extern "C" fn submit_decode_unit_cb(decode_unit: PDECODE_UNIT) -> std::os::raw::
             return ret;
         }
     }
+    //debug!("dispatched decode unit: {:?}", std::time::Instant::now());
     0
 }
 

gamestream-webtransport-proxy/src/gamestream/mod.rs

@@ -2,7 +2,7 @@ use anyhow::{Result, anyhow};
 use tokio::sync::mpsc;
 
 mod config;
-mod decoder;
+pub mod decoder;
 
 #[derive(Debug)]
 pub struct GamestreamChannels {
@@ -20,7 +20,7 @@ pub fn start_connection(
         address,
         stream.server_codec_mode_support,
     )?;
-    let mut stream_config = config::stream_config(&stream);
+    let mut stream_config = config::stream_config(stream);
     let mut listener_callbacks = config::listener_callbacks();
     let (mut decoder_callbacks, decoder_rx) = decoder::decoder_callbacks()?;
     let ret;
@@ -47,3 +47,31 @@ pub fn start_connection(
 pub fn stop_connection() {
     unsafe { moonlight_common_c_sys::LiStopConnection() };
 }
+
+pub fn send_keyboard_event(keycode: i16, keyaction: i8, modifiers: u8) -> Result<()> {
+    let ret =
+        unsafe { moonlight_common_c_sys::LiSendKeyboardEvent(keycode, keyaction, modifiers as i8) };
+
+    match ret {
+        0 => Ok(()),
+        _ => Err(anyhow!("Could not send keyboard event: {ret}")),
+    }
+}
+
+pub fn send_mouse_move_event(movement_x: i16, movement_y: i16) -> Result<()> {
+    let ret = unsafe { moonlight_common_c_sys::LiSendMouseMoveEvent(movement_x, movement_y) };
+
+    match ret {
+        0 => Ok(()),
+        _ => Err(anyhow!("Could not send mouse movement event: {ret}")),
+    }
+}
+
+pub fn send_mouse_input_event(action: i8, button: i32) -> Result<()> {
+    let ret = unsafe { moonlight_common_c_sys::LiSendMouseButtonEvent(action, button) };
+
+    match ret {
+        0 => Ok(()),
+        _ => Err(anyhow!("Could not send mouse input event: {ret}")),
+    }
+}

gamestream-webtransport-proxy/src/main.rs

@@ -9,6 +9,7 @@ mod common;
 mod gamestream;
 mod pair;
 mod proxy;
+mod responses;
 mod state;
 mod stream;
 

gamestream-webtransport-proxy/src/proxy/handler.rs

@@ -29,6 +29,12 @@ async fn setup_webtransport(
 ) -> Result<(
     impl tokio::io::AsyncWrite + Send + Sync + 'static,
     impl tokio::io::AsyncRead + Send + Sync + 'static,
+    h3_datagram::datagram_handler::DatagramSender<
+        <h3_quinn::Connection as h3_datagram::quic_traits::DatagramConnectionExt<
+            salvo::hyper::body::Bytes,
+        >>::SendDatagramHandler,
+        salvo::hyper::body::Bytes,
+    >,
     //salvo::webtransport::stream::SendStream<
     //    impl salvo::proto::quic::SendStream<salvo::hyper::body::Bytes>,
     //    salvo::hyper::body::Bytes,
@@ -39,13 +45,17 @@ async fn setup_webtransport(
     //>,
 )> {
     let session = req.web_transport_mut().await?;
+
+    let datagram_send = session.datagram_sender();
+
     let bidirectional_stream = session
         .accept_bi()
         .await?
         .ok_or(anyhow!("No bidirectional stream"))?;
 
     if let webtransport::server::AcceptedBi::BidiStream(_, stream) = bidirectional_stream {
-        Ok(stream.split())
+        let (stream_send, stream_recv) = stream.split();
+        Ok((stream_send, stream_recv, datagram_send))
     } else {
         Err(anyhow!("bidirectional stream was of the wrong type"))
     }
@@ -76,7 +86,8 @@ impl crate::proxy::Proxy {
         });
 
         info!("WebTransport connection initiated");
-        let (wt_send, wt_recv) = match setup_webtransport(req).await {
+        let (wt_stream_send, wt_stream_recv, wt_datagram_send) = match setup_webtransport(req).await
+        {
             Ok(w) => w,
             Err(e) => {
                 error!("Could not upgrade connection to WebTransport: {e}");
@@ -95,7 +106,7 @@ impl crate::proxy::Proxy {
             }
         };
 
-        match super::proxy_main(stream, wt_send, wt_recv).await {
+        match super::proxy_main(stream, wt_stream_send, wt_stream_recv, wt_datagram_send).await {
             Ok(()) => Ok(()),
             Err(e) => {
                 error!("Proxy main loop failed: {e}");

gamestream-webtransport-proxy/src/proxy/input/mod.rs

@@ -0,0 +1 @@
+pub mod input_generated;

gamestream-webtransport-proxy/src/proxy/mod.rs

@@ -1,11 +1,13 @@
 use anyhow::{Context, Result};
-use salvo::{conn::http1::Connection, hyper::body::Bytes, proto::WebTransportSession};
 use tokio::{io::AsyncReadExt, io::AsyncWriteExt, select, sync::RwLock};
 use tracing::{debug, error, info};
 
 use crate::{backend, gamestream};
 
 pub mod handler;
+mod input;
+mod packet_parser;
+mod video;
 
 pub struct Proxy {
     pub cert_hash: [u8; 32],
@@ -32,6 +34,12 @@ async fn proxy_main(
     stream: backend::Stream,
     mut wt_send: impl tokio::io::AsyncWrite + Send + Sync + 'static + std::marker::Unpin,
     mut wt_recv: impl tokio::io::AsyncRead + Send + Sync + 'static + std::marker::Unpin,
+    mut wt_datagram_send: h3_datagram::datagram_handler::DatagramSender<
+        <h3_quinn::Connection as h3_datagram::quic_traits::DatagramConnectionExt<
+            salvo::hyper::body::Bytes,
+        >>::SendDatagramHandler,
+        salvo::hyper::body::Bytes,
+    >,
 ) -> Result<()> {
     debug!(
         "Connecting to stream at address {} with stream config {:?}",
@@ -40,19 +48,15 @@ async fn proxy_main(
 
     let mut channels = spawn_gamestream(stream).await?;
 
-    let mut buffer = vec![0; 65536].into_boxed_slice();
+    let mut packet_buffer = packet_parser::PacketBuffer::new();
+    let mut buffer = [0u8; 65536];
 
     loop {
         select! {
             gamestream_packet = channels.gamestream_channels.decoder_rx.recv() => {
                 match gamestream_packet {
                     Some(frame) => {
-                        info!("Got decoder packet");
+                        video::send_video_update(&frame, &mut wt_datagram_send).await?;
-                        let frame_json = serde_json::to_vec(&frame)?;
-                        let frame_json_len: u32 = <u32>::try_from(frame_json.len())?;
-
-                        wt_send.write_all(&frame_json_len.to_le_bytes()).await?;
-                        wt_send.write_all(&frame_json).await?;
                     }
                     None => {
                         error!("Decoder channel is None");
@@ -60,8 +64,11 @@ async fn proxy_main(
                     }
                 }
             },
-            webtransport_packet = wt_recv.read(&mut buffer) => {
+            ret = wt_recv.read(&mut buffer) => {
-                info!("Got packet from client");
+                let bytes_read = ret?;
+
+                packet_parser::handle_client_packet(&mut packet_buffer, &buffer[..bytes_read])?;
+
             }
         }
     }

gamestream-webtransport-proxy/src/proxy/packet_parser.rs

@@ -0,0 +1,158 @@
+use anyhow::{Result, anyhow};
+use tracing::{debug, error, info};
+
+use super::input::input_generated::input_event;
+use crate::gamestream;
+
+pub struct PacketBuffer {
+    buffer: Vec<u8>,
+    expected_length: Option<usize>,
+    length_bytes_needed: usize,
+}
+
+impl PacketBuffer {
+    pub fn new() -> Self {
+        Self {
+            buffer: Vec::new(),
+            expected_length: None,
+            length_bytes_needed: 4, // Assuming u32 length (4 bytes)
+        }
+    }
+
+    /// Process incoming data and return complete packets
+    pub fn process_data(&mut self, data: &[u8]) -> Vec<Vec<u8>> {
+        let mut complete_packets = Vec::new();
+        self.buffer.extend_from_slice(data);
+
+        loop {
+            // If we don't know the expected length yet, try to read it
+            if self.expected_length.is_none() {
+                if self.buffer.len() >= self.length_bytes_needed {
+                    // Read length from first 4 bytes (adjust as needed)
+                    let length = u32::from_be_bytes([
+                        self.buffer[0],
+                        self.buffer[1],
+                        self.buffer[2],
+                        self.buffer[3],
+                    ]) as usize;
+
+                    self.expected_length = Some(length);
+                } else {
+                    // Not enough data to read length yet
+                    break;
+                }
+            }
+
+            // We know the expected length, check if we have a complete packet
+            if let Some(expected_len) = self.expected_length {
+                let total_packet_size = self.length_bytes_needed + expected_len;
+
+                if self.buffer.len() >= total_packet_size {
+                    // We have a complete packet - extract just the data portion
+                    let packet_data =
+                        self.buffer[self.length_bytes_needed..total_packet_size].to_vec();
+                    complete_packets.push(packet_data);
+
+                    // Remove the processed packet from buffer
+                    self.buffer.drain(0..total_packet_size);
+                    self.expected_length = None;
+                } else {
+                    // Packet is incomplete
+                    break;
+                }
+            }
+        }
+
+        complete_packets
+    }
+}
+
+pub fn process_keyboard_event(input_event: input_event::InputEvent) -> Result<()> {
+    let Some(table) = input_event.input() else {
+        debug!("Keyboard event table was empty, ignoring");
+        return Err(anyhow!("Keyboard event table was empty"));
+    };
+    let keyboard_event = unsafe { input_event::KeyboardInput::init_from_table(table) };
+
+    let key_action_i8 = match keyboard_event.key_action() {
+        input_event::KeyAction::DOWN => moonlight_common_c_sys::KEY_ACTION_DOWN,
+        input_event::KeyAction::UP => moonlight_common_c_sys::KEY_ACTION_UP,
+        _ => {
+            debug!("Invalid KeyAction value, ignoring");
+            return Err(anyhow!("Invalid KeyAction value"));
+        }
+    };
+
+    gamestream::send_keyboard_event(keyboard_event.key_code(), key_action_i8, 0)
+}
+
+pub fn process_mouse_move_event(input_event: input_event::InputEvent) -> Result<()> {
+    let Some(table) = input_event.input() else {
+        debug!("MouseMovement event table was empty, ignoring");
+        return Err(anyhow!("MouseMovement event table was empty"));
+    };
+    let mouse_event = unsafe { input_event::MouseMovement::init_from_table(table) };
+
+    gamestream::send_mouse_move_event(mouse_event.movement_x(), mouse_event.movement_y())
+}
+
+pub fn process_mouse_input_event(input_event: input_event::InputEvent) -> Result<()> {
+    let Some(table) = input_event.input() else {
+        debug!("MouseInput event table was empty, ignoring");
+        return Err(anyhow!("MouseInput event table was empty"));
+    };
+    let mouse_event = unsafe { input_event::MouseInput::init_from_table(table) };
+
+    let button_action_i8 = match mouse_event.button_action() {
+        input_event::KeyAction::DOWN => moonlight_common_c_sys::BUTTON_ACTION_PRESS,
+        input_event::KeyAction::UP => moonlight_common_c_sys::BUTTON_ACTION_RELEASE,
+        _ => {
+            error!("Invalid KeyAction value, ignoring",);
+            return Err(anyhow!("Invalid KeyAction value"));
+        }
+    };
+
+    let mouse_button_i32 = match mouse_event.button() {
+        input_event::MouseButton::LEFT => moonlight_common_c_sys::BUTTON_LEFT,
+        input_event::MouseButton::MIDDLE => moonlight_common_c_sys::BUTTON_MIDDLE,
+        input_event::MouseButton::RIGHT => moonlight_common_c_sys::BUTTON_RIGHT,
+        input_event::MouseButton::X1 => moonlight_common_c_sys::BUTTON_X1,
+        input_event::MouseButton::X2 => moonlight_common_c_sys::BUTTON_X2,
+        _ => {
+            error!("Invalid MouseButton value, ignoring",);
+            return Err(anyhow!("Invalid MouseButton value"));
+        }
+    };
+
+    gamestream::send_mouse_input_event(button_action_i8, mouse_button_i32)
+}
+
+pub fn handle_client_packet(packet_buffer: &mut PacketBuffer, buffer: &[u8]) -> anyhow::Result<()> {
+    let complete_packets = packet_buffer.process_data(buffer);
+
+    // TODO: only supports input packets. this should use a union wrapper or something
+    for packet_data in complete_packets {
+        let input_event = input_event::root_as_input_event(&packet_data)?;
+        match input_event.input_type() {
+            input_event::Input::Keyboard => {
+                process_keyboard_event(input_event)?;
+            }
+            input_event::Input::MouseMovement => {
+                process_mouse_move_event(input_event)?;
+            }
+            input_event::Input::MouseInput => {
+                process_mouse_input_event(input_event)?;
+            }
+
+            input_event::Input::NONE => {
+                debug!("Input event was empty, ignoring.")
+            }
+            _ => {
+                error!("Unknown InputEvent type");
+                return Err(anyhow!("Unknown InputEvent type"));
+            }
+        }
+    }
+
+    Ok(())
+}

gamestream-webtransport-proxy/src/proxy/video/mod.rs

@@ -0,0 +1,185 @@
+use anyhow::Result;
+use tracing::debug;
+
+use crate::gamestream;
+use video_generated::video_update;
+
+mod video_generated;
+
+fn create_setup_videoupdate(
+    video_format: &gamestream::decoder::VideoFormat,
+    width: u64,
+    height: u64,
+    redraw_rate: u64,
+) -> Vec<u8> {
+    let mut builder = flatbuffers::FlatBufferBuilder::with_capacity(1024);
+
+    //TODO: this is hardcoded to h264 main profile
+    let video_format = Some(builder.create_string("avc1.4D401E"));
+
+    let setup = video_update::Setup::create(
+        &mut builder,
+        &video_update::SetupArgs {
+            video_format,
+            width: width as u16,
+            height: height as u16,
+            redraw_rate: redraw_rate as u16,
+        },
+    );
+
+    let video_update = video_update::VideoUpdate::create(
+        &mut builder,
+        &video_update::VideoUpdateArgs {
+            update_type: video_update::Update::Setup,
+            update: Some(setup.as_union_value()),
+        },
+    );
+
+    builder.finish(video_update, None);
+
+    builder.finished_data().to_vec()
+}
+
+fn create_decodeunitstart_videoupdate(
+    frame_number: u64,
+    frame_type: &gamestream::decoder::FrameType,
+    num_buffers: u64,
+    receive_time_ms: u64,
+    full_length: u64,
+) -> Vec<u8> {
+    let mut builder = flatbuffers::FlatBufferBuilder::with_capacity(1024);
+
+    let frame_type_fb = match frame_type {
+        gamestream::decoder::FrameType::IDR => video_update::FrameType::IDR,
+        gamestream::decoder::FrameType::PFRAME => video_update::FrameType::PFRAME,
+    };
+
+    let decode_unit_start = video_update::DecodeUnitStart::create(
+        &mut builder,
+        &video_update::DecodeUnitStartArgs {
+            frame_number,
+            frame_type: frame_type_fb,
+            num_buffers,
+            receive_time_ms: receive_time_ms as u16,
+            full_length,
+        },
+    );
+
+    let video_update = video_update::VideoUpdate::create(
+        &mut builder,
+        &video_update::VideoUpdateArgs {
+            update_type: video_update::Update::DecodeUnitStart,
+            update: Some(decode_unit_start.as_union_value()),
+        },
+    );
+
+    builder.finish(video_update, None);
+
+    builder.finished_data().to_vec()
+}
+
+fn create_decodeunitbuffer_videoupdate(
+    frame_number: u64,
+    buffer_index: u64,
+    buffer_offset: u64,
+    buffer: &gamestream::decoder::Buffer,
+) -> Vec<u8> {
+    let mut builder = flatbuffers::FlatBufferBuilder::with_capacity(1024);
+
+    let data_vector = builder.create_vector(&buffer.data);
+
+    let decode_unit_buffer = video_update::DecodeUnitBuffer::create(
+        &mut builder,
+        &video_update::DecodeUnitBufferArgs {
+            frame_number,
+            buffer_index,
+            buffer_offset,
+            data: Some(data_vector),
+        },
+    );
+
+    let video_update = video_update::VideoUpdate::create(
+        &mut builder,
+        &video_update::VideoUpdateArgs {
+            update_type: video_update::Update::DecodeUnitBuffer,
+            update: Some(decode_unit_buffer.as_union_value()),
+        },
+    );
+
+    builder.finish(video_update, None);
+
+    builder.finished_data().to_vec()
+}
+
+pub async fn send_video_update(
+    frame: &gamestream::decoder::RendererMessage,
+    wt_datagram_send: &mut h3_datagram::datagram_handler::DatagramSender<
+        <h3_quinn::Connection as h3_datagram::quic_traits::DatagramConnectionExt<
+            salvo::hyper::body::Bytes,
+        >>::SendDatagramHandler,
+        salvo::hyper::body::Bytes,
+    >,
+) -> Result<()> {
+    let mut print_time = false;
+    let buffer = match frame {
+        gamestream::decoder::RendererMessage::Setup {
+            video_format,
+            width,
+            height,
+            redraw_rate,
+            dr_flags,
+        } => create_setup_videoupdate(video_format, *width, *height, *redraw_rate),
+        gamestream::decoder::RendererMessage::DecodeUnitStart {
+            frame_number,
+            frame_type,
+            num_buffers,
+            host_processing_latency,
+            receive_time_ms,
+            enqueue_time_ms,
+            presentation_time,
+            full_length,
+            hdr_active,
+            colorspace,
+        } => {
+            //debug!(
+            //    "sending decodeunitstart {}: {:?}",
+            //    *frame_number,
+            //    std::time::Instant::now()
+            //);
+            create_decodeunitstart_videoupdate(
+                *frame_number,
+                frame_type,
+                *num_buffers,
+                *receive_time_ms,
+                *full_length,
+            )
+        }
+        gamestream::decoder::RendererMessage::DecodeUnitBuffer {
+            frame_number,
+            buffer_index,
+            buffer_offset,
+            buffer,
+        } => {
+            //debug!(
+            //    "sending decodeunitbuffer {}/{}: {:?}",
+            //    *frame_number,
+            //    *buffer_index,
+            //    std::time::Instant::now()
+            //);
+            create_decodeunitbuffer_videoupdate(
+                *frame_number,
+                *buffer_index,
+                *buffer_offset,
+                buffer,
+            )
+        }
+    };
+
+    let bytes = salvo::hyper::body::Bytes::copy_from_slice(&buffer);
+
+    wt_datagram_send.send_datagram(bytes)?;
+    debug!("sent start: {:?}", std::time::Instant::now());
+    //if (print_time) {
+    //}
+    Ok(())
+}

gamestream-webtransport-proxy/src/proxy/video/video_generated.rs

@@ -0,0 +1,925 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+
+// @generated
+
+use core::mem;
+use core::cmp::Ordering;
+
+extern crate flatbuffers;
+use self::flatbuffers::{EndianScalar, Follow};
+
+#[allow(unused_imports, dead_code)]
+pub mod video_update {
+
+  use core::mem;
+  use core::cmp::Ordering;
+
+  extern crate flatbuffers;
+  use self::flatbuffers::{EndianScalar, Follow};
+
+#[deprecated(since = "2.0.0", note = "Use associated constants instead. This will no longer be generated in 2021.")]
+pub const ENUM_MIN_FRAME_TYPE: i8 = 0;
+#[deprecated(since = "2.0.0", note = "Use associated constants instead. This will no longer be generated in 2021.")]
+pub const ENUM_MAX_FRAME_TYPE: i8 = 1;
+#[deprecated(since = "2.0.0", note = "Use associated constants instead. This will no longer be generated in 2021.")]
+#[allow(non_camel_case_types)]
+pub const ENUM_VALUES_FRAME_TYPE: [FrameType; 2] = [
+  FrameType::PFRAME,
+  FrameType::IDR,
+];
+
+#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Default)]
+#[repr(transparent)]
+pub struct FrameType(pub i8);
+#[allow(non_upper_case_globals)]
+impl FrameType {
+  pub const PFRAME: Self = Self(0);
+  pub const IDR: Self = Self(1);
+
+  pub const ENUM_MIN: i8 = 0;
+  pub const ENUM_MAX: i8 = 1;
+  pub const ENUM_VALUES: &'static [Self] = &[
+    Self::PFRAME,
+    Self::IDR,
+  ];
+  /// Returns the variant's name or "" if unknown.
+  pub fn variant_name(self) -> Option<&'static str> {
+    match self {
+      Self::PFRAME => Some("PFRAME"),
+      Self::IDR => Some("IDR"),
+      _ => None,
+    }
+  }
+}
+impl core::fmt::Debug for FrameType {
+  fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result {
+    if let Some(name) = self.variant_name() {
+      f.write_str(name)
+    } else {
+      f.write_fmt(format_args!("<UNKNOWN {:?}>", self.0))
+    }
+  }
+}
+impl<'a> flatbuffers::Follow<'a> for FrameType {
+  type Inner = Self;
+  #[inline]
+  unsafe fn follow(buf: &'a [u8], loc: usize) -> Self::Inner {
+    let b = flatbuffers::read_scalar_at::<i8>(buf, loc);
+    Self(b)
+  }
+}
+
+impl flatbuffers::Push for FrameType {
+    type Output = FrameType;
+    #[inline]
+    unsafe fn push(&self, dst: &mut [u8], _written_len: usize) {
+        flatbuffers::emplace_scalar::<i8>(dst, self.0);
+    }
+}
+
+impl flatbuffers::EndianScalar for FrameType {
+  type Scalar = i8;
+  #[inline]
+  fn to_little_endian(self) -> i8 {
+    self.0.to_le()
+  }
+  #[inline]
+  #[allow(clippy::wrong_self_convention)]
+  fn from_little_endian(v: i8) -> Self {
+    let b = i8::from_le(v);
+    Self(b)
+  }
+}
+
+impl<'a> flatbuffers::Verifiable for FrameType {
+  #[inline]
+  fn run_verifier(
+    v: &mut flatbuffers::Verifier, pos: usize
+  ) -> Result<(), flatbuffers::InvalidFlatbuffer> {
+    use self::flatbuffers::Verifiable;
+    i8::run_verifier(v, pos)
+  }
+}
+
+impl flatbuffers::SimpleToVerifyInSlice for FrameType {}
+#[deprecated(since = "2.0.0", note = "Use associated constants instead. This will no longer be generated in 2021.")]
+pub const ENUM_MIN_UPDATE: u8 = 0;
+#[deprecated(since = "2.0.0", note = "Use associated constants instead. This will no longer be generated in 2021.")]
+pub const ENUM_MAX_UPDATE: u8 = 3;
+#[deprecated(since = "2.0.0", note = "Use associated constants instead. This will no longer be generated in 2021.")]
+#[allow(non_camel_case_types)]
+pub const ENUM_VALUES_UPDATE: [Update; 4] = [
+  Update::NONE,
+  Update::Setup,
+  Update::DecodeUnitStart,
+  Update::DecodeUnitBuffer,
+];
+
+#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Default)]
+#[repr(transparent)]
+pub struct Update(pub u8);
+#[allow(non_upper_case_globals)]
+impl Update {
+  pub const NONE: Self = Self(0);
+  pub const Setup: Self = Self(1);
+  pub const DecodeUnitStart: Self = Self(2);
+  pub const DecodeUnitBuffer: Self = Self(3);
+
+  pub const ENUM_MIN: u8 = 0;
+  pub const ENUM_MAX: u8 = 3;
+  pub const ENUM_VALUES: &'static [Self] = &[
+    Self::NONE,
+    Self::Setup,
+    Self::DecodeUnitStart,
+    Self::DecodeUnitBuffer,
+  ];
+  /// Returns the variant's name or "" if unknown.
+  pub fn variant_name(self) -> Option<&'static str> {
+    match self {
+      Self::NONE => Some("NONE"),
+      Self::Setup => Some("Setup"),
+      Self::DecodeUnitStart => Some("DecodeUnitStart"),
+      Self::DecodeUnitBuffer => Some("DecodeUnitBuffer"),
+      _ => None,
+    }
+  }
+}
+impl core::fmt::Debug for Update {
+  fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result {
+    if let Some(name) = self.variant_name() {
+      f.write_str(name)
+    } else {
+      f.write_fmt(format_args!("<UNKNOWN {:?}>", self.0))
+    }
+  }
+}
+impl<'a> flatbuffers::Follow<'a> for Update {
+  type Inner = Self;
+  #[inline]
+  unsafe fn follow(buf: &'a [u8], loc: usize) -> Self::Inner {
+    let b = flatbuffers::read_scalar_at::<u8>(buf, loc);
+    Self(b)
+  }
+}
+
+impl flatbuffers::Push for Update {
+    type Output = Update;
+    #[inline]
+    unsafe fn push(&self, dst: &mut [u8], _written_len: usize) {
+        flatbuffers::emplace_scalar::<u8>(dst, self.0);
+    }
+}
+
+impl flatbuffers::EndianScalar for Update {
+  type Scalar = u8;
+  #[inline]
+  fn to_little_endian(self) -> u8 {
+    self.0.to_le()
+  }
+  #[inline]
+  #[allow(clippy::wrong_self_convention)]
+  fn from_little_endian(v: u8) -> Self {
+    let b = u8::from_le(v);
+    Self(b)
+  }
+}
+
+impl<'a> flatbuffers::Verifiable for Update {
+  #[inline]
+  fn run_verifier(
+    v: &mut flatbuffers::Verifier, pos: usize
+  ) -> Result<(), flatbuffers::InvalidFlatbuffer> {
+    use self::flatbuffers::Verifiable;
+    u8::run_verifier(v, pos)
+  }
+}
+
+impl flatbuffers::SimpleToVerifyInSlice for Update {}
+pub struct UpdateUnionTableOffset {}
+
+pub enum SetupOffset {}
+#[derive(Copy, Clone, PartialEq)]
+
+pub struct Setup<'a> {
+  pub _tab: flatbuffers::Table<'a>,
+}
+
+impl<'a> flatbuffers::Follow<'a> for Setup<'a> {
+  type Inner = Setup<'a>;
+  #[inline]
+  unsafe fn follow(buf: &'a [u8], loc: usize) -> Self::Inner {
+    Self { _tab: flatbuffers::Table::new(buf, loc) }
+  }
+}
+
+impl<'a> Setup<'a> {
+  pub const VT_VIDEO_FORMAT: flatbuffers::VOffsetT = 4;
+  pub const VT_WIDTH: flatbuffers::VOffsetT = 6;
+  pub const VT_HEIGHT: flatbuffers::VOffsetT = 8;
+  pub const VT_REDRAW_RATE: flatbuffers::VOffsetT = 10;
+
+  #[inline]
+  pub unsafe fn init_from_table(table: flatbuffers::Table<'a>) -> Self {
+    Setup { _tab: table }
+  }
+  #[allow(unused_mut)]
+  pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr, A: flatbuffers::Allocator + 'bldr>(
+    _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr, A>,
+    args: &'args SetupArgs<'args>
+  ) -> flatbuffers::WIPOffset<Setup<'bldr>> {
+    let mut builder = SetupBuilder::new(_fbb);
+    if let Some(x) = args.video_format { builder.add_video_format(x); }
+    builder.add_redraw_rate(args.redraw_rate);
+    builder.add_height(args.height);
+    builder.add_width(args.width);
+    builder.finish()
+  }
+
+
+  #[inline]
+  pub fn video_format(&self) -> Option<&'a str> {
+    // Safety:
+    // Created from valid Table for this object
+    // which contains a valid value in this slot
+    unsafe { self._tab.get::<flatbuffers::ForwardsUOffset<&str>>(Setup::VT_VIDEO_FORMAT, None)}
+  }
+  #[inline]
+  pub fn width(&self) -> u16 {
+    // Safety:
+    // Created from valid Table for this object
+    // which contains a valid value in this slot
+    unsafe { self._tab.get::<u16>(Setup::VT_WIDTH, Some(0)).unwrap()}
+  }
+  #[inline]
+  pub fn height(&self) -> u16 {
+    // Safety:
+    // Created from valid Table for this object
+    // which contains a valid value in this slot
+    unsafe { self._tab.get::<u16>(Setup::VT_HEIGHT, Some(0)).unwrap()}
+  }
+  #[inline]
+  pub fn redraw_rate(&self) -> u16 {
+    // Safety:
+    // Created from valid Table for this object
+    // which contains a valid value in this slot
+    unsafe { self._tab.get::<u16>(Setup::VT_REDRAW_RATE, Some(0)).unwrap()}
+  }
+}
+
+impl flatbuffers::Verifiable for Setup<'_> {
+  #[inline]
+  fn run_verifier(
+    v: &mut flatbuffers::Verifier, pos: usize
+  ) -> Result<(), flatbuffers::InvalidFlatbuffer> {
+    use self::flatbuffers::Verifiable;
+    v.visit_table(pos)?
+     .visit_field::<flatbuffers::ForwardsUOffset<&str>>("video_format", Self::VT_VIDEO_FORMAT, false)?
+     .visit_field::<u16>("width", Self::VT_WIDTH, false)?
+     .visit_field::<u16>("height", Self::VT_HEIGHT, false)?
+     .visit_field::<u16>("redraw_rate", Self::VT_REDRAW_RATE, false)?
+     .finish();
+    Ok(())
+  }
+}
+pub struct SetupArgs<'a> {
+    pub video_format: Option<flatbuffers::WIPOffset<&'a str>>,
+    pub width: u16,
+    pub height: u16,
+    pub redraw_rate: u16,
+}
+impl<'a> Default for SetupArgs<'a> {
+  #[inline]
+  fn default() -> Self {
+    SetupArgs {
+      video_format: None,
+      width: 0,
+      height: 0,
+      redraw_rate: 0,
+    }
+  }
+}
+
+pub struct SetupBuilder<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> {
+  fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a, A>,
+  start_: flatbuffers::WIPOffset<flatbuffers::TableUnfinishedWIPOffset>,
+}
+impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> SetupBuilder<'a, 'b, A> {
+  #[inline]
+  pub fn add_video_format(&mut self, video_format: flatbuffers::WIPOffset<&'b  str>) {
+    self.fbb_.push_slot_always::<flatbuffers::WIPOffset<_>>(Setup::VT_VIDEO_FORMAT, video_format);
+  }
+  #[inline]
+  pub fn add_width(&mut self, width: u16) {
+    self.fbb_.push_slot::<u16>(Setup::VT_WIDTH, width, 0);
+  }
+  #[inline]
+  pub fn add_height(&mut self, height: u16) {
+    self.fbb_.push_slot::<u16>(Setup::VT_HEIGHT, height, 0);
+  }
+  #[inline]
+  pub fn add_redraw_rate(&mut self, redraw_rate: u16) {
+    self.fbb_.push_slot::<u16>(Setup::VT_REDRAW_RATE, redraw_rate, 0);
+  }
+  #[inline]
+  pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>) -> SetupBuilder<'a, 'b, A> {
+    let start = _fbb.start_table();
+    SetupBuilder {
+      fbb_: _fbb,
+      start_: start,
+    }
+  }
+  #[inline]
+  pub fn finish(self) -> flatbuffers::WIPOffset<Setup<'a>> {
+    let o = self.fbb_.end_table(self.start_);
+    flatbuffers::WIPOffset::new(o.value())
+  }
+}
+
+impl core::fmt::Debug for Setup<'_> {
+  fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
+    let mut ds = f.debug_struct("Setup");
+      ds.field("video_format", &self.video_format());
+      ds.field("width", &self.width());
+      ds.field("height", &self.height());
+      ds.field("redraw_rate", &self.redraw_rate());
+      ds.finish()
+  }
+}
+pub enum DecodeUnitStartOffset {}
+#[derive(Copy, Clone, PartialEq)]
+
+pub struct DecodeUnitStart<'a> {
+  pub _tab: flatbuffers::Table<'a>,
+}
+
+impl<'a> flatbuffers::Follow<'a> for DecodeUnitStart<'a> {
+  type Inner = DecodeUnitStart<'a>;
+  #[inline]
+  unsafe fn follow(buf: &'a [u8], loc: usize) -> Self::Inner {
+    Self { _tab: flatbuffers::Table::new(buf, loc) }
+  }
+}
+
+impl<'a> DecodeUnitStart<'a> {
+  pub const VT_FRAME_NUMBER: flatbuffers::VOffsetT = 4;
+  pub const VT_FRAME_TYPE: flatbuffers::VOffsetT = 6;
+  pub const VT_NUM_BUFFERS: flatbuffers::VOffsetT = 8;
+  pub const VT_RECEIVE_TIME_MS: flatbuffers::VOffsetT = 10;
+  pub const VT_FULL_LENGTH: flatbuffers::VOffsetT = 12;
+
+  #[inline]
+  pub unsafe fn init_from_table(table: flatbuffers::Table<'a>) -> Self {
+    DecodeUnitStart { _tab: table }
+  }
+  #[allow(unused_mut)]
+  pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr, A: flatbuffers::Allocator + 'bldr>(
+    _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr, A>,
+    args: &'args DecodeUnitStartArgs
+  ) -> flatbuffers::WIPOffset<DecodeUnitStart<'bldr>> {
+    let mut builder = DecodeUnitStartBuilder::new(_fbb);
+    builder.add_full_length(args.full_length);
+    builder.add_num_buffers(args.num_buffers);
+    builder.add_frame_number(args.frame_number);
+    builder.add_receive_time_ms(args.receive_time_ms);
+    builder.add_frame_type(args.frame_type);
+    builder.finish()
+  }
+
+
+  #[inline]
+  pub fn frame_number(&self) -> u64 {
+    // Safety:
+    // Created from valid Table for this object
+    // which contains a valid value in this slot
+    unsafe { self._tab.get::<u64>(DecodeUnitStart::VT_FRAME_NUMBER, Some(0)).unwrap()}
+  }
+  #[inline]
+  pub fn frame_type(&self) -> FrameType {
+    // Safety:
+    // Created from valid Table for this object
+    // which contains a valid value in this slot
+    unsafe { self._tab.get::<FrameType>(DecodeUnitStart::VT_FRAME_TYPE, Some(FrameType::PFRAME)).unwrap()}
+  }
+  #[inline]
+  pub fn num_buffers(&self) -> u64 {
+    // Safety:
+    // Created from valid Table for this object
+    // which contains a valid value in this slot
+    unsafe { self._tab.get::<u64>(DecodeUnitStart::VT_NUM_BUFFERS, Some(0)).unwrap()}
+  }
+  #[inline]
+  pub fn receive_time_ms(&self) -> u16 {
+    // Safety:
+    // Created from valid Table for this object
+    // which contains a valid value in this slot
+    unsafe { self._tab.get::<u16>(DecodeUnitStart::VT_RECEIVE_TIME_MS, Some(0)).unwrap()}
+  }
+  #[inline]
+  pub fn full_length(&self) -> u64 {
+    // Safety:
+    // Created from valid Table for this object
+    // which contains a valid value in this slot
+    unsafe { self._tab.get::<u64>(DecodeUnitStart::VT_FULL_LENGTH, Some(0)).unwrap()}
+  }
+}
+
+impl flatbuffers::Verifiable for DecodeUnitStart<'_> {
+  #[inline]
+  fn run_verifier(
+    v: &mut flatbuffers::Verifier, pos: usize
+  ) -> Result<(), flatbuffers::InvalidFlatbuffer> {
+    use self::flatbuffers::Verifiable;
+    v.visit_table(pos)?
+     .visit_field::<u64>("frame_number", Self::VT_FRAME_NUMBER, false)?
+     .visit_field::<FrameType>("frame_type", Self::VT_FRAME_TYPE, false)?
+     .visit_field::<u64>("num_buffers", Self::VT_NUM_BUFFERS, false)?
+     .visit_field::<u16>("receive_time_ms", Self::VT_RECEIVE_TIME_MS, false)?
+     .visit_field::<u64>("full_length", Self::VT_FULL_LENGTH, false)?
+     .finish();
+    Ok(())
+  }
+}
+pub struct DecodeUnitStartArgs {
+    pub frame_number: u64,
+    pub frame_type: FrameType,
+    pub num_buffers: u64,
+    pub receive_time_ms: u16,
+    pub full_length: u64,
+}
+impl<'a> Default for DecodeUnitStartArgs {
+  #[inline]
+  fn default() -> Self {
+    DecodeUnitStartArgs {
+      frame_number: 0,
+      frame_type: FrameType::PFRAME,
+      num_buffers: 0,
+      receive_time_ms: 0,
+      full_length: 0,
+    }
+  }
+}
+
+pub struct DecodeUnitStartBuilder<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> {
+  fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a, A>,
+  start_: flatbuffers::WIPOffset<flatbuffers::TableUnfinishedWIPOffset>,
+}
+impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> DecodeUnitStartBuilder<'a, 'b, A> {
+  #[inline]
+  pub fn add_frame_number(&mut self, frame_number: u64) {
+    self.fbb_.push_slot::<u64>(DecodeUnitStart::VT_FRAME_NUMBER, frame_number, 0);
+  }
+  #[inline]
+  pub fn add_frame_type(&mut self, frame_type: FrameType) {
+    self.fbb_.push_slot::<FrameType>(DecodeUnitStart::VT_FRAME_TYPE, frame_type, FrameType::PFRAME);
+  }
+  #[inline]
+  pub fn add_num_buffers(&mut self, num_buffers: u64) {
+    self.fbb_.push_slot::<u64>(DecodeUnitStart::VT_NUM_BUFFERS, num_buffers, 0);
+  }
+  #[inline]
+  pub fn add_receive_time_ms(&mut self, receive_time_ms: u16) {
+    self.fbb_.push_slot::<u16>(DecodeUnitStart::VT_RECEIVE_TIME_MS, receive_time_ms, 0);
+  }
+  #[inline]
+  pub fn add_full_length(&mut self, full_length: u64) {
+    self.fbb_.push_slot::<u64>(DecodeUnitStart::VT_FULL_LENGTH, full_length, 0);
+  }
+  #[inline]
+  pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>) -> DecodeUnitStartBuilder<'a, 'b, A> {
+    let start = _fbb.start_table();
+    DecodeUnitStartBuilder {
+      fbb_: _fbb,
+      start_: start,
+    }
+  }
+  #[inline]
+  pub fn finish(self) -> flatbuffers::WIPOffset<DecodeUnitStart<'a>> {
+    let o = self.fbb_.end_table(self.start_);
+    flatbuffers::WIPOffset::new(o.value())
+  }
+}
+
+impl core::fmt::Debug for DecodeUnitStart<'_> {
+  fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
+    let mut ds = f.debug_struct("DecodeUnitStart");
+      ds.field("frame_number", &self.frame_number());
+      ds.field("frame_type", &self.frame_type());
+      ds.field("num_buffers", &self.num_buffers());
+      ds.field("receive_time_ms", &self.receive_time_ms());
+      ds.field("full_length", &self.full_length());
+      ds.finish()
+  }
+}
+pub enum DecodeUnitBufferOffset {}
+#[derive(Copy, Clone, PartialEq)]
+
+pub struct DecodeUnitBuffer<'a> {
+  pub _tab: flatbuffers::Table<'a>,
+}
+
+impl<'a> flatbuffers::Follow<'a> for DecodeUnitBuffer<'a> {
+  type Inner = DecodeUnitBuffer<'a>;
+  #[inline]
+  unsafe fn follow(buf: &'a [u8], loc: usize) -> Self::Inner {
+    Self { _tab: flatbuffers::Table::new(buf, loc) }
+  }
+}
+
+impl<'a> DecodeUnitBuffer<'a> {
+  pub const VT_FRAME_NUMBER: flatbuffers::VOffsetT = 4;
+  pub const VT_BUFFER_INDEX: flatbuffers::VOffsetT = 6;
+  pub const VT_BUFFER_OFFSET: flatbuffers::VOffsetT = 8;
+  pub const VT_DATA: flatbuffers::VOffsetT = 10;
+
+  #[inline]
+  pub unsafe fn init_from_table(table: flatbuffers::Table<'a>) -> Self {
+    DecodeUnitBuffer { _tab: table }
+  }
+  #[allow(unused_mut)]
+  pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr, A: flatbuffers::Allocator + 'bldr>(
+    _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr, A>,
+    args: &'args DecodeUnitBufferArgs<'args>
+  ) -> flatbuffers::WIPOffset<DecodeUnitBuffer<'bldr>> {
+    let mut builder = DecodeUnitBufferBuilder::new(_fbb);
+    builder.add_buffer_offset(args.buffer_offset);
+    builder.add_buffer_index(args.buffer_index);
+    builder.add_frame_number(args.frame_number);
+    if let Some(x) = args.data { builder.add_data(x); }
+    builder.finish()
+  }
+
+
+  #[inline]
+  pub fn frame_number(&self) -> u64 {
+    // Safety:
+    // Created from valid Table for this object
+    // which contains a valid value in this slot
+    unsafe { self._tab.get::<u64>(DecodeUnitBuffer::VT_FRAME_NUMBER, Some(0)).unwrap()}
+  }
+  #[inline]
+  pub fn buffer_index(&self) -> u64 {
+    // Safety:
+    // Created from valid Table for this object
+    // which contains a valid value in this slot
+    unsafe { self._tab.get::<u64>(DecodeUnitBuffer::VT_BUFFER_INDEX, Some(0)).unwrap()}
+  }
+  #[inline]
+  pub fn buffer_offset(&self) -> u64 {
+    // Safety:
+    // Created from valid Table for this object
+    // which contains a valid value in this slot
+    unsafe { self._tab.get::<u64>(DecodeUnitBuffer::VT_BUFFER_OFFSET, Some(0)).unwrap()}
+  }
+  #[inline]
+  pub fn data(&self) -> Option<flatbuffers::Vector<'a, u8>> {
+    // Safety:
+    // Created from valid Table for this object
+    // which contains a valid value in this slot
+    unsafe { self._tab.get::<flatbuffers::ForwardsUOffset<flatbuffers::Vector<'a, u8>>>(DecodeUnitBuffer::VT_DATA, None)}
+  }
+}
+
+impl flatbuffers::Verifiable for DecodeUnitBuffer<'_> {
+  #[inline]
+  fn run_verifier(
+    v: &mut flatbuffers::Verifier, pos: usize
+  ) -> Result<(), flatbuffers::InvalidFlatbuffer> {
+    use self::flatbuffers::Verifiable;
+    v.visit_table(pos)?
+     .visit_field::<u64>("frame_number", Self::VT_FRAME_NUMBER, false)?
+     .visit_field::<u64>("buffer_index", Self::VT_BUFFER_INDEX, false)?
+     .visit_field::<u64>("buffer_offset", Self::VT_BUFFER_OFFSET, false)?
+     .visit_field::<flatbuffers::ForwardsUOffset<flatbuffers::Vector<'_, u8>>>("data", Self::VT_DATA, false)?
+     .finish();
+    Ok(())
+  }
+}
+pub struct DecodeUnitBufferArgs<'a> {
+    pub frame_number: u64,
+    pub buffer_index: u64,
+    pub buffer_offset: u64,
+    pub data: Option<flatbuffers::WIPOffset<flatbuffers::Vector<'a, u8>>>,
+}
+impl<'a> Default for DecodeUnitBufferArgs<'a> {
+  #[inline]
+  fn default() -> Self {
+    DecodeUnitBufferArgs {
+      frame_number: 0,
+      buffer_index: 0,
+      buffer_offset: 0,
+      data: None,
+    }
+  }
+}
+
+pub struct DecodeUnitBufferBuilder<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> {
+  fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a, A>,
+  start_: flatbuffers::WIPOffset<flatbuffers::TableUnfinishedWIPOffset>,
+}
+impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> DecodeUnitBufferBuilder<'a, 'b, A> {
+  #[inline]
+  pub fn add_frame_number(&mut self, frame_number: u64) {
+    self.fbb_.push_slot::<u64>(DecodeUnitBuffer::VT_FRAME_NUMBER, frame_number, 0);
+  }
+  #[inline]
+  pub fn add_buffer_index(&mut self, buffer_index: u64) {
+    self.fbb_.push_slot::<u64>(DecodeUnitBuffer::VT_BUFFER_INDEX, buffer_index, 0);
+  }
+  #[inline]
+  pub fn add_buffer_offset(&mut self, buffer_offset: u64) {
+    self.fbb_.push_slot::<u64>(DecodeUnitBuffer::VT_BUFFER_OFFSET, buffer_offset, 0);
+  }
+  #[inline]
+  pub fn add_data(&mut self, data: flatbuffers::WIPOffset<flatbuffers::Vector<'b , u8>>) {
+    self.fbb_.push_slot_always::<flatbuffers::WIPOffset<_>>(DecodeUnitBuffer::VT_DATA, data);
+  }
+  #[inline]
+  pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>) -> DecodeUnitBufferBuilder<'a, 'b, A> {
+    let start = _fbb.start_table();
+    DecodeUnitBufferBuilder {
+      fbb_: _fbb,
+      start_: start,
+    }
+  }
+  #[inline]
+  pub fn finish(self) -> flatbuffers::WIPOffset<DecodeUnitBuffer<'a>> {
+    let o = self.fbb_.end_table(self.start_);
+    flatbuffers::WIPOffset::new(o.value())
+  }
+}
+
+impl core::fmt::Debug for DecodeUnitBuffer<'_> {
+  fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
+    let mut ds = f.debug_struct("DecodeUnitBuffer");
+      ds.field("frame_number", &self.frame_number());
+      ds.field("buffer_index", &self.buffer_index());
+      ds.field("buffer_offset", &self.buffer_offset());
+      ds.field("data", &self.data());
+      ds.finish()
+  }
+}
+pub enum VideoUpdateOffset {}
+#[derive(Copy, Clone, PartialEq)]
+
+pub struct VideoUpdate<'a> {
+  pub _tab: flatbuffers::Table<'a>,
+}
+
+impl<'a> flatbuffers::Follow<'a> for VideoUpdate<'a> {
+  type Inner = VideoUpdate<'a>;
+  #[inline]
+  unsafe fn follow(buf: &'a [u8], loc: usize) -> Self::Inner {
+    Self { _tab: flatbuffers::Table::new(buf, loc) }
+  }
+}
+
+impl<'a> VideoUpdate<'a> {
+  pub const VT_UPDATE_TYPE: flatbuffers::VOffsetT = 4;
+  pub const VT_UPDATE: flatbuffers::VOffsetT = 6;
+
+  #[inline]
+  pub unsafe fn init_from_table(table: flatbuffers::Table<'a>) -> Self {
+    VideoUpdate { _tab: table }
+  }
+  #[allow(unused_mut)]
+  pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr, A: flatbuffers::Allocator + 'bldr>(
+    _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr, A>,
+    args: &'args VideoUpdateArgs
+  ) -> flatbuffers::WIPOffset<VideoUpdate<'bldr>> {
+    let mut builder = VideoUpdateBuilder::new(_fbb);
+    if let Some(x) = args.update { builder.add_update(x); }
+    builder.add_update_type(args.update_type);
+    builder.finish()
+  }
+
+
+  #[inline]
+  pub fn update_type(&self) -> Update {
+    // Safety:
+    // Created from valid Table for this object
+    // which contains a valid value in this slot
+    unsafe { self._tab.get::<Update>(VideoUpdate::VT_UPDATE_TYPE, Some(Update::NONE)).unwrap()}
+  }
+  #[inline]
+  pub fn update(&self) -> Option<flatbuffers::Table<'a>> {
+    // Safety:
+    // Created from valid Table for this object
+    // which contains a valid value in this slot
+    unsafe { self._tab.get::<flatbuffers::ForwardsUOffset<flatbuffers::Table<'a>>>(VideoUpdate::VT_UPDATE, None)}
+  }
+  #[inline]
+  #[allow(non_snake_case)]
+  pub fn update_as_setup(&self) -> Option<Setup<'a>> {
+    if self.update_type() == Update::Setup {
+      self.update().map(|t| {
+       // Safety:
+       // Created from a valid Table for this object
+       // Which contains a valid union in this slot
+       unsafe { Setup::init_from_table(t) }
+     })
+    } else {
+      None
+    }
+  }
+
+  #[inline]
+  #[allow(non_snake_case)]
+  pub fn update_as_decode_unit_start(&self) -> Option<DecodeUnitStart<'a>> {
+    if self.update_type() == Update::DecodeUnitStart {
+      self.update().map(|t| {
+       // Safety:
+       // Created from a valid Table for this object
+       // Which contains a valid union in this slot
+       unsafe { DecodeUnitStart::init_from_table(t) }
+     })
+    } else {
+      None
+    }
+  }
+
+  #[inline]
+  #[allow(non_snake_case)]
+  pub fn update_as_decode_unit_buffer(&self) -> Option<DecodeUnitBuffer<'a>> {
+    if self.update_type() == Update::DecodeUnitBuffer {
+      self.update().map(|t| {
+       // Safety:
+       // Created from a valid Table for this object
+       // Which contains a valid union in this slot
+       unsafe { DecodeUnitBuffer::init_from_table(t) }
+     })
+    } else {
+      None
+    }
+  }
+
+}
+
+impl flatbuffers::Verifiable for VideoUpdate<'_> {
+  #[inline]
+  fn run_verifier(
+    v: &mut flatbuffers::Verifier, pos: usize
+  ) -> Result<(), flatbuffers::InvalidFlatbuffer> {
+    use self::flatbuffers::Verifiable;
+    v.visit_table(pos)?
+     .visit_union::<Update, _>("update_type", Self::VT_UPDATE_TYPE, "update", Self::VT_UPDATE, false, |key, v, pos| {
+        match key {
+          Update::Setup => v.verify_union_variant::<flatbuffers::ForwardsUOffset<Setup>>("Update::Setup", pos),
+          Update::DecodeUnitStart => v.verify_union_variant::<flatbuffers::ForwardsUOffset<DecodeUnitStart>>("Update::DecodeUnitStart", pos),
+          Update::DecodeUnitBuffer => v.verify_union_variant::<flatbuffers::ForwardsUOffset<DecodeUnitBuffer>>("Update::DecodeUnitBuffer", pos),
+          _ => Ok(()),
+        }
+     })?
+     .finish();
+    Ok(())
+  }
+}
+pub struct VideoUpdateArgs {
+    pub update_type: Update,
+    pub update: Option<flatbuffers::WIPOffset<flatbuffers::UnionWIPOffset>>,
+}
+impl<'a> Default for VideoUpdateArgs {
+  #[inline]
+  fn default() -> Self {
+    VideoUpdateArgs {
+      update_type: Update::NONE,
+      update: None,
+    }
+  }
+}
+
+pub struct VideoUpdateBuilder<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> {
+  fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a, A>,
+  start_: flatbuffers::WIPOffset<flatbuffers::TableUnfinishedWIPOffset>,
+}
+impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> VideoUpdateBuilder<'a, 'b, A> {
+  #[inline]
+  pub fn add_update_type(&mut self, update_type: Update) {
+    self.fbb_.push_slot::<Update>(VideoUpdate::VT_UPDATE_TYPE, update_type, Update::NONE);
+  }
+  #[inline]
+  pub fn add_update(&mut self, update: flatbuffers::WIPOffset<flatbuffers::UnionWIPOffset>) {
+    self.fbb_.push_slot_always::<flatbuffers::WIPOffset<_>>(VideoUpdate::VT_UPDATE, update);
+  }
+  #[inline]
+  pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>) -> VideoUpdateBuilder<'a, 'b, A> {
+    let start = _fbb.start_table();
+    VideoUpdateBuilder {
+      fbb_: _fbb,
+      start_: start,
+    }
+  }
+  #[inline]
+  pub fn finish(self) -> flatbuffers::WIPOffset<VideoUpdate<'a>> {
+    let o = self.fbb_.end_table(self.start_);
+    flatbuffers::WIPOffset::new(o.value())
+  }
+}
+
+impl core::fmt::Debug for VideoUpdate<'_> {
+  fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
+    let mut ds = f.debug_struct("VideoUpdate");
+      ds.field("update_type", &self.update_type());
+      match self.update_type() {
+        Update::Setup => {
+          if let Some(x) = self.update_as_setup() {
+            ds.field("update", &x)
+          } else {
+            ds.field("update", &"InvalidFlatbuffer: Union discriminant does not match value.")
+          }
+        },
+        Update::DecodeUnitStart => {
+          if let Some(x) = self.update_as_decode_unit_start() {
+            ds.field("update", &x)
+          } else {
+            ds.field("update", &"InvalidFlatbuffer: Union discriminant does not match value.")
+          }
+        },
+        Update::DecodeUnitBuffer => {
+          if let Some(x) = self.update_as_decode_unit_buffer() {
+            ds.field("update", &x)
+          } else {
+            ds.field("update", &"InvalidFlatbuffer: Union discriminant does not match value.")
+          }
+        },
+        _ => {
+          let x: Option<()> = None;
+          ds.field("update", &x)
+        },
+      };
+      ds.finish()
+  }
+}
+#[inline]
+/// Verifies that a buffer of bytes contains a `VideoUpdate`
+/// and returns it.
+/// Note that verification is still experimental and may not
+/// catch every error, or be maximally performant. For the
+/// previous, unchecked, behavior use
+/// `root_as_video_update_unchecked`.
+pub fn root_as_video_update(buf: &[u8]) -> Result<VideoUpdate, flatbuffers::InvalidFlatbuffer> {
+  flatbuffers::root::<VideoUpdate>(buf)
+}
+#[inline]
+/// Verifies that a buffer of bytes contains a size prefixed
+/// `VideoUpdate` and returns it.
+/// Note that verification is still experimental and may not
+/// catch every error, or be maximally performant. For the
+/// previous, unchecked, behavior use
+/// `size_prefixed_root_as_video_update_unchecked`.
+pub fn size_prefixed_root_as_video_update(buf: &[u8]) -> Result<VideoUpdate, flatbuffers::InvalidFlatbuffer> {
+  flatbuffers::size_prefixed_root::<VideoUpdate>(buf)
+}
+#[inline]
+/// Verifies, with the given options, that a buffer of bytes
+/// contains a `VideoUpdate` and returns it.
+/// Note that verification is still experimental and may not
+/// catch every error, or be maximally performant. For the
+/// previous, unchecked, behavior use
+/// `root_as_video_update_unchecked`.
+pub fn root_as_video_update_with_opts<'b, 'o>(
+  opts: &'o flatbuffers::VerifierOptions,
+  buf: &'b [u8],
+) -> Result<VideoUpdate<'b>, flatbuffers::InvalidFlatbuffer> {
+  flatbuffers::root_with_opts::<VideoUpdate<'b>>(opts, buf)
+}
+#[inline]
+/// Verifies, with the given verifier options, that a buffer of
+/// bytes contains a size prefixed `VideoUpdate` and returns
+/// it. Note that verification is still experimental and may not
+/// catch every error, or be maximally performant. For the
+/// previous, unchecked, behavior use
+/// `root_as_video_update_unchecked`.
+pub fn size_prefixed_root_as_video_update_with_opts<'b, 'o>(
+  opts: &'o flatbuffers::VerifierOptions,
+  buf: &'b [u8],
+) -> Result<VideoUpdate<'b>, flatbuffers::InvalidFlatbuffer> {
+  flatbuffers::size_prefixed_root_with_opts::<VideoUpdate<'b>>(opts, buf)
+}
+#[inline]
+/// Assumes, without verification, that a buffer of bytes contains a VideoUpdate and returns it.
+/// # Safety
+/// Callers must trust the given bytes do indeed contain a valid `VideoUpdate`.
+pub unsafe fn root_as_video_update_unchecked(buf: &[u8]) -> VideoUpdate {
+  flatbuffers::root_unchecked::<VideoUpdate>(buf)
+}
+#[inline]
+/// Assumes, without verification, that a buffer of bytes contains a size prefixed VideoUpdate and returns it.
+/// # Safety
+/// Callers must trust the given bytes do indeed contain a valid size prefixed `VideoUpdate`.
+pub unsafe fn size_prefixed_root_as_video_update_unchecked(buf: &[u8]) -> VideoUpdate {
+  flatbuffers::size_prefixed_root_unchecked::<VideoUpdate>(buf)
+}
+#[inline]
+pub fn finish_video_update_buffer<'a, 'b, A: flatbuffers::Allocator + 'a>(
+    fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>,
+    root: flatbuffers::WIPOffset<VideoUpdate<'a>>) {
+  fbb.finish(root, None);
+}
+
+#[inline]
+pub fn finish_size_prefixed_video_update_buffer<'a, 'b, A: flatbuffers::Allocator + 'a>(fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>, root: flatbuffers::WIPOffset<VideoUpdate<'a>>) {
+  fbb.finish_size_prefixed(root, None);
+}
+}  // pub mod VideoUpdate
+

gamestream-webtransport-proxy/src/responses.rs

@@ -0,0 +1,23 @@
+use serde::Deserialize;
+
+#[derive(Deserialize)]
+pub struct ServerInfoResponse {
+    pub hostname: String,
+    pub appversion: String,
+    pub GfeVersion: String,
+    pub uniqueid: uuid::Uuid,
+    pub HttpsPort: u16,
+    pub ExternalPort: u16,
+    pub MaxLumaPixelsHEVC: u64,
+    pub mac: String,
+    //pub ServerCommand: String,
+    //pub Permission: u64,
+    //pub VirtualDisplayCapable: bool,
+    //pub VirtualDisplayDriverReady: bool,
+    pub LocalIP: String,
+    pub ServerCodecModeSupport: i32,
+    pub PairStatus: u64,
+    pub currentgame: u64,
+    pub state: String,
+}
+

gamestream-webtransport-proxy/src/stream.rs

@@ -1,3 +1,5 @@
+use std::str::FromStr;
+
 use anyhow::{Context, Result};
 use moonlight_common_c_sys::SCM_H264;
 use salvo::prelude::*;
@@ -6,7 +8,7 @@ use tracing::{debug, error, info};
 
 use crate::{
     common::{AppError, AppResult, get_url},
-    proxy,
+    proxy, responses,
     state::{GamestreamServer, StateReadAccess, StateReader},
 };
 
@@ -25,33 +27,16 @@ struct PostStreamStartResponse {
     //cert_hash: String,
 }
 
-#[derive(Deserialize)]
-struct ServerInfoResponse {
-    hostname: String,
-    appversion: String,
-    GfeVersion: String,
-    uniqueid: uuid::Uuid,
-    HttpsPort: u16,
-    ExternalPort: u16,
-    MaxLumaPixelsHEVC: u64,
-    mac: String,
-    //ServerCommand: String,
-    //Permission: u64,
-    //VirtualDisplayCapable: bool,
-    //VirtualDisplayDriverReady: bool,
-    LocalIP: String,
-    ServerCodecModeSupport: i32,
-    PairStatus: u64,
-    currentgame: u64,
-    state: String,
-}
-
 #[derive(Deserialize)]
 struct LaunchResponse {
+    #[serde(rename = "@status_code")]
+    status_code: String,
+    #[serde(rename = "@status_message")]
+    status_message: Option<String>,
     #[serde(rename = "sessionUrl0")]
-    session_url_0: url::Url,
+    session_url_0: Option<url::Url>,
     #[serde(rename = "gamesession")]
-    game_session: u64,
+    game_session: Option<u64>,
 }
 
 fn get_server(reader: &StateReadAccess, server: &String) -> Result<Option<GamestreamServer>> {
@@ -108,7 +93,7 @@ impl crate::backend::Backend {
             Ok(s) => match s {
                 Some(s) => s,
                 None => {
-                    info!("No server with name: {}", body.server);
+                    error!("No server with name: {}", body.server);
                     return Err(AppError {
                         status_code: StatusCode::BAD_REQUEST,
                         description: "No server configured with that name.".to_string(),
@@ -139,8 +124,9 @@ impl crate::backend::Backend {
         );
 
         let server_info = crate::common::get_url(&mut base_url, true).await.unwrap();
-        debug!("app_info: {server_info}");
+        debug!("server_info: {server_info}");
-        let server_info: ServerInfoResponse = match serde_xml_rs::from_str(&server_info) {
+        let server_info: responses::ServerInfoResponse = match serde_xml_rs::from_str(&server_info)
+        {
             Ok(s) => s,
             Err(e) => {
                 error!("Could not parse serverinfo response: {e}");
@@ -227,6 +213,36 @@ impl crate::backend::Backend {
             }
         };
 
+        let Ok(launch_resp_status) = http::StatusCode::from_str(&launch_response.status_code)
+        else {
+            error!(
+                "Server returned invalid status code: {}",
+                launch_response.status_code
+            );
+            return standard_error;
+        };
+        if !launch_resp_status.is_success() {
+            let status_message = launch_response
+                .status_message
+                .unwrap_or("Server did not provide a status message".to_string());
+
+            info!("Could not launch game: {}", status_message,);
+            return Err(AppError {
+                status_code: StatusCode::INTERNAL_SERVER_ERROR,
+                description: format!("Could not launch game: {}", status_message),
+            });
+        }
+
+        let Some(url) = launch_response.session_url_0 else {
+            error!("Server response was missing session_url_0");
+            return standard_error;
+        };
+
+        let Some(game_session) = launch_response.game_session else {
+            error!("Server response was missing game_session");
+            return standard_error;
+        };
+
         let stream_id = uuid::Uuid::new_v4();
 
         let server_codec_mode_support = if server_info.ServerCodecModeSupport == 0 {
@@ -237,8 +253,8 @@ impl crate::backend::Backend {
 
         let stream = crate::backend::Stream {
             id: stream_id,
-            url: launch_response.session_url_0,
+            url,
-            game_session: launch_response.game_session,
+            game_session,
             server_address: server.host.clone(),
             stream_config: body.stream_config.clone(),
             app_version: server_info.appversion,
@@ -299,7 +315,10 @@ impl crate::backend::Backend {
 
         let webtransport_url = url_constructor::UrlConstructor::new()
             .scheme("https")
-            .host(host)
+            // TODO: this is hardcoded to 127.0.0.1 to fix problems with
+            // tls certificates and IPv6 in chrome. This needs to eventually be fixed
+            // but I don't actually know what the fix is
+            .host("127.0.0.1")
             .port(port)
             .subdir("api/stream/connect")
             .build();

moonlight-common-c-sys/build.rs

@@ -12,9 +12,13 @@ struct CustomCallbacks;
 impl ParseCallbacks for CustomCallbacks {
     fn int_macro(&self, name: &str, _value: i64) -> Option<IntKind> {
         match name {
-            "STREAM_CFG_LOCAL" => Some(IntKind::I32),
+            //"STREAM_CFG_LOCAL" => Some(IntKind::I32),
-            "STREAM_CFG_REMOTE" => Some(IntKind::I32),
+            //"STREAM_CFG_REMOTE" => Some(IntKind::I32),
-            "STREAM_CFG_AUTO" => Some(IntKind::I32),
+            //"STREAM_CFG_AUTO" => Some(IntKind::I32),
+            "KEY_ACTION_DOWN" => Some(IntKind::I8),
+            "KEY_ACTION_UP" => Some(IntKind::I8),
+            "BUTTON_ACTION_PRESS" => Some(IntKind::I8),
+            "BUTTON_ACTION_RELEASE" => Some(IntKind::I8),
             _ => None, // Default behavior for all others
         }
     }
@@ -34,7 +38,7 @@ fn main() {
     let bindings = bindgen::Builder::default()
         .header("moonlight-common-c/src/Limelight.h")
         .clang_arg(format!("-I{}/src", dst.display())) // Include built headers
-        //.parse_callbacks(Box::new(CustomCallbacks))
+        .parse_callbacks(Box::new(CustomCallbacks))
         .default_macro_constant_type(bindgen::MacroTypeVariation::Signed)
         .generate()
         .expect("Failed to generate bindings");

schema/input.fbs

@@ -0,0 +1,53 @@
+namespace InputEvent;
+
+
+enum KeyAction: byte {
+  DOWN,
+  UP,
+}
+
+
+table MouseMovement {
+  movement_x: int16;
+  movement_y: int16;
+}
+
+enum MouseButton: byte {
+  LEFT,
+  MIDDLE,
+  RIGHT,
+  X1,
+  X2
+}
+
+table MouseInput {
+  button: MouseButton;
+  button_action: KeyAction;
+}
+
+
+struct ModifierState {
+  shift: bool;
+  ctrl: bool;
+  alt: bool;
+  meta: bool;
+}
+
+table KeyboardInput {
+  key_code: int16;
+  key_action: KeyAction;
+  modifiers: ModifierState;
+}
+
+union Input {
+  Keyboard:KeyboardInput,
+  MouseMovement:MouseMovement,
+  MouseInput:MouseInput,
+}
+
+table InputEvent {
+  input: Input;
+}
+
+root_type InputEvent;
+

schema/video.fbs

@@ -0,0 +1,43 @@
+namespace VideoUpdate;
+
+
+table Setup {
+  video_format: string;
+  width: uint16;
+  height: uint16;
+  redraw_rate: uint16;
+}
+
+enum FrameType: byte {
+  PFRAME,
+  IDR,
+}
+
+table DecodeUnitStart {
+  frame_number: uint64;
+  frame_type: FrameType;
+  num_buffers: uint64;
+
+  receive_time_ms: uint16;
+  full_length: uint64;
+
+}
+
+table DecodeUnitBuffer {
+  frame_number: uint64;
+  buffer_index: uint64;
+  buffer_offset: uint64;
+  data: [ubyte]; 
+}
+
+union Update {
+    Setup:Setup,
+    DecodeUnitStart:DecodeUnitStart,
+    DecodeUnitBuffer:DecodeUnitBuffer,
+}
+
+table VideoUpdate {
+    update: Update; 
+}
+
+root_type VideoUpdate;