backend and frontend: support out of order chunks + now it's performant on chrome

- changed the backend to append the buffers together and dispatch them
  as such (this is what moonlight-embedded does)
- fixed the frontend trying to playback an empty buffer because types
  are hard (this is why I should have used protobuf...)

Cargo.lock

@@ -537,6 +537,16 @@ version = "2.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
 
+[[package]]
+name = "flatbuffers"
+version = "25.2.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1045398c1bfd89168b5fd3f1fc11f6e70b34f6f66300c87d44d3de849463abf1"
+dependencies = [
+ "bitflags",
+ "rustc_version",
+]
+
 [[package]]
 name = "fnv"
 version = "1.0.7"
@@ -668,9 +678,13 @@ version = "0.1.0"
 dependencies = [
  "anyhow",
  "directories",
+ "flatbuffers",
  "getrandom 0.3.3",
+ "h3-datagram",
+ "h3-quinn",
  "hex",
  "hmac-sha256",
+ "http",
  "libc",
  "moonlight-common-c-sys",
  "openssl",
@@ -2037,6 +2051,15 @@ version = "2.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d"
 
+[[package]]
+name = "rustc_version"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92"
+dependencies = [
+ "semver",
+]
+
 [[package]]
 name = "rustix"
 version = "1.0.7"
@@ -2442,6 +2465,12 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "semver"
+version = "1.0.26"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "56e6fa9c48d24d85fb3de5ad847117517440f6beceb7798af16b4a87d616b8d0"
+
 [[package]]
 name = "serde"
 version = "1.0.219"

docker/Caddyfile

@@ -0,0 +1,15 @@
+localhost:64444 {
+    tls internal
+
+    # Proxy /config path to proxy
+    reverse_proxy /api/* https://127.0.0.1:3000 {
+        transport http {
+                tls
+                tls_insecure_skip_verify
+        }
+      }
+
+    # Proxy root path to dev server
+    reverse_proxy http://localhost:5173
+
+}

docker/docker-compose.yaml

@@ -0,0 +1,36 @@
+services:
+  caddy:
+    image: caddy:latest
+    ports:
+      - "64444:64444/tcp"
+      - "64444:64444/udp"
+    volumes:
+      - ./Caddyfile:/etc/caddy/Caddyfile
+    network_mode: host
+##  mumble-web2-proxy:
+##    image: rust:latest
+##    working_dir: /app
+##    volumes:
+##      - ..:/app
+##      - ./proxy-config.toml:/app/config.toml
+##    ports:
+##      - "4433:4433/tcp"
+##      - "4433:4433/udp"
+##    command: ["cargo", "run", "-p", "mumble-web2-proxy"]
+##    network_mode: host
+##
+##  mumble-server:
+##    image: mumblevoip/mumble-server:latest
+##    ports:
+##      - "64738:64738/tcp"
+##      - "64738:64738/udp"
+##    environment:
+##      - MUMBLE_CONFIG_WELCOMETEXT=Welcome to the Mumble server
+##    network_mode: host
+#volumes:
+#  caddy_data:
+#  caddy_config:
+#
+#networks:
+#  app-network:
+#    driver: bridge

frontend/.gitignore

@@ -0,0 +1,23 @@
+node_modules
+
+# Output
+.output
+.vercel
+.netlify
+.wrangler
+/.svelte-kit
+/build
+
+# OS
+.DS_Store
+Thumbs.db
+
+# Env
+.env
+.env.*
+!.env.example
+!.env.test
+
+# Vite
+vite.config.js.timestamp-*
+vite.config.ts.timestamp-*

frontend/.npmrc

@@ -0,0 +1 @@
+engine-strict=true

frontend/.prettierignore

@@ -0,0 +1,9 @@
+# Package Managers
+package-lock.json
+pnpm-lock.yaml
+yarn.lock
+bun.lock
+bun.lockb
+
+# Miscellaneous
+/static/

frontend/.prettierrc

@@ -0,0 +1,16 @@
+{
+	"useTabs": true,
+	"singleQuote": true,
+	"trailingComma": "none",
+	"printWidth": 100,
+	"plugins": ["prettier-plugin-svelte", "prettier-plugin-tailwindcss"],
+	"overrides": [
+		{
+			"files": "*.svelte",
+			"options": {
+				"parser": "svelte"
+			}
+		}
+	],
+	"tailwindStylesheet": "./src/app.css"
+}

frontend/README.md

@@ -0,0 +1,38 @@
+# sv
+
+Everything you need to build a Svelte project, powered by [`sv`](https://github.com/sveltejs/cli).
+
+## Creating a project
+
+If you're seeing this, you've probably already done this step. Congrats!
+
+```bash
+# create a new project in the current directory
+npx sv create
+
+# create a new project in my-app
+npx sv create my-app
+```
+
+## Developing
+
+Once you've created a project and installed dependencies with `npm install` (or `pnpm install` or `yarn`), start a development server:
+
+```bash
+npm run dev
+
+# or start the server and open the app in a new browser tab
+npm run dev -- --open
+```
+
+## Building
+
+To create a production version of your app:
+
+```bash
+npm run build
+```
+
+You can preview the production build with `npm run preview`.
+
+> To deploy your app, you may need to install an [adapter](https://svelte.dev/docs/kit/adapters) for your target environment.

frontend/eslint.config.js

@@ -0,0 +1,40 @@
+import prettier from 'eslint-config-prettier';
+import { includeIgnoreFile } from '@eslint/compat';
+import js from '@eslint/js';
+import svelte from 'eslint-plugin-svelte';
+import globals from 'globals';
+import { fileURLToPath } from 'node:url';
+import ts from 'typescript-eslint';
+import svelteConfig from './svelte.config.js';
+
+const gitignorePath = fileURLToPath(new URL('./.gitignore', import.meta.url));
+
+export default ts.config(
+	includeIgnoreFile(gitignorePath),
+	js.configs.recommended,
+	...ts.configs.recommended,
+	...svelte.configs.recommended,
+	prettier,
+	...svelte.configs.prettier,
+	{
+		languageOptions: {
+			globals: { ...globals.browser, ...globals.node }
+		},
+		rules: {
+			// typescript-eslint strongly recommend that you do not use the no-undef lint rule on TypeScript projects.
+			// see: https://typescript-eslint.io/troubleshooting/faqs/eslint/#i-get-errors-from-the-no-undef-rule-about-global-variables-not-being-defined-even-though-there-are-no-typescript-errors
+			'no-undef': 'off'
+		}
+	},
+	{
+		files: ['**/*.svelte', '**/*.svelte.ts', '**/*.svelte.js'],
+		languageOptions: {
+			parserOptions: {
+				projectService: true,
+				extraFileExtensions: ['.svelte'],
+				parser: ts.parser,
+				svelteConfig
+			}
+		}
+	}
+);

frontend/package.json

@@ -0,0 +1,49 @@
+{
+	"name": "frontend",
+	"private": true,
+	"version": "0.0.1",
+	"type": "module",
+	"scripts": {
+		"dev": "vite dev",
+		"build": "vite build",
+		"preview": "vite preview",
+		"prepare": "svelte-kit sync || echo ''",
+		"check": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json",
+		"check:watch": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json --watch",
+		"format": "prettier --write .",
+		"lint": "prettier --check . && eslint ."
+	},
+	"devDependencies": {
+		"@eslint/compat": "^1.2.5",
+		"@eslint/js": "^9.18.0",
+		"@fontsource/fira-mono": "^5.0.0",
+		"@neoconfetti/svelte": "^2.0.0",
+		"@sveltejs/adapter-static": "^3.0.8",
+		"@sveltejs/kit": "^2.22.0",
+		"@sveltejs/vite-plugin-svelte": "^6.0.0",
+		"@tailwindcss/vite": "^4.0.0",
+		"eslint": "^9.18.0",
+		"eslint-config-prettier": "^10.0.1",
+		"eslint-plugin-svelte": "^3.0.0",
+		"globals": "^16.0.0",
+		"prettier": "^3.4.2",
+		"prettier-plugin-svelte": "^3.3.3",
+		"prettier-plugin-tailwindcss": "^0.6.11",
+		"svelte": "^5.25.0",
+		"svelte-check": "^4.0.0",
+		"tailwindcss": "^4.0.0",
+		"typescript": "^5.0.0",
+		"typescript-eslint": "^8.20.0",
+		"vite": "^7.0.4"
+	},
+	"pnpm": {
+		"onlyBuiltDependencies": [
+			"esbuild"
+		]
+	},
+	"dependencies": {
+		"flatbuffers": "^25.2.10",
+		"lucide-svelte": "^0.539.0",
+		"svelte-loading-spinners": "^0.3.6"
+	}
+}

frontend/src/app.css

@@ -0,0 +1,106 @@
+@import 'tailwindcss';
+@import '@fontsource/fira-mono';
+
+:root {
+	--font-body:
+		Arial, -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen, Ubuntu, Cantarell,
+		'Open Sans', 'Helvetica Neue', sans-serif;
+	--font-mono: 'Fira Mono', monospace;
+	--color-bg-0: rgb(202, 216, 228);
+	--color-bg-1: hsl(209, 36%, 86%);
+	--color-bg-2: hsl(224, 44%, 95%);
+	--color-theme-1: #ff3e00;
+	--color-theme-2: #4075a6;
+	--color-text: rgba(0, 0, 0, 0.7);
+	--column-width: 42rem;
+	--column-margin-top: 4rem;
+	font-family: var(--font-body);
+	color: var(--color-text);
+}
+
+body {
+	min-height: 100vh;
+	margin: 0;
+	background-attachment: fixed;
+	background-color: var(--color-bg-1);
+	background-size: 100vw 100vh;
+	background-image:
+		radial-gradient(50% 50% at 50% 50%, rgba(255, 255, 255, 0.75) 0%, rgba(255, 255, 255, 0) 100%),
+		linear-gradient(180deg, var(--color-bg-0) 0%, var(--color-bg-1) 15%, var(--color-bg-2) 50%);
+}
+
+h1,
+h2,
+p {
+	font-weight: 400;
+}
+
+p {
+	line-height: 1.5;
+}
+
+a {
+	color: var(--color-theme-1);
+	text-decoration: none;
+}
+
+a:hover {
+	text-decoration: underline;
+}
+
+h1 {
+	font-size: 2rem;
+	text-align: center;
+}
+
+h2 {
+	font-size: 1rem;
+}
+
+pre {
+	font-size: 16px;
+	font-family: var(--font-mono);
+	background-color: rgba(255, 255, 255, 0.45);
+	border-radius: 3px;
+	box-shadow: 2px 2px 6px rgb(255 255 255 / 25%);
+	padding: 0.5em;
+	overflow-x: auto;
+	color: var(--color-text);
+}
+
+.text-column {
+	display: flex;
+	max-width: 48rem;
+	flex: 0.6;
+	flex-direction: column;
+	justify-content: center;
+	margin: 0 auto;
+}
+
+input,
+button {
+	font-size: inherit;
+	font-family: inherit;
+}
+
+button:focus:not(:focus-visible) {
+	outline: none;
+}
+
+@media (min-width: 720px) {
+	h1 {
+		font-size: 2.4rem;
+	}
+}
+
+.visually-hidden {
+	border: 0;
+	clip: rect(0 0 0 0);
+	height: auto;
+	margin: 0;
+	overflow: hidden;
+	padding: 0;
+	position: absolute;
+	width: 1px;
+	white-space: nowrap;
+}

frontend/src/app.d.ts

@@ -0,0 +1,13 @@
+// See https://svelte.dev/docs/kit/types#app.d.ts
+// for information about these interfaces
+declare global {
+	namespace App {
+		// interface Error {}
+		// interface Locals {}
+		// interface PageData {}
+		// interface PageState {}
+		// interface Platform {}
+	}
+}
+
+export {};

frontend/src/app.html

@@ -0,0 +1,12 @@
+<!doctype html>
+<html lang="en">
+	<head>
+		<meta charset="utf-8" />
+		<link rel="icon" href="%sveltekit.assets%/favicon.svg" />
+		<meta name="viewport" content="width=device-width, initial-scale=1" />
+		%sveltekit.head%
+	</head>
+	<body data-sveltekit-preload-data="hover">
+		<div style="display: contents">%sveltekit.body%</div>
+	</body>
+</html>

frontend/src/lib/canvas.worker.ts

@@ -0,0 +1,24 @@
+import { streamVideoFromReader } from "./video";
+
+let canvas: OffscreenCanvas | undefined = undefined;
+let reader: ReadableStream | undefined = undefined;
+let resolver: any = undefined;
+const messagePromise = new Promise((resolve) => resolver = resolve);
+
+
+self.onmessage = async ({ data: payload }: { data: any }) => {
+  canvas = payload["canvas"];
+  reader = payload["reader"];
+  self.onmessage = null;
+  resolver?.();
+};
+
+console.log(`Worker spawned successfully`)
+await messagePromise;
+console.log("Worker received required objects")
+console.log("Starting video streaming")
+
+await streamVideoFromReader(reader!.getReader(), canvas!);
+
+
+export { }; // This makes TypeScript happy

frontend/src/lib/images/github.svg

@@ -0,0 +1 @@
+<svg width="98" height="96" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" clip-rule="evenodd" d="M48.854 0C21.839 0 0 22 0 49.217c0 21.756 13.993 40.172 33.405 46.69 2.427.49 3.316-1.059 3.316-2.362 0-1.141-.08-5.052-.08-9.127-13.59 2.934-16.42-5.867-16.42-5.867-2.184-5.704-5.42-7.17-5.42-7.17-4.448-3.015.324-3.015.324-3.015 4.934.326 7.523 5.052 7.523 5.052 4.367 7.496 11.404 5.378 14.235 4.074.404-3.178 1.699-5.378 3.074-6.6-10.839-1.141-22.243-5.378-22.243-24.283 0-5.378 1.94-9.778 5.014-13.2-.485-1.222-2.184-6.275.486-13.038 0 0 4.125-1.304 13.426 5.052a46.97 46.97 0 0 1 12.214-1.63c4.125 0 8.33.571 12.213 1.63 9.302-6.356 13.427-5.052 13.427-5.052 2.67 6.763.97 11.816.485 13.038 3.155 3.422 5.015 7.822 5.015 13.2 0 18.905-11.404 23.06-22.324 24.283 1.78 1.548 3.316 4.481 3.316 9.126 0 6.6-.08 11.897-.08 13.526 0 1.304.89 2.853 3.316 2.364 19.412-6.52 33.405-24.935 33.405-46.691C97.707 22 75.788 0 48.854 0z" fill="#24292f"/></svg>

frontend/src/lib/images/svelte-logo.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="107" height="128" viewBox="0 0 107 128"><title>svelte-logo</title><path d="M94.1566,22.8189c-10.4-14.8851-30.94-19.2971-45.7914-9.8348L22.2825,29.6078A29.9234,29.9234,0,0,0,8.7639,49.6506a31.5136,31.5136,0,0,0,3.1076,20.2318A30.0061,30.0061,0,0,0,7.3953,81.0653a31.8886,31.8886,0,0,0,5.4473,24.1157c10.4022,14.8865,30.9423,19.2966,45.7914,9.8348L84.7167,98.3921A29.9177,29.9177,0,0,0,98.2353,78.3493,31.5263,31.5263,0,0,0,95.13,58.117a30,30,0,0,0,4.4743-11.1824,31.88,31.88,0,0,0-5.4473-24.1157" style="fill:#ff3e00"/><path d="M45.8171,106.5815A20.7182,20.7182,0,0,1,23.58,98.3389a19.1739,19.1739,0,0,1-3.2766-14.5025,18.1886,18.1886,0,0,1,.6233-2.4357l.4912-1.4978,1.3363.9815a33.6443,33.6443,0,0,0,10.203,5.0978l.9694.2941-.0893.9675a5.8474,5.8474,0,0,0,1.052,3.8781,6.2389,6.2389,0,0,0,6.6952,2.485,5.7449,5.7449,0,0,0,1.6021-.7041L69.27,76.281a5.4306,5.4306,0,0,0,2.4506-3.631,5.7948,5.7948,0,0,0-.9875-4.3712,6.2436,6.2436,0,0,0-6.6978-2.4864,5.7427,5.7427,0,0,0-1.6.7036l-9.9532,6.3449a19.0329,19.0329,0,0,1-5.2965,2.3259,20.7181,20.7181,0,0,1-22.2368-8.2427,19.1725,19.1725,0,0,1-3.2766-14.5024,17.9885,17.9885,0,0,1,8.13-12.0513L55.8833,23.7472a19.0038,19.0038,0,0,1,5.3-2.3287A20.7182,20.7182,0,0,1,83.42,29.6611a19.1739,19.1739,0,0,1,3.2766,14.5025,18.4,18.4,0,0,1-.6233,2.4357l-.4912,1.4978-1.3356-.98a33.6175,33.6175,0,0,0-10.2037-5.1l-.9694-.2942.0893-.9675a5.8588,5.8588,0,0,0-1.052-3.878,6.2389,6.2389,0,0,0-6.6952-2.485,5.7449,5.7449,0,0,0-1.6021.7041L37.73,51.719a5.4218,5.4218,0,0,0-2.4487,3.63,5.7862,5.7862,0,0,0,.9856,4.3717,6.2437,6.2437,0,0,0,6.6978,2.4864,5.7652,5.7652,0,0,0,1.602-.7041l9.9519-6.3425a18.978,18.978,0,0,1,5.2959-2.3278,20.7181,20.7181,0,0,1,22.2368,8.2427,19.1725,19.1725,0,0,1,3.2766,14.5024,17.9977,17.9977,0,0,1-8.13,12.0532L51.1167,104.2528a19.0038,19.0038,0,0,1-5.3,2.3287" style="fill:#fff"/></svg>

frontend/src/lib/input.ts

@@ -0,0 +1,152 @@
+import { Input } from "$lib/proto/input-event"
+import { InputEvent } from "$lib/proto/input"
+import { Builder } from "flatbuffers";
+
+export enum KeyAction {
+  DOWN,
+  UP,
+}
+
+export async function sendKeyboardEvent(
+  writer: WritableStreamDefaultWriter,
+  event: KeyboardEvent,
+  keyAction: KeyAction,
+) {
+  const builder = new Builder(1024);
+
+  InputEvent.KeyboardInput.startKeyboardInput(builder);
+  // Have to use keyCode even though it's deprecated beacuse .code is a unhelpful string :sigh:
+  InputEvent.KeyboardInput.addKeyCode(builder, event.keyCode);
+  InputEvent.KeyboardInput.addKeyAction(builder, keyAction);
+  const modifierState = InputEvent.ModifierState.createModifierState(
+    builder,
+    event.shiftKey,
+    event.ctrlKey,
+    event.altKey,
+    event.metaKey
+  );
+
+  InputEvent.KeyboardInput.addModifiers(builder, modifierState);
+  const keyboardInput = InputEvent.KeyboardInput.endKeyboardInput(builder);
+
+  // Create InputEvent table with union
+  InputEvent.InputEvent.startInputEvent(builder);
+  InputEvent.InputEvent.addInputType(builder, Input.Keyboard);
+  InputEvent.InputEvent.addInput(builder, keyboardInput);
+  const inputEvent = InputEvent.InputEvent.endInputEvent(builder);
+
+  // Finish the buffer
+  builder.finish(inputEvent);
+
+  // Get the buffer data
+  const bufferData = builder.asUint8Array();
+
+  // Create size prefix as big-endian u32
+  const sizePrefix = new Uint8Array(4);
+  const size = bufferData.length;
+  sizePrefix[0] = (size >> 24) & 0xFF;  // Most significant byte
+  sizePrefix[1] = (size >> 16) & 0xFF;
+  sizePrefix[2] = (size >> 8) & 0xFF;
+  sizePrefix[3] = size & 0xFF;          // Least significant byte
+
+  // Create combined buffer with size prefix + flatbuffer data
+  const combined = new Uint8Array(sizePrefix.length + bufferData.length);
+  combined.set(sizePrefix, 0);
+  combined.set(bufferData, sizePrefix.length);
+
+  // Write to stream
+  await writer.write(combined);
+}
+
+export async function sendMouseMoveEvent(
+  writer: WritableStreamDefaultWriter,
+  event: MouseEvent,
+) {
+  const builder = new Builder(1024);
+
+  const mouseMovement = InputEvent.MouseMovement.createMouseMovement(builder, event.movementX, event.movementY);
+
+  // Create InputEvent table with union
+  InputEvent.InputEvent.startInputEvent(builder);
+  InputEvent.InputEvent.addInputType(builder, Input.MouseMovement);
+  InputEvent.InputEvent.addInput(builder, mouseMovement);
+  const inputEvent = InputEvent.InputEvent.endInputEvent(builder);
+
+  // Finish the buffer
+  builder.finish(inputEvent);
+
+  // Get the buffer data
+  const bufferData = builder.asUint8Array();
+
+  // Create size prefix as big-endian u32
+  const sizePrefix = new Uint8Array(4);
+  const size = bufferData.length;
+  sizePrefix[0] = (size >> 24) & 0xFF;  // Most significant byte
+  sizePrefix[1] = (size >> 16) & 0xFF;
+  sizePrefix[2] = (size >> 8) & 0xFF;
+  sizePrefix[3] = size & 0xFF;          // Least significant byte
+
+  // Create combined buffer with size prefix + flatbuffer data
+  const combined = new Uint8Array(sizePrefix.length + bufferData.length);
+  combined.set(sizePrefix, 0);
+  combined.set(bufferData, sizePrefix.length);
+
+  // Write to stream
+  await writer.write(combined);
+}
+
+export async function sendMouseInputEvent(
+  writer: WritableStreamDefaultWriter,
+  event: MouseEvent,
+  keyAction: KeyAction,
+) {
+  const builder = new Builder(1024);
+
+  var mouseButton: InputEvent.MouseButton;
+
+  switch (event.button) {
+    case 0:
+      mouseButton = InputEvent.MouseButton.LEFT;
+      break;
+    case 1:
+      mouseButton = InputEvent.MouseButton.MIDDLE;
+      break;
+    case 2:
+      mouseButton = InputEvent.MouseButton.RIGHT;
+      break;
+    default:
+      console.log(`Unknown mouse button ${event.button}, defaulting to LEFT`);
+      mouseButton = InputEvent.MouseButton.LEFT;
+      break;
+  }
+
+  const mouseInput = InputEvent.MouseInput.createMouseInput(builder, mouseButton, keyAction);
+
+  // Create InputEvent table with union
+  InputEvent.InputEvent.startInputEvent(builder);
+  InputEvent.InputEvent.addInputType(builder, Input.MouseInput);
+  InputEvent.InputEvent.addInput(builder, mouseInput);
+  const inputEvent = InputEvent.InputEvent.endInputEvent(builder);
+
+  // Finish the buffer
+  builder.finish(inputEvent);
+
+  // Get the buffer data
+  const bufferData = builder.asUint8Array();
+
+  // Create size prefix as big-endian u32
+  const sizePrefix = new Uint8Array(4);
+  const size = bufferData.length;
+  sizePrefix[0] = (size >> 24) & 0xFF;  // Most significant byte
+  sizePrefix[1] = (size >> 16) & 0xFF;
+  sizePrefix[2] = (size >> 8) & 0xFF;
+  sizePrefix[3] = size & 0xFF;          // Least significant byte
+
+  // Create combined buffer with size prefix + flatbuffer data
+  const combined = new Uint8Array(sizePrefix.length + bufferData.length);
+  combined.set(sizePrefix, 0);
+  combined.set(bufferData, sizePrefix.length);
+
+  // Write to stream
+  await writer.write(combined);
+}

frontend/src/lib/proto/input-event.ts

@@ -0,0 +1,12 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */
+
+export { Input } from './input-event/input.js';
+export { InputEvent } from './input-event/input-event.js';
+export { KeyAction } from './input-event/key-action.js';
+export { KeyboardInput } from './input-event/keyboard-input.js';
+export { ModifierState } from './input-event/modifier-state.js';
+export { MouseButton } from './input-event/mouse-button.js';
+export { MouseInput } from './input-event/mouse-input.js';
+export { MouseMovement } from './input-event/mouse-movement.js';

frontend/src/lib/proto/input-event/input-event.ts

@@ -0,0 +1,69 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */
+
+import * as flatbuffers from 'flatbuffers';
+
+import { Input, unionToInput, unionListToInput } from '../input-event/input.js';
+
+
+export class InputEvent {
+  bb: flatbuffers.ByteBuffer|null = null;
+  bb_pos = 0;
+  __init(i:number, bb:flatbuffers.ByteBuffer):InputEvent {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+}
+
+static getRootAsInputEvent(bb:flatbuffers.ByteBuffer, obj?:InputEvent):InputEvent {
+  return (obj || new InputEvent()).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+}
+
+static getSizePrefixedRootAsInputEvent(bb:flatbuffers.ByteBuffer, obj?:InputEvent):InputEvent {
+  bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH);
+  return (obj || new InputEvent()).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+}
+
+inputType():Input {
+  const offset = this.bb!.__offset(this.bb_pos, 4);
+  return offset ? this.bb!.readUint8(this.bb_pos + offset) : Input.NONE;
+}
+
+input<T extends flatbuffers.Table>(obj:any):any|null {
+  const offset = this.bb!.__offset(this.bb_pos, 6);
+  return offset ? this.bb!.__union(obj, this.bb_pos + offset) : null;
+}
+
+static startInputEvent(builder:flatbuffers.Builder) {
+  builder.startObject(2);
+}
+
+static addInputType(builder:flatbuffers.Builder, inputType:Input) {
+  builder.addFieldInt8(0, inputType, Input.NONE);
+}
+
+static addInput(builder:flatbuffers.Builder, inputOffset:flatbuffers.Offset) {
+  builder.addFieldOffset(1, inputOffset, 0);
+}
+
+static endInputEvent(builder:flatbuffers.Builder):flatbuffers.Offset {
+  const offset = builder.endObject();
+  return offset;
+}
+
+static finishInputEventBuffer(builder:flatbuffers.Builder, offset:flatbuffers.Offset) {
+  builder.finish(offset);
+}
+
+static finishSizePrefixedInputEventBuffer(builder:flatbuffers.Builder, offset:flatbuffers.Offset) {
+  builder.finish(offset, undefined, true);
+}
+
+static createInputEvent(builder:flatbuffers.Builder, inputType:Input, inputOffset:flatbuffers.Offset):flatbuffers.Offset {
+  InputEvent.startInputEvent(builder);
+  InputEvent.addInputType(builder, inputType);
+  InputEvent.addInput(builder, inputOffset);
+  return InputEvent.endInputEvent(builder);
+}
+}

frontend/src/lib/proto/input-event/input.ts

@@ -0,0 +1,42 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */
+
+import { KeyboardInput } from '../input-event/keyboard-input.js';
+import { MouseInput } from '../input-event/mouse-input.js';
+import { MouseMovement } from '../input-event/mouse-movement.js';
+
+
+export enum Input {
+  NONE = 0,
+  Keyboard = 1,
+  MouseMovement = 2,
+  MouseInput = 3
+}
+
+export function unionToInput(
+  type: Input,
+  accessor: (obj:KeyboardInput|MouseInput|MouseMovement) => KeyboardInput|MouseInput|MouseMovement|null
+): KeyboardInput|MouseInput|MouseMovement|null {
+  switch(Input[type]) {
+    case 'NONE': return null; 
+    case 'Keyboard': return accessor(new KeyboardInput())! as KeyboardInput;
+    case 'MouseMovement': return accessor(new MouseMovement())! as MouseMovement;
+    case 'MouseInput': return accessor(new MouseInput())! as MouseInput;
+    default: return null;
+  }
+}
+
+export function unionListToInput(
+  type: Input, 
+  accessor: (index: number, obj:KeyboardInput|MouseInput|MouseMovement) => KeyboardInput|MouseInput|MouseMovement|null, 
+  index: number
+): KeyboardInput|MouseInput|MouseMovement|null {
+  switch(Input[type]) {
+    case 'NONE': return null; 
+    case 'Keyboard': return accessor(index, new KeyboardInput())! as KeyboardInput;
+    case 'MouseMovement': return accessor(index, new MouseMovement())! as MouseMovement;
+    case 'MouseInput': return accessor(index, new MouseInput())! as MouseInput;
+    default: return null;
+  }
+}

frontend/src/lib/proto/input-event/key-action.ts

@@ -0,0 +1,8 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */
+
+export enum KeyAction {
+  DOWN = 0,
+  UP = 1
+}

frontend/src/lib/proto/input-event/keyboard-input.ts

@@ -0,0 +1,65 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */
+
+import * as flatbuffers from 'flatbuffers';
+
+import { KeyAction } from '../input-event/key-action.js';
+import { ModifierState } from '../input-event/modifier-state.js';
+
+
+export class KeyboardInput {
+  bb: flatbuffers.ByteBuffer|null = null;
+  bb_pos = 0;
+  __init(i:number, bb:flatbuffers.ByteBuffer):KeyboardInput {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+}
+
+static getRootAsKeyboardInput(bb:flatbuffers.ByteBuffer, obj?:KeyboardInput):KeyboardInput {
+  return (obj || new KeyboardInput()).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+}
+
+static getSizePrefixedRootAsKeyboardInput(bb:flatbuffers.ByteBuffer, obj?:KeyboardInput):KeyboardInput {
+  bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH);
+  return (obj || new KeyboardInput()).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+}
+
+keyCode():number {
+  const offset = this.bb!.__offset(this.bb_pos, 4);
+  return offset ? this.bb!.readInt16(this.bb_pos + offset) : 0;
+}
+
+keyAction():KeyAction {
+  const offset = this.bb!.__offset(this.bb_pos, 6);
+  return offset ? this.bb!.readInt8(this.bb_pos + offset) : KeyAction.DOWN;
+}
+
+modifiers(obj?:ModifierState):ModifierState|null {
+  const offset = this.bb!.__offset(this.bb_pos, 8);
+  return offset ? (obj || new ModifierState()).__init(this.bb_pos + offset, this.bb!) : null;
+}
+
+static startKeyboardInput(builder:flatbuffers.Builder) {
+  builder.startObject(3);
+}
+
+static addKeyCode(builder:flatbuffers.Builder, keyCode:number) {
+  builder.addFieldInt16(0, keyCode, 0);
+}
+
+static addKeyAction(builder:flatbuffers.Builder, keyAction:KeyAction) {
+  builder.addFieldInt8(1, keyAction, KeyAction.DOWN);
+}
+
+static addModifiers(builder:flatbuffers.Builder, modifiersOffset:flatbuffers.Offset) {
+  builder.addFieldStruct(2, modifiersOffset, 0);
+}
+
+static endKeyboardInput(builder:flatbuffers.Builder):flatbuffers.Offset {
+  const offset = builder.endObject();
+  return offset;
+}
+
+}

frontend/src/lib/proto/input-event/modifier-state.ts

@@ -0,0 +1,45 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */
+
+import * as flatbuffers from 'flatbuffers';
+
+export class ModifierState {
+  bb: flatbuffers.ByteBuffer|null = null;
+  bb_pos = 0;
+  __init(i:number, bb:flatbuffers.ByteBuffer):ModifierState {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+}
+
+shift():boolean {
+  return !!this.bb!.readInt8(this.bb_pos);
+}
+
+ctrl():boolean {
+  return !!this.bb!.readInt8(this.bb_pos + 1);
+}
+
+alt():boolean {
+  return !!this.bb!.readInt8(this.bb_pos + 2);
+}
+
+meta():boolean {
+  return !!this.bb!.readInt8(this.bb_pos + 3);
+}
+
+static sizeOf():number {
+  return 4;
+}
+
+static createModifierState(builder:flatbuffers.Builder, shift: boolean, ctrl: boolean, alt: boolean, meta: boolean):flatbuffers.Offset {
+  builder.prep(1, 4);
+  builder.writeInt8(Number(Boolean(meta)));
+  builder.writeInt8(Number(Boolean(alt)));
+  builder.writeInt8(Number(Boolean(ctrl)));
+  builder.writeInt8(Number(Boolean(shift)));
+  return builder.offset();
+}
+
+}

frontend/src/lib/proto/input-event/mouse-button.ts

@@ -0,0 +1,11 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */
+
+export enum MouseButton {
+  LEFT = 0,
+  MIDDLE = 1,
+  RIGHT = 2,
+  X1 = 3,
+  X2 = 4
+}

frontend/src/lib/proto/input-event/mouse-input.ts

@@ -0,0 +1,62 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */
+
+import * as flatbuffers from 'flatbuffers';
+
+import { KeyAction } from '../input-event/key-action.js';
+import { MouseButton } from '../input-event/mouse-button.js';
+
+
+export class MouseInput {
+  bb: flatbuffers.ByteBuffer|null = null;
+  bb_pos = 0;
+  __init(i:number, bb:flatbuffers.ByteBuffer):MouseInput {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+}
+
+static getRootAsMouseInput(bb:flatbuffers.ByteBuffer, obj?:MouseInput):MouseInput {
+  return (obj || new MouseInput()).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+}
+
+static getSizePrefixedRootAsMouseInput(bb:flatbuffers.ByteBuffer, obj?:MouseInput):MouseInput {
+  bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH);
+  return (obj || new MouseInput()).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+}
+
+button():MouseButton {
+  const offset = this.bb!.__offset(this.bb_pos, 4);
+  return offset ? this.bb!.readInt8(this.bb_pos + offset) : MouseButton.LEFT;
+}
+
+buttonAction():KeyAction {
+  const offset = this.bb!.__offset(this.bb_pos, 6);
+  return offset ? this.bb!.readInt8(this.bb_pos + offset) : KeyAction.DOWN;
+}
+
+static startMouseInput(builder:flatbuffers.Builder) {
+  builder.startObject(2);
+}
+
+static addButton(builder:flatbuffers.Builder, button:MouseButton) {
+  builder.addFieldInt8(0, button, MouseButton.LEFT);
+}
+
+static addButtonAction(builder:flatbuffers.Builder, buttonAction:KeyAction) {
+  builder.addFieldInt8(1, buttonAction, KeyAction.DOWN);
+}
+
+static endMouseInput(builder:flatbuffers.Builder):flatbuffers.Offset {
+  const offset = builder.endObject();
+  return offset;
+}
+
+static createMouseInput(builder:flatbuffers.Builder, button:MouseButton, buttonAction:KeyAction):flatbuffers.Offset {
+  MouseInput.startMouseInput(builder);
+  MouseInput.addButton(builder, button);
+  MouseInput.addButtonAction(builder, buttonAction);
+  return MouseInput.endMouseInput(builder);
+}
+}

frontend/src/lib/proto/input-event/mouse-movement.ts

@@ -0,0 +1,58 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */
+
+import * as flatbuffers from 'flatbuffers';
+
+export class MouseMovement {
+  bb: flatbuffers.ByteBuffer|null = null;
+  bb_pos = 0;
+  __init(i:number, bb:flatbuffers.ByteBuffer):MouseMovement {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+}
+
+static getRootAsMouseMovement(bb:flatbuffers.ByteBuffer, obj?:MouseMovement):MouseMovement {
+  return (obj || new MouseMovement()).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+}
+
+static getSizePrefixedRootAsMouseMovement(bb:flatbuffers.ByteBuffer, obj?:MouseMovement):MouseMovement {
+  bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH);
+  return (obj || new MouseMovement()).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+}
+
+movementX():number {
+  const offset = this.bb!.__offset(this.bb_pos, 4);
+  return offset ? this.bb!.readInt16(this.bb_pos + offset) : 0;
+}
+
+movementY():number {
+  const offset = this.bb!.__offset(this.bb_pos, 6);
+  return offset ? this.bb!.readInt16(this.bb_pos + offset) : 0;
+}
+
+static startMouseMovement(builder:flatbuffers.Builder) {
+  builder.startObject(2);
+}
+
+static addMovementX(builder:flatbuffers.Builder, movementX:number) {
+  builder.addFieldInt16(0, movementX, 0);
+}
+
+static addMovementY(builder:flatbuffers.Builder, movementY:number) {
+  builder.addFieldInt16(1, movementY, 0);
+}
+
+static endMouseMovement(builder:flatbuffers.Builder):flatbuffers.Offset {
+  const offset = builder.endObject();
+  return offset;
+}
+
+static createMouseMovement(builder:flatbuffers.Builder, movementX:number, movementY:number):flatbuffers.Offset {
+  MouseMovement.startMouseMovement(builder);
+  MouseMovement.addMovementX(builder, movementX);
+  MouseMovement.addMovementY(builder, movementY);
+  return MouseMovement.endMouseMovement(builder);
+}
+}

frontend/src/lib/proto/input.ts

@@ -0,0 +1,5 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */
+
+export * as InputEvent from './input-event.js';

frontend/src/lib/proto/video-update.ts

@@ -0,0 +1,10 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */
+
+export { DecodeUnitBuffer } from './video-update/decode-unit-buffer.js';
+export { DecodeUnitStart } from './video-update/decode-unit-start.js';
+export { FrameType } from './video-update/frame-type.js';
+export { Setup } from './video-update/setup.js';
+export { Update } from './video-update/update.js';
+export { VideoUpdate } from './video-update/video-update.js';

frontend/src/lib/proto/video-update/decode-unit-buffer.ts

@@ -0,0 +1,100 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */
+
+import * as flatbuffers from 'flatbuffers';
+
+export class DecodeUnitBuffer {
+  bb: flatbuffers.ByteBuffer|null = null;
+  bb_pos = 0;
+  __init(i:number, bb:flatbuffers.ByteBuffer):DecodeUnitBuffer {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+}
+
+static getRootAsDecodeUnitBuffer(bb:flatbuffers.ByteBuffer, obj?:DecodeUnitBuffer):DecodeUnitBuffer {
+  return (obj || new DecodeUnitBuffer()).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+}
+
+static getSizePrefixedRootAsDecodeUnitBuffer(bb:flatbuffers.ByteBuffer, obj?:DecodeUnitBuffer):DecodeUnitBuffer {
+  bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH);
+  return (obj || new DecodeUnitBuffer()).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+}
+
+frameNumber():bigint {
+  const offset = this.bb!.__offset(this.bb_pos, 4);
+  return offset ? this.bb!.readUint64(this.bb_pos + offset) : BigInt('0');
+}
+
+bufferIndex():bigint {
+  const offset = this.bb!.__offset(this.bb_pos, 6);
+  return offset ? this.bb!.readUint64(this.bb_pos + offset) : BigInt('0');
+}
+
+bufferOffset():bigint {
+  const offset = this.bb!.__offset(this.bb_pos, 8);
+  return offset ? this.bb!.readUint64(this.bb_pos + offset) : BigInt('0');
+}
+
+data(index: number):number|null {
+  const offset = this.bb!.__offset(this.bb_pos, 10);
+  return offset ? this.bb!.readUint8(this.bb!.__vector(this.bb_pos + offset) + index) : 0;
+}
+
+dataLength():number {
+  const offset = this.bb!.__offset(this.bb_pos, 10);
+  return offset ? this.bb!.__vector_len(this.bb_pos + offset) : 0;
+}
+
+dataArray():Uint8Array|null {
+  const offset = this.bb!.__offset(this.bb_pos, 10);
+  return offset ? new Uint8Array(this.bb!.bytes().buffer, this.bb!.bytes().byteOffset + this.bb!.__vector(this.bb_pos + offset), this.bb!.__vector_len(this.bb_pos + offset)) : null;
+}
+
+static startDecodeUnitBuffer(builder:flatbuffers.Builder) {
+  builder.startObject(4);
+}
+
+static addFrameNumber(builder:flatbuffers.Builder, frameNumber:bigint) {
+  builder.addFieldInt64(0, frameNumber, BigInt('0'));
+}
+
+static addBufferIndex(builder:flatbuffers.Builder, bufferIndex:bigint) {
+  builder.addFieldInt64(1, bufferIndex, BigInt('0'));
+}
+
+static addBufferOffset(builder:flatbuffers.Builder, bufferOffset:bigint) {
+  builder.addFieldInt64(2, bufferOffset, BigInt('0'));
+}
+
+static addData(builder:flatbuffers.Builder, dataOffset:flatbuffers.Offset) {
+  builder.addFieldOffset(3, dataOffset, 0);
+}
+
+static createDataVector(builder:flatbuffers.Builder, data:number[]|Uint8Array):flatbuffers.Offset {
+  builder.startVector(1, data.length, 1);
+  for (let i = data.length - 1; i >= 0; i--) {
+    builder.addInt8(data[i]!);
+  }
+  return builder.endVector();
+}
+
+static startDataVector(builder:flatbuffers.Builder, numElems:number) {
+  builder.startVector(1, numElems, 1);
+}
+
+static endDecodeUnitBuffer(builder:flatbuffers.Builder):flatbuffers.Offset {
+  const offset = builder.endObject();
+  return offset;
+}
+
+static createDecodeUnitBuffer(builder:flatbuffers.Builder, frameNumber:bigint, bufferIndex:bigint, bufferOffset:bigint, dataOffset:flatbuffers.Offset):flatbuffers.Offset {
+  DecodeUnitBuffer.startDecodeUnitBuffer(builder);
+  DecodeUnitBuffer.addFrameNumber(builder, frameNumber);
+  DecodeUnitBuffer.addBufferIndex(builder, bufferIndex);
+  DecodeUnitBuffer.addBufferOffset(builder, bufferOffset);
+  DecodeUnitBuffer.addData(builder, dataOffset);
+  return DecodeUnitBuffer.endDecodeUnitBuffer(builder);
+}
+}

frontend/src/lib/proto/video-update/decode-unit-start.ts

@@ -0,0 +1,91 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */
+
+import * as flatbuffers from 'flatbuffers';
+
+import { FrameType } from '../video-update/frame-type.js';
+
+
+export class DecodeUnitStart {
+  bb: flatbuffers.ByteBuffer|null = null;
+  bb_pos = 0;
+  __init(i:number, bb:flatbuffers.ByteBuffer):DecodeUnitStart {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+}
+
+static getRootAsDecodeUnitStart(bb:flatbuffers.ByteBuffer, obj?:DecodeUnitStart):DecodeUnitStart {
+  return (obj || new DecodeUnitStart()).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+}
+
+static getSizePrefixedRootAsDecodeUnitStart(bb:flatbuffers.ByteBuffer, obj?:DecodeUnitStart):DecodeUnitStart {
+  bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH);
+  return (obj || new DecodeUnitStart()).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+}
+
+frameNumber():bigint {
+  const offset = this.bb!.__offset(this.bb_pos, 4);
+  return offset ? this.bb!.readUint64(this.bb_pos + offset) : BigInt('0');
+}
+
+frameType():FrameType {
+  const offset = this.bb!.__offset(this.bb_pos, 6);
+  return offset ? this.bb!.readInt8(this.bb_pos + offset) : FrameType.PFRAME;
+}
+
+numBuffers():bigint {
+  const offset = this.bb!.__offset(this.bb_pos, 8);
+  return offset ? this.bb!.readUint64(this.bb_pos + offset) : BigInt('0');
+}
+
+receiveTimeMs():number {
+  const offset = this.bb!.__offset(this.bb_pos, 10);
+  return offset ? this.bb!.readUint16(this.bb_pos + offset) : 0;
+}
+
+fullLength():bigint {
+  const offset = this.bb!.__offset(this.bb_pos, 12);
+  return offset ? this.bb!.readUint64(this.bb_pos + offset) : BigInt('0');
+}
+
+static startDecodeUnitStart(builder:flatbuffers.Builder) {
+  builder.startObject(5);
+}
+
+static addFrameNumber(builder:flatbuffers.Builder, frameNumber:bigint) {
+  builder.addFieldInt64(0, frameNumber, BigInt('0'));
+}
+
+static addFrameType(builder:flatbuffers.Builder, frameType:FrameType) {
+  builder.addFieldInt8(1, frameType, FrameType.PFRAME);
+}
+
+static addNumBuffers(builder:flatbuffers.Builder, numBuffers:bigint) {
+  builder.addFieldInt64(2, numBuffers, BigInt('0'));
+}
+
+static addReceiveTimeMs(builder:flatbuffers.Builder, receiveTimeMs:number) {
+  builder.addFieldInt16(3, receiveTimeMs, 0);
+}
+
+static addFullLength(builder:flatbuffers.Builder, fullLength:bigint) {
+  builder.addFieldInt64(4, fullLength, BigInt('0'));
+}
+
+static endDecodeUnitStart(builder:flatbuffers.Builder):flatbuffers.Offset {
+  const offset = builder.endObject();
+  return offset;
+}
+
+static createDecodeUnitStart(builder:flatbuffers.Builder, frameNumber:bigint, frameType:FrameType, numBuffers:bigint, receiveTimeMs:number, fullLength:bigint):flatbuffers.Offset {
+  DecodeUnitStart.startDecodeUnitStart(builder);
+  DecodeUnitStart.addFrameNumber(builder, frameNumber);
+  DecodeUnitStart.addFrameType(builder, frameType);
+  DecodeUnitStart.addNumBuffers(builder, numBuffers);
+  DecodeUnitStart.addReceiveTimeMs(builder, receiveTimeMs);
+  DecodeUnitStart.addFullLength(builder, fullLength);
+  return DecodeUnitStart.endDecodeUnitStart(builder);
+}
+}

frontend/src/lib/proto/video-update/decode-unit.ts

@@ -0,0 +1,103 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */
+
+import * as flatbuffers from 'flatbuffers';
+
+import { FrameType } from '../video-update/frame-type.js';
+
+
+export class DecodeUnit {
+  bb: flatbuffers.ByteBuffer|null = null;
+  bb_pos = 0;
+  __init(i:number, bb:flatbuffers.ByteBuffer):DecodeUnit {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+}
+
+static getRootAsDecodeUnit(bb:flatbuffers.ByteBuffer, obj?:DecodeUnit):DecodeUnit {
+  return (obj || new DecodeUnit()).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+}
+
+static getSizePrefixedRootAsDecodeUnit(bb:flatbuffers.ByteBuffer, obj?:DecodeUnit):DecodeUnit {
+  bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH);
+  return (obj || new DecodeUnit()).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+}
+
+frameNumber():number {
+  const offset = this.bb!.__offset(this.bb_pos, 4);
+  return offset ? this.bb!.readUint16(this.bb_pos + offset) : 0;
+}
+
+frameType():FrameType {
+  const offset = this.bb!.__offset(this.bb_pos, 6);
+  return offset ? this.bb!.readInt8(this.bb_pos + offset) : FrameType.PFRAME;
+}
+
+receiveTimeMs():number {
+  const offset = this.bb!.__offset(this.bb_pos, 8);
+  return offset ? this.bb!.readUint16(this.bb_pos + offset) : 0;
+}
+
+data(index: number):number|null {
+  const offset = this.bb!.__offset(this.bb_pos, 10);
+  return offset ? this.bb!.readUint8(this.bb!.__vector(this.bb_pos + offset) + index) : 0;
+}
+
+dataLength():number {
+  const offset = this.bb!.__offset(this.bb_pos, 10);
+  return offset ? this.bb!.__vector_len(this.bb_pos + offset) : 0;
+}
+
+dataArray():Uint8Array|null {
+  const offset = this.bb!.__offset(this.bb_pos, 10);
+  return offset ? new Uint8Array(this.bb!.bytes().buffer, this.bb!.bytes().byteOffset + this.bb!.__vector(this.bb_pos + offset), this.bb!.__vector_len(this.bb_pos + offset)) : null;
+}
+
+static startDecodeUnit(builder:flatbuffers.Builder) {
+  builder.startObject(4);
+}
+
+static addFrameNumber(builder:flatbuffers.Builder, frameNumber:number) {
+  builder.addFieldInt16(0, frameNumber, 0);
+}
+
+static addFrameType(builder:flatbuffers.Builder, frameType:FrameType) {
+  builder.addFieldInt8(1, frameType, FrameType.PFRAME);
+}
+
+static addReceiveTimeMs(builder:flatbuffers.Builder, receiveTimeMs:number) {
+  builder.addFieldInt16(2, receiveTimeMs, 0);
+}
+
+static addData(builder:flatbuffers.Builder, dataOffset:flatbuffers.Offset) {
+  builder.addFieldOffset(3, dataOffset, 0);
+}
+
+static createDataVector(builder:flatbuffers.Builder, data:number[]|Uint8Array):flatbuffers.Offset {
+  builder.startVector(1, data.length, 1);
+  for (let i = data.length - 1; i >= 0; i--) {
+    builder.addInt8(data[i]!);
+  }
+  return builder.endVector();
+}
+
+static startDataVector(builder:flatbuffers.Builder, numElems:number) {
+  builder.startVector(1, numElems, 1);
+}
+
+static endDecodeUnit(builder:flatbuffers.Builder):flatbuffers.Offset {
+  const offset = builder.endObject();
+  return offset;
+}
+
+static createDecodeUnit(builder:flatbuffers.Builder, frameNumber:number, frameType:FrameType, receiveTimeMs:number, dataOffset:flatbuffers.Offset):flatbuffers.Offset {
+  DecodeUnit.startDecodeUnit(builder);
+  DecodeUnit.addFrameNumber(builder, frameNumber);
+  DecodeUnit.addFrameType(builder, frameType);
+  DecodeUnit.addReceiveTimeMs(builder, receiveTimeMs);
+  DecodeUnit.addData(builder, dataOffset);
+  return DecodeUnit.endDecodeUnit(builder);
+}
+}

frontend/src/lib/proto/video-update/frame-type.ts

@@ -0,0 +1,8 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */
+
+export enum FrameType {
+  PFRAME = 0,
+  IDR = 1
+}

frontend/src/lib/proto/video-update/setup.ts

@@ -0,0 +1,80 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */
+
+import * as flatbuffers from 'flatbuffers';
+
+export class Setup {
+  bb: flatbuffers.ByteBuffer|null = null;
+  bb_pos = 0;
+  __init(i:number, bb:flatbuffers.ByteBuffer):Setup {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+}
+
+static getRootAsSetup(bb:flatbuffers.ByteBuffer, obj?:Setup):Setup {
+  return (obj || new Setup()).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+}
+
+static getSizePrefixedRootAsSetup(bb:flatbuffers.ByteBuffer, obj?:Setup):Setup {
+  bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH);
+  return (obj || new Setup()).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+}
+
+videoFormat():string|null
+videoFormat(optionalEncoding:flatbuffers.Encoding):string|Uint8Array|null
+videoFormat(optionalEncoding?:any):string|Uint8Array|null {
+  const offset = this.bb!.__offset(this.bb_pos, 4);
+  return offset ? this.bb!.__string(this.bb_pos + offset, optionalEncoding) : null;
+}
+
+width():number {
+  const offset = this.bb!.__offset(this.bb_pos, 6);
+  return offset ? this.bb!.readUint16(this.bb_pos + offset) : 0;
+}
+
+height():number {
+  const offset = this.bb!.__offset(this.bb_pos, 8);
+  return offset ? this.bb!.readUint16(this.bb_pos + offset) : 0;
+}
+
+redrawRate():number {
+  const offset = this.bb!.__offset(this.bb_pos, 10);
+  return offset ? this.bb!.readUint16(this.bb_pos + offset) : 0;
+}
+
+static startSetup(builder:flatbuffers.Builder) {
+  builder.startObject(4);
+}
+
+static addVideoFormat(builder:flatbuffers.Builder, videoFormatOffset:flatbuffers.Offset) {
+  builder.addFieldOffset(0, videoFormatOffset, 0);
+}
+
+static addWidth(builder:flatbuffers.Builder, width:number) {
+  builder.addFieldInt16(1, width, 0);
+}
+
+static addHeight(builder:flatbuffers.Builder, height:number) {
+  builder.addFieldInt16(2, height, 0);
+}
+
+static addRedrawRate(builder:flatbuffers.Builder, redrawRate:number) {
+  builder.addFieldInt16(3, redrawRate, 0);
+}
+
+static endSetup(builder:flatbuffers.Builder):flatbuffers.Offset {
+  const offset = builder.endObject();
+  return offset;
+}
+
+static createSetup(builder:flatbuffers.Builder, videoFormatOffset:flatbuffers.Offset, width:number, height:number, redrawRate:number):flatbuffers.Offset {
+  Setup.startSetup(builder);
+  Setup.addVideoFormat(builder, videoFormatOffset);
+  Setup.addWidth(builder, width);
+  Setup.addHeight(builder, height);
+  Setup.addRedrawRate(builder, redrawRate);
+  return Setup.endSetup(builder);
+}
+}

frontend/src/lib/proto/video-update/update.ts

@@ -0,0 +1,42 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */
+
+import { DecodeUnitBuffer } from '../video-update/decode-unit-buffer.js';
+import { DecodeUnitStart } from '../video-update/decode-unit-start.js';
+import { Setup } from '../video-update/setup.js';
+
+
+export enum Update {
+  NONE = 0,
+  Setup = 1,
+  DecodeUnitStart = 2,
+  DecodeUnitBuffer = 3
+}
+
+export function unionToUpdate(
+  type: Update,
+  accessor: (obj:DecodeUnitBuffer|DecodeUnitStart|Setup) => DecodeUnitBuffer|DecodeUnitStart|Setup|null
+): DecodeUnitBuffer|DecodeUnitStart|Setup|null {
+  switch(Update[type]) {
+    case 'NONE': return null; 
+    case 'Setup': return accessor(new Setup())! as Setup;
+    case 'DecodeUnitStart': return accessor(new DecodeUnitStart())! as DecodeUnitStart;
+    case 'DecodeUnitBuffer': return accessor(new DecodeUnitBuffer())! as DecodeUnitBuffer;
+    default: return null;
+  }
+}
+
+export function unionListToUpdate(
+  type: Update, 
+  accessor: (index: number, obj:DecodeUnitBuffer|DecodeUnitStart|Setup) => DecodeUnitBuffer|DecodeUnitStart|Setup|null, 
+  index: number
+): DecodeUnitBuffer|DecodeUnitStart|Setup|null {
+  switch(Update[type]) {
+    case 'NONE': return null; 
+    case 'Setup': return accessor(index, new Setup())! as Setup;
+    case 'DecodeUnitStart': return accessor(index, new DecodeUnitStart())! as DecodeUnitStart;
+    case 'DecodeUnitBuffer': return accessor(index, new DecodeUnitBuffer())! as DecodeUnitBuffer;
+    default: return null;
+  }
+}

frontend/src/lib/proto/video-update/video-update.ts

@@ -0,0 +1,69 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */
+
+import * as flatbuffers from 'flatbuffers';
+
+import { Update, unionToUpdate, unionListToUpdate } from '../video-update/update.js';
+
+
+export class VideoUpdate {
+  bb: flatbuffers.ByteBuffer|null = null;
+  bb_pos = 0;
+  __init(i:number, bb:flatbuffers.ByteBuffer):VideoUpdate {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+}
+
+static getRootAsVideoUpdate(bb:flatbuffers.ByteBuffer, obj?:VideoUpdate):VideoUpdate {
+  return (obj || new VideoUpdate()).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+}
+
+static getSizePrefixedRootAsVideoUpdate(bb:flatbuffers.ByteBuffer, obj?:VideoUpdate):VideoUpdate {
+  bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH);
+  return (obj || new VideoUpdate()).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+}
+
+updateType():Update {
+  const offset = this.bb!.__offset(this.bb_pos, 4);
+  return offset ? this.bb!.readUint8(this.bb_pos + offset) : Update.NONE;
+}
+
+update<T extends flatbuffers.Table>(obj:any):any|null {
+  const offset = this.bb!.__offset(this.bb_pos, 6);
+  return offset ? this.bb!.__union(obj, this.bb_pos + offset) : null;
+}
+
+static startVideoUpdate(builder:flatbuffers.Builder) {
+  builder.startObject(2);
+}
+
+static addUpdateType(builder:flatbuffers.Builder, updateType:Update) {
+  builder.addFieldInt8(0, updateType, Update.NONE);
+}
+
+static addUpdate(builder:flatbuffers.Builder, updateOffset:flatbuffers.Offset) {
+  builder.addFieldOffset(1, updateOffset, 0);
+}
+
+static endVideoUpdate(builder:flatbuffers.Builder):flatbuffers.Offset {
+  const offset = builder.endObject();
+  return offset;
+}
+
+static finishVideoUpdateBuffer(builder:flatbuffers.Builder, offset:flatbuffers.Offset) {
+  builder.finish(offset);
+}
+
+static finishSizePrefixedVideoUpdateBuffer(builder:flatbuffers.Builder, offset:flatbuffers.Offset) {
+  builder.finish(offset, undefined, true);
+}
+
+static createVideoUpdate(builder:flatbuffers.Builder, updateType:Update, updateOffset:flatbuffers.Offset):flatbuffers.Offset {
+  VideoUpdate.startVideoUpdate(builder);
+  VideoUpdate.addUpdateType(builder, updateType);
+  VideoUpdate.addUpdate(builder, updateOffset);
+  return VideoUpdate.endVideoUpdate(builder);
+}
+}

frontend/src/lib/proto/video.ts

@@ -0,0 +1,5 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */
+
+export * as VideoUpdate from './video-update.js';

frontend/src/lib/video.ts

@@ -0,0 +1,189 @@
+import { VideoUpdate } from "$lib/proto/video";
+import { ByteBuffer } from "flatbuffers";
+import { DecodeUnitBuffer } from "./proto/video-update";
+import { DoorClosed, Video } from "lucide-svelte";
+
+function getVideoDecoder(canvasElement: OffscreenCanvas): VideoDecoder {
+  const canvasCtx: OffscreenCanvasRenderingContext2D | null = canvasElement.getContext('2d');
+  if (canvasCtx == null) {
+    throw new Error(`Could not get 2d canvas context`);
+  }
+
+  const videoDecoder = new VideoDecoder({
+    output: (frame) => {
+      //console.log(`rendering frame start: ${performance.now()}`);
+      //canvasElement.width = frame.displayWidth;
+      //canvasElement.height = frame.displayHeight;
+
+      //console.log(`rendering frame drawImage: ${performance.now()}`);
+      canvasCtx.drawImage(frame, 0, 0);
+
+      //console.log(`rendering frame end: ${performance.now()}`);
+      frame.close();
+      //console.log(`rendering frame close: ${performance.now()}`);
+    },
+    error: (e) => {
+      console.error('Decode error:', e);
+    }
+  });
+  return videoDecoder;
+}
+
+async function configureDecoder(videoDecoder: VideoDecoder, videoFormat: string, width: number, height: number) {
+  let config: VideoDecoderConfig = {
+    codec: videoFormat,
+    codedWidth: width,
+    codedHeight: height,
+    optimizeForLatency: true,
+    //hardwareAcceleration: "prefer-hardware",
+  };
+
+  const codecSupport = await VideoDecoder.isConfigSupported(config);
+  console.log(codecSupport);
+  if (codecSupport.supported) {
+    videoDecoder.configure(config);
+  } else {
+    throw new Error(`Could not configure decoder`);
+  }
+}
+
+class Decoder {
+  videoDecoder: VideoDecoder;
+
+  frameNumber: bigint | undefined;
+  frameType: EncodedAudioChunkType = "delta";
+  fullLength: bigint = 0n;
+  receiveTimeMs: number = 0;
+
+  //frameBroken: boolean = false;
+  //lastBufferIndex: bigint = 0n;
+  frameCollector: boolean[] = new Array<boolean>();
+  dataOffset: number = 0;
+  data: Uint8Array = new Uint8Array();
+
+  constructor(videoDecoder: VideoDecoder) {
+    this.videoDecoder = videoDecoder;
+  }
+
+  print() {
+    console.log(
+      `
+    frameNumber: ${this.frameNumber}
+    frameType: ${this.frameType}
+    fullLength: ${this.fullLength}
+    receiveTimeMs: ${this.receiveTimeMs}
+    frameCollector: ${this.frameCollector}
+    ts: ${performance.now()}
+    `
+    );
+  }
+
+  processStart(decodeUnitStart: VideoUpdate.DecodeUnitStart) {
+    //this.print();
+    const frameCompleted = !this.frameCollector.includes(false);
+    if (!frameCompleted) {
+      console.log(`Got setup packet for frame ${decodeUnitStart.frameNumber()} but the last frame has not been completed`);
+    }
+
+    this.frameNumber = decodeUnitStart.frameNumber();
+    this.frameType = "delta";
+    if (decodeUnitStart.frameType() == VideoUpdate.FrameType.IDR) {
+      this.frameType = "key";
+    }
+    this.fullLength = decodeUnitStart.fullLength();
+    this.receiveTimeMs = decodeUnitStart.receiveTimeMs();
+
+    //this.frameBroken = false;
+    //this.lastBufferIndex = -1n;
+    //this.dataOffset = 0;
+    this.frameCollector = new Array(Number(decodeUnitStart.numBuffers())).fill(false);
+
+    this.data = new Uint8Array(Number(this.fullLength));
+
+
+    //this.print();
+    //console.log(`start: `, this);
+    //console.log(performance.now());
+  }
+
+  processBuffer(decodeUnitBuffer: VideoUpdate.DecodeUnitBuffer) {
+    //console.log(`buffer: `, this);
+    //console.log(performance.now());
+    //this.print();
+
+
+    const frameNumber = decodeUnitBuffer.frameNumber();
+    if (this.frameNumber === undefined) {
+      console.log("frameNumber is undefined but we got a buffer, ignoring...");
+      return;
+    }
+    if (this.frameNumber != frameNumber) {
+      console.log(`Got buffer for frame ${frameNumber} but we are processing frame ${this.frameNumber}, ignoring...`);
+      return;
+    }
+
+    let offset = decodeUnitBuffer.bufferOffset();
+
+    for (var i = 0; i < decodeUnitBuffer.dataLength(); i++) {
+      this.data[Number(offset) + i] = decodeUnitBuffer.data(i)!;
+    }
+
+    this.frameCollector[Number(decodeUnitBuffer.bufferIndex())] = true;
+
+
+    const gotAllframes = !this.frameCollector.includes(false);
+
+    if (gotAllframes) {
+      const chunk = new EncodedVideoChunk({
+        //timestamp: this.receiveTimeMs,
+        timestamp: 0,
+        type: this.frameType,
+        data: this.data,
+      });
+
+      //console.log(`${ performance.now() }: Enqueing a new decode request, current queue size ${ this.videoDecoder.decodeQueueSize } `);
+      this.videoDecoder.decode(chunk);
+    }
+    //this.print();
+  }
+}
+
+export async function streamVideoFromReader(reader: ReadableStreamDefaultReader, canvasElement: OffscreenCanvas) {
+  const videoDecoder = getVideoDecoder(canvasElement);
+  try {
+    let decodeUnitBuffer: VideoUpdate.DecodeUnitBuffer = new VideoUpdate.DecodeUnitBuffer();
+    let decoder = new Decoder(videoDecoder);
+
+    while (true) {
+      const { value, done } = await reader.read();
+      if (done) break;
+
+      const dataToParse = new ByteBuffer(value);
+      const videoUpdate = VideoUpdate.VideoUpdate.getRootAsVideoUpdate(dataToParse);
+
+      if (videoUpdate.updateType() == VideoUpdate.Update.Setup) {
+        let setup = videoUpdate.update(new VideoUpdate.Setup());
+        await configureDecoder(videoDecoder, setup.videoFormat(), setup.width(), setup.height());
+
+      } else if (videoUpdate.updateType() == VideoUpdate.Update.DecodeUnitStart) {
+        let decodeUnitStart: VideoUpdate.DecodeUnitStart = new VideoUpdate.DecodeUnitStart();
+        videoUpdate.update(decodeUnitStart);
+        decoder.processStart(decodeUnitStart);
+
+      } else if (videoUpdate.updateType() == VideoUpdate.Update.DecodeUnitBuffer) {
+        videoUpdate.update(decodeUnitBuffer);
+        decoder.processBuffer(decodeUnitBuffer);
+
+      } else {
+        throw new Error(`Got packet of unknown type`);
+      }
+    }
+
+  } catch (e) {
+    var error = <Error>e;
+    console.error('Error connecting to stream:', error);
+    alert('Failed to connect to stream: ' + error.message);
+  }
+}
+
+

frontend/src/routes/+layout.svelte

@@ -0,0 +1,57 @@
+<script lang="ts">
+	//import Header from './Header.svelte';
+	import '../app.css';
+
+	let { children } = $props();
+</script>
+
+<div class="app">
+	<!--<Header />-->
+
+	<main>
+		{@render children()}
+	</main>
+
+	<!--<footer>
+		<p>
+			visit <a href="https://svelte.dev/docs/kit">svelte.dev/docs/kit</a> to learn about SvelteKit
+		</p>
+	</footer>-->
+</div>
+
+<style>
+	.app {
+		display: flex;
+		flex-direction: column;
+		min-height: 100vh;
+	}
+
+	main {
+		/*flex: 1;*/
+		/*display: flex;*/
+		/*flex-direction: column;*/
+		/*padding: 1rem;*/
+		width: 100%;
+		/*max-width: 64rem;*/
+		margin: 0 auto;
+		box-sizing: border-box;
+	}
+
+	footer {
+		display: flex;
+		flex-direction: column;
+		justify-content: center;
+		align-items: center;
+		padding: 12px;
+	}
+
+	footer a {
+		font-weight: bold;
+	}
+
+	@media (min-width: 480px) {
+		footer {
+			padding: 12px 0;
+		}
+	}
+</style>

frontend/src/routes/+layout.ts

@@ -0,0 +1 @@
+export const ssr = false;

frontend/src/routes/+page.svelte

@@ -0,0 +1,24 @@
+<script lang="ts">
+	import Covers from './Covers.svelte';
+	import welcome from '$lib/images/svelte-welcome.webp';
+	import welcomeFallback from '$lib/images/svelte-welcome.png';
+</script>
+
+<svelte:head>
+	<title>Home</title>
+	<meta name="description" content="Svelte demo app" />
+</svelte:head>
+
+<section>
+	<Covers />
+</section>
+
+<style>
+	section {
+		display: flex;
+		flex-direction: column;
+		justify-content: center;
+		align-items: center;
+		flex: 0.6;
+	}
+</style>

frontend/src/routes/+page.ts

@@ -0,0 +1,3 @@
+// since there's no dynamic data here, we can prerender
+// it so that it gets served as a static asset in production
+//export const prerender = true;

frontend/src/routes/Cover.svelte

@@ -0,0 +1,179 @@
+<script lang="ts">
+	import type { HTMLButtonAttributes } from 'svelte/elements';
+	import { getStreamData } from './getStreamData';
+	import { type App } from './apps';
+	import { Circle } from 'svelte-loading-spinners';
+	import { goto } from '$app/navigation';
+	import { streamStore } from './stores/streamStore.svelte';
+
+	interface Props {
+		app: App;
+		server_name: string;
+		tab_index: number;
+	}
+
+	let { app, server_name, tab_index }: Props = $props();
+
+	let loading = $state(false);
+
+	async function streamApp() {
+		loading = true;
+
+		console.log(`Getting stream data for ${app.title} on ${server_name}`);
+		let streamData = await getStreamData(app.id, server_name);
+		streamStore.Url = streamData.Url;
+		streamStore.CertHash = streamData.CertHash;
+		streamStore.Width = streamData.Width;
+		streamStore.Height = streamData.Height;
+
+		console.log(`Stream data retrieved. Navigating to /stream.`);
+		await goto('/stream');
+	}
+</script>
+
+{#if loading}
+	<div class="app-box">
+		<div class="app-artwork">
+			<div class="app-loading-box">
+				<Circle size="60" color="#FF3E00" unit="px" duration="1s" />
+			</div>
+			{#if app.active}
+				<div class="running-pill">Running</div>
+			{/if}
+		</div>
+		<div class="app-title">{app.title}</div>
+		<div class="app-server">{server_name}</div>
+	</div>
+{:else}
+	<div class="app-box" onclick={streamApp} onkeydown={streamApp} role="button" tabindex={tab_index}>
+		<div class="app-artwork">
+			{#if app.active}
+				<div class="app-control-box">
+					<div class="resume-button"></div>
+					<div class="stop-button"></div>
+				</div>
+				<div class="running-pill">Running</div>
+			{:else}
+				<div class="app-control-box">
+					<div class="resume-button"></div>
+				</div>
+			{/if}
+		</div>
+		<div class="app-title">{app.title}</div>
+		<div class="app-server">{server_name}</div>
+	</div>
+{/if}
+
+<style>
+	.app-box {
+		position: relative;
+		width: 200px;
+		height: 280px;
+		cursor: pointer;
+		transition: all 0.3s ease;
+	}
+
+	.app-box:hover {
+		transform: translateY(-5px);
+		box-shadow: 0 10px 20px rgba(255, 255, 255, 0.1);
+	}
+
+	.app-artwork {
+		width: 200px;
+		height: 280px;
+		background-color: #333;
+		border: 2px solid #555;
+		border-radius: 8px;
+		position: relative;
+		overflow: hidden;
+		transition: border-color 0.3s ease;
+	}
+
+	.app-box:hover .app-artwork {
+		border-color: #00aaff;
+	}
+
+	.app-loading-box,
+	.app-control-box {
+		width: 200px;
+		height: 60px;
+		position: relative;
+		top: 50%;
+		transform: translate(0%, -50%);
+		opacity: 0;
+
+		display: flex;
+		justify-content: space-evenly;
+	}
+
+	.app-loading-box {
+		opacity: 1;
+	}
+
+	.app-box:hover .app-control-box {
+		opacity: 1;
+	}
+
+	.stop-button,
+	.resume-button {
+		width: 60px;
+		height: 60px;
+		border-radius: 50%;
+
+		background-color: rgba(0, 170, 255, 0.9);
+		display: flex;
+		align-items: center;
+		justify-content: center;
+		transition: opacity 0.3s ease;
+		pointer-events: none;
+	}
+
+	.resume-button::after {
+		content: '';
+		width: 0;
+		height: 0;
+		border-left: 20px solid white;
+		border-top: 12px solid transparent;
+		border-bottom: 12px solid transparent;
+		margin-left: 4px;
+	}
+
+	.stop-button::after {
+		content: '';
+		width: 24px;
+		height: 24px;
+		background-color: white;
+	}
+
+	.running-pill {
+		position: absolute;
+		bottom: 12px;
+		left: 50%;
+		transform: translateX(-50%);
+		background-color: #22c55e;
+		color: white;
+		padding: 6px 16px;
+		border-radius: 20px;
+		font-size: 20px;
+		font-weight: 500;
+		text-align: center;
+		box-shadow: 0 2px 4px rgba(0, 0, 0, 0.2);
+	}
+
+	.app-title {
+		text-align: center;
+		margin: 10px 0 5px 0;
+		font-size: 16px;
+		font-weight: bold;
+		white-space: nowrap;
+		overflow: hidden;
+		text-overflow: ellipsis;
+	}
+
+	.app-server {
+		text-align: center;
+		font-size: 12px;
+		color: #aaa;
+		margin: 0;
+	}
+</style>

frontend/src/routes/Covers.svelte

@@ -0,0 +1,36 @@
+<script lang="ts">
+	import Cover from './Cover.svelte';
+	import { fetchApps } from './apps';
+	import type { App } from './apps';
+
+	interface AppsResponse {
+		apps: Record<string, App[]>;
+	}
+
+	let appsPromise: Promise<AppsResponse>;
+
+	appsPromise = fetchApps();
+</script>
+
+{#await appsPromise}
+	<p>Loading...</p>
+{:then resp}
+	<div class="apps-container">
+		{#each Object.entries(resp.apps) as [server_name, apps]}
+			{#each apps as app, tab_index}
+				<Cover {app} {server_name} {tab_index}></Cover>
+			{/each}
+		{/each}
+	</div>
+{:catch error}
+	<p>Error: {error.message}</p>
+{/await}
+
+<style>
+	.apps-container {
+		display: flex;
+		flex-wrap: wrap;
+		gap: 20px;
+		padding: 20px 0;
+	}
+</style>

frontend/src/routes/Header.svelte

@@ -0,0 +1,129 @@
+<script lang="ts">
+	import { page } from '$app/state';
+	import logo from '$lib/images/svelte-logo.svg';
+	import github from '$lib/images/github.svg';
+</script>
+
+<header>
+	<div class="corner">
+		<a href="https://svelte.dev/docs/kit">
+			<img src={logo} alt="SvelteKit" />
+		</a>
+	</div>
+
+	<nav>
+		<svg viewBox="0 0 2 3" aria-hidden="true">
+			<path d="M0,0 L1,2 C1.5,3 1.5,3 2,3 L2,0 Z" />
+		</svg>
+		<ul>
+			<li aria-current={page.url.pathname === '/' ? 'page' : undefined}>
+				<a href="/">Home</a>
+			</li>
+			<li aria-current={page.url.pathname === '/about' ? 'page' : undefined}>
+				<a href="/about">About</a>
+			</li>
+			<li aria-current={page.url.pathname.startsWith('/sverdle') ? 'page' : undefined}>
+				<a href="/sverdle">Sverdle</a>
+			</li>
+		</ul>
+		<svg viewBox="0 0 2 3" aria-hidden="true">
+			<path d="M0,0 L0,3 C0.5,3 0.5,3 1,2 L2,0 Z" />
+		</svg>
+	</nav>
+
+	<div class="corner">
+		<a href="https://github.com/sveltejs/kit">
+			<img src={github} alt="GitHub" />
+		</a>
+	</div>
+</header>
+
+<style>
+	header {
+		display: flex;
+		justify-content: space-between;
+	}
+
+	.corner {
+		width: 3em;
+		height: 3em;
+	}
+
+	.corner a {
+		display: flex;
+		align-items: center;
+		justify-content: center;
+		width: 100%;
+		height: 100%;
+	}
+
+	.corner img {
+		width: 2em;
+		height: 2em;
+		object-fit: contain;
+	}
+
+	nav {
+		display: flex;
+		justify-content: center;
+		--background: rgba(255, 255, 255, 0.7);
+	}
+
+	svg {
+		width: 2em;
+		height: 3em;
+		display: block;
+	}
+
+	path {
+		fill: var(--background);
+	}
+
+	ul {
+		position: relative;
+		padding: 0;
+		margin: 0;
+		height: 3em;
+		display: flex;
+		justify-content: center;
+		align-items: center;
+		list-style: none;
+		background: var(--background);
+		background-size: contain;
+	}
+
+	li {
+		position: relative;
+		height: 100%;
+	}
+
+	li[aria-current='page']::before {
+		--size: 6px;
+		content: '';
+		width: 0;
+		height: 0;
+		position: absolute;
+		top: 0;
+		left: calc(50% - var(--size));
+		border: var(--size) solid transparent;
+		border-top: var(--size) solid var(--color-theme-1);
+	}
+
+	nav a {
+		display: flex;
+		height: 100%;
+		align-items: center;
+		padding: 0 0.5rem;
+		color: var(--color-text);
+		font-weight: 700;
+		font-size: 0.8rem;
+		text-transform: uppercase;
+		letter-spacing: 0.1em;
+		text-decoration: none;
+		transition: color 0.2s linear;
+	}
+
+	a:hover {
+		color: var(--color-theme-1);
+	}
+</style>

frontend/src/routes/about/+page.svelte

@@ -0,0 +1,26 @@
+<svelte:head>
+	<title>About</title>
+	<meta name="description" content="About this app" />
+</svelte:head>
+
+<div class="text-column">
+	<h1>About this app</h1>
+
+	<p>
+		This is a <a href="https://svelte.dev/docs/kit">SvelteKit</a> app. You can make your own by typing
+		the following into your command line and following the prompts:
+	</p>
+
+	<pre>npx sv create</pre>
+
+	<p>
+		The page you're looking at is purely static HTML, with no client-side interactivity needed.
+		Because of that, we don't need to load any JavaScript. Try viewing the page's source, or opening
+		the devtools network panel and reloading.
+	</p>
+
+	<p>
+		The <a href="/sverdle">Sverdle</a> page illustrates SvelteKit's data loading and form handling. Try
+		using it with JavaScript disabled!
+	</p>
+</div>

frontend/src/routes/about/+page.ts

@@ -0,0 +1,9 @@
+import { dev } from '$app/environment';
+
+// we don't need any JS on this page, though we'll load
+// it in dev so that we get hot module replacement
+export const csr = dev;
+
+// since there's no dynamic data here, we can prerender
+// it so that it gets served as a static asset in production
+export const prerender = true;

frontend/src/routes/apps.ts

@@ -0,0 +1,20 @@
+export interface App {
+  title: string;
+  id: number;
+  hdr_supported: boolean;
+  active: boolean;
+}
+
+export interface AppsResponse {
+  apps: Record<string, App[]>;
+}
+
+
+export async function fetchApps() {
+  console.log('Getting apps');
+  const response = await fetch('/api/apps');
+  console.log(response);
+  const data = (await response.json()) as AppsResponse;
+  console.log(data);
+  return data;
+}

frontend/src/routes/getStreamData.ts

@@ -0,0 +1,59 @@
+type StreamData = {
+  Url: string,
+  CertHash: Array<number>,
+  Width: number,
+  Height: number,
+}
+
+export async function getStreamData(appId: number, server_name: string): Promise<StreamData> {
+  try {
+    // Create the POST request payload
+    const width = 1920;
+    const height = 1080;
+
+    const payload = {
+      id: appId,
+      server: server_name,
+      server_mode: {
+        fps: 60,
+        width: width,
+        height: height,
+      },
+      stream_config: {
+        bitrate_kbps: 1024 * 10 * 5,
+        mode: {
+          fps: 60,
+          width: width,
+          height: height,
+        }
+      }
+    };
+
+    // Make POST request to start stream
+    const response = await fetch('/api/stream/start', {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+      },
+      body: JSON.stringify(payload)
+    });
+
+    if (!response.ok) {
+      throw new Error(`HTTP error! status: ${response.status}: ${await response.text()}`);
+    }
+
+    const streamDataResp = await response.json();
+    console.log('Stream started:', streamDataResp);
+
+    let streamData: StreamData = { Url: streamDataResp.url, CertHash: streamDataResp.cert_hash, Width: width, Height: height };
+    return streamData;
+
+
+  } catch (error) {
+    console.error('Error getting stream data: ', error);
+    throw new Error('Failed to start stream: ' + error);
+  }
+}
+
+
+

frontend/src/routes/stores/streamStore.svelte.ts

@@ -0,0 +1,6 @@
+export const streamStore = $state({
+  Url: '',
+  CertHash: [0],
+  Width: 0,
+  Height: 0,
+});

frontend/src/routes/stream/+page.svelte

@@ -0,0 +1,32 @@
+<script lang="ts">
+	import Stream from './Stream.svelte';
+	import { streamStore } from '../stores/streamStore.svelte';
+
+	$: url = streamStore.Url;
+	$: certHash = streamStore.CertHash;
+	$: width = streamStore.Width;
+	$: height = streamStore.Height;
+</script>
+
+<svelte:head>
+	<title>Stream</title>
+	<meta name="description" content="Streaming game" />
+</svelte:head>
+
+<!--<section>
+</section>-->
+<Stream {url} {certHash} {width} {height} />
+
+<style>
+	section {
+		min-height: 100vh;
+		min-width: 100vw;
+	}
+	/*section {
+		display: flex;
+		flex-direction: column;
+		justify-content: center;
+		align-items: center;
+		flex: 0.6;
+	}*/
+</style>

frontend/src/routes/stream/Stream.svelte

@@ -0,0 +1,81 @@
+<script lang="ts">
+	import { onMount } from 'svelte';
+	import { startWebtransportStream } from './stream';
+	import StreamUi from './StreamUi.svelte';
+
+	interface Props {
+		url: string;
+		certHash: Array<number>;
+		width: number;
+		height: number;
+	}
+
+	let { url, certHash, width, height }: Props = $props();
+	let loading = $state(true);
+	let fullscreen = $state(false);
+	let gameplayView: HTMLDivElement;
+	let gameplayCanvas: HTMLCanvasElement;
+
+	async function startStream() {
+		await startWebtransportStream(
+			url,
+			certHash,
+			width,
+			height,
+			gameplayCanvas,
+			gameplayCanvas,
+			gameplayCanvas
+		);
+	}
+
+	async function requestFullscreen() {
+		// Update fullscreen var if fullscreen was exited outside our control
+		if (document.fullscreenElement == null) {
+			fullscreen = false;
+		}
+
+		if (fullscreen) {
+			await document.exitFullscreen();
+			fullscreen = false;
+		} else {
+			await gameplayCanvas.requestFullscreen();
+			fullscreen = true;
+		}
+	}
+
+	onMount(async () => {
+		await startStream();
+	});
+</script>
+
+<div id="gameplay-view" class="gameplay-view" bind:this={gameplayView}>
+	<canvas id="gamestream-canvas" class="gamestream-canvas" bind:this={gameplayCanvas} tabindex="0"
+	></canvas>
+	<StreamUi fullscreenFunc={requestFullscreen}></StreamUi>
+</div>
+
+<style>
+	.gameplay-view {
+		height: 100vh;
+		aspect-ratio: 16 / 9;
+
+		margin-left: auto;
+		margin-right: auto;
+		background-color: black;
+
+		position: relative;
+
+		display: flex;
+		flex-direction: column;
+	}
+
+	.gamestream-canvas {
+		height: 100%;
+		aspect-ratio: 16 / 9;
+
+		position: relative;
+		margin: auto;
+
+		/*z-index: 1;*/
+	}
+</style>

frontend/src/routes/stream/StreamUi.svelte

@@ -0,0 +1,57 @@
+<script lang="ts">
+	import { Maximize, Settings } from 'lucide-svelte';
+
+	interface Props {
+		fullscreenFunc: () => void;
+	}
+
+	let { fullscreenFunc }: Props = $props();
+</script>
+
+<div class="stream-controls">
+	<div style="flex-grow: 1"></div>
+	<div class="bar-button" role="button">
+		<Settings size="100%" />
+	</div>
+	<div style="width: 0.5%"></div>
+	<div class="bar-button" onclick={fullscreenFunc} role="button">
+		<Maximize size="100%" />
+	</div>
+	<div style="width: 0.5%"></div>
+</div>
+
+<style>
+	.bar-button {
+		height: 85%;
+		aspect-ratio: 1 / 1;
+		color: oklch(0.454 0 360);
+
+		transition:
+			transform 0.3s ease,
+			filter 0.3s ease;
+		/* Set transform origin to center so it grows from the middle */
+		transform-origin: center;
+	}
+	.bar-button:hover {
+		transform: scale(1.05);
+		filter: brightness(1.2);
+	}
+
+	.stream-controls {
+		height: 5%;
+		width: 100%;
+
+		/*position: absolute;*/
+
+		bottom: 0;
+		left: 0;
+
+		/*z-index: 2;*/
+
+		border-top-color: oklch(0.454 0 360);
+		border-top-width: 2px;
+
+		display: flex;
+		align-items: center;
+	}
+</style>

frontend/src/routes/stream/stream.ts

@@ -0,0 +1,99 @@
+import { sendKeyboardEvent, sendMouseInputEvent, sendMouseMoveEvent, KeyAction } from "$lib/input"
+import CanvasWorker from "$lib/canvas.worker?worker";
+
+export async function getStreamTransport(url: string, certHash: Array<number>): Promise<WebTransport> {
+  let certHashArray = new Uint8Array(certHash);
+
+  if (!window.WebTransport) {
+    throw new Error('WebTransport is not supported in this browser');
+  }
+
+  const transport = new WebTransport(url, {
+    //congestionControl: "low-latency",
+    serverCertificateHashes: [
+      {
+        algorithm: "sha-256",
+        value: certHashArray,
+      }
+    ]
+  });
+
+  console.log('Connecting to WebTransport at ', url);
+  await transport.ready;
+  console.log(`WebTransport connection established`);
+
+  return transport;
+}
+
+
+export async function spawnWorker(gameplayCanvas: HTMLCanvasElement, reader: ReadableStream) {
+  const offscreenCanvas = gameplayCanvas.transferControlToOffscreen();
+
+  const worker = new CanvasWorker();
+
+  worker.postMessage({
+    canvas: offscreenCanvas,
+    reader: reader,
+  }, [offscreenCanvas, reader]);
+
+}
+
+
+export async function startWebtransportStream(
+  url: string,
+  certHash: Array<number>,
+  width: number,
+  height: number,
+  gameplayCanvas: HTMLCanvasElement,
+  keyEventElement: HTMLElement,
+  mouseElement: HTMLElement,
+) {
+  console.log(width, height);
+  gameplayCanvas.width = width;
+  gameplayCanvas.height = height;
+
+
+  console.log(`Connecting to stream at ${url} with cert_hash ${certHash}`);
+  const transport = await getStreamTransport(url, certHash);
+
+  const datagrams = transport.datagrams;
+  datagrams.incomingHighWaterMark = 20000;
+  const stream = await transport.createBidirectionalStream();
+
+  //const reader = stream.readable
+  const datagramReader = datagrams.readable
+  const writer = stream.writable.getWriter();
+
+  spawnWorker(gameplayCanvas, datagramReader);
+
+  keyEventElement.addEventListener("keydown", (event: KeyboardEvent) => { sendKeyboardEvent(writer, event, KeyAction.DOWN) });
+  keyEventElement.addEventListener("keyup", (event: KeyboardEvent) => { sendKeyboardEvent(writer, event, KeyAction.UP) });
+
+  mouseElement.addEventListener("mousedown", (event: MouseEvent) => { sendMouseInputEvent(writer, event, KeyAction.DOWN) });
+  mouseElement.addEventListener("mouseup", (event: MouseEvent) => { sendMouseInputEvent(writer, event, KeyAction.UP) });
+
+
+  mouseElement.addEventListener("click", async () => {
+    console.log("Requesting pointer lock");
+    await mouseElement.requestPointerLock({
+      //unadjustedMovement: true,
+    });
+    console.log("Pointer lock aquired");
+  })
+
+  mouseElement.addEventListener("mousemove", (event: MouseEvent) => {
+    if (document.pointerLockElement === mouseElement) {
+      sendMouseMoveEvent(writer, event);
+    }
+  });
+
+
+  // Handle connection close
+  transport.closed
+    .then(() => {
+      console.log('WebTransport connection closed');
+    })
+    .catch((error) => {
+      console.error('WebTransport connection closed with error:', error);
+    });
+}

frontend/static/favicon.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="107" height="128" viewBox="0 0 107 128"><title>svelte-logo</title><path d="M94.157 22.819c-10.4-14.885-30.94-19.297-45.792-9.835L22.282 29.608A29.92 29.92 0 0 0 8.764 49.65a31.5 31.5 0 0 0 3.108 20.231 30 30 0 0 0-4.477 11.183 31.9 31.9 0 0 0 5.448 24.116c10.402 14.887 30.942 19.297 45.791 9.835l26.083-16.624A29.92 29.92 0 0 0 98.235 78.35a31.53 31.53 0 0 0-3.105-20.232 30 30 0 0 0 4.474-11.182 31.88 31.88 0 0 0-5.447-24.116" style="fill:#ff3e00"/><path d="M45.817 106.582a20.72 20.72 0 0 1-22.237-8.243 19.17 19.17 0 0 1-3.277-14.503 18 18 0 0 1 .624-2.435l.49-1.498 1.337.981a33.6 33.6 0 0 0 10.203 5.098l.97.294-.09.968a5.85 5.85 0 0 0 1.052 3.878 6.24 6.24 0 0 0 6.695 2.485 5.8 5.8 0 0 0 1.603-.704L69.27 76.28a5.43 5.43 0 0 0 2.45-3.631 5.8 5.8 0 0 0-.987-4.371 6.24 6.24 0 0 0-6.698-2.487 5.7 5.7 0 0 0-1.6.704l-9.953 6.345a19 19 0 0 1-5.296 2.326 20.72 20.72 0 0 1-22.237-8.243 19.17 19.17 0 0 1-3.277-14.502 17.99 17.99 0 0 1 8.13-12.052l26.081-16.623a19 19 0 0 1 5.3-2.329 20.72 20.72 0 0 1 22.237 8.243 19.17 19.17 0 0 1 3.277 14.503 18 18 0 0 1-.624 2.435l-.49 1.498-1.337-.98a33.6 33.6 0 0 0-10.203-5.1l-.97-.294.09-.968a5.86 5.86 0 0 0-1.052-3.878 6.24 6.24 0 0 0-6.696-2.485 5.8 5.8 0 0 0-1.602.704L37.73 51.72a5.42 5.42 0 0 0-2.449 3.63 5.79 5.79 0 0 0 .986 4.372 6.24 6.24 0 0 0 6.698 2.486 5.8 5.8 0 0 0 1.602-.704l9.952-6.342a19 19 0 0 1 5.295-2.328 20.72 20.72 0 0 1 22.237 8.242 19.17 19.17 0 0 1 3.277 14.503 18 18 0 0 1-8.13 12.053l-26.081 16.622a19 19 0 0 1-5.3 2.328" style="fill:#fff"/></svg>

frontend/static/robots.txt

@@ -0,0 +1,3 @@
+# https://www.robotstxt.org/robotstxt.html
+User-agent: *
+Disallow:

frontend/svelte.config.js

@@ -0,0 +1,12 @@
+import adapter from '@sveltejs/adapter-static';
+import { vitePreprocess } from '@sveltejs/vite-plugin-svelte';
+
+/** @type {import('@sveltejs/kit').Config} */
+const config = {
+	// Consult https://svelte.dev/docs/kit/integrations
+	// for more information about preprocessors
+	preprocess: vitePreprocess(),
+	kit: { adapter: adapter() }
+};
+
+export default config;

frontend/tsconfig.json

@@ -0,0 +1,19 @@
+{
+	"extends": "./.svelte-kit/tsconfig.json",
+	"compilerOptions": {
+		"allowJs": true,
+		"checkJs": true,
+		"esModuleInterop": true,
+		"forceConsistentCasingInFileNames": true,
+		"resolveJsonModule": true,
+		"skipLibCheck": true,
+		"sourceMap": true,
+		"strict": true,
+		"moduleResolution": "bundler"
+	}
+	// Path aliases are handled by https://svelte.dev/docs/kit/configuration#alias
+	// except $lib which is handled by https://svelte.dev/docs/kit/configuration#files
+	//
+	// If you want to overwrite includes/excludes, make sure to copy over the relevant includes/excludes
+	// from the referenced tsconfig.json - TypeScript does not merge them in
+}

frontend/vite.config.ts

@@ -0,0 +1,7 @@
+import tailwindcss from '@tailwindcss/vite';
+import { sveltekit } from '@sveltejs/kit/vite';
+import { defineConfig } from 'vite';
+
+export default defineConfig({
+	plugins: [tailwindcss(), sveltekit()]
+});

gamestream-webtransport-proxy/Cargo.toml

@@ -6,9 +6,13 @@ edition = "2024"
 [dependencies]
 anyhow = "1.0.98"
 directories = "6.0.0"
+flatbuffers = "25.2.10"
 getrandom = { version = "0.3.3", features = ["std"] }
+h3-datagram = "0.0.2"
+h3-quinn = "0.0.10"
 hex = "0.4.3"
 hmac-sha256 = "1.1.12"
+http = "1.3.1"
 libc = "0.2.174"
 moonlight-common-c-sys = { path = "../moonlight-common-c-sys" }
 openssl = "0.10.73"

gamestream-webtransport-proxy/src/apps.rs

@@ -4,7 +4,12 @@ use salvo::prelude::*;
 use serde::{Deserialize, Serialize};
 use tracing::{debug, error};
 
-use crate::{common::AppResult, state::StateReader};
+use crate::{
+    common,
+    common::{AppError, AppResult},
+    responses,
+    state::StateReader,
+};
 
 #[derive(Deserialize)]
 struct AppListRespApp {
@@ -29,6 +34,7 @@ struct App {
     title: String,
     id: u64,
     hdr_supported: bool,
+    active: bool,
 }
 
 #[derive(Debug, Serialize, ToSchema)]
@@ -40,7 +46,7 @@ struct GetAppsResponse {
 impl crate::backend::Backend {
     #[craft(endpoint(status_codes(StatusCode::OK, StatusCode::INTERNAL_SERVER_ERROR)))]
     pub async fn get_apps(self: ::std::sync::Arc<Self>) -> AppResult<Json<GetAppsResponse>> {
-        let standard_error = Err(crate::common::AppError {
+        let standard_error = Err(AppError {
             status_code: StatusCode::INTERNAL_SERVER_ERROR,
             description: "failed to get available apps".to_string(),
         });
@@ -67,7 +73,33 @@ impl crate::backend::Backend {
         };
 
         for (_, server) in servers.into_iter() {
-            let mut base_url = crate::common::base_url(
+            let mut server_info_base_url = common::base_url(
+                "https",
+                &server.host,
+                server.https_port(),
+                &unique_id,
+                "serverinfo",
+                None,
+            );
+
+            let server_info = match common::get_url(&mut server_info_base_url, true).await {
+                Ok(s) => s,
+                Err(e) => {
+                    error!("Could not get server_info: {}", e);
+                    return standard_error;
+                }
+            };
+            debug!("server_info: {server_info}");
+            let server_info: responses::ServerInfoResponse =
+                match serde_xml_rs::from_str(&server_info) {
+                    Ok(s) => s,
+                    Err(e) => {
+                        error!("Could not parse serverinfo response: {e}");
+                        return standard_error;
+                    }
+                };
+
+            let mut base_url = common::base_url(
                 "https",
                 &server.host,
                 server.https_port(),
@@ -76,7 +108,7 @@ impl crate::backend::Backend {
                 None,
             );
 
-            let resp = match crate::common::get_url(&mut base_url, true).await {
+            let resp = match common::get_url(&mut base_url, true).await {
                 Ok(r) => r,
                 Err(e) => {
                     error!("could not get applist from server {}: {}", server.name, e);
@@ -103,6 +135,7 @@ impl crate::backend::Backend {
                     title: a.app_title,
                     hdr_supported: a.is_hdr_supported,
                     id: a.id,
+                    active: a.id == server_info.currentgame,
                 })
                 .rev()
                 .collect();

gamestream-webtransport-proxy/src/certs.rs

@@ -120,15 +120,41 @@ fn generate_http_cert_and_key(
 
     cert_builder.set_version(2)?;
 
+    let serial = openssl::bn::BigNum::from_u32(1)?;
+    let asn_serial = openssl::asn1::Asn1Integer::from_bn(&serial)?;
+    cert_builder.set_serial_number(&asn_serial)?;
+
     // Set subject (Distinguished Name)
     let mut name_builder = X509NameBuilder::new()?;
-    name_builder.append_entry_by_text("CN", "mumble-web self-signed")?;
+    name_builder.append_entry_by_text("CN", "localhost")?;
     let subject_name = name_builder.build();
     cert_builder.set_subject_name(&subject_name)?;
-
-    // Set issuer (same as subject for self-signed)
     cert_builder.set_issuer_name(&subject_name)?;
 
+    let context = cert_builder.x509v3_context(None, None);
+
+    let mut san = openssl::x509::extension::SubjectAlternativeName::new();
+    san.dns("localhost");
+
+    let san_extension = san.build(&context)?;
+
+    let key_usage = openssl::x509::extension::KeyUsage::new()
+        .digital_signature()
+        .key_encipherment()
+        .build()?;
+
+    let ext_key_usage = openssl::x509::extension::ExtendedKeyUsage::new()
+        .server_auth()
+        .build()?;
+
+    // Add Subject Key Identifier
+    let subject_key_id = openssl::x509::extension::SubjectKeyIdentifier::new().build(&context)?;
+
+    cert_builder.append_extension(san_extension)?;
+    cert_builder.append_extension(key_usage)?;
+    cert_builder.append_extension(ext_key_usage)?;
+    cert_builder.append_extension(subject_key_id)?;
+
     cert_builder.set_not_before(&now)?;
     cert_builder.set_not_after(&expiration_time)?;
     cert_builder.set_pubkey(&key)?;

gamestream-webtransport-proxy/src/gamestream/config.rs

@@ -40,10 +40,10 @@ pub fn stream_config(stream: &crate::backend::Stream) -> _STREAM_CONFIGURATION {
         height: stream.stream_config.mode.height,
         fps: stream.stream_config.mode.fps,
         bitrate: stream.stream_config.bitrate_kbps,
-        packetSize: 512,
+        packetSize: 1024,
         streamingRemotely: STREAM_CFG_AUTO,
         audioConfiguration: (0x3 << 16) | (2 << 8) | 0xCA,
-        supportedVideoFormats: VIDEO_FORMAT_H265,
+        supportedVideoFormats: VIDEO_FORMAT_H264,
         clientRefreshRateX100: 0,
         colorSpace: COLORSPACE_REC_601,
         colorRange: COLOR_RANGE_LIMITED,

gamestream-webtransport-proxy/src/gamestream/decoder.rs

@@ -7,10 +7,12 @@ use moonlight_common_c_sys::{
     VIDEO_FORMAT_H265_REXT8_444, VIDEO_FRAME_HANDLE,
 };
 use salvo::{http::body::Frame, hyper::body::Buf};
+use serde::Serialize;
 use tokio::sync::mpsc;
 use tracing::{debug, error};
 
-enum FrameType {
+#[derive(Serialize)]
+pub enum FrameType {
     PFRAME,
     IDR,
 }
@@ -29,7 +31,8 @@ impl TryFrom<i32> for FrameType {
     }
 }
 
-enum VideoFormat {
+#[derive(Serialize)]
+pub enum VideoFormat {
     H264,
     H264_HIGH8_444,
     H265,
@@ -64,6 +67,7 @@ impl TryFrom<i32> for VideoFormat {
     }
 }
 
+#[derive(Serialize)]
 enum BufferType {
     PICDATA,
     SPS,
@@ -87,8 +91,9 @@ impl TryFrom<i32> for BufferType {
     }
 }
 
-struct Buffer {
+#[derive(Serialize)]
-    data: Vec<u8>,
+pub struct Buffer {
+    pub data: Vec<u8>,
     buffer_type: BufferType,
 }
 
@@ -105,6 +110,7 @@ impl TryFrom<_LENTRY> for Buffer {
     }
 }
 
+#[derive(Serialize)]
 pub enum RendererMessage {
     Setup {
         video_format: VideoFormat,
@@ -113,21 +119,27 @@ pub enum RendererMessage {
         redraw_rate: u64,
         dr_flags: i32,
     },
-    DecodeUnit {
+    DecodeUnitStart {
         frame_number: u64,
         frame_type: FrameType,
+        num_buffers: u64,
 
         host_processing_latency: u16,
-        receieve_time_ms: u64,
+        receive_time_ms: u64,
         enqueue_time_ms: u64,
         presentation_time: u64,
 
-        full_length: usize,
+        full_length: u64,
-        buffers: Vec<Buffer>,
 
         hdr_active: bool,
         colorspace: u8,
     },
+    DecodeUnitBuffer {
+        frame_number: u64,
+        buffer_index: u64,
+        buffer_offset: u64,
+        buffer: Buffer,
+    },
 }
 
 impl RendererMessage {
@@ -147,37 +159,60 @@ impl RendererMessage {
         })
     }
 
-    fn from_decode_unit(decode_unit: _DECODE_UNIT) -> Result<Self> {
+    fn from_decode_unit(decode_unit: _DECODE_UNIT) -> Result<Vec<Self>> {
-        let mut buffers = Vec::new();
+        let mut messages = Vec::new();
 
         if decode_unit.bufferList.is_null() {
             return Err(anyhow!("DecodeUnit bufferList is null"));
         }
+        let frame_number = <u64>::try_from(decode_unit.frameNumber)?;
+
+        messages.push(RendererMessage::DecodeUnitStart {
+            frame_number,
+            frame_type: FrameType::try_from(decode_unit.frameType)?,
+            num_buffers: 0,
+            host_processing_latency: decode_unit.frameHostProcessingLatency,
+            receive_time_ms: decode_unit.receiveTimeMs,
+            enqueue_time_ms: decode_unit.enqueueTimeMs,
+            presentation_time: decode_unit.presentationTimeMs as u64,
+            full_length: <u64>::try_from(decode_unit.fullLength)?,
+            hdr_active: decode_unit.hdrActive,
+            colorspace: decode_unit.colorspace,
+        });
+
         let mut next = unsafe { *decode_unit.bufferList };
 
+        let mut index = 0;
+        let mut offset = 0;
         loop {
+            let b = Buffer::try_from(next)?;
+            let buffer_len = b.data.len() as u64;
+
+            messages.push(RendererMessage::DecodeUnitBuffer {
+                frame_number,
+                buffer_index: index,
+                buffer_offset: offset,
+                buffer: b,
+            });
+
+            offset = offset + buffer_len;
+            index = index + 1;
             if next.next.is_null() {
                 break;
             }
-            let buffer = Buffer::try_from(next)?;
-
-            buffers.push(buffer);
 
             next = unsafe { *next.next };
         }
 
-        Ok(RendererMessage::DecodeUnit {
+        if let RendererMessage::DecodeUnitStart {
-            frame_number: <u64>::try_from(decode_unit.frameNumber)?,
+            ref mut num_buffers,
-            frame_type: FrameType::try_from(decode_unit.frameType)?,
+            ..
-            host_processing_latency: decode_unit.frameHostProcessingLatency,
+        } = messages[0]
-            receieve_time_ms: decode_unit.receiveTimeMs,
+        {
-            enqueue_time_ms: decode_unit.enqueueTimeMs,
+            *num_buffers = index;
-            presentation_time: decode_unit.presentationTimeMs as u64,
+        }
-            full_length: <usize>::try_from(decode_unit.fullLength)?,
+
-            buffers,
+        Ok(messages)
-            hdr_active: decode_unit.hdrActive,
-            colorspace: decode_unit.colorspace,
-        })
     }
 }
 
@@ -238,22 +273,34 @@ extern "C" fn start_cb() {
 }
 
 extern "C" fn submit_decode_unit_cb(decode_unit: PDECODE_UNIT) -> std::os::raw::c_int {
-    debug!("SUBMIT DECODE UNIT CB");
     if decode_unit.is_null() {
         error!("Decode unit pointer was null");
         return -1;
     }
     let decode_unit = unsafe { *decode_unit };
+    //debug!("decode unit bytes: {}", decode_unit.fullLength);
 
-    let msg = match RendererMessage::from_decode_unit(decode_unit) {
+    let messages = match RendererMessage::from_decode_unit(decode_unit) {
         Ok(m) => m,
         Err(e) => {
             error!("Cannot construct RendererMessage: {e}");
             return -1;
         }
     };
+    debug!(
+        "got decode unit with {} buffers: {:?}",
+        messages.len() - 1,
+        std::time::Instant::now()
+    );
 
-    send_message(msg)
+    for msg in messages {
+        let ret = send_message(msg);
+        if ret != 0 {
+            return ret;
+        }
+    }
+    //debug!("dispatched decode unit: {:?}", std::time::Instant::now());
+    0
 }
 
 pub fn decoder_callbacks() -> Result<(DECODER_RENDERER_CALLBACKS, mpsc::Receiver<RendererMessage>)>

gamestream-webtransport-proxy/src/gamestream/mod.rs

@@ -2,7 +2,7 @@ use anyhow::{Result, anyhow};
 use tokio::sync::mpsc;
 
 mod config;
-mod decoder;
+pub mod decoder;
 
 #[derive(Debug)]
 pub struct GamestreamChannels {
@@ -20,7 +20,7 @@ pub fn start_connection(
         address,
         stream.server_codec_mode_support,
     )?;
-    let mut stream_config = config::stream_config(&stream);
+    let mut stream_config = config::stream_config(stream);
     let mut listener_callbacks = config::listener_callbacks();
     let (mut decoder_callbacks, decoder_rx) = decoder::decoder_callbacks()?;
     let ret;
@@ -47,3 +47,31 @@ pub fn start_connection(
 pub fn stop_connection() {
     unsafe { moonlight_common_c_sys::LiStopConnection() };
 }
+
+pub fn send_keyboard_event(keycode: i16, keyaction: i8, modifiers: u8) -> Result<()> {
+    let ret =
+        unsafe { moonlight_common_c_sys::LiSendKeyboardEvent(keycode, keyaction, modifiers as i8) };
+
+    match ret {
+        0 => Ok(()),
+        _ => Err(anyhow!("Could not send keyboard event: {ret}")),
+    }
+}
+
+pub fn send_mouse_move_event(movement_x: i16, movement_y: i16) -> Result<()> {
+    let ret = unsafe { moonlight_common_c_sys::LiSendMouseMoveEvent(movement_x, movement_y) };
+
+    match ret {
+        0 => Ok(()),
+        _ => Err(anyhow!("Could not send mouse movement event: {ret}")),
+    }
+}
+
+pub fn send_mouse_input_event(action: i8, button: i32) -> Result<()> {
+    let ret = unsafe { moonlight_common_c_sys::LiSendMouseButtonEvent(action, button) };
+
+    match ret {
+        0 => Ok(()),
+        _ => Err(anyhow!("Could not send mouse input event: {ret}")),
+    }
+}

gamestream-webtransport-proxy/src/main.rs

@@ -9,6 +9,7 @@ mod common;
 mod gamestream;
 mod pair;
 mod proxy;
+mod responses;
 mod state;
 mod stream;
 

gamestream-webtransport-proxy/src/proxy/handler.rs

@@ -29,6 +29,12 @@ async fn setup_webtransport(
 ) -> Result<(
     impl tokio::io::AsyncWrite + Send + Sync + 'static,
     impl tokio::io::AsyncRead + Send + Sync + 'static,
+    h3_datagram::datagram_handler::DatagramSender<
+        <h3_quinn::Connection as h3_datagram::quic_traits::DatagramConnectionExt<
+            salvo::hyper::body::Bytes,
+        >>::SendDatagramHandler,
+        salvo::hyper::body::Bytes,
+    >,
     //salvo::webtransport::stream::SendStream<
     //    impl salvo::proto::quic::SendStream<salvo::hyper::body::Bytes>,
     //    salvo::hyper::body::Bytes,
@@ -39,13 +45,17 @@ async fn setup_webtransport(
     //>,
 )> {
     let session = req.web_transport_mut().await?;
+
+    let datagram_send = session.datagram_sender();
+
     let bidirectional_stream = session
         .accept_bi()
         .await?
         .ok_or(anyhow!("No bidirectional stream"))?;
 
     if let webtransport::server::AcceptedBi::BidiStream(_, stream) = bidirectional_stream {
-        Ok(stream.split())
+        let (stream_send, stream_recv) = stream.split();
+        Ok((stream_send, stream_recv, datagram_send))
     } else {
         Err(anyhow!("bidirectional stream was of the wrong type"))
     }
@@ -76,7 +86,8 @@ impl crate::proxy::Proxy {
         });
 
         info!("WebTransport connection initiated");
-        let (wt_send, wt_recv) = match setup_webtransport(req).await {
+        let (wt_stream_send, wt_stream_recv, wt_datagram_send) = match setup_webtransport(req).await
+        {
             Ok(w) => w,
             Err(e) => {
                 error!("Could not upgrade connection to WebTransport: {e}");
@@ -95,7 +106,7 @@ impl crate::proxy::Proxy {
             }
         };
 
-        match super::proxy_main(stream, wt_send, wt_recv).await {
+        match super::proxy_main(stream, wt_stream_send, wt_stream_recv, wt_datagram_send).await {
             Ok(()) => Ok(()),
             Err(e) => {
                 error!("Proxy main loop failed: {e}");

gamestream-webtransport-proxy/src/proxy/input/mod.rs

@@ -0,0 +1 @@
+pub mod input_generated;

gamestream-webtransport-proxy/src/proxy/mod.rs

@@ -1,11 +1,13 @@
 use anyhow::{Context, Result};
-use salvo::{conn::http1::Connection, hyper::body::Bytes, proto::WebTransportSession};
 use tokio::{io::AsyncReadExt, io::AsyncWriteExt, select, sync::RwLock};
 use tracing::{debug, error, info};
 
 use crate::{backend, gamestream};
 
 pub mod handler;
+mod input;
+mod packet_parser;
+mod video;
 
 pub struct Proxy {
     pub cert_hash: [u8; 32],
@@ -32,6 +34,12 @@ async fn proxy_main(
     stream: backend::Stream,
     mut wt_send: impl tokio::io::AsyncWrite + Send + Sync + 'static + std::marker::Unpin,
     mut wt_recv: impl tokio::io::AsyncRead + Send + Sync + 'static + std::marker::Unpin,
+    mut wt_datagram_send: h3_datagram::datagram_handler::DatagramSender<
+        <h3_quinn::Connection as h3_datagram::quic_traits::DatagramConnectionExt<
+            salvo::hyper::body::Bytes,
+        >>::SendDatagramHandler,
+        salvo::hyper::body::Bytes,
+    >,
 ) -> Result<()> {
     debug!(
         "Connecting to stream at address {} with stream config {:?}",
@@ -40,15 +48,15 @@ async fn proxy_main(
 
     let mut channels = spawn_gamestream(stream).await?;
 
-    let mut buffer = vec![0; 65536].into_boxed_slice();
+    let mut packet_buffer = packet_parser::PacketBuffer::new();
+    let mut buffer = [0u8; 65536];
 
     loop {
         select! {
             gamestream_packet = channels.gamestream_channels.decoder_rx.recv() => {
                 match gamestream_packet {
                     Some(frame) => {
-                        info!("Got decoder packet");
+                        video::send_video_update(&frame, &mut wt_datagram_send).await?;
-                        wt_send.write_all(&[0;32]).await?;
                     }
                     None => {
                         error!("Decoder channel is None");
@@ -56,11 +64,17 @@ async fn proxy_main(
                     }
                 }
             },
-            webtransport_packet = wt_recv.read(&mut buffer) => {
+            ret = wt_recv.read(&mut buffer) => {
-                info!("Got packet from client");
+                let bytes_read = ret?;
+
+                packet_parser::handle_client_packet(&mut packet_buffer, &buffer[..bytes_read])?;
+
             }
         }
     }
+
+    channels.stop_tx.send(());
+
     Ok(())
 }
 

gamestream-webtransport-proxy/src/proxy/packet_parser.rs

@@ -0,0 +1,158 @@
+use anyhow::{Result, anyhow};
+use tracing::{debug, error, info};
+
+use super::input::input_generated::input_event;
+use crate::gamestream;
+
+pub struct PacketBuffer {
+    buffer: Vec<u8>,
+    expected_length: Option<usize>,
+    length_bytes_needed: usize,
+}
+
+impl PacketBuffer {
+    pub fn new() -> Self {
+        Self {
+            buffer: Vec::new(),
+            expected_length: None,
+            length_bytes_needed: 4, // Assuming u32 length (4 bytes)
+        }
+    }
+
+    /// Process incoming data and return complete packets
+    pub fn process_data(&mut self, data: &[u8]) -> Vec<Vec<u8>> {
+        let mut complete_packets = Vec::new();
+        self.buffer.extend_from_slice(data);
+
+        loop {
+            // If we don't know the expected length yet, try to read it
+            if self.expected_length.is_none() {
+                if self.buffer.len() >= self.length_bytes_needed {
+                    // Read length from first 4 bytes (adjust as needed)
+                    let length = u32::from_be_bytes([
+                        self.buffer[0],
+                        self.buffer[1],
+                        self.buffer[2],
+                        self.buffer[3],
+                    ]) as usize;
+
+                    self.expected_length = Some(length);
+                } else {
+                    // Not enough data to read length yet
+                    break;
+                }
+            }
+
+            // We know the expected length, check if we have a complete packet
+            if let Some(expected_len) = self.expected_length {
+                let total_packet_size = self.length_bytes_needed + expected_len;
+
+                if self.buffer.len() >= total_packet_size {
+                    // We have a complete packet - extract just the data portion
+                    let packet_data =
+                        self.buffer[self.length_bytes_needed..total_packet_size].to_vec();
+                    complete_packets.push(packet_data);
+
+                    // Remove the processed packet from buffer
+                    self.buffer.drain(0..total_packet_size);
+                    self.expected_length = None;
+                } else {
+                    // Packet is incomplete
+                    break;
+                }
+            }
+        }
+
+        complete_packets
+    }
+}
+
+pub fn process_keyboard_event(input_event: input_event::InputEvent) -> Result<()> {
+    let Some(table) = input_event.input() else {
+        debug!("Keyboard event table was empty, ignoring");
+        return Err(anyhow!("Keyboard event table was empty"));
+    };
+    let keyboard_event = unsafe { input_event::KeyboardInput::init_from_table(table) };
+
+    let key_action_i8 = match keyboard_event.key_action() {
+        input_event::KeyAction::DOWN => moonlight_common_c_sys::KEY_ACTION_DOWN,
+        input_event::KeyAction::UP => moonlight_common_c_sys::KEY_ACTION_UP,
+        _ => {
+            debug!("Invalid KeyAction value, ignoring");
+            return Err(anyhow!("Invalid KeyAction value"));
+        }
+    };
+
+    gamestream::send_keyboard_event(keyboard_event.key_code(), key_action_i8, 0)
+}
+
+pub fn process_mouse_move_event(input_event: input_event::InputEvent) -> Result<()> {
+    let Some(table) = input_event.input() else {
+        debug!("MouseMovement event table was empty, ignoring");
+        return Err(anyhow!("MouseMovement event table was empty"));
+    };
+    let mouse_event = unsafe { input_event::MouseMovement::init_from_table(table) };
+
+    gamestream::send_mouse_move_event(mouse_event.movement_x(), mouse_event.movement_y())
+}
+
+pub fn process_mouse_input_event(input_event: input_event::InputEvent) -> Result<()> {
+    let Some(table) = input_event.input() else {
+        debug!("MouseInput event table was empty, ignoring");
+        return Err(anyhow!("MouseInput event table was empty"));
+    };
+    let mouse_event = unsafe { input_event::MouseInput::init_from_table(table) };
+
+    let button_action_i8 = match mouse_event.button_action() {
+        input_event::KeyAction::DOWN => moonlight_common_c_sys::BUTTON_ACTION_PRESS,
+        input_event::KeyAction::UP => moonlight_common_c_sys::BUTTON_ACTION_RELEASE,
+        _ => {
+            error!("Invalid KeyAction value, ignoring",);
+            return Err(anyhow!("Invalid KeyAction value"));
+        }
+    };
+
+    let mouse_button_i32 = match mouse_event.button() {
+        input_event::MouseButton::LEFT => moonlight_common_c_sys::BUTTON_LEFT,
+        input_event::MouseButton::MIDDLE => moonlight_common_c_sys::BUTTON_MIDDLE,
+        input_event::MouseButton::RIGHT => moonlight_common_c_sys::BUTTON_RIGHT,
+        input_event::MouseButton::X1 => moonlight_common_c_sys::BUTTON_X1,
+        input_event::MouseButton::X2 => moonlight_common_c_sys::BUTTON_X2,
+        _ => {
+            error!("Invalid MouseButton value, ignoring",);
+            return Err(anyhow!("Invalid MouseButton value"));
+        }
+    };
+
+    gamestream::send_mouse_input_event(button_action_i8, mouse_button_i32)
+}
+
+pub fn handle_client_packet(packet_buffer: &mut PacketBuffer, buffer: &[u8]) -> anyhow::Result<()> {
+    let complete_packets = packet_buffer.process_data(buffer);
+
+    // TODO: only supports input packets. this should use a union wrapper or something
+    for packet_data in complete_packets {
+        let input_event = input_event::root_as_input_event(&packet_data)?;
+        match input_event.input_type() {
+            input_event::Input::Keyboard => {
+                process_keyboard_event(input_event)?;
+            }
+            input_event::Input::MouseMovement => {
+                process_mouse_move_event(input_event)?;
+            }
+            input_event::Input::MouseInput => {
+                process_mouse_input_event(input_event)?;
+            }
+
+            input_event::Input::NONE => {
+                debug!("Input event was empty, ignoring.")
+            }
+            _ => {
+                error!("Unknown InputEvent type");
+                return Err(anyhow!("Unknown InputEvent type"));
+            }
+        }
+    }
+
+    Ok(())
+}

gamestream-webtransport-proxy/src/proxy/video/mod.rs

@@ -0,0 +1,185 @@
+use anyhow::Result;
+use tracing::debug;
+
+use crate::gamestream;
+use video_generated::video_update;
+
+mod video_generated;
+
+fn create_setup_videoupdate(
+    video_format: &gamestream::decoder::VideoFormat,
+    width: u64,
+    height: u64,
+    redraw_rate: u64,
+) -> Vec<u8> {
+    let mut builder = flatbuffers::FlatBufferBuilder::with_capacity(1024);
+
+    //TODO: this is hardcoded to h264 main profile
+    let video_format = Some(builder.create_string("avc1.4D401E"));
+
+    let setup = video_update::Setup::create(
+        &mut builder,
+        &video_update::SetupArgs {
+            video_format,
+            width: width as u16,
+            height: height as u16,
+            redraw_rate: redraw_rate as u16,
+        },
+    );
+
+    let video_update = video_update::VideoUpdate::create(
+        &mut builder,
+        &video_update::VideoUpdateArgs {
+            update_type: video_update::Update::Setup,
+            update: Some(setup.as_union_value()),
+        },
+    );
+
+    builder.finish(video_update, None);
+
+    builder.finished_data().to_vec()
+}
+
+fn create_decodeunitstart_videoupdate(
+    frame_number: u64,
+    frame_type: &gamestream::decoder::FrameType,
+    num_buffers: u64,
+    receive_time_ms: u64,
+    full_length: u64,
+) -> Vec<u8> {
+    let mut builder = flatbuffers::FlatBufferBuilder::with_capacity(1024);
+
+    let frame_type_fb = match frame_type {
+        gamestream::decoder::FrameType::IDR => video_update::FrameType::IDR,
+        gamestream::decoder::FrameType::PFRAME => video_update::FrameType::PFRAME,
+    };
+
+    let decode_unit_start = video_update::DecodeUnitStart::create(
+        &mut builder,
+        &video_update::DecodeUnitStartArgs {
+            frame_number,
+            frame_type: frame_type_fb,
+            num_buffers,
+            receive_time_ms: receive_time_ms as u16,
+            full_length,
+        },
+    );
+
+    let video_update = video_update::VideoUpdate::create(
+        &mut builder,
+        &video_update::VideoUpdateArgs {
+            update_type: video_update::Update::DecodeUnitStart,
+            update: Some(decode_unit_start.as_union_value()),
+        },
+    );
+
+    builder.finish(video_update, None);
+
+    builder.finished_data().to_vec()
+}
+
+fn create_decodeunitbuffer_videoupdate(
+    frame_number: u64,
+    buffer_index: u64,
+    buffer_offset: u64,
+    buffer: &gamestream::decoder::Buffer,
+) -> Vec<u8> {
+    let mut builder = flatbuffers::FlatBufferBuilder::with_capacity(1024);
+
+    let data_vector = builder.create_vector(&buffer.data);
+
+    let decode_unit_buffer = video_update::DecodeUnitBuffer::create(
+        &mut builder,
+        &video_update::DecodeUnitBufferArgs {
+            frame_number,
+            buffer_index,
+            buffer_offset,
+            data: Some(data_vector),
+        },
+    );
+
+    let video_update = video_update::VideoUpdate::create(
+        &mut builder,
+        &video_update::VideoUpdateArgs {
+            update_type: video_update::Update::DecodeUnitBuffer,
+            update: Some(decode_unit_buffer.as_union_value()),
+        },
+    );
+
+    builder.finish(video_update, None);
+
+    builder.finished_data().to_vec()
+}
+
+pub async fn send_video_update(
+    frame: &gamestream::decoder::RendererMessage,
+    wt_datagram_send: &mut h3_datagram::datagram_handler::DatagramSender<
+        <h3_quinn::Connection as h3_datagram::quic_traits::DatagramConnectionExt<
+            salvo::hyper::body::Bytes,
+        >>::SendDatagramHandler,
+        salvo::hyper::body::Bytes,
+    >,
+) -> Result<()> {
+    let mut print_time = false;
+    let buffer = match frame {
+        gamestream::decoder::RendererMessage::Setup {
+            video_format,
+            width,
+            height,
+            redraw_rate,
+            dr_flags,
+        } => create_setup_videoupdate(video_format, *width, *height, *redraw_rate),
+        gamestream::decoder::RendererMessage::DecodeUnitStart {
+            frame_number,
+            frame_type,
+            num_buffers,
+            host_processing_latency,
+            receive_time_ms,
+            enqueue_time_ms,
+            presentation_time,
+            full_length,
+            hdr_active,
+            colorspace,
+        } => {
+            //debug!(
+            //    "sending decodeunitstart {}: {:?}",
+            //    *frame_number,
+            //    std::time::Instant::now()
+            //);
+            create_decodeunitstart_videoupdate(
+                *frame_number,
+                frame_type,
+                *num_buffers,
+                *receive_time_ms,
+                *full_length,
+            )
+        }
+        gamestream::decoder::RendererMessage::DecodeUnitBuffer {
+            frame_number,
+            buffer_index,
+            buffer_offset,
+            buffer,
+        } => {
+            //debug!(
+            //    "sending decodeunitbuffer {}/{}: {:?}",
+            //    *frame_number,
+            //    *buffer_index,
+            //    std::time::Instant::now()
+            //);
+            create_decodeunitbuffer_videoupdate(
+                *frame_number,
+                *buffer_index,
+                *buffer_offset,
+                buffer,
+            )
+        }
+    };
+
+    let bytes = salvo::hyper::body::Bytes::copy_from_slice(&buffer);
+
+    wt_datagram_send.send_datagram(bytes)?;
+    debug!("sent start: {:?}", std::time::Instant::now());
+    //if (print_time) {
+    //}
+    Ok(())
+}

gamestream-webtransport-proxy/src/proxy/video/video_generated.rs

@@ -0,0 +1,925 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+
+// @generated
+
+use core::mem;
+use core::cmp::Ordering;
+
+extern crate flatbuffers;
+use self::flatbuffers::{EndianScalar, Follow};
+
+#[allow(unused_imports, dead_code)]
+pub mod video_update {
+
+  use core::mem;
+  use core::cmp::Ordering;
+
+  extern crate flatbuffers;
+  use self::flatbuffers::{EndianScalar, Follow};
+
+#[deprecated(since = "2.0.0", note = "Use associated constants instead. This will no longer be generated in 2021.")]
+pub const ENUM_MIN_FRAME_TYPE: i8 = 0;
+#[deprecated(since = "2.0.0", note = "Use associated constants instead. This will no longer be generated in 2021.")]
+pub const ENUM_MAX_FRAME_TYPE: i8 = 1;
+#[deprecated(since = "2.0.0", note = "Use associated constants instead. This will no longer be generated in 2021.")]
+#[allow(non_camel_case_types)]
+pub const ENUM_VALUES_FRAME_TYPE: [FrameType; 2] = [
+  FrameType::PFRAME,
+  FrameType::IDR,
+];
+
+#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Default)]
+#[repr(transparent)]
+pub struct FrameType(pub i8);
+#[allow(non_upper_case_globals)]
+impl FrameType {
+  pub const PFRAME: Self = Self(0);
+  pub const IDR: Self = Self(1);
+
+  pub const ENUM_MIN: i8 = 0;
+  pub const ENUM_MAX: i8 = 1;
+  pub const ENUM_VALUES: &'static [Self] = &[
+    Self::PFRAME,
+    Self::IDR,
+  ];
+  /// Returns the variant's name or "" if unknown.
+  pub fn variant_name(self) -> Option<&'static str> {
+    match self {
+      Self::PFRAME => Some("PFRAME"),
+      Self::IDR => Some("IDR"),
+      _ => None,
+    }
+  }
+}
+impl core::fmt::Debug for FrameType {
+  fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result {
+    if let Some(name) = self.variant_name() {
+      f.write_str(name)
+    } else {
+      f.write_fmt(format_args!("<UNKNOWN {:?}>", self.0))
+    }
+  }
+}
+impl<'a> flatbuffers::Follow<'a> for FrameType {
+  type Inner = Self;
+  #[inline]
+  unsafe fn follow(buf: &'a [u8], loc: usize) -> Self::Inner {
+    let b = flatbuffers::read_scalar_at::<i8>(buf, loc);
+    Self(b)
+  }
+}
+
+impl flatbuffers::Push for FrameType {
+    type Output = FrameType;
+    #[inline]
+    unsafe fn push(&self, dst: &mut [u8], _written_len: usize) {
+        flatbuffers::emplace_scalar::<i8>(dst, self.0);
+    }
+}
+
+impl flatbuffers::EndianScalar for FrameType {
+  type Scalar = i8;
+  #[inline]
+  fn to_little_endian(self) -> i8 {
+    self.0.to_le()
+  }
+  #[inline]
+  #[allow(clippy::wrong_self_convention)]
+  fn from_little_endian(v: i8) -> Self {
+    let b = i8::from_le(v);
+    Self(b)
+  }
+}
+
+impl<'a> flatbuffers::Verifiable for FrameType {
+  #[inline]
+  fn run_verifier(
+    v: &mut flatbuffers::Verifier, pos: usize
+  ) -> Result<(), flatbuffers::InvalidFlatbuffer> {
+    use self::flatbuffers::Verifiable;
+    i8::run_verifier(v, pos)
+  }
+}
+
+impl flatbuffers::SimpleToVerifyInSlice for FrameType {}
+#[deprecated(since = "2.0.0", note = "Use associated constants instead. This will no longer be generated in 2021.")]
+pub const ENUM_MIN_UPDATE: u8 = 0;
+#[deprecated(since = "2.0.0", note = "Use associated constants instead. This will no longer be generated in 2021.")]
+pub const ENUM_MAX_UPDATE: u8 = 3;
+#[deprecated(since = "2.0.0", note = "Use associated constants instead. This will no longer be generated in 2021.")]
+#[allow(non_camel_case_types)]
+pub const ENUM_VALUES_UPDATE: [Update; 4] = [
+  Update::NONE,
+  Update::Setup,
+  Update::DecodeUnitStart,
+  Update::DecodeUnitBuffer,
+];
+
+#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Default)]
+#[repr(transparent)]
+pub struct Update(pub u8);
+#[allow(non_upper_case_globals)]
+impl Update {
+  pub const NONE: Self = Self(0);
+  pub const Setup: Self = Self(1);
+  pub const DecodeUnitStart: Self = Self(2);
+  pub const DecodeUnitBuffer: Self = Self(3);
+
+  pub const ENUM_MIN: u8 = 0;
+  pub const ENUM_MAX: u8 = 3;
+  pub const ENUM_VALUES: &'static [Self] = &[
+    Self::NONE,
+    Self::Setup,
+    Self::DecodeUnitStart,
+    Self::DecodeUnitBuffer,
+  ];
+  /// Returns the variant's name or "" if unknown.
+  pub fn variant_name(self) -> Option<&'static str> {
+    match self {
+      Self::NONE => Some("NONE"),
+      Self::Setup => Some("Setup"),
+      Self::DecodeUnitStart => Some("DecodeUnitStart"),
+      Self::DecodeUnitBuffer => Some("DecodeUnitBuffer"),
+      _ => None,
+    }
+  }
+}
+impl core::fmt::Debug for Update {
+  fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result {
+    if let Some(name) = self.variant_name() {
+      f.write_str(name)
+    } else {
+      f.write_fmt(format_args!("<UNKNOWN {:?}>", self.0))
+    }
+  }
+}
+impl<'a> flatbuffers::Follow<'a> for Update {
+  type Inner = Self;
+  #[inline]
+  unsafe fn follow(buf: &'a [u8], loc: usize) -> Self::Inner {
+    let b = flatbuffers::read_scalar_at::<u8>(buf, loc);
+    Self(b)
+  }
+}
+
+impl flatbuffers::Push for Update {
+    type Output = Update;
+    #[inline]
+    unsafe fn push(&self, dst: &mut [u8], _written_len: usize) {
+        flatbuffers::emplace_scalar::<u8>(dst, self.0);
+    }
+}
+
+impl flatbuffers::EndianScalar for Update {
+  type Scalar = u8;
+  #[inline]
+  fn to_little_endian(self) -> u8 {
+    self.0.to_le()
+  }
+  #[inline]
+  #[allow(clippy::wrong_self_convention)]
+  fn from_little_endian(v: u8) -> Self {
+    let b = u8::from_le(v);
+    Self(b)
+  }
+}
+
+impl<'a> flatbuffers::Verifiable for Update {
+  #[inline]
+  fn run_verifier(
+    v: &mut flatbuffers::Verifier, pos: usize
+  ) -> Result<(), flatbuffers::InvalidFlatbuffer> {
+    use self::flatbuffers::Verifiable;
+    u8::run_verifier(v, pos)
+  }
+}
+
+impl flatbuffers::SimpleToVerifyInSlice for Update {}
+pub struct UpdateUnionTableOffset {}
+
+pub enum SetupOffset {}
+#[derive(Copy, Clone, PartialEq)]
+
+pub struct Setup<'a> {
+  pub _tab: flatbuffers::Table<'a>,
+}
+
+impl<'a> flatbuffers::Follow<'a> for Setup<'a> {
+  type Inner = Setup<'a>;
+  #[inline]
+  unsafe fn follow(buf: &'a [u8], loc: usize) -> Self::Inner {
+    Self { _tab: flatbuffers::Table::new(buf, loc) }
+  }
+}
+
+impl<'a> Setup<'a> {
+  pub const VT_VIDEO_FORMAT: flatbuffers::VOffsetT = 4;
+  pub const VT_WIDTH: flatbuffers::VOffsetT = 6;
+  pub const VT_HEIGHT: flatbuffers::VOffsetT = 8;
+  pub const VT_REDRAW_RATE: flatbuffers::VOffsetT = 10;
+
+  #[inline]
+  pub unsafe fn init_from_table(table: flatbuffers::Table<'a>) -> Self {
+    Setup { _tab: table }
+  }
+  #[allow(unused_mut)]
+  pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr, A: flatbuffers::Allocator + 'bldr>(
+    _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr, A>,
+    args: &'args SetupArgs<'args>
+  ) -> flatbuffers::WIPOffset<Setup<'bldr>> {
+    let mut builder = SetupBuilder::new(_fbb);
+    if let Some(x) = args.video_format { builder.add_video_format(x); }
+    builder.add_redraw_rate(args.redraw_rate);
+    builder.add_height(args.height);
+    builder.add_width(args.width);
+    builder.finish()
+  }
+
+
+  #[inline]
+  pub fn video_format(&self) -> Option<&'a str> {
+    // Safety:
+    // Created from valid Table for this object
+    // which contains a valid value in this slot
+    unsafe { self._tab.get::<flatbuffers::ForwardsUOffset<&str>>(Setup::VT_VIDEO_FORMAT, None)}
+  }
+  #[inline]
+  pub fn width(&self) -> u16 {
+    // Safety:
+    // Created from valid Table for this object
+    // which contains a valid value in this slot
+    unsafe { self._tab.get::<u16>(Setup::VT_WIDTH, Some(0)).unwrap()}
+  }
+  #[inline]
+  pub fn height(&self) -> u16 {
+    // Safety:
+    // Created from valid Table for this object
+    // which contains a valid value in this slot
+    unsafe { self._tab.get::<u16>(Setup::VT_HEIGHT, Some(0)).unwrap()}
+  }
+  #[inline]
+  pub fn redraw_rate(&self) -> u16 {
+    // Safety:
+    // Created from valid Table for this object
+    // which contains a valid value in this slot
+    unsafe { self._tab.get::<u16>(Setup::VT_REDRAW_RATE, Some(0)).unwrap()}
+  }
+}
+
+impl flatbuffers::Verifiable for Setup<'_> {
+  #[inline]
+  fn run_verifier(
+    v: &mut flatbuffers::Verifier, pos: usize
+  ) -> Result<(), flatbuffers::InvalidFlatbuffer> {
+    use self::flatbuffers::Verifiable;
+    v.visit_table(pos)?
+     .visit_field::<flatbuffers::ForwardsUOffset<&str>>("video_format", Self::VT_VIDEO_FORMAT, false)?
+     .visit_field::<u16>("width", Self::VT_WIDTH, false)?
+     .visit_field::<u16>("height", Self::VT_HEIGHT, false)?
+     .visit_field::<u16>("redraw_rate", Self::VT_REDRAW_RATE, false)?
+     .finish();
+    Ok(())
+  }
+}
+pub struct SetupArgs<'a> {
+    pub video_format: Option<flatbuffers::WIPOffset<&'a str>>,
+    pub width: u16,
+    pub height: u16,
+    pub redraw_rate: u16,
+}
+impl<'a> Default for SetupArgs<'a> {
+  #[inline]
+  fn default() -> Self {
+    SetupArgs {
+      video_format: None,
+      width: 0,
+      height: 0,
+      redraw_rate: 0,
+    }
+  }
+}
+
+pub struct SetupBuilder<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> {
+  fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a, A>,
+  start_: flatbuffers::WIPOffset<flatbuffers::TableUnfinishedWIPOffset>,
+}
+impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> SetupBuilder<'a, 'b, A> {
+  #[inline]
+  pub fn add_video_format(&mut self, video_format: flatbuffers::WIPOffset<&'b  str>) {
+    self.fbb_.push_slot_always::<flatbuffers::WIPOffset<_>>(Setup::VT_VIDEO_FORMAT, video_format);
+  }
+  #[inline]
+  pub fn add_width(&mut self, width: u16) {
+    self.fbb_.push_slot::<u16>(Setup::VT_WIDTH, width, 0);
+  }
+  #[inline]
+  pub fn add_height(&mut self, height: u16) {
+    self.fbb_.push_slot::<u16>(Setup::VT_HEIGHT, height, 0);
+  }
+  #[inline]
+  pub fn add_redraw_rate(&mut self, redraw_rate: u16) {
+    self.fbb_.push_slot::<u16>(Setup::VT_REDRAW_RATE, redraw_rate, 0);
+  }
+  #[inline]
+  pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>) -> SetupBuilder<'a, 'b, A> {
+    let start = _fbb.start_table();
+    SetupBuilder {
+      fbb_: _fbb,
+      start_: start,
+    }
+  }
+  #[inline]
+  pub fn finish(self) -> flatbuffers::WIPOffset<Setup<'a>> {
+    let o = self.fbb_.end_table(self.start_);
+    flatbuffers::WIPOffset::new(o.value())
+  }
+}
+
+impl core::fmt::Debug for Setup<'_> {
+  fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
+    let mut ds = f.debug_struct("Setup");
+      ds.field("video_format", &self.video_format());
+      ds.field("width", &self.width());
+      ds.field("height", &self.height());
+      ds.field("redraw_rate", &self.redraw_rate());
+      ds.finish()
+  }
+}
+pub enum DecodeUnitStartOffset {}
+#[derive(Copy, Clone, PartialEq)]
+
+pub struct DecodeUnitStart<'a> {
+  pub _tab: flatbuffers::Table<'a>,
+}
+
+impl<'a> flatbuffers::Follow<'a> for DecodeUnitStart<'a> {
+  type Inner = DecodeUnitStart<'a>;
+  #[inline]
+  unsafe fn follow(buf: &'a [u8], loc: usize) -> Self::Inner {
+    Self { _tab: flatbuffers::Table::new(buf, loc) }
+  }
+}
+
+impl<'a> DecodeUnitStart<'a> {
+  pub const VT_FRAME_NUMBER: flatbuffers::VOffsetT = 4;
+  pub const VT_FRAME_TYPE: flatbuffers::VOffsetT = 6;
+  pub const VT_NUM_BUFFERS: flatbuffers::VOffsetT = 8;
+  pub const VT_RECEIVE_TIME_MS: flatbuffers::VOffsetT = 10;
+  pub const VT_FULL_LENGTH: flatbuffers::VOffsetT = 12;
+
+  #[inline]
+  pub unsafe fn init_from_table(table: flatbuffers::Table<'a>) -> Self {
+    DecodeUnitStart { _tab: table }
+  }
+  #[allow(unused_mut)]
+  pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr, A: flatbuffers::Allocator + 'bldr>(
+    _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr, A>,
+    args: &'args DecodeUnitStartArgs
+  ) -> flatbuffers::WIPOffset<DecodeUnitStart<'bldr>> {
+    let mut builder = DecodeUnitStartBuilder::new(_fbb);
+    builder.add_full_length(args.full_length);
+    builder.add_num_buffers(args.num_buffers);
+    builder.add_frame_number(args.frame_number);
+    builder.add_receive_time_ms(args.receive_time_ms);
+    builder.add_frame_type(args.frame_type);
+    builder.finish()
+  }
+
+
+  #[inline]
+  pub fn frame_number(&self) -> u64 {
+    // Safety:
+    // Created from valid Table for this object
+    // which contains a valid value in this slot
+    unsafe { self._tab.get::<u64>(DecodeUnitStart::VT_FRAME_NUMBER, Some(0)).unwrap()}
+  }
+  #[inline]
+  pub fn frame_type(&self) -> FrameType {
+    // Safety:
+    // Created from valid Table for this object
+    // which contains a valid value in this slot
+    unsafe { self._tab.get::<FrameType>(DecodeUnitStart::VT_FRAME_TYPE, Some(FrameType::PFRAME)).unwrap()}
+  }
+  #[inline]
+  pub fn num_buffers(&self) -> u64 {
+    // Safety:
+    // Created from valid Table for this object
+    // which contains a valid value in this slot
+    unsafe { self._tab.get::<u64>(DecodeUnitStart::VT_NUM_BUFFERS, Some(0)).unwrap()}
+  }
+  #[inline]
+  pub fn receive_time_ms(&self) -> u16 {
+    // Safety:
+    // Created from valid Table for this object
+    // which contains a valid value in this slot
+    unsafe { self._tab.get::<u16>(DecodeUnitStart::VT_RECEIVE_TIME_MS, Some(0)).unwrap()}
+  }
+  #[inline]
+  pub fn full_length(&self) -> u64 {
+    // Safety:
+    // Created from valid Table for this object
+    // which contains a valid value in this slot
+    unsafe { self._tab.get::<u64>(DecodeUnitStart::VT_FULL_LENGTH, Some(0)).unwrap()}
+  }
+}
+
+impl flatbuffers::Verifiable for DecodeUnitStart<'_> {
+  #[inline]
+  fn run_verifier(
+    v: &mut flatbuffers::Verifier, pos: usize
+  ) -> Result<(), flatbuffers::InvalidFlatbuffer> {
+    use self::flatbuffers::Verifiable;
+    v.visit_table(pos)?
+     .visit_field::<u64>("frame_number", Self::VT_FRAME_NUMBER, false)?
+     .visit_field::<FrameType>("frame_type", Self::VT_FRAME_TYPE, false)?
+     .visit_field::<u64>("num_buffers", Self::VT_NUM_BUFFERS, false)?
+     .visit_field::<u16>("receive_time_ms", Self::VT_RECEIVE_TIME_MS, false)?
+     .visit_field::<u64>("full_length", Self::VT_FULL_LENGTH, false)?
+     .finish();
+    Ok(())
+  }
+}
+pub struct DecodeUnitStartArgs {
+    pub frame_number: u64,
+    pub frame_type: FrameType,
+    pub num_buffers: u64,
+    pub receive_time_ms: u16,
+    pub full_length: u64,
+}
+impl<'a> Default for DecodeUnitStartArgs {
+  #[inline]
+  fn default() -> Self {
+    DecodeUnitStartArgs {
+      frame_number: 0,
+      frame_type: FrameType::PFRAME,
+      num_buffers: 0,
+      receive_time_ms: 0,
+      full_length: 0,
+    }
+  }
+}
+
+pub struct DecodeUnitStartBuilder<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> {
+  fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a, A>,
+  start_: flatbuffers::WIPOffset<flatbuffers::TableUnfinishedWIPOffset>,
+}
+impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> DecodeUnitStartBuilder<'a, 'b, A> {
+  #[inline]
+  pub fn add_frame_number(&mut self, frame_number: u64) {
+    self.fbb_.push_slot::<u64>(DecodeUnitStart::VT_FRAME_NUMBER, frame_number, 0);
+  }
+  #[inline]
+  pub fn add_frame_type(&mut self, frame_type: FrameType) {
+    self.fbb_.push_slot::<FrameType>(DecodeUnitStart::VT_FRAME_TYPE, frame_type, FrameType::PFRAME);
+  }
+  #[inline]
+  pub fn add_num_buffers(&mut self, num_buffers: u64) {
+    self.fbb_.push_slot::<u64>(DecodeUnitStart::VT_NUM_BUFFERS, num_buffers, 0);
+  }
+  #[inline]
+  pub fn add_receive_time_ms(&mut self, receive_time_ms: u16) {
+    self.fbb_.push_slot::<u16>(DecodeUnitStart::VT_RECEIVE_TIME_MS, receive_time_ms, 0);
+  }
+  #[inline]
+  pub fn add_full_length(&mut self, full_length: u64) {
+    self.fbb_.push_slot::<u64>(DecodeUnitStart::VT_FULL_LENGTH, full_length, 0);
+  }
+  #[inline]
+  pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>) -> DecodeUnitStartBuilder<'a, 'b, A> {
+    let start = _fbb.start_table();
+    DecodeUnitStartBuilder {
+      fbb_: _fbb,
+      start_: start,
+    }
+  }
+  #[inline]
+  pub fn finish(self) -> flatbuffers::WIPOffset<DecodeUnitStart<'a>> {
+    let o = self.fbb_.end_table(self.start_);
+    flatbuffers::WIPOffset::new(o.value())
+  }
+}
+
+impl core::fmt::Debug for DecodeUnitStart<'_> {
+  fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
+    let mut ds = f.debug_struct("DecodeUnitStart");
+      ds.field("frame_number", &self.frame_number());
+      ds.field("frame_type", &self.frame_type());
+      ds.field("num_buffers", &self.num_buffers());
+      ds.field("receive_time_ms", &self.receive_time_ms());
+      ds.field("full_length", &self.full_length());
+      ds.finish()
+  }
+}
+pub enum DecodeUnitBufferOffset {}
+#[derive(Copy, Clone, PartialEq)]
+
+pub struct DecodeUnitBuffer<'a> {
+  pub _tab: flatbuffers::Table<'a>,
+}
+
+impl<'a> flatbuffers::Follow<'a> for DecodeUnitBuffer<'a> {
+  type Inner = DecodeUnitBuffer<'a>;
+  #[inline]
+  unsafe fn follow(buf: &'a [u8], loc: usize) -> Self::Inner {
+    Self { _tab: flatbuffers::Table::new(buf, loc) }
+  }
+}
+
+impl<'a> DecodeUnitBuffer<'a> {
+  pub const VT_FRAME_NUMBER: flatbuffers::VOffsetT = 4;
+  pub const VT_BUFFER_INDEX: flatbuffers::VOffsetT = 6;
+  pub const VT_BUFFER_OFFSET: flatbuffers::VOffsetT = 8;
+  pub const VT_DATA: flatbuffers::VOffsetT = 10;
+
+  #[inline]
+  pub unsafe fn init_from_table(table: flatbuffers::Table<'a>) -> Self {
+    DecodeUnitBuffer { _tab: table }
+  }
+  #[allow(unused_mut)]
+  pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr, A: flatbuffers::Allocator + 'bldr>(
+    _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr, A>,
+    args: &'args DecodeUnitBufferArgs<'args>
+  ) -> flatbuffers::WIPOffset<DecodeUnitBuffer<'bldr>> {
+    let mut builder = DecodeUnitBufferBuilder::new(_fbb);
+    builder.add_buffer_offset(args.buffer_offset);
+    builder.add_buffer_index(args.buffer_index);
+    builder.add_frame_number(args.frame_number);
+    if let Some(x) = args.data { builder.add_data(x); }
+    builder.finish()
+  }
+
+
+  #[inline]
+  pub fn frame_number(&self) -> u64 {
+    // Safety:
+    // Created from valid Table for this object
+    // which contains a valid value in this slot
+    unsafe { self._tab.get::<u64>(DecodeUnitBuffer::VT_FRAME_NUMBER, Some(0)).unwrap()}
+  }
+  #[inline]
+  pub fn buffer_index(&self) -> u64 {
+    // Safety:
+    // Created from valid Table for this object
+    // which contains a valid value in this slot
+    unsafe { self._tab.get::<u64>(DecodeUnitBuffer::VT_BUFFER_INDEX, Some(0)).unwrap()}
+  }
+  #[inline]
+  pub fn buffer_offset(&self) -> u64 {
+    // Safety:
+    // Created from valid Table for this object
+    // which contains a valid value in this slot
+    unsafe { self._tab.get::<u64>(DecodeUnitBuffer::VT_BUFFER_OFFSET, Some(0)).unwrap()}
+  }
+  #[inline]
+  pub fn data(&self) -> Option<flatbuffers::Vector<'a, u8>> {
+    // Safety:
+    // Created from valid Table for this object
+    // which contains a valid value in this slot
+    unsafe { self._tab.get::<flatbuffers::ForwardsUOffset<flatbuffers::Vector<'a, u8>>>(DecodeUnitBuffer::VT_DATA, None)}
+  }
+}
+
+impl flatbuffers::Verifiable for DecodeUnitBuffer<'_> {
+  #[inline]
+  fn run_verifier(
+    v: &mut flatbuffers::Verifier, pos: usize
+  ) -> Result<(), flatbuffers::InvalidFlatbuffer> {
+    use self::flatbuffers::Verifiable;
+    v.visit_table(pos)?
+     .visit_field::<u64>("frame_number", Self::VT_FRAME_NUMBER, false)?
+     .visit_field::<u64>("buffer_index", Self::VT_BUFFER_INDEX, false)?
+     .visit_field::<u64>("buffer_offset", Self::VT_BUFFER_OFFSET, false)?
+     .visit_field::<flatbuffers::ForwardsUOffset<flatbuffers::Vector<'_, u8>>>("data", Self::VT_DATA, false)?
+     .finish();
+    Ok(())
+  }
+}
+pub struct DecodeUnitBufferArgs<'a> {
+    pub frame_number: u64,
+    pub buffer_index: u64,
+    pub buffer_offset: u64,
+    pub data: Option<flatbuffers::WIPOffset<flatbuffers::Vector<'a, u8>>>,
+}
+impl<'a> Default for DecodeUnitBufferArgs<'a> {
+  #[inline]
+  fn default() -> Self {
+    DecodeUnitBufferArgs {
+      frame_number: 0,
+      buffer_index: 0,
+      buffer_offset: 0,
+      data: None,
+    }
+  }
+}
+
+pub struct DecodeUnitBufferBuilder<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> {
+  fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a, A>,
+  start_: flatbuffers::WIPOffset<flatbuffers::TableUnfinishedWIPOffset>,
+}
+impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> DecodeUnitBufferBuilder<'a, 'b, A> {
+  #[inline]
+  pub fn add_frame_number(&mut self, frame_number: u64) {
+    self.fbb_.push_slot::<u64>(DecodeUnitBuffer::VT_FRAME_NUMBER, frame_number, 0);
+  }
+  #[inline]
+  pub fn add_buffer_index(&mut self, buffer_index: u64) {
+    self.fbb_.push_slot::<u64>(DecodeUnitBuffer::VT_BUFFER_INDEX, buffer_index, 0);
+  }
+  #[inline]
+  pub fn add_buffer_offset(&mut self, buffer_offset: u64) {
+    self.fbb_.push_slot::<u64>(DecodeUnitBuffer::VT_BUFFER_OFFSET, buffer_offset, 0);
+  }
+  #[inline]
+  pub fn add_data(&mut self, data: flatbuffers::WIPOffset<flatbuffers::Vector<'b , u8>>) {
+    self.fbb_.push_slot_always::<flatbuffers::WIPOffset<_>>(DecodeUnitBuffer::VT_DATA, data);
+  }
+  #[inline]
+  pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>) -> DecodeUnitBufferBuilder<'a, 'b, A> {
+    let start = _fbb.start_table();
+    DecodeUnitBufferBuilder {
+      fbb_: _fbb,
+      start_: start,
+    }
+  }
+  #[inline]
+  pub fn finish(self) -> flatbuffers::WIPOffset<DecodeUnitBuffer<'a>> {
+    let o = self.fbb_.end_table(self.start_);
+    flatbuffers::WIPOffset::new(o.value())
+  }
+}
+
+impl core::fmt::Debug for DecodeUnitBuffer<'_> {
+  fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
+    let mut ds = f.debug_struct("DecodeUnitBuffer");
+      ds.field("frame_number", &self.frame_number());
+      ds.field("buffer_index", &self.buffer_index());
+      ds.field("buffer_offset", &self.buffer_offset());
+      ds.field("data", &self.data());
+      ds.finish()
+  }
+}
+pub enum VideoUpdateOffset {}
+#[derive(Copy, Clone, PartialEq)]
+
+pub struct VideoUpdate<'a> {
+  pub _tab: flatbuffers::Table<'a>,
+}
+
+impl<'a> flatbuffers::Follow<'a> for VideoUpdate<'a> {
+  type Inner = VideoUpdate<'a>;
+  #[inline]
+  unsafe fn follow(buf: &'a [u8], loc: usize) -> Self::Inner {
+    Self { _tab: flatbuffers::Table::new(buf, loc) }
+  }
+}
+
+impl<'a> VideoUpdate<'a> {
+  pub const VT_UPDATE_TYPE: flatbuffers::VOffsetT = 4;
+  pub const VT_UPDATE: flatbuffers::VOffsetT = 6;
+
+  #[inline]
+  pub unsafe fn init_from_table(table: flatbuffers::Table<'a>) -> Self {
+    VideoUpdate { _tab: table }
+  }
+  #[allow(unused_mut)]
+  pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr, A: flatbuffers::Allocator + 'bldr>(
+    _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr, A>,
+    args: &'args VideoUpdateArgs
+  ) -> flatbuffers::WIPOffset<VideoUpdate<'bldr>> {
+    let mut builder = VideoUpdateBuilder::new(_fbb);
+    if let Some(x) = args.update { builder.add_update(x); }
+    builder.add_update_type(args.update_type);
+    builder.finish()
+  }
+
+
+  #[inline]
+  pub fn update_type(&self) -> Update {
+    // Safety:
+    // Created from valid Table for this object
+    // which contains a valid value in this slot
+    unsafe { self._tab.get::<Update>(VideoUpdate::VT_UPDATE_TYPE, Some(Update::NONE)).unwrap()}
+  }
+  #[inline]
+  pub fn update(&self) -> Option<flatbuffers::Table<'a>> {
+    // Safety:
+    // Created from valid Table for this object
+    // which contains a valid value in this slot
+    unsafe { self._tab.get::<flatbuffers::ForwardsUOffset<flatbuffers::Table<'a>>>(VideoUpdate::VT_UPDATE, None)}
+  }
+  #[inline]
+  #[allow(non_snake_case)]
+  pub fn update_as_setup(&self) -> Option<Setup<'a>> {
+    if self.update_type() == Update::Setup {
+      self.update().map(|t| {
+       // Safety:
+       // Created from a valid Table for this object
+       // Which contains a valid union in this slot
+       unsafe { Setup::init_from_table(t) }
+     })
+    } else {
+      None
+    }
+  }
+
+  #[inline]
+  #[allow(non_snake_case)]
+  pub fn update_as_decode_unit_start(&self) -> Option<DecodeUnitStart<'a>> {
+    if self.update_type() == Update::DecodeUnitStart {
+      self.update().map(|t| {
+       // Safety:
+       // Created from a valid Table for this object
+       // Which contains a valid union in this slot
+       unsafe { DecodeUnitStart::init_from_table(t) }
+     })
+    } else {
+      None
+    }
+  }
+
+  #[inline]
+  #[allow(non_snake_case)]
+  pub fn update_as_decode_unit_buffer(&self) -> Option<DecodeUnitBuffer<'a>> {
+    if self.update_type() == Update::DecodeUnitBuffer {
+      self.update().map(|t| {
+       // Safety:
+       // Created from a valid Table for this object
+       // Which contains a valid union in this slot
+       unsafe { DecodeUnitBuffer::init_from_table(t) }
+     })
+    } else {
+      None
+    }
+  }
+
+}
+
+impl flatbuffers::Verifiable for VideoUpdate<'_> {
+  #[inline]
+  fn run_verifier(
+    v: &mut flatbuffers::Verifier, pos: usize
+  ) -> Result<(), flatbuffers::InvalidFlatbuffer> {
+    use self::flatbuffers::Verifiable;
+    v.visit_table(pos)?
+     .visit_union::<Update, _>("update_type", Self::VT_UPDATE_TYPE, "update", Self::VT_UPDATE, false, |key, v, pos| {
+        match key {
+          Update::Setup => v.verify_union_variant::<flatbuffers::ForwardsUOffset<Setup>>("Update::Setup", pos),
+          Update::DecodeUnitStart => v.verify_union_variant::<flatbuffers::ForwardsUOffset<DecodeUnitStart>>("Update::DecodeUnitStart", pos),
+          Update::DecodeUnitBuffer => v.verify_union_variant::<flatbuffers::ForwardsUOffset<DecodeUnitBuffer>>("Update::DecodeUnitBuffer", pos),
+          _ => Ok(()),
+        }
+     })?
+     .finish();
+    Ok(())
+  }
+}
+pub struct VideoUpdateArgs {
+    pub update_type: Update,
+    pub update: Option<flatbuffers::WIPOffset<flatbuffers::UnionWIPOffset>>,
+}
+impl<'a> Default for VideoUpdateArgs {
+  #[inline]
+  fn default() -> Self {
+    VideoUpdateArgs {
+      update_type: Update::NONE,
+      update: None,
+    }
+  }
+}
+
+pub struct VideoUpdateBuilder<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> {
+  fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a, A>,
+  start_: flatbuffers::WIPOffset<flatbuffers::TableUnfinishedWIPOffset>,
+}
+impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> VideoUpdateBuilder<'a, 'b, A> {
+  #[inline]
+  pub fn add_update_type(&mut self, update_type: Update) {
+    self.fbb_.push_slot::<Update>(VideoUpdate::VT_UPDATE_TYPE, update_type, Update::NONE);
+  }
+  #[inline]
+  pub fn add_update(&mut self, update: flatbuffers::WIPOffset<flatbuffers::UnionWIPOffset>) {
+    self.fbb_.push_slot_always::<flatbuffers::WIPOffset<_>>(VideoUpdate::VT_UPDATE, update);
+  }
+  #[inline]
+  pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>) -> VideoUpdateBuilder<'a, 'b, A> {
+    let start = _fbb.start_table();
+    VideoUpdateBuilder {
+      fbb_: _fbb,
+      start_: start,
+    }
+  }
+  #[inline]
+  pub fn finish(self) -> flatbuffers::WIPOffset<VideoUpdate<'a>> {
+    let o = self.fbb_.end_table(self.start_);
+    flatbuffers::WIPOffset::new(o.value())
+  }
+}
+
+impl core::fmt::Debug for VideoUpdate<'_> {
+  fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
+    let mut ds = f.debug_struct("VideoUpdate");
+      ds.field("update_type", &self.update_type());
+      match self.update_type() {
+        Update::Setup => {
+          if let Some(x) = self.update_as_setup() {
+            ds.field("update", &x)
+          } else {
+            ds.field("update", &"InvalidFlatbuffer: Union discriminant does not match value.")
+          }
+        },
+        Update::DecodeUnitStart => {
+          if let Some(x) = self.update_as_decode_unit_start() {
+            ds.field("update", &x)
+          } else {
+            ds.field("update", &"InvalidFlatbuffer: Union discriminant does not match value.")
+          }
+        },
+        Update::DecodeUnitBuffer => {
+          if let Some(x) = self.update_as_decode_unit_buffer() {
+            ds.field("update", &x)
+          } else {
+            ds.field("update", &"InvalidFlatbuffer: Union discriminant does not match value.")
+          }
+        },
+        _ => {
+          let x: Option<()> = None;
+          ds.field("update", &x)
+        },
+      };
+      ds.finish()
+  }
+}
+#[inline]
+/// Verifies that a buffer of bytes contains a `VideoUpdate`
+/// and returns it.
+/// Note that verification is still experimental and may not
+/// catch every error, or be maximally performant. For the
+/// previous, unchecked, behavior use
+/// `root_as_video_update_unchecked`.
+pub fn root_as_video_update(buf: &[u8]) -> Result<VideoUpdate, flatbuffers::InvalidFlatbuffer> {
+  flatbuffers::root::<VideoUpdate>(buf)
+}
+#[inline]
+/// Verifies that a buffer of bytes contains a size prefixed
+/// `VideoUpdate` and returns it.
+/// Note that verification is still experimental and may not
+/// catch every error, or be maximally performant. For the
+/// previous, unchecked, behavior use
+/// `size_prefixed_root_as_video_update_unchecked`.
+pub fn size_prefixed_root_as_video_update(buf: &[u8]) -> Result<VideoUpdate, flatbuffers::InvalidFlatbuffer> {
+  flatbuffers::size_prefixed_root::<VideoUpdate>(buf)
+}
+#[inline]
+/// Verifies, with the given options, that a buffer of bytes
+/// contains a `VideoUpdate` and returns it.
+/// Note that verification is still experimental and may not
+/// catch every error, or be maximally performant. For the
+/// previous, unchecked, behavior use
+/// `root_as_video_update_unchecked`.
+pub fn root_as_video_update_with_opts<'b, 'o>(
+  opts: &'o flatbuffers::VerifierOptions,
+  buf: &'b [u8],
+) -> Result<VideoUpdate<'b>, flatbuffers::InvalidFlatbuffer> {
+  flatbuffers::root_with_opts::<VideoUpdate<'b>>(opts, buf)
+}
+#[inline]
+/// Verifies, with the given verifier options, that a buffer of
+/// bytes contains a size prefixed `VideoUpdate` and returns
+/// it. Note that verification is still experimental and may not
+/// catch every error, or be maximally performant. For the
+/// previous, unchecked, behavior use
+/// `root_as_video_update_unchecked`.
+pub fn size_prefixed_root_as_video_update_with_opts<'b, 'o>(
+  opts: &'o flatbuffers::VerifierOptions,
+  buf: &'b [u8],
+) -> Result<VideoUpdate<'b>, flatbuffers::InvalidFlatbuffer> {
+  flatbuffers::size_prefixed_root_with_opts::<VideoUpdate<'b>>(opts, buf)
+}
+#[inline]
+/// Assumes, without verification, that a buffer of bytes contains a VideoUpdate and returns it.
+/// # Safety
+/// Callers must trust the given bytes do indeed contain a valid `VideoUpdate`.
+pub unsafe fn root_as_video_update_unchecked(buf: &[u8]) -> VideoUpdate {
+  flatbuffers::root_unchecked::<VideoUpdate>(buf)
+}
+#[inline]
+/// Assumes, without verification, that a buffer of bytes contains a size prefixed VideoUpdate and returns it.
+/// # Safety
+/// Callers must trust the given bytes do indeed contain a valid size prefixed `VideoUpdate`.
+pub unsafe fn size_prefixed_root_as_video_update_unchecked(buf: &[u8]) -> VideoUpdate {
+  flatbuffers::size_prefixed_root_unchecked::<VideoUpdate>(buf)
+}
+#[inline]
+pub fn finish_video_update_buffer<'a, 'b, A: flatbuffers::Allocator + 'a>(
+    fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>,
+    root: flatbuffers::WIPOffset<VideoUpdate<'a>>) {
+  fbb.finish(root, None);
+}
+
+#[inline]
+pub fn finish_size_prefixed_video_update_buffer<'a, 'b, A: flatbuffers::Allocator + 'a>(fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>, root: flatbuffers::WIPOffset<VideoUpdate<'a>>) {
+  fbb.finish_size_prefixed(root, None);
+}
+}  // pub mod VideoUpdate
+

gamestream-webtransport-proxy/src/responses.rs

@@ -0,0 +1,23 @@
+use serde::Deserialize;
+
+#[derive(Deserialize)]
+pub struct ServerInfoResponse {
+    pub hostname: String,
+    pub appversion: String,
+    pub GfeVersion: String,
+    pub uniqueid: uuid::Uuid,
+    pub HttpsPort: u16,
+    pub ExternalPort: u16,
+    pub MaxLumaPixelsHEVC: u64,
+    pub mac: String,
+    //pub ServerCommand: String,
+    //pub Permission: u64,
+    //pub VirtualDisplayCapable: bool,
+    //pub VirtualDisplayDriverReady: bool,
+    pub LocalIP: String,
+    pub ServerCodecModeSupport: i32,
+    pub PairStatus: u64,
+    pub currentgame: u64,
+    pub state: String,
+}
+

gamestream-webtransport-proxy/src/stream.rs

@@ -1,3 +1,5 @@
+use std::str::FromStr;
+
 use anyhow::{Context, Result};
 use moonlight_common_c_sys::SCM_H264;
 use salvo::prelude::*;
@@ -6,7 +8,7 @@ use tracing::{debug, error, info};
 
 use crate::{
     common::{AppError, AppResult, get_url},
-    proxy,
+    proxy, responses,
     state::{GamestreamServer, StateReadAccess, StateReader},
 };
 
@@ -25,33 +27,16 @@ struct PostStreamStartResponse {
     //cert_hash: String,
 }
 
-#[derive(Deserialize)]
-struct ServerInfoResponse {
-    hostname: String,
-    appversion: String,
-    GfeVersion: String,
-    uniqueid: uuid::Uuid,
-    HttpsPort: u16,
-    ExternalPort: u16,
-    MaxLumaPixelsHEVC: u64,
-    mac: String,
-    //ServerCommand: String,
-    //Permission: u64,
-    //VirtualDisplayCapable: bool,
-    //VirtualDisplayDriverReady: bool,
-    LocalIP: String,
-    ServerCodecModeSupport: i32,
-    PairStatus: u64,
-    currentgame: u64,
-    state: String,
-}
-
 #[derive(Deserialize)]
 struct LaunchResponse {
+    #[serde(rename = "@status_code")]
+    status_code: String,
+    #[serde(rename = "@status_message")]
+    status_message: Option<String>,
     #[serde(rename = "sessionUrl0")]
-    session_url_0: url::Url,
+    session_url_0: Option<url::Url>,
     #[serde(rename = "gamesession")]
-    game_session: u64,
+    game_session: Option<u64>,
 }
 
 fn get_server(reader: &StateReadAccess, server: &String) -> Result<Option<GamestreamServer>> {
@@ -108,7 +93,7 @@ impl crate::backend::Backend {
             Ok(s) => match s {
                 Some(s) => s,
                 None => {
-                    info!("No server with name: {}", body.server);
+                    error!("No server with name: {}", body.server);
                     return Err(AppError {
                         status_code: StatusCode::BAD_REQUEST,
                         description: "No server configured with that name.".to_string(),
@@ -139,8 +124,9 @@ impl crate::backend::Backend {
         );
 
         let server_info = crate::common::get_url(&mut base_url, true).await.unwrap();
-        debug!("app_info: {server_info}");
+        debug!("server_info: {server_info}");
-        let server_info: ServerInfoResponse = match serde_xml_rs::from_str(&server_info) {
+        let server_info: responses::ServerInfoResponse = match serde_xml_rs::from_str(&server_info)
+        {
             Ok(s) => s,
             Err(e) => {
                 error!("Could not parse serverinfo response: {e}");
@@ -227,6 +213,36 @@ impl crate::backend::Backend {
             }
         };
 
+        let Ok(launch_resp_status) = http::StatusCode::from_str(&launch_response.status_code)
+        else {
+            error!(
+                "Server returned invalid status code: {}",
+                launch_response.status_code
+            );
+            return standard_error;
+        };
+        if !launch_resp_status.is_success() {
+            let status_message = launch_response
+                .status_message
+                .unwrap_or("Server did not provide a status message".to_string());
+
+            info!("Could not launch game: {}", status_message,);
+            return Err(AppError {
+                status_code: StatusCode::INTERNAL_SERVER_ERROR,
+                description: format!("Could not launch game: {}", status_message),
+            });
+        }
+
+        let Some(url) = launch_response.session_url_0 else {
+            error!("Server response was missing session_url_0");
+            return standard_error;
+        };
+
+        let Some(game_session) = launch_response.game_session else {
+            error!("Server response was missing game_session");
+            return standard_error;
+        };
+
         let stream_id = uuid::Uuid::new_v4();
 
         let server_codec_mode_support = if server_info.ServerCodecModeSupport == 0 {
@@ -237,8 +253,8 @@ impl crate::backend::Backend {
 
         let stream = crate::backend::Stream {
             id: stream_id,
-            url: launch_response.session_url_0,
+            url,
-            game_session: launch_response.game_session,
+            game_session,
             server_address: server.host.clone(),
             stream_config: body.stream_config.clone(),
             app_version: server_info.appversion,
@@ -299,7 +315,10 @@ impl crate::backend::Backend {
 
         let webtransport_url = url_constructor::UrlConstructor::new()
             .scheme("https")
-            .host(host)
+            // TODO: this is hardcoded to 127.0.0.1 to fix problems with
+            // tls certificates and IPv6 in chrome. This needs to eventually be fixed
+            // but I don't actually know what the fix is
+            .host("127.0.0.1")
             .port(port)
             .subdir("api/stream/connect")
             .build();

gamestream-webtransport-proxy/webroot/index.html

@@ -1,125 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-<head>
-    <meta charset="UTF-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <title>Game Streaming App</title>
-    <style>
-        body {
-            font-family: Arial, sans-serif;
-            margin: 20px;
-            background-color: #1a1a1a;
-            color: white;
-        }
-
-        .apps-container {
-            display: flex;
-            flex-wrap: wrap;
-            gap: 20px;
-            padding: 20px 0;
-        }
-
-        .app-box {
-            position: relative;
-            width: 200px;
-            height: 280px;
-            cursor: pointer;
-            transition: all 0.3s ease;
-        }
-
-        .app-box:hover {
-            transform: translateY(-5px);
-            box-shadow: 0 10px 20px rgba(255, 255, 255, 0.1);
-        }
-
-        .app-artwork {
-            width: 200px;
-            height: 200px;
-            background-color: #333;
-            border: 2px solid #555;
-            border-radius: 8px;
-            position: relative;
-            overflow: hidden;
-            transition: border-color 0.3s ease;
-        }
-
-        .app-box:hover .app-artwork {
-            border-color: #00aaff;
-        }
-
-        .app-title {
-            text-align: center;
-            margin: 10px 0 5px 0;
-            font-size: 16px;
-            font-weight: bold;
-            white-space: nowrap;
-            overflow: hidden;
-            text-overflow: ellipsis;
-        }
-
-        .app-server {
-            text-align: center;
-            font-size: 12px;
-            color: #aaa;
-            margin: 0;
-        }
-
-        .play-button {
-            position: absolute;
-            top: 50%;
-            left: 50%;
-            transform: translate(-50%, -50%);
-            width: 60px;
-            height: 60px;
-            background-color: rgba(0, 170, 255, 0.9);
-            border-radius: 50%;
-            display: flex;
-            align-items: center;
-            justify-content: center;
-            opacity: 0;
-            transition: opacity 0.3s ease;
-            pointer-events: none;
-        }
-
-        .play-button::after {
-            content: '';
-            width: 0;
-            height: 0;
-            border-left: 20px solid white;
-            border-top: 12px solid transparent;
-            border-bottom: 12px solid transparent;
-            margin-left: 4px;
-        }
-
-        .app-box:hover .play-button {
-            opacity: 1;
-        }
-
-        .app-box.clicked {
-            transform: scale(0.95);
-        }
-
-        .loading {
-            text-align: center;
-            font-size: 18px;
-            margin-top: 50px;
-        }
-
-        .error {
-            color: #ff4444;
-            text-align: center;
-            font-size: 18px;
-            margin-top: 50px;
-        }
-    </style>
-</head>
-<body>
-    <h1>Game Streaming Platform</h1>
-    <div id="content">
-        <div class="loading">Loading applications...</div>
-    </div>
-
-    <script src="index.js"></script>
-</body>
-</html>
-

gamestream-webtransport-proxy/webroot/index.js

@@ -1,205 +0,0 @@
-class GameStreamingApp {
-  constructor() {
-    this.apps = [];
-    this.init();
-  }
-
-  async init() {
-    try {
-      await this.loadApps();
-      this.renderApps();
-    } catch (error) {
-      this.showError('Failed to load applications: ' + error.message);
-    }
-  }
-
-  async loadApps() {
-    try {
-      const response = await fetch('/api/apps');
-      if (!response.ok) {
-        throw new Error(`HTTP error! status: ${response.status}`);
-      }
-      const data = await response.json();
-      this.apps = this.processAppsData(data);
-    } catch (error) {
-      console.error('Error loading apps:', error);
-      throw error;
-    }
-  }
-
-  processAppsData(data) {
-    const apps = [];
-    if (data.apps) {
-      Object.keys(data.apps).forEach(serverName => {
-        data.apps[serverName].forEach(app => {
-          apps.push({
-            ...app,
-            server: serverName
-          });
-        });
-      });
-    }
-    return apps;
-  }
-
-  renderApps() {
-    const contentDiv = document.getElementById('content');
-
-    if (this.apps.length === 0) {
-      contentDiv.innerHTML = '<div class="error">No applications found.</div>';
-      return;
-    }
-
-    const appsContainer = document.createElement('div');
-    appsContainer.className = 'apps-container';
-
-    this.apps.forEach(app => {
-      const appBox = this.createAppBox(app);
-      appsContainer.appendChild(appBox);
-    });
-
-    contentDiv.innerHTML = '';
-    contentDiv.appendChild(appsContainer);
-  }
-
-  createAppBox(app) {
-    const appBox = document.createElement('div');
-    appBox.className = 'app-box';
-    appBox.dataset.id = app.id;
-    appBox.dataset.server = app.server;
-
-    appBox.innerHTML = `
-            <div class="app-artwork">
-                <div class="play-button"></div>
-            </div>
-            <div class="app-title">${app.title}</div>
-            <div class="app-server">${app.server}</div>
-        `;
-
-    // Add click event listener
-    appBox.addEventListener('click', (e) => this.handleAppClick(e, app));
-
-    // Add click animation
-    appBox.addEventListener('mousedown', () => {
-      appBox.classList.add('clicked');
-    });
-
-    appBox.addEventListener('mouseup', () => {
-      setTimeout(() => {
-        appBox.classList.remove('clicked');
-      }, 150);
-    });
-
-    appBox.addEventListener('mouseleave', () => {
-      appBox.classList.remove('clicked');
-    });
-
-    return appBox;
-  }
-
-  async handleAppClick(event, app) {
-    event.preventDefault();
-
-    try {
-      console.log(`Starting stream for ${app.title} on ${app.server}`);
-
-      // Create the POST request payload
-      const payload = {
-        id: app.id,
-        server: app.server,
-        server_mode: {
-          fps: 60,
-          height: 1280,
-          width: 720
-        },
-        stream_config: {
-          bitrate_kbps: 5120,
-          mode: {
-            fps: 60,
-            height: 1280,
-            width: 720
-          }
-        }
-      };
-
-      // Make POST request to start stream
-      const response = await fetch('/api/stream/start', {
-        method: 'POST',
-        headers: {
-          'Content-Type': 'application/json',
-        },
-        body: JSON.stringify(payload)
-      });
-
-      if (!response.ok) {
-        throw new Error(`HTTP error! status: ${response.status}`);
-      }
-
-      const streamData = await response.json();
-      console.log('Stream started:', streamData);
-
-      if (streamData.url && streamData.cert_hash) {
-        await this.connectToStream(streamData.url, streamData.cert_hash);
-      } else {
-        throw new Error('Response was missing required parameters');
-      }
-
-    } catch (error) {
-      console.error('Error starting stream:', error);
-      alert('Failed to start stream: ' + error.message);
-    }
-  }
-
-  async connectToStream(url, cert_hash) {
-    const buffer = new Uint8Array(cert_hash);
-    console.log('Hash: ', buffer);
-    try {
-      console.log(`Connecting to stream`);
-
-      // Check if WebTransport is supported
-      if (!window.WebTransport) {
-        throw new Error('WebTransport is not supported in this browser');
-      }
-
-      //const url = new URL();
-      const transport = new WebTransport(url, {
-        serverCertificateHashes: [
-          {
-            algorithm: "sha-256",
-            value: buffer,
-          }
-        ]
-      });
-
-      console.log('Connecting to WebTransport at ', url);
-      // Wait for the connection to be ready
-      await transport.ready;
-      console.log('WebTransport connection established');
-
-      // Handle connection close
-      transport.closed.then(() => {
-        console.log('WebTransport connection closed');
-      }).catch((error) => {
-        console.error('WebTransport connection closed with error:', error);
-      });
-
-      // You can add more WebTransport handling logic here
-      // For example, handling incoming streams, sending data, etc.
-
-    } catch (error) {
-      console.error('Error connecting to stream:', error);
-      alert('Failed to connect to stream: ' + error.message);
-    }
-  }
-
-  showError(message) {
-    const contentDiv = document.getElementById('content');
-    contentDiv.innerHTML = `<div class="error">${message}</div>`;
-  }
-}
-
-// Initialize the app when the page loads
-document.addEventListener('DOMContentLoaded', () => {
-  new GameStreamingApp();
-});
-

moonlight-common-c-sys/build.rs

@@ -12,9 +12,13 @@ struct CustomCallbacks;
 impl ParseCallbacks for CustomCallbacks {
     fn int_macro(&self, name: &str, _value: i64) -> Option<IntKind> {
         match name {
-            "STREAM_CFG_LOCAL" => Some(IntKind::I32),
+            //"STREAM_CFG_LOCAL" => Some(IntKind::I32),
-            "STREAM_CFG_REMOTE" => Some(IntKind::I32),
+            //"STREAM_CFG_REMOTE" => Some(IntKind::I32),
-            "STREAM_CFG_AUTO" => Some(IntKind::I32),
+            //"STREAM_CFG_AUTO" => Some(IntKind::I32),
+            "KEY_ACTION_DOWN" => Some(IntKind::I8),
+            "KEY_ACTION_UP" => Some(IntKind::I8),
+            "BUTTON_ACTION_PRESS" => Some(IntKind::I8),
+            "BUTTON_ACTION_RELEASE" => Some(IntKind::I8),
             _ => None, // Default behavior for all others
         }
     }
@@ -34,7 +38,7 @@ fn main() {
     let bindings = bindgen::Builder::default()
         .header("moonlight-common-c/src/Limelight.h")
         .clang_arg(format!("-I{}/src", dst.display())) // Include built headers
-        //.parse_callbacks(Box::new(CustomCallbacks))
+        .parse_callbacks(Box::new(CustomCallbacks))
         .default_macro_constant_type(bindgen::MacroTypeVariation::Signed)
         .generate()
         .expect("Failed to generate bindings");

schema/input.fbs

@@ -0,0 +1,53 @@
+namespace InputEvent;
+
+
+enum KeyAction: byte {
+  DOWN,
+  UP,
+}
+
+
+table MouseMovement {
+  movement_x: int16;
+  movement_y: int16;
+}
+
+enum MouseButton: byte {
+  LEFT,
+  MIDDLE,
+  RIGHT,
+  X1,
+  X2
+}
+
+table MouseInput {
+  button: MouseButton;
+  button_action: KeyAction;
+}
+
+
+struct ModifierState {
+  shift: bool;
+  ctrl: bool;
+  alt: bool;
+  meta: bool;
+}
+
+table KeyboardInput {
+  key_code: int16;
+  key_action: KeyAction;
+  modifiers: ModifierState;
+}
+
+union Input {
+  Keyboard:KeyboardInput,
+  MouseMovement:MouseMovement,
+  MouseInput:MouseInput,
+}
+
+table InputEvent {
+  input: Input;
+}
+
+root_type InputEvent;
+

schema/video.fbs

@@ -0,0 +1,43 @@
+namespace VideoUpdate;
+
+
+table Setup {
+  video_format: string;
+  width: uint16;
+  height: uint16;
+  redraw_rate: uint16;
+}
+
+enum FrameType: byte {
+  PFRAME,
+  IDR,
+}
+
+table DecodeUnitStart {
+  frame_number: uint64;
+  frame_type: FrameType;
+  num_buffers: uint64;
+
+  receive_time_ms: uint16;
+  full_length: uint64;
+
+}
+
+table DecodeUnitBuffer {
+  frame_number: uint64;
+  buffer_index: uint64;
+  buffer_offset: uint64;
+  data: [ubyte]; 
+}
+
+union Update {
+    Setup:Setup,
+    DecodeUnitStart:DecodeUnitStart,
+    DecodeUnitBuffer:DecodeUnitBuffer,
+}
+
+table VideoUpdate {
+    update: Update; 
+}
+
+root_type VideoUpdate;