frontend: support input + refactoring

frontend/package.json

@@ -42,6 +42,7 @@
 		]
 	},
 	"dependencies": {
+		"flatbuffers": "^25.2.10",
 		"lucide-svelte": "^0.539.0",
 		"svelte-loading-spinners": "^0.3.6"
 	}

frontend/pnpm-lock.yaml

@@ -8,6 +8,9 @@ importers:
 
   .:
     dependencies:
+      flatbuffers:
+        specifier: ^25.2.10
+        version: 25.2.10
       lucide-svelte:
         specifier: ^0.539.0
         version: 0.539.0(svelte@5.36.12)
@@ -864,6 +867,9 @@ packages:
     resolution: {integrity: sha512-f7ccFPK3SXFHpx15UIGyRJ/FJQctuKZ0zVuN3frBo4HnK3cay9VEW0R6yPYFHC0AgqhukPzKjq22t5DmAyqGyw==}
     engines: {node: '>=16'}
 
+  flatbuffers@25.2.10:
+    resolution: {integrity: sha512-7JlN9ZvLDG1McO3kbX0k4v+SUAg48L1rIwEvN6ZQl/eCtgJz9UylTMzE9wrmYrcorgxm3CX/3T/w5VAub99UUw==}
+
   flatted@3.3.3:
     resolution: {integrity: sha512-GX+ysw4PBCz0PzosHDepZGANEuFCMLrnRTiEy9McGjmkCQYwRq4A/X786G/fjM/+OjsWSU1ZrY5qyARZmO/uwg==}
 
@@ -2167,6 +2173,8 @@ snapshots:
       flatted: 3.3.3
       keyv: 4.5.4
 
+  flatbuffers@25.2.10: {}
+
   flatted@3.3.3: {}
 
   fsevents@2.3.3:

frontend/src/lib/proto/input-event.ts

@@ -0,0 +1,12 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */
+
+export { Input } from './input-event/input.js';
+export { InputEvent } from './input-event/input-event.js';
+export { KeyAction } from './input-event/key-action.js';
+export { KeyboardInput } from './input-event/keyboard-input.js';
+export { ModifierState } from './input-event/modifier-state.js';
+export { MouseButton } from './input-event/mouse-button.js';
+export { MouseInput } from './input-event/mouse-input.js';
+export { MouseMovement } from './input-event/mouse-movement.js';

frontend/src/lib/proto/input-event/input-event.ts

@@ -0,0 +1,69 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */
+
+import * as flatbuffers from 'flatbuffers';
+
+import { Input, unionToInput, unionListToInput } from '../input-event/input.js';
+
+
+export class InputEvent {
+  bb: flatbuffers.ByteBuffer|null = null;
+  bb_pos = 0;
+  __init(i:number, bb:flatbuffers.ByteBuffer):InputEvent {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+}
+
+static getRootAsInputEvent(bb:flatbuffers.ByteBuffer, obj?:InputEvent):InputEvent {
+  return (obj || new InputEvent()).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+}
+
+static getSizePrefixedRootAsInputEvent(bb:flatbuffers.ByteBuffer, obj?:InputEvent):InputEvent {
+  bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH);
+  return (obj || new InputEvent()).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+}
+
+inputType():Input {
+  const offset = this.bb!.__offset(this.bb_pos, 4);
+  return offset ? this.bb!.readUint8(this.bb_pos + offset) : Input.NONE;
+}
+
+input<T extends flatbuffers.Table>(obj:any):any|null {
+  const offset = this.bb!.__offset(this.bb_pos, 6);
+  return offset ? this.bb!.__union(obj, this.bb_pos + offset) : null;
+}
+
+static startInputEvent(builder:flatbuffers.Builder) {
+  builder.startObject(2);
+}
+
+static addInputType(builder:flatbuffers.Builder, inputType:Input) {
+  builder.addFieldInt8(0, inputType, Input.NONE);
+}
+
+static addInput(builder:flatbuffers.Builder, inputOffset:flatbuffers.Offset) {
+  builder.addFieldOffset(1, inputOffset, 0);
+}
+
+static endInputEvent(builder:flatbuffers.Builder):flatbuffers.Offset {
+  const offset = builder.endObject();
+  return offset;
+}
+
+static finishInputEventBuffer(builder:flatbuffers.Builder, offset:flatbuffers.Offset) {
+  builder.finish(offset);
+}
+
+static finishSizePrefixedInputEventBuffer(builder:flatbuffers.Builder, offset:flatbuffers.Offset) {
+  builder.finish(offset, undefined, true);
+}
+
+static createInputEvent(builder:flatbuffers.Builder, inputType:Input, inputOffset:flatbuffers.Offset):flatbuffers.Offset {
+  InputEvent.startInputEvent(builder);
+  InputEvent.addInputType(builder, inputType);
+  InputEvent.addInput(builder, inputOffset);
+  return InputEvent.endInputEvent(builder);
+}
+}

frontend/src/lib/proto/input-event/input.ts

@@ -0,0 +1,42 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */
+
+import { KeyboardInput } from '../input-event/keyboard-input.js';
+import { MouseInput } from '../input-event/mouse-input.js';
+import { MouseMovement } from '../input-event/mouse-movement.js';
+
+
+export enum Input {
+  NONE = 0,
+  Keyboard = 1,
+  MouseMovement = 2,
+  MouseInput = 3
+}
+
+export function unionToInput(
+  type: Input,
+  accessor: (obj:KeyboardInput|MouseInput|MouseMovement) => KeyboardInput|MouseInput|MouseMovement|null
+): KeyboardInput|MouseInput|MouseMovement|null {
+  switch(Input[type]) {
+    case 'NONE': return null; 
+    case 'Keyboard': return accessor(new KeyboardInput())! as KeyboardInput;
+    case 'MouseMovement': return accessor(new MouseMovement())! as MouseMovement;
+    case 'MouseInput': return accessor(new MouseInput())! as MouseInput;
+    default: return null;
+  }
+}
+
+export function unionListToInput(
+  type: Input, 
+  accessor: (index: number, obj:KeyboardInput|MouseInput|MouseMovement) => KeyboardInput|MouseInput|MouseMovement|null, 
+  index: number
+): KeyboardInput|MouseInput|MouseMovement|null {
+  switch(Input[type]) {
+    case 'NONE': return null; 
+    case 'Keyboard': return accessor(index, new KeyboardInput())! as KeyboardInput;
+    case 'MouseMovement': return accessor(index, new MouseMovement())! as MouseMovement;
+    case 'MouseInput': return accessor(index, new MouseInput())! as MouseInput;
+    default: return null;
+  }
+}

frontend/src/lib/proto/input-event/key-action.ts

@@ -0,0 +1,8 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */
+
+export enum KeyAction {
+  DOWN = 0,
+  UP = 1
+}

frontend/src/lib/proto/input-event/keyboard-input.ts

@@ -0,0 +1,65 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */
+
+import * as flatbuffers from 'flatbuffers';
+
+import { KeyAction } from '../input-event/key-action.js';
+import { ModifierState } from '../input-event/modifier-state.js';
+
+
+export class KeyboardInput {
+  bb: flatbuffers.ByteBuffer|null = null;
+  bb_pos = 0;
+  __init(i:number, bb:flatbuffers.ByteBuffer):KeyboardInput {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+}
+
+static getRootAsKeyboardInput(bb:flatbuffers.ByteBuffer, obj?:KeyboardInput):KeyboardInput {
+  return (obj || new KeyboardInput()).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+}
+
+static getSizePrefixedRootAsKeyboardInput(bb:flatbuffers.ByteBuffer, obj?:KeyboardInput):KeyboardInput {
+  bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH);
+  return (obj || new KeyboardInput()).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+}
+
+keyCode():number {
+  const offset = this.bb!.__offset(this.bb_pos, 4);
+  return offset ? this.bb!.readInt16(this.bb_pos + offset) : 0;
+}
+
+keyAction():KeyAction {
+  const offset = this.bb!.__offset(this.bb_pos, 6);
+  return offset ? this.bb!.readInt8(this.bb_pos + offset) : KeyAction.DOWN;
+}
+
+modifiers(obj?:ModifierState):ModifierState|null {
+  const offset = this.bb!.__offset(this.bb_pos, 8);
+  return offset ? (obj || new ModifierState()).__init(this.bb_pos + offset, this.bb!) : null;
+}
+
+static startKeyboardInput(builder:flatbuffers.Builder) {
+  builder.startObject(3);
+}
+
+static addKeyCode(builder:flatbuffers.Builder, keyCode:number) {
+  builder.addFieldInt16(0, keyCode, 0);
+}
+
+static addKeyAction(builder:flatbuffers.Builder, keyAction:KeyAction) {
+  builder.addFieldInt8(1, keyAction, KeyAction.DOWN);
+}
+
+static addModifiers(builder:flatbuffers.Builder, modifiersOffset:flatbuffers.Offset) {
+  builder.addFieldStruct(2, modifiersOffset, 0);
+}
+
+static endKeyboardInput(builder:flatbuffers.Builder):flatbuffers.Offset {
+  const offset = builder.endObject();
+  return offset;
+}
+
+}

frontend/src/lib/proto/input-event/modifier-state.ts

@@ -0,0 +1,45 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */
+
+import * as flatbuffers from 'flatbuffers';
+
+export class ModifierState {
+  bb: flatbuffers.ByteBuffer|null = null;
+  bb_pos = 0;
+  __init(i:number, bb:flatbuffers.ByteBuffer):ModifierState {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+}
+
+shift():boolean {
+  return !!this.bb!.readInt8(this.bb_pos);
+}
+
+ctrl():boolean {
+  return !!this.bb!.readInt8(this.bb_pos + 1);
+}
+
+alt():boolean {
+  return !!this.bb!.readInt8(this.bb_pos + 2);
+}
+
+meta():boolean {
+  return !!this.bb!.readInt8(this.bb_pos + 3);
+}
+
+static sizeOf():number {
+  return 4;
+}
+
+static createModifierState(builder:flatbuffers.Builder, shift: boolean, ctrl: boolean, alt: boolean, meta: boolean):flatbuffers.Offset {
+  builder.prep(1, 4);
+  builder.writeInt8(Number(Boolean(meta)));
+  builder.writeInt8(Number(Boolean(alt)));
+  builder.writeInt8(Number(Boolean(ctrl)));
+  builder.writeInt8(Number(Boolean(shift)));
+  return builder.offset();
+}
+
+}

frontend/src/lib/proto/input-event/mouse-button.ts

@@ -0,0 +1,11 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */
+
+export enum MouseButton {
+  LEFT = 0,
+  MIDDLE = 1,
+  RIGHT = 2,
+  X1 = 3,
+  X2 = 4
+}

frontend/src/lib/proto/input-event/mouse-input.ts

@@ -0,0 +1,62 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */
+
+import * as flatbuffers from 'flatbuffers';
+
+import { KeyAction } from '../input-event/key-action.js';
+import { MouseButton } from '../input-event/mouse-button.js';
+
+
+export class MouseInput {
+  bb: flatbuffers.ByteBuffer|null = null;
+  bb_pos = 0;
+  __init(i:number, bb:flatbuffers.ByteBuffer):MouseInput {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+}
+
+static getRootAsMouseInput(bb:flatbuffers.ByteBuffer, obj?:MouseInput):MouseInput {
+  return (obj || new MouseInput()).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+}
+
+static getSizePrefixedRootAsMouseInput(bb:flatbuffers.ByteBuffer, obj?:MouseInput):MouseInput {
+  bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH);
+  return (obj || new MouseInput()).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+}
+
+button():MouseButton {
+  const offset = this.bb!.__offset(this.bb_pos, 4);
+  return offset ? this.bb!.readInt8(this.bb_pos + offset) : MouseButton.LEFT;
+}
+
+buttonAction():KeyAction {
+  const offset = this.bb!.__offset(this.bb_pos, 6);
+  return offset ? this.bb!.readInt8(this.bb_pos + offset) : KeyAction.DOWN;
+}
+
+static startMouseInput(builder:flatbuffers.Builder) {
+  builder.startObject(2);
+}
+
+static addButton(builder:flatbuffers.Builder, button:MouseButton) {
+  builder.addFieldInt8(0, button, MouseButton.LEFT);
+}
+
+static addButtonAction(builder:flatbuffers.Builder, buttonAction:KeyAction) {
+  builder.addFieldInt8(1, buttonAction, KeyAction.DOWN);
+}
+
+static endMouseInput(builder:flatbuffers.Builder):flatbuffers.Offset {
+  const offset = builder.endObject();
+  return offset;
+}
+
+static createMouseInput(builder:flatbuffers.Builder, button:MouseButton, buttonAction:KeyAction):flatbuffers.Offset {
+  MouseInput.startMouseInput(builder);
+  MouseInput.addButton(builder, button);
+  MouseInput.addButtonAction(builder, buttonAction);
+  return MouseInput.endMouseInput(builder);
+}
+}

frontend/src/lib/proto/input-event/mouse-movement.ts

@@ -0,0 +1,58 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */
+
+import * as flatbuffers from 'flatbuffers';
+
+export class MouseMovement {
+  bb: flatbuffers.ByteBuffer|null = null;
+  bb_pos = 0;
+  __init(i:number, bb:flatbuffers.ByteBuffer):MouseMovement {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+}
+
+static getRootAsMouseMovement(bb:flatbuffers.ByteBuffer, obj?:MouseMovement):MouseMovement {
+  return (obj || new MouseMovement()).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+}
+
+static getSizePrefixedRootAsMouseMovement(bb:flatbuffers.ByteBuffer, obj?:MouseMovement):MouseMovement {
+  bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH);
+  return (obj || new MouseMovement()).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+}
+
+movementX():number {
+  const offset = this.bb!.__offset(this.bb_pos, 4);
+  return offset ? this.bb!.readInt16(this.bb_pos + offset) : 0;
+}
+
+movementY():number {
+  const offset = this.bb!.__offset(this.bb_pos, 6);
+  return offset ? this.bb!.readInt16(this.bb_pos + offset) : 0;
+}
+
+static startMouseMovement(builder:flatbuffers.Builder) {
+  builder.startObject(2);
+}
+
+static addMovementX(builder:flatbuffers.Builder, movementX:number) {
+  builder.addFieldInt16(0, movementX, 0);
+}
+
+static addMovementY(builder:flatbuffers.Builder, movementY:number) {
+  builder.addFieldInt16(1, movementY, 0);
+}
+
+static endMouseMovement(builder:flatbuffers.Builder):flatbuffers.Offset {
+  const offset = builder.endObject();
+  return offset;
+}
+
+static createMouseMovement(builder:flatbuffers.Builder, movementX:number, movementY:number):flatbuffers.Offset {
+  MouseMovement.startMouseMovement(builder);
+  MouseMovement.addMovementX(builder, movementX);
+  MouseMovement.addMovementY(builder, movementY);
+  return MouseMovement.endMouseMovement(builder);
+}
+}

frontend/src/lib/proto/input.ts

@@ -0,0 +1,5 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */
+
+export * as InputEvent from './input-event.js';

frontend/src/routes/getStreamData.ts

@@ -37,15 +37,15 @@ export async function getStreamData(appId: number, server_name: string): Promise
       server: server_name,
       server_mode: {
         fps: 60,
-        width: 1280,
-        height: 720,
+        width: 1920,
+        height: 1080,
       },
       stream_config: {
-        bitrate_kbps: 5120,
+        bitrate_kbps: 10240,
         mode: {
           fps: 60,
-          width: 1280,
-          height: 720,
+          width: 1920,
+          height: 1080,
         }
       }
     };

frontend/src/routes/stream/Stream.svelte

@@ -1,6 +1,6 @@
 <script lang="ts">
 	import { onMount } from 'svelte';
-	import { streamUrl } from './stream';
+	import { startWebtransportStream } from './stream';
 	import StreamUi from './StreamUi.svelte';
 
 	interface Props {
@@ -15,8 +15,7 @@
 	let gameplayCanvas: HTMLCanvasElement;
 
 	async function startStream() {
-		console.log(`Connecting to stream at ${url} with cert_hash ${certHash}`);
-		await streamUrl(url, certHash, gameplayCanvas);
+		await startWebtransportStream(url, certHash, gameplayCanvas, gameplayCanvas, gameplayCanvas);
 	}
 
 	async function requestFullscreen() {
@@ -29,10 +28,8 @@
 			await document.exitFullscreen();
 			fullscreen = false;
 		} else {
-			if (gameplayView.requestFullscreen) {
-				await gameplayView.requestFullscreen();
-				fullscreen = true;
-			}
+			await gameplayCanvas.requestFullscreen();
+			fullscreen = true;
 		}
 	}
 
@@ -42,7 +39,8 @@
 </script>
 
 <div id="gameplay-view" class="gameplay-view" bind:this={gameplayView}>
-	<canvas id="gamestream-canvas" class="gamestream-canvas" bind:this={gameplayCanvas}></canvas>
+	<canvas id="gamestream-canvas" class="gamestream-canvas" bind:this={gameplayCanvas} tabindex="0"
+	></canvas>
 	<StreamUi fullscreenFunc={requestFullscreen}></StreamUi>
 </div>
 
@@ -56,6 +54,9 @@
 		background-color: black;
 
 		position: relative;
+
+		display: flex;
+		flex-direction: column;
 	}
 
 	.gamestream-canvas {
@@ -65,6 +66,6 @@
 		position: relative;
 		margin: auto;
 
-		z-index: 1;
+		/*z-index: 1;*/
 	}
 </style>

frontend/src/routes/stream/StreamUi.svelte

@@ -10,11 +10,11 @@
 
 <div class="stream-controls">
 	<div style="flex-grow: 1"></div>
-	<div class="bar-button" role="button" tabindex="0">
+	<div class="bar-button" role="button">
 		<Settings size="100%" />
 	</div>
 	<div style="width: 0.5%"></div>
-	<div class="bar-button" onclick={fullscreenFunc} role="button" tabindex="0">
+	<div class="bar-button" onclick={fullscreenFunc} role="button">
 		<Maximize size="100%" />
 	</div>
 	<div style="width: 0.5%"></div>
@@ -41,12 +41,12 @@
 		height: 5%;
 		width: 100%;
 
-		position: absolute;
+		/*position: absolute;*/
 
 		bottom: 0;
 		left: 0;
 
-		z-index: 2;
+		/*z-index: 2;*/
 
 		border-top-color: oklch(0.454 0 360);
 		border-top-width: 2px;

frontend/src/routes/stream/stream.ts

@@ -1,3 +1,7 @@
+import { Input } from "$lib/proto/input-event"
+import { InputEvent } from "../../lib/proto/input"
+import { Builder } from "flatbuffers"
+
 type Setup = {
   video_format: string,
   width: number,
@@ -26,9 +30,8 @@ type DecodeUnitPacket = {
   DecodeUnit: DecodeUnit
 }
 
-async function connectToUrl(url: string, certHash: Uint8Array): Promise<WebTransport> {
-  console.log('Hash: ', certHash);
-  console.log(`Connecting to stream`);
+export async function getStreamTransport(url: string, certHash: Array<number>): Promise<WebTransport> {
+  let certHashArray = new Uint8Array(certHash);
 
   // Check if WebTransport is supported
   if (!window.WebTransport) {
@@ -39,7 +42,7 @@ async function connectToUrl(url: string, certHash: Uint8Array): Promise<WebTrans
     serverCertificateHashes: [
       {
         algorithm: "sha-256",
-        value: certHash,
+        value: certHashArray,
       }
     ]
   });
@@ -84,20 +87,12 @@ function parseData(newBuffer: Uint8Array, oldBuffer: Uint8Array): [Array<Object>
   return [packets, unparsedData];
 }
 
-export async function streamUrl(url: string, certHash: Array<number>, canvasElement: HTMLCanvasElement) {
-  const certHashArray = new Uint8Array(certHash);
-
-  const transport = await connectToUrl(url, certHashArray);
-  console.log('Creating WebTransport bidirectional stream');
-  const stream = await transport.createBidirectionalStream();
-  console.log('Bidirectional stream created');
-
+export async function streamVideoFromReader(reader: ReadableStreamDefaultReader, canvasElement: HTMLCanvasElement) {
   const canvasCtx: CanvasRenderingContext2D | null = canvasElement.getContext('2d');
   if (canvasCtx == null) {
     throw new Error(`Could not get 2d canvas context`);
   }
 
-  const reader = stream.readable.getReader();
   try {
     let unparsedData = new Uint8Array();
 
@@ -171,16 +166,6 @@ export async function streamUrl(url: string, certHash: Array<number>, canvasElem
 
     }
 
-    // Handle connection close
-    transport.closed.then(() => {
-      console.log('WebTransport connection closed');
-    }).catch((error) => {
-      console.error('WebTransport connection closed with error:', error);
-    });
-
-    // You can add more WebTransport handling logic here
-    // For example, handling incoming streams, sending data, etc.
-
   } catch (e) {
     var error = <Error>e;
     console.error('Error connecting to stream:', error);
@@ -188,3 +173,202 @@ export async function streamUrl(url: string, certHash: Array<number>, canvasElem
   }
 }
 
+enum KeyAction {
+  DOWN,
+  UP,
+}
+
+async function sendKeyboardEvent(
+  writer: WritableStreamDefaultWriter,
+  event: KeyboardEvent,
+  keyAction: KeyAction,
+) {
+  const builder = new Builder(1024);
+
+  InputEvent.KeyboardInput.startKeyboardInput(builder);
+  // Have to use keyCode even though it's deprecated beacuse .code is a unhelpful string :sigh:
+  InputEvent.KeyboardInput.addKeyCode(builder, event.keyCode);
+  InputEvent.KeyboardInput.addKeyAction(builder, keyAction);
+  const modifierState = InputEvent.ModifierState.createModifierState(
+    builder,
+    event.shiftKey,
+    event.ctrlKey,
+    event.altKey,
+    event.metaKey
+  );
+
+  InputEvent.KeyboardInput.addModifiers(builder, modifierState);
+  const keyboardInput = InputEvent.KeyboardInput.endKeyboardInput(builder);
+
+  // Create InputEvent table with union
+  InputEvent.InputEvent.startInputEvent(builder);
+  InputEvent.InputEvent.addInputType(builder, Input.Keyboard);
+  InputEvent.InputEvent.addInput(builder, keyboardInput);
+  const inputEvent = InputEvent.InputEvent.endInputEvent(builder);
+
+  // Finish the buffer
+  builder.finish(inputEvent);
+
+  // Get the buffer data
+  const bufferData = builder.asUint8Array();
+
+  // Create size prefix as big-endian u32
+  const sizePrefix = new Uint8Array(4);
+  const size = bufferData.length;
+  sizePrefix[0] = (size >> 24) & 0xFF;  // Most significant byte
+  sizePrefix[1] = (size >> 16) & 0xFF;
+  sizePrefix[2] = (size >> 8) & 0xFF;
+  sizePrefix[3] = size & 0xFF;          // Least significant byte
+
+  // Create combined buffer with size prefix + flatbuffer data
+  const combined = new Uint8Array(sizePrefix.length + bufferData.length);
+  combined.set(sizePrefix, 0);
+  combined.set(bufferData, sizePrefix.length);
+
+  // Write to stream
+  await writer.write(combined);
+}
+
+async function sendMouseMoveEvent(
+  writer: WritableStreamDefaultWriter,
+  event: MouseEvent,
+) {
+  const builder = new Builder(1024);
+
+  const mouseMovement = InputEvent.MouseMovement.createMouseMovement(builder, event.movementX, event.movementY);
+
+  // Create InputEvent table with union
+  InputEvent.InputEvent.startInputEvent(builder);
+  InputEvent.InputEvent.addInputType(builder, Input.MouseMovement);
+  InputEvent.InputEvent.addInput(builder, mouseMovement);
+  const inputEvent = InputEvent.InputEvent.endInputEvent(builder);
+
+  // Finish the buffer
+  builder.finish(inputEvent);
+
+  // Get the buffer data
+  const bufferData = builder.asUint8Array();
+
+  // Create size prefix as big-endian u32
+  const sizePrefix = new Uint8Array(4);
+  const size = bufferData.length;
+  sizePrefix[0] = (size >> 24) & 0xFF;  // Most significant byte
+  sizePrefix[1] = (size >> 16) & 0xFF;
+  sizePrefix[2] = (size >> 8) & 0xFF;
+  sizePrefix[3] = size & 0xFF;          // Least significant byte
+
+  // Create combined buffer with size prefix + flatbuffer data
+  const combined = new Uint8Array(sizePrefix.length + bufferData.length);
+  combined.set(sizePrefix, 0);
+  combined.set(bufferData, sizePrefix.length);
+
+  // Write to stream
+  await writer.write(combined);
+}
+
+async function sendMouseInputEvent(
+  writer: WritableStreamDefaultWriter,
+  event: MouseEvent,
+  keyAction: KeyAction,
+) {
+  const builder = new Builder(1024);
+
+  var mouseButton: InputEvent.MouseButton;
+
+  switch (event.button) {
+    case 0:
+      mouseButton = InputEvent.MouseButton.LEFT;
+      break;
+    case 1:
+      mouseButton = InputEvent.MouseButton.MIDDLE;
+      break;
+    case 2:
+      mouseButton = InputEvent.MouseButton.RIGHT;
+      break;
+    default:
+      console.log(`Unknown mouse button ${event.button}, defaulting to LEFT`);
+      mouseButton = InputEvent.MouseButton.LEFT;
+      break;
+  }
+
+  const mouseInput = InputEvent.MouseInput.createMouseInput(builder, mouseButton, keyAction);
+
+  // Create InputEvent table with union
+  InputEvent.InputEvent.startInputEvent(builder);
+  InputEvent.InputEvent.addInputType(builder, Input.MouseInput);
+  InputEvent.InputEvent.addInput(builder, mouseInput);
+  const inputEvent = InputEvent.InputEvent.endInputEvent(builder);
+
+  // Finish the buffer
+  builder.finish(inputEvent);
+
+  // Get the buffer data
+  const bufferData = builder.asUint8Array();
+
+  // Create size prefix as big-endian u32
+  const sizePrefix = new Uint8Array(4);
+  const size = bufferData.length;
+  sizePrefix[0] = (size >> 24) & 0xFF;  // Most significant byte
+  sizePrefix[1] = (size >> 16) & 0xFF;
+  sizePrefix[2] = (size >> 8) & 0xFF;
+  sizePrefix[3] = size & 0xFF;          // Least significant byte
+
+  // Create combined buffer with size prefix + flatbuffer data
+  const combined = new Uint8Array(sizePrefix.length + bufferData.length);
+  combined.set(sizePrefix, 0);
+  combined.set(bufferData, sizePrefix.length);
+
+  // Write to stream
+  await writer.write(combined);
+}
+
+
+
+export async function startWebtransportStream(
+  url: string,
+  certHash: Array<number>,
+  gameplayCanvas: HTMLCanvasElement,
+  keyEventElement: HTMLElement,
+  mouseElement: HTMLElement,
+) {
+  console.log(`Connecting to stream at ${url} with cert_hash ${certHash}`);
+  const transport = await getStreamTransport(url, certHash);
+
+  const stream = await transport.createBidirectionalStream();
+
+  const reader = stream.readable.getReader();
+  const writer = stream.writable.getWriter();
+
+  keyEventElement.addEventListener("keydown", (event: KeyboardEvent) => { sendKeyboardEvent(writer, event, KeyAction.DOWN) });
+  keyEventElement.addEventListener("keyup", (event: KeyboardEvent) => { sendKeyboardEvent(writer, event, KeyAction.UP) });
+
+  mouseElement.addEventListener("mousedown", (event: MouseEvent) => { sendMouseInputEvent(writer, event, KeyAction.DOWN) });
+  mouseElement.addEventListener("mouseup", (event: MouseEvent) => { sendMouseInputEvent(writer, event, KeyAction.UP) });
+
+
+  mouseElement.addEventListener("click", async () => {
+    console.log("Requesting pointer lock");
+    await mouseElement.requestPointerLock({
+      unadjustedMovement: true,
+    });
+    console.log("Pointer lock aquired");
+  })
+
+  mouseElement.addEventListener("mousemove", (event: MouseEvent) => {
+    if (document.pointerLockElement === mouseElement) {
+      sendMouseMoveEvent(writer, event);
+    }
+  });
+
+
+  await streamVideoFromReader(reader, gameplayCanvas);
+
+  // Handle connection close
+  transport.closed
+    .then(() => {
+      console.log('WebTransport connection closed');
+    })
+    .catch((error) => {
+      console.error('WebTransport connection closed with error:', error);
+    });
+}