diff --git a/frontend/package.json b/frontend/package.json index 071203c..0254621 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -42,6 +42,7 @@ ] }, "dependencies": { + "flatbuffers": "^25.2.10", "lucide-svelte": "^0.539.0", "svelte-loading-spinners": "^0.3.6" } diff --git a/frontend/pnpm-lock.yaml b/frontend/pnpm-lock.yaml index 5e75462..8f0c817 100644 --- a/frontend/pnpm-lock.yaml +++ b/frontend/pnpm-lock.yaml @@ -8,6 +8,9 @@ importers: .: dependencies: + flatbuffers: + specifier: ^25.2.10 + version: 25.2.10 lucide-svelte: specifier: ^0.539.0 version: 0.539.0(svelte@5.36.12) @@ -864,6 +867,9 @@ packages: resolution: {integrity: sha512-f7ccFPK3SXFHpx15UIGyRJ/FJQctuKZ0zVuN3frBo4HnK3cay9VEW0R6yPYFHC0AgqhukPzKjq22t5DmAyqGyw==} engines: {node: '>=16'} + flatbuffers@25.2.10: + resolution: {integrity: sha512-7JlN9ZvLDG1McO3kbX0k4v+SUAg48L1rIwEvN6ZQl/eCtgJz9UylTMzE9wrmYrcorgxm3CX/3T/w5VAub99UUw==} + flatted@3.3.3: resolution: {integrity: sha512-GX+ysw4PBCz0PzosHDepZGANEuFCMLrnRTiEy9McGjmkCQYwRq4A/X786G/fjM/+OjsWSU1ZrY5qyARZmO/uwg==} @@ -2167,6 +2173,8 @@ snapshots: flatted: 3.3.3 keyv: 4.5.4 + flatbuffers@25.2.10: {} + flatted@3.3.3: {} fsevents@2.3.3: diff --git a/frontend/src/lib/proto/input-event.ts b/frontend/src/lib/proto/input-event.ts new file mode 100644 index 0000000..746fda0 --- /dev/null +++ b/frontend/src/lib/proto/input-event.ts @@ -0,0 +1,12 @@ +// automatically generated by the FlatBuffers compiler, do not modify + +/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */ + +export { Input } from './input-event/input.js'; +export { InputEvent } from './input-event/input-event.js'; +export { KeyAction } from './input-event/key-action.js'; +export { KeyboardInput } from './input-event/keyboard-input.js'; +export { ModifierState } from './input-event/modifier-state.js'; +export { MouseButton } from './input-event/mouse-button.js'; +export { MouseInput } from './input-event/mouse-input.js'; +export { MouseMovement } from './input-event/mouse-movement.js'; diff --git a/frontend/src/lib/proto/input-event/input-event.ts b/frontend/src/lib/proto/input-event/input-event.ts new file mode 100644 index 0000000..42feb3c --- /dev/null +++ b/frontend/src/lib/proto/input-event/input-event.ts @@ -0,0 +1,69 @@ +// automatically generated by the FlatBuffers compiler, do not modify + +/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */ + +import * as flatbuffers from 'flatbuffers'; + +import { Input, unionToInput, unionListToInput } from '../input-event/input.js'; + + +export class InputEvent { + bb: flatbuffers.ByteBuffer|null = null; + bb_pos = 0; + __init(i:number, bb:flatbuffers.ByteBuffer):InputEvent { + this.bb_pos = i; + this.bb = bb; + return this; +} + +static getRootAsInputEvent(bb:flatbuffers.ByteBuffer, obj?:InputEvent):InputEvent { + return (obj || new InputEvent()).__init(bb.readInt32(bb.position()) + bb.position(), bb); +} + +static getSizePrefixedRootAsInputEvent(bb:flatbuffers.ByteBuffer, obj?:InputEvent):InputEvent { + bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH); + return (obj || new InputEvent()).__init(bb.readInt32(bb.position()) + bb.position(), bb); +} + +inputType():Input { + const offset = this.bb!.__offset(this.bb_pos, 4); + return offset ? this.bb!.readUint8(this.bb_pos + offset) : Input.NONE; +} + +input(obj:any):any|null { + const offset = this.bb!.__offset(this.bb_pos, 6); + return offset ? this.bb!.__union(obj, this.bb_pos + offset) : null; +} + +static startInputEvent(builder:flatbuffers.Builder) { + builder.startObject(2); +} + +static addInputType(builder:flatbuffers.Builder, inputType:Input) { + builder.addFieldInt8(0, inputType, Input.NONE); +} + +static addInput(builder:flatbuffers.Builder, inputOffset:flatbuffers.Offset) { + builder.addFieldOffset(1, inputOffset, 0); +} + +static endInputEvent(builder:flatbuffers.Builder):flatbuffers.Offset { + const offset = builder.endObject(); + return offset; +} + +static finishInputEventBuffer(builder:flatbuffers.Builder, offset:flatbuffers.Offset) { + builder.finish(offset); +} + +static finishSizePrefixedInputEventBuffer(builder:flatbuffers.Builder, offset:flatbuffers.Offset) { + builder.finish(offset, undefined, true); +} + +static createInputEvent(builder:flatbuffers.Builder, inputType:Input, inputOffset:flatbuffers.Offset):flatbuffers.Offset { + InputEvent.startInputEvent(builder); + InputEvent.addInputType(builder, inputType); + InputEvent.addInput(builder, inputOffset); + return InputEvent.endInputEvent(builder); +} +} diff --git a/frontend/src/lib/proto/input-event/input.ts b/frontend/src/lib/proto/input-event/input.ts new file mode 100644 index 0000000..84b6931 --- /dev/null +++ b/frontend/src/lib/proto/input-event/input.ts @@ -0,0 +1,42 @@ +// automatically generated by the FlatBuffers compiler, do not modify + +/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */ + +import { KeyboardInput } from '../input-event/keyboard-input.js'; +import { MouseInput } from '../input-event/mouse-input.js'; +import { MouseMovement } from '../input-event/mouse-movement.js'; + + +export enum Input { + NONE = 0, + Keyboard = 1, + MouseMovement = 2, + MouseInput = 3 +} + +export function unionToInput( + type: Input, + accessor: (obj:KeyboardInput|MouseInput|MouseMovement) => KeyboardInput|MouseInput|MouseMovement|null +): KeyboardInput|MouseInput|MouseMovement|null { + switch(Input[type]) { + case 'NONE': return null; + case 'Keyboard': return accessor(new KeyboardInput())! as KeyboardInput; + case 'MouseMovement': return accessor(new MouseMovement())! as MouseMovement; + case 'MouseInput': return accessor(new MouseInput())! as MouseInput; + default: return null; + } +} + +export function unionListToInput( + type: Input, + accessor: (index: number, obj:KeyboardInput|MouseInput|MouseMovement) => KeyboardInput|MouseInput|MouseMovement|null, + index: number +): KeyboardInput|MouseInput|MouseMovement|null { + switch(Input[type]) { + case 'NONE': return null; + case 'Keyboard': return accessor(index, new KeyboardInput())! as KeyboardInput; + case 'MouseMovement': return accessor(index, new MouseMovement())! as MouseMovement; + case 'MouseInput': return accessor(index, new MouseInput())! as MouseInput; + default: return null; + } +} diff --git a/frontend/src/lib/proto/input-event/key-action.ts b/frontend/src/lib/proto/input-event/key-action.ts new file mode 100644 index 0000000..e9d3938 --- /dev/null +++ b/frontend/src/lib/proto/input-event/key-action.ts @@ -0,0 +1,8 @@ +// automatically generated by the FlatBuffers compiler, do not modify + +/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */ + +export enum KeyAction { + DOWN = 0, + UP = 1 +} diff --git a/frontend/src/lib/proto/input-event/keyboard-input.ts b/frontend/src/lib/proto/input-event/keyboard-input.ts new file mode 100644 index 0000000..ccd3230 --- /dev/null +++ b/frontend/src/lib/proto/input-event/keyboard-input.ts @@ -0,0 +1,65 @@ +// automatically generated by the FlatBuffers compiler, do not modify + +/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */ + +import * as flatbuffers from 'flatbuffers'; + +import { KeyAction } from '../input-event/key-action.js'; +import { ModifierState } from '../input-event/modifier-state.js'; + + +export class KeyboardInput { + bb: flatbuffers.ByteBuffer|null = null; + bb_pos = 0; + __init(i:number, bb:flatbuffers.ByteBuffer):KeyboardInput { + this.bb_pos = i; + this.bb = bb; + return this; +} + +static getRootAsKeyboardInput(bb:flatbuffers.ByteBuffer, obj?:KeyboardInput):KeyboardInput { + return (obj || new KeyboardInput()).__init(bb.readInt32(bb.position()) + bb.position(), bb); +} + +static getSizePrefixedRootAsKeyboardInput(bb:flatbuffers.ByteBuffer, obj?:KeyboardInput):KeyboardInput { + bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH); + return (obj || new KeyboardInput()).__init(bb.readInt32(bb.position()) + bb.position(), bb); +} + +keyCode():number { + const offset = this.bb!.__offset(this.bb_pos, 4); + return offset ? this.bb!.readInt16(this.bb_pos + offset) : 0; +} + +keyAction():KeyAction { + const offset = this.bb!.__offset(this.bb_pos, 6); + return offset ? this.bb!.readInt8(this.bb_pos + offset) : KeyAction.DOWN; +} + +modifiers(obj?:ModifierState):ModifierState|null { + const offset = this.bb!.__offset(this.bb_pos, 8); + return offset ? (obj || new ModifierState()).__init(this.bb_pos + offset, this.bb!) : null; +} + +static startKeyboardInput(builder:flatbuffers.Builder) { + builder.startObject(3); +} + +static addKeyCode(builder:flatbuffers.Builder, keyCode:number) { + builder.addFieldInt16(0, keyCode, 0); +} + +static addKeyAction(builder:flatbuffers.Builder, keyAction:KeyAction) { + builder.addFieldInt8(1, keyAction, KeyAction.DOWN); +} + +static addModifiers(builder:flatbuffers.Builder, modifiersOffset:flatbuffers.Offset) { + builder.addFieldStruct(2, modifiersOffset, 0); +} + +static endKeyboardInput(builder:flatbuffers.Builder):flatbuffers.Offset { + const offset = builder.endObject(); + return offset; +} + +} diff --git a/frontend/src/lib/proto/input-event/modifier-state.ts b/frontend/src/lib/proto/input-event/modifier-state.ts new file mode 100644 index 0000000..11ef2a0 --- /dev/null +++ b/frontend/src/lib/proto/input-event/modifier-state.ts @@ -0,0 +1,45 @@ +// automatically generated by the FlatBuffers compiler, do not modify + +/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */ + +import * as flatbuffers from 'flatbuffers'; + +export class ModifierState { + bb: flatbuffers.ByteBuffer|null = null; + bb_pos = 0; + __init(i:number, bb:flatbuffers.ByteBuffer):ModifierState { + this.bb_pos = i; + this.bb = bb; + return this; +} + +shift():boolean { + return !!this.bb!.readInt8(this.bb_pos); +} + +ctrl():boolean { + return !!this.bb!.readInt8(this.bb_pos + 1); +} + +alt():boolean { + return !!this.bb!.readInt8(this.bb_pos + 2); +} + +meta():boolean { + return !!this.bb!.readInt8(this.bb_pos + 3); +} + +static sizeOf():number { + return 4; +} + +static createModifierState(builder:flatbuffers.Builder, shift: boolean, ctrl: boolean, alt: boolean, meta: boolean):flatbuffers.Offset { + builder.prep(1, 4); + builder.writeInt8(Number(Boolean(meta))); + builder.writeInt8(Number(Boolean(alt))); + builder.writeInt8(Number(Boolean(ctrl))); + builder.writeInt8(Number(Boolean(shift))); + return builder.offset(); +} + +} diff --git a/frontend/src/lib/proto/input-event/mouse-button.ts b/frontend/src/lib/proto/input-event/mouse-button.ts new file mode 100644 index 0000000..0518d42 --- /dev/null +++ b/frontend/src/lib/proto/input-event/mouse-button.ts @@ -0,0 +1,11 @@ +// automatically generated by the FlatBuffers compiler, do not modify + +/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */ + +export enum MouseButton { + LEFT = 0, + MIDDLE = 1, + RIGHT = 2, + X1 = 3, + X2 = 4 +} diff --git a/frontend/src/lib/proto/input-event/mouse-input.ts b/frontend/src/lib/proto/input-event/mouse-input.ts new file mode 100644 index 0000000..df61357 --- /dev/null +++ b/frontend/src/lib/proto/input-event/mouse-input.ts @@ -0,0 +1,62 @@ +// automatically generated by the FlatBuffers compiler, do not modify + +/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */ + +import * as flatbuffers from 'flatbuffers'; + +import { KeyAction } from '../input-event/key-action.js'; +import { MouseButton } from '../input-event/mouse-button.js'; + + +export class MouseInput { + bb: flatbuffers.ByteBuffer|null = null; + bb_pos = 0; + __init(i:number, bb:flatbuffers.ByteBuffer):MouseInput { + this.bb_pos = i; + this.bb = bb; + return this; +} + +static getRootAsMouseInput(bb:flatbuffers.ByteBuffer, obj?:MouseInput):MouseInput { + return (obj || new MouseInput()).__init(bb.readInt32(bb.position()) + bb.position(), bb); +} + +static getSizePrefixedRootAsMouseInput(bb:flatbuffers.ByteBuffer, obj?:MouseInput):MouseInput { + bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH); + return (obj || new MouseInput()).__init(bb.readInt32(bb.position()) + bb.position(), bb); +} + +button():MouseButton { + const offset = this.bb!.__offset(this.bb_pos, 4); + return offset ? this.bb!.readInt8(this.bb_pos + offset) : MouseButton.LEFT; +} + +buttonAction():KeyAction { + const offset = this.bb!.__offset(this.bb_pos, 6); + return offset ? this.bb!.readInt8(this.bb_pos + offset) : KeyAction.DOWN; +} + +static startMouseInput(builder:flatbuffers.Builder) { + builder.startObject(2); +} + +static addButton(builder:flatbuffers.Builder, button:MouseButton) { + builder.addFieldInt8(0, button, MouseButton.LEFT); +} + +static addButtonAction(builder:flatbuffers.Builder, buttonAction:KeyAction) { + builder.addFieldInt8(1, buttonAction, KeyAction.DOWN); +} + +static endMouseInput(builder:flatbuffers.Builder):flatbuffers.Offset { + const offset = builder.endObject(); + return offset; +} + +static createMouseInput(builder:flatbuffers.Builder, button:MouseButton, buttonAction:KeyAction):flatbuffers.Offset { + MouseInput.startMouseInput(builder); + MouseInput.addButton(builder, button); + MouseInput.addButtonAction(builder, buttonAction); + return MouseInput.endMouseInput(builder); +} +} diff --git a/frontend/src/lib/proto/input-event/mouse-movement.ts b/frontend/src/lib/proto/input-event/mouse-movement.ts new file mode 100644 index 0000000..1b31e85 --- /dev/null +++ b/frontend/src/lib/proto/input-event/mouse-movement.ts @@ -0,0 +1,58 @@ +// automatically generated by the FlatBuffers compiler, do not modify + +/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */ + +import * as flatbuffers from 'flatbuffers'; + +export class MouseMovement { + bb: flatbuffers.ByteBuffer|null = null; + bb_pos = 0; + __init(i:number, bb:flatbuffers.ByteBuffer):MouseMovement { + this.bb_pos = i; + this.bb = bb; + return this; +} + +static getRootAsMouseMovement(bb:flatbuffers.ByteBuffer, obj?:MouseMovement):MouseMovement { + return (obj || new MouseMovement()).__init(bb.readInt32(bb.position()) + bb.position(), bb); +} + +static getSizePrefixedRootAsMouseMovement(bb:flatbuffers.ByteBuffer, obj?:MouseMovement):MouseMovement { + bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH); + return (obj || new MouseMovement()).__init(bb.readInt32(bb.position()) + bb.position(), bb); +} + +movementX():number { + const offset = this.bb!.__offset(this.bb_pos, 4); + return offset ? this.bb!.readInt16(this.bb_pos + offset) : 0; +} + +movementY():number { + const offset = this.bb!.__offset(this.bb_pos, 6); + return offset ? this.bb!.readInt16(this.bb_pos + offset) : 0; +} + +static startMouseMovement(builder:flatbuffers.Builder) { + builder.startObject(2); +} + +static addMovementX(builder:flatbuffers.Builder, movementX:number) { + builder.addFieldInt16(0, movementX, 0); +} + +static addMovementY(builder:flatbuffers.Builder, movementY:number) { + builder.addFieldInt16(1, movementY, 0); +} + +static endMouseMovement(builder:flatbuffers.Builder):flatbuffers.Offset { + const offset = builder.endObject(); + return offset; +} + +static createMouseMovement(builder:flatbuffers.Builder, movementX:number, movementY:number):flatbuffers.Offset { + MouseMovement.startMouseMovement(builder); + MouseMovement.addMovementX(builder, movementX); + MouseMovement.addMovementY(builder, movementY); + return MouseMovement.endMouseMovement(builder); +} +} diff --git a/frontend/src/lib/proto/input.ts b/frontend/src/lib/proto/input.ts new file mode 100644 index 0000000..ec6568f --- /dev/null +++ b/frontend/src/lib/proto/input.ts @@ -0,0 +1,5 @@ +// automatically generated by the FlatBuffers compiler, do not modify + +/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */ + +export * as InputEvent from './input-event.js'; diff --git a/frontend/src/routes/getStreamData.ts b/frontend/src/routes/getStreamData.ts index 5566741..e3f652c 100644 --- a/frontend/src/routes/getStreamData.ts +++ b/frontend/src/routes/getStreamData.ts @@ -37,15 +37,15 @@ export async function getStreamData(appId: number, server_name: string): Promise server: server_name, server_mode: { fps: 60, - width: 1280, - height: 720, + width: 1920, + height: 1080, }, stream_config: { - bitrate_kbps: 5120, + bitrate_kbps: 10240, mode: { fps: 60, - width: 1280, - height: 720, + width: 1920, + height: 1080, } } }; diff --git a/frontend/src/routes/stream/Stream.svelte b/frontend/src/routes/stream/Stream.svelte index 311fa95..13dcaf5 100644 --- a/frontend/src/routes/stream/Stream.svelte +++ b/frontend/src/routes/stream/Stream.svelte @@ -1,6 +1,6 @@
- +
@@ -56,6 +54,9 @@ background-color: black; position: relative; + + display: flex; + flex-direction: column; } .gamestream-canvas { @@ -65,6 +66,6 @@ position: relative; margin: auto; - z-index: 1; + /*z-index: 1;*/ } diff --git a/frontend/src/routes/stream/StreamUi.svelte b/frontend/src/routes/stream/StreamUi.svelte index d4d7eca..4e6497d 100644 --- a/frontend/src/routes/stream/StreamUi.svelte +++ b/frontend/src/routes/stream/StreamUi.svelte @@ -10,11 +10,11 @@
-
+
-
+
@@ -41,12 +41,12 @@ height: 5%; width: 100%; - position: absolute; + /*position: absolute;*/ bottom: 0; left: 0; - z-index: 2; + /*z-index: 2;*/ border-top-color: oklch(0.454 0 360); border-top-width: 2px; diff --git a/frontend/src/routes/stream/stream.ts b/frontend/src/routes/stream/stream.ts index 7423171..abafcd3 100644 --- a/frontend/src/routes/stream/stream.ts +++ b/frontend/src/routes/stream/stream.ts @@ -1,3 +1,7 @@ +import { Input } from "$lib/proto/input-event" +import { InputEvent } from "../../lib/proto/input" +import { Builder } from "flatbuffers" + type Setup = { video_format: string, width: number, @@ -26,9 +30,8 @@ type DecodeUnitPacket = { DecodeUnit: DecodeUnit } -async function connectToUrl(url: string, certHash: Uint8Array): Promise { - console.log('Hash: ', certHash); - console.log(`Connecting to stream`); +export async function getStreamTransport(url: string, certHash: Array): Promise { + let certHashArray = new Uint8Array(certHash); // Check if WebTransport is supported if (!window.WebTransport) { @@ -39,7 +42,7 @@ async function connectToUrl(url: string, certHash: Uint8Array): Promise return [packets, unparsedData]; } -export async function streamUrl(url: string, certHash: Array, canvasElement: HTMLCanvasElement) { - const certHashArray = new Uint8Array(certHash); - - const transport = await connectToUrl(url, certHashArray); - console.log('Creating WebTransport bidirectional stream'); - const stream = await transport.createBidirectionalStream(); - console.log('Bidirectional stream created'); - +export async function streamVideoFromReader(reader: ReadableStreamDefaultReader, canvasElement: HTMLCanvasElement) { const canvasCtx: CanvasRenderingContext2D | null = canvasElement.getContext('2d'); if (canvasCtx == null) { throw new Error(`Could not get 2d canvas context`); } - const reader = stream.readable.getReader(); try { let unparsedData = new Uint8Array(); @@ -171,16 +166,6 @@ export async function streamUrl(url: string, certHash: Array, canvasElem } - // Handle connection close - transport.closed.then(() => { - console.log('WebTransport connection closed'); - }).catch((error) => { - console.error('WebTransport connection closed with error:', error); - }); - - // You can add more WebTransport handling logic here - // For example, handling incoming streams, sending data, etc. - } catch (e) { var error = e; console.error('Error connecting to stream:', error); @@ -188,3 +173,202 @@ export async function streamUrl(url: string, certHash: Array, canvasElem } } +enum KeyAction { + DOWN, + UP, +} + +async function sendKeyboardEvent( + writer: WritableStreamDefaultWriter, + event: KeyboardEvent, + keyAction: KeyAction, +) { + const builder = new Builder(1024); + + InputEvent.KeyboardInput.startKeyboardInput(builder); + // Have to use keyCode even though it's deprecated beacuse .code is a unhelpful string :sigh: + InputEvent.KeyboardInput.addKeyCode(builder, event.keyCode); + InputEvent.KeyboardInput.addKeyAction(builder, keyAction); + const modifierState = InputEvent.ModifierState.createModifierState( + builder, + event.shiftKey, + event.ctrlKey, + event.altKey, + event.metaKey + ); + + InputEvent.KeyboardInput.addModifiers(builder, modifierState); + const keyboardInput = InputEvent.KeyboardInput.endKeyboardInput(builder); + + // Create InputEvent table with union + InputEvent.InputEvent.startInputEvent(builder); + InputEvent.InputEvent.addInputType(builder, Input.Keyboard); + InputEvent.InputEvent.addInput(builder, keyboardInput); + const inputEvent = InputEvent.InputEvent.endInputEvent(builder); + + // Finish the buffer + builder.finish(inputEvent); + + // Get the buffer data + const bufferData = builder.asUint8Array(); + + // Create size prefix as big-endian u32 + const sizePrefix = new Uint8Array(4); + const size = bufferData.length; + sizePrefix[0] = (size >> 24) & 0xFF; // Most significant byte + sizePrefix[1] = (size >> 16) & 0xFF; + sizePrefix[2] = (size >> 8) & 0xFF; + sizePrefix[3] = size & 0xFF; // Least significant byte + + // Create combined buffer with size prefix + flatbuffer data + const combined = new Uint8Array(sizePrefix.length + bufferData.length); + combined.set(sizePrefix, 0); + combined.set(bufferData, sizePrefix.length); + + // Write to stream + await writer.write(combined); +} + +async function sendMouseMoveEvent( + writer: WritableStreamDefaultWriter, + event: MouseEvent, +) { + const builder = new Builder(1024); + + const mouseMovement = InputEvent.MouseMovement.createMouseMovement(builder, event.movementX, event.movementY); + + // Create InputEvent table with union + InputEvent.InputEvent.startInputEvent(builder); + InputEvent.InputEvent.addInputType(builder, Input.MouseMovement); + InputEvent.InputEvent.addInput(builder, mouseMovement); + const inputEvent = InputEvent.InputEvent.endInputEvent(builder); + + // Finish the buffer + builder.finish(inputEvent); + + // Get the buffer data + const bufferData = builder.asUint8Array(); + + // Create size prefix as big-endian u32 + const sizePrefix = new Uint8Array(4); + const size = bufferData.length; + sizePrefix[0] = (size >> 24) & 0xFF; // Most significant byte + sizePrefix[1] = (size >> 16) & 0xFF; + sizePrefix[2] = (size >> 8) & 0xFF; + sizePrefix[3] = size & 0xFF; // Least significant byte + + // Create combined buffer with size prefix + flatbuffer data + const combined = new Uint8Array(sizePrefix.length + bufferData.length); + combined.set(sizePrefix, 0); + combined.set(bufferData, sizePrefix.length); + + // Write to stream + await writer.write(combined); +} + +async function sendMouseInputEvent( + writer: WritableStreamDefaultWriter, + event: MouseEvent, + keyAction: KeyAction, +) { + const builder = new Builder(1024); + + var mouseButton: InputEvent.MouseButton; + + switch (event.button) { + case 0: + mouseButton = InputEvent.MouseButton.LEFT; + break; + case 1: + mouseButton = InputEvent.MouseButton.MIDDLE; + break; + case 2: + mouseButton = InputEvent.MouseButton.RIGHT; + break; + default: + console.log(`Unknown mouse button ${event.button}, defaulting to LEFT`); + mouseButton = InputEvent.MouseButton.LEFT; + break; + } + + const mouseInput = InputEvent.MouseInput.createMouseInput(builder, mouseButton, keyAction); + + // Create InputEvent table with union + InputEvent.InputEvent.startInputEvent(builder); + InputEvent.InputEvent.addInputType(builder, Input.MouseInput); + InputEvent.InputEvent.addInput(builder, mouseInput); + const inputEvent = InputEvent.InputEvent.endInputEvent(builder); + + // Finish the buffer + builder.finish(inputEvent); + + // Get the buffer data + const bufferData = builder.asUint8Array(); + + // Create size prefix as big-endian u32 + const sizePrefix = new Uint8Array(4); + const size = bufferData.length; + sizePrefix[0] = (size >> 24) & 0xFF; // Most significant byte + sizePrefix[1] = (size >> 16) & 0xFF; + sizePrefix[2] = (size >> 8) & 0xFF; + sizePrefix[3] = size & 0xFF; // Least significant byte + + // Create combined buffer with size prefix + flatbuffer data + const combined = new Uint8Array(sizePrefix.length + bufferData.length); + combined.set(sizePrefix, 0); + combined.set(bufferData, sizePrefix.length); + + // Write to stream + await writer.write(combined); +} + + + +export async function startWebtransportStream( + url: string, + certHash: Array, + gameplayCanvas: HTMLCanvasElement, + keyEventElement: HTMLElement, + mouseElement: HTMLElement, +) { + console.log(`Connecting to stream at ${url} with cert_hash ${certHash}`); + const transport = await getStreamTransport(url, certHash); + + const stream = await transport.createBidirectionalStream(); + + const reader = stream.readable.getReader(); + const writer = stream.writable.getWriter(); + + keyEventElement.addEventListener("keydown", (event: KeyboardEvent) => { sendKeyboardEvent(writer, event, KeyAction.DOWN) }); + keyEventElement.addEventListener("keyup", (event: KeyboardEvent) => { sendKeyboardEvent(writer, event, KeyAction.UP) }); + + mouseElement.addEventListener("mousedown", (event: MouseEvent) => { sendMouseInputEvent(writer, event, KeyAction.DOWN) }); + mouseElement.addEventListener("mouseup", (event: MouseEvent) => { sendMouseInputEvent(writer, event, KeyAction.UP) }); + + + mouseElement.addEventListener("click", async () => { + console.log("Requesting pointer lock"); + await mouseElement.requestPointerLock({ + unadjustedMovement: true, + }); + console.log("Pointer lock aquired"); + }) + + mouseElement.addEventListener("mousemove", (event: MouseEvent) => { + if (document.pointerLockElement === mouseElement) { + sendMouseMoveEvent(writer, event); + } + }); + + + await streamVideoFromReader(reader, gameplayCanvas); + + // Handle connection close + transport.closed + .then(() => { + console.log('WebTransport connection closed'); + }) + .catch((error) => { + console.error('WebTransport connection closed with error:', error); + }); +}