backend and frontend: port frame sending to flatbuffers

frontend/src/lib/proto/video-update.ts

@@ -0,0 +1,9 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */
+
+export { DecodeUnit } from './video-update/decode-unit.js';
+export { FrameType } from './video-update/frame-type.js';
+export { Setup } from './video-update/setup.js';
+export { Update } from './video-update/update.js';
+export { VideoUpdate } from './video-update/video-update.js';

frontend/src/lib/proto/video-update/decode-unit.ts

@@ -0,0 +1,103 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */
+
+import * as flatbuffers from 'flatbuffers';
+
+import { FrameType } from '../video-update/frame-type.js';
+
+
+export class DecodeUnit {
+  bb: flatbuffers.ByteBuffer|null = null;
+  bb_pos = 0;
+  __init(i:number, bb:flatbuffers.ByteBuffer):DecodeUnit {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+}
+
+static getRootAsDecodeUnit(bb:flatbuffers.ByteBuffer, obj?:DecodeUnit):DecodeUnit {
+  return (obj || new DecodeUnit()).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+}
+
+static getSizePrefixedRootAsDecodeUnit(bb:flatbuffers.ByteBuffer, obj?:DecodeUnit):DecodeUnit {
+  bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH);
+  return (obj || new DecodeUnit()).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+}
+
+frameNumber():number {
+  const offset = this.bb!.__offset(this.bb_pos, 4);
+  return offset ? this.bb!.readUint16(this.bb_pos + offset) : 0;
+}
+
+frameType():FrameType {
+  const offset = this.bb!.__offset(this.bb_pos, 6);
+  return offset ? this.bb!.readInt8(this.bb_pos + offset) : FrameType.PFRAME;
+}
+
+receiveTimeMs():number {
+  const offset = this.bb!.__offset(this.bb_pos, 8);
+  return offset ? this.bb!.readUint16(this.bb_pos + offset) : 0;
+}
+
+data(index: number):number|null {
+  const offset = this.bb!.__offset(this.bb_pos, 10);
+  return offset ? this.bb!.readUint8(this.bb!.__vector(this.bb_pos + offset) + index) : 0;
+}
+
+dataLength():number {
+  const offset = this.bb!.__offset(this.bb_pos, 10);
+  return offset ? this.bb!.__vector_len(this.bb_pos + offset) : 0;
+}
+
+dataArray():Uint8Array|null {
+  const offset = this.bb!.__offset(this.bb_pos, 10);
+  return offset ? new Uint8Array(this.bb!.bytes().buffer, this.bb!.bytes().byteOffset + this.bb!.__vector(this.bb_pos + offset), this.bb!.__vector_len(this.bb_pos + offset)) : null;
+}
+
+static startDecodeUnit(builder:flatbuffers.Builder) {
+  builder.startObject(4);
+}
+
+static addFrameNumber(builder:flatbuffers.Builder, frameNumber:number) {
+  builder.addFieldInt16(0, frameNumber, 0);
+}
+
+static addFrameType(builder:flatbuffers.Builder, frameType:FrameType) {
+  builder.addFieldInt8(1, frameType, FrameType.PFRAME);
+}
+
+static addReceiveTimeMs(builder:flatbuffers.Builder, receiveTimeMs:number) {
+  builder.addFieldInt16(2, receiveTimeMs, 0);
+}
+
+static addData(builder:flatbuffers.Builder, dataOffset:flatbuffers.Offset) {
+  builder.addFieldOffset(3, dataOffset, 0);
+}
+
+static createDataVector(builder:flatbuffers.Builder, data:number[]|Uint8Array):flatbuffers.Offset {
+  builder.startVector(1, data.length, 1);
+  for (let i = data.length - 1; i >= 0; i--) {
+    builder.addInt8(data[i]!);
+  }
+  return builder.endVector();
+}
+
+static startDataVector(builder:flatbuffers.Builder, numElems:number) {
+  builder.startVector(1, numElems, 1);
+}
+
+static endDecodeUnit(builder:flatbuffers.Builder):flatbuffers.Offset {
+  const offset = builder.endObject();
+  return offset;
+}
+
+static createDecodeUnit(builder:flatbuffers.Builder, frameNumber:number, frameType:FrameType, receiveTimeMs:number, dataOffset:flatbuffers.Offset):flatbuffers.Offset {
+  DecodeUnit.startDecodeUnit(builder);
+  DecodeUnit.addFrameNumber(builder, frameNumber);
+  DecodeUnit.addFrameType(builder, frameType);
+  DecodeUnit.addReceiveTimeMs(builder, receiveTimeMs);
+  DecodeUnit.addData(builder, dataOffset);
+  return DecodeUnit.endDecodeUnit(builder);
+}
+}

frontend/src/lib/proto/video-update/frame-type.ts

@@ -0,0 +1,8 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */
+
+export enum FrameType {
+  PFRAME = 0,
+  IDR = 1
+}

frontend/src/lib/proto/video-update/setup.ts

@@ -0,0 +1,80 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */
+
+import * as flatbuffers from 'flatbuffers';
+
+export class Setup {
+  bb: flatbuffers.ByteBuffer|null = null;
+  bb_pos = 0;
+  __init(i:number, bb:flatbuffers.ByteBuffer):Setup {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+}
+
+static getRootAsSetup(bb:flatbuffers.ByteBuffer, obj?:Setup):Setup {
+  return (obj || new Setup()).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+}
+
+static getSizePrefixedRootAsSetup(bb:flatbuffers.ByteBuffer, obj?:Setup):Setup {
+  bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH);
+  return (obj || new Setup()).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+}
+
+videoFormat():string|null
+videoFormat(optionalEncoding:flatbuffers.Encoding):string|Uint8Array|null
+videoFormat(optionalEncoding?:any):string|Uint8Array|null {
+  const offset = this.bb!.__offset(this.bb_pos, 4);
+  return offset ? this.bb!.__string(this.bb_pos + offset, optionalEncoding) : null;
+}
+
+width():number {
+  const offset = this.bb!.__offset(this.bb_pos, 6);
+  return offset ? this.bb!.readUint16(this.bb_pos + offset) : 0;
+}
+
+height():number {
+  const offset = this.bb!.__offset(this.bb_pos, 8);
+  return offset ? this.bb!.readUint16(this.bb_pos + offset) : 0;
+}
+
+redrawRate():number {
+  const offset = this.bb!.__offset(this.bb_pos, 10);
+  return offset ? this.bb!.readUint16(this.bb_pos + offset) : 0;
+}
+
+static startSetup(builder:flatbuffers.Builder) {
+  builder.startObject(4);
+}
+
+static addVideoFormat(builder:flatbuffers.Builder, videoFormatOffset:flatbuffers.Offset) {
+  builder.addFieldOffset(0, videoFormatOffset, 0);
+}
+
+static addWidth(builder:flatbuffers.Builder, width:number) {
+  builder.addFieldInt16(1, width, 0);
+}
+
+static addHeight(builder:flatbuffers.Builder, height:number) {
+  builder.addFieldInt16(2, height, 0);
+}
+
+static addRedrawRate(builder:flatbuffers.Builder, redrawRate:number) {
+  builder.addFieldInt16(3, redrawRate, 0);
+}
+
+static endSetup(builder:flatbuffers.Builder):flatbuffers.Offset {
+  const offset = builder.endObject();
+  return offset;
+}
+
+static createSetup(builder:flatbuffers.Builder, videoFormatOffset:flatbuffers.Offset, width:number, height:number, redrawRate:number):flatbuffers.Offset {
+  Setup.startSetup(builder);
+  Setup.addVideoFormat(builder, videoFormatOffset);
+  Setup.addWidth(builder, width);
+  Setup.addHeight(builder, height);
+  Setup.addRedrawRate(builder, redrawRate);
+  return Setup.endSetup(builder);
+}
+}

frontend/src/lib/proto/video-update/update.ts

@@ -0,0 +1,38 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */
+
+import { DecodeUnit } from '../video-update/decode-unit.js';
+import { Setup } from '../video-update/setup.js';
+
+
+export enum Update {
+  NONE = 0,
+  Setup = 1,
+  DecodeUnit = 2
+}
+
+export function unionToUpdate(
+  type: Update,
+  accessor: (obj:DecodeUnit|Setup) => DecodeUnit|Setup|null
+): DecodeUnit|Setup|null {
+  switch(Update[type]) {
+    case 'NONE': return null; 
+    case 'Setup': return accessor(new Setup())! as Setup;
+    case 'DecodeUnit': return accessor(new DecodeUnit())! as DecodeUnit;
+    default: return null;
+  }
+}
+
+export function unionListToUpdate(
+  type: Update, 
+  accessor: (index: number, obj:DecodeUnit|Setup) => DecodeUnit|Setup|null, 
+  index: number
+): DecodeUnit|Setup|null {
+  switch(Update[type]) {
+    case 'NONE': return null; 
+    case 'Setup': return accessor(index, new Setup())! as Setup;
+    case 'DecodeUnit': return accessor(index, new DecodeUnit())! as DecodeUnit;
+    default: return null;
+  }
+}

frontend/src/lib/proto/video-update/video-update.ts

@@ -0,0 +1,69 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */
+
+import * as flatbuffers from 'flatbuffers';
+
+import { Update, unionToUpdate, unionListToUpdate } from '../video-update/update.js';
+
+
+export class VideoUpdate {
+  bb: flatbuffers.ByteBuffer|null = null;
+  bb_pos = 0;
+  __init(i:number, bb:flatbuffers.ByteBuffer):VideoUpdate {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+}
+
+static getRootAsVideoUpdate(bb:flatbuffers.ByteBuffer, obj?:VideoUpdate):VideoUpdate {
+  return (obj || new VideoUpdate()).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+}
+
+static getSizePrefixedRootAsVideoUpdate(bb:flatbuffers.ByteBuffer, obj?:VideoUpdate):VideoUpdate {
+  bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH);
+  return (obj || new VideoUpdate()).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+}
+
+updateType():Update {
+  const offset = this.bb!.__offset(this.bb_pos, 4);
+  return offset ? this.bb!.readUint8(this.bb_pos + offset) : Update.NONE;
+}
+
+update<T extends flatbuffers.Table>(obj:any):any|null {
+  const offset = this.bb!.__offset(this.bb_pos, 6);
+  return offset ? this.bb!.__union(obj, this.bb_pos + offset) : null;
+}
+
+static startVideoUpdate(builder:flatbuffers.Builder) {
+  builder.startObject(2);
+}
+
+static addUpdateType(builder:flatbuffers.Builder, updateType:Update) {
+  builder.addFieldInt8(0, updateType, Update.NONE);
+}
+
+static addUpdate(builder:flatbuffers.Builder, updateOffset:flatbuffers.Offset) {
+  builder.addFieldOffset(1, updateOffset, 0);
+}
+
+static endVideoUpdate(builder:flatbuffers.Builder):flatbuffers.Offset {
+  const offset = builder.endObject();
+  return offset;
+}
+
+static finishVideoUpdateBuffer(builder:flatbuffers.Builder, offset:flatbuffers.Offset) {
+  builder.finish(offset);
+}
+
+static finishSizePrefixedVideoUpdateBuffer(builder:flatbuffers.Builder, offset:flatbuffers.Offset) {
+  builder.finish(offset, undefined, true);
+}
+
+static createVideoUpdate(builder:flatbuffers.Builder, updateType:Update, updateOffset:flatbuffers.Offset):flatbuffers.Offset {
+  VideoUpdate.startVideoUpdate(builder);
+  VideoUpdate.addUpdateType(builder, updateType);
+  VideoUpdate.addUpdate(builder, updateOffset);
+  return VideoUpdate.endVideoUpdate(builder);
+}
+}

frontend/src/lib/proto/video.ts

@@ -0,0 +1,5 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */
+
+export * as VideoUpdate from './video-update.js';

frontend/src/lib/video.ts

@@ -1,33 +1,8 @@
-type Setup = {
-  video_format: string,
-  width: number,
-  height: number,
-  redraw_rate: number,
-  dr_flags: number,
-}
+import { VideoUpdate } from "$lib/proto/video";
+import { ByteBuffer } from "flatbuffers";
 
-type SetupPacket = {
-  Setup: Setup
-}
-
-type DecodeBuffer = {
-  buffer_bype: string,
-  data: Array<number>,
-}
-
-type DecodeUnit = {
-  frame_number: number,
-  frame_type: string,
-  buffer: DecodeBuffer,
-  receieve_time_ms: number,
-}
-
-type DecodeUnitPacket = {
-  DecodeUnit: DecodeUnit
-}
-
-function parseData(newBuffer: Uint8Array, oldBuffer: Uint8Array): [Array<Object>, Uint8Array<ArrayBuffer>] {
-  let packets = new Array<Object>();
+function parseData(newBuffer: Uint8Array, oldBuffer: Uint8Array): [Array<VideoUpdate.VideoUpdate>, Uint8Array<ArrayBuffer>] {
+  let packets = new Array<VideoUpdate.VideoUpdate>();
   let unparsedData = new Uint8Array();
 
   let data = new Uint8Array([...oldBuffer, ...newBuffer]);
@@ -43,15 +18,14 @@ function parseData(newBuffer: Uint8Array, oldBuffer: Uint8Array): [Array<Object>
     const slice_end_index = index + 4 + dataLength;
 
     if (data.length < slice_end_index) {
-      unparsedData = new Uint8Array(data.buffer.slice(index, data.length));
+      unparsedData = data.slice(index, data.length);
       break;
     }
 
-    const dataToParse = data.buffer.slice(slice_start_index, slice_end_index);
-    const decoder = new TextDecoder('utf-8');
-    const jsonString = decoder.decode(dataToParse);
+    const dataToParse = new ByteBuffer(data.slice(slice_start_index, slice_end_index));
 
-    packets.push(JSON.parse(jsonString));
+    const videoUpdate = VideoUpdate.VideoUpdate.getRootAsVideoUpdate(dataToParse);
+    packets.push(videoUpdate);
 
     index += 4 + dataLength;
   }
@@ -70,21 +44,18 @@ export async function streamVideoFromReader(reader: ReadableStreamDefaultReader,
 
     const videoDecoder = new VideoDecoder({
       output: (frame) => {
-        // Set canvas dimensions to match the frame
         canvasElement.width = frame.displayWidth;
         canvasElement.height = frame.displayHeight;
 
-        // Draw the decoded frame to canvas
         canvasCtx.drawImage(frame, 0, 0);
 
-        // Important: close the frame to free memory
         frame.close();
       },
       error: (e) => {
         console.error('Decode error:', e);
       }
     });
-
+    let decodeUnit: VideoUpdate.DecodeUnit = new VideoUpdate.DecodeUnit();
     while (true) {
       const { value, done } = await reader.read();
       if (done) break;
@@ -93,20 +64,13 @@ export async function streamVideoFromReader(reader: ReadableStreamDefaultReader,
       unparsedData = remainingData;
 
       for (let i = 0; i < packets.length; i++) {
-        if (Object.hasOwn(packets[i], "Setup")) {
-          let packet = packets[i] as SetupPacket;
-
-          let config: VideoDecoderConfig | undefined = undefined;
-          if (packet.Setup.video_format == "H264") {
-            config = {
-              //codec: 'avc1.42E01E', // H.264 codec
-              codec: 'avc1.4D002A', // H.264 codec
-              codedWidth: packet.Setup.width,
-              codedHeight: packet.Setup.height,
-            };
-          } else {
-            throw new Error(`Unsupported video codec ${packet.Setup.video_format}`);
-          }
+        if (packets[i].updateType() == VideoUpdate.Update.Setup) {
+          let setup = packets[i].update(new VideoUpdate.Setup());
+          let config: VideoDecoderConfig = {
+            codec: setup.videoFormat(),
+            codedWidth: setup.width(),
+            codedHeight: setup.height(),
+          };
 
           const codecSupport = await VideoDecoder.isConfigSupported(config);
           if (codecSupport.supported) {
@@ -115,18 +79,18 @@ export async function streamVideoFromReader(reader: ReadableStreamDefaultReader,
             throw new Error(`Could not configure decoder`);
           }
 
-        } else if (Object.hasOwn(packets[i], "DecodeUnit")) {
-          let packet = packets[i] as DecodeUnitPacket;
+        } else if (packets[i].updateType() == VideoUpdate.Update.DecodeUnit) {
+          packets[i].update(decodeUnit);
 
-
-          let frame_type: EncodedAudioChunkType = "delta";
-          if (packet.DecodeUnit.frame_type == "IDR") {
-            frame_type = "key";
+          let frameType: EncodedAudioChunkType = "delta";
+          if (decodeUnit.frameType() == VideoUpdate.FrameType.IDR) {
+            console.log("GOT KEYFRAME");
+            frameType = "key";
           }
           const chunk = new EncodedVideoChunk({
-            timestamp: packet.DecodeUnit.receieve_time_ms,
-            type: frame_type,
-            data: new Uint8Array(packet.DecodeUnit.buffer.data),
+            timestamp: decodeUnit.receiveTimeMs(),
+            type: frameType,
+            data: decodeUnit.dataArray()!,
           });
 
           videoDecoder.decode(chunk);

frontend/src/routes/getStreamData.ts

@@ -41,7 +41,7 @@ export async function getStreamData(appId: number, server_name: string): Promise
         height: 1080,
       },
       stream_config: {
-        bitrate_kbps: 1024 * 10,
+        bitrate_kbps: 1024 * 10 * 2,
         mode: {
           fps: 60,
           width: 1920,

frontend/src/routes/stream/stream.ts

@@ -4,12 +4,12 @@ import CanvasWorker from "$lib/canvas.worker?worker";
 export async function getStreamTransport(url: string, certHash: Array<number>): Promise<WebTransport> {
   let certHashArray = new Uint8Array(certHash);
 
-  // Check if WebTransport is supported
   if (!window.WebTransport) {
     throw new Error('WebTransport is not supported in this browser');
   }
 
   const transport = new WebTransport(url, {
+    congestionControl: "low-latency",
     serverCertificateHashes: [
       {
         algorithm: "sha-256",
@@ -19,7 +19,6 @@ export async function getStreamTransport(url: string, certHash: Array<number>):
   });
 
   console.log('Connecting to WebTransport at ', url);
-  // Wait for the connection to be ready
   await transport.ready;
   console.log('WebTransport connection established');