backend and frontend: support out of order chunks + now it's performant on chrome

Cargo.lock

@@ -680,6 +680,8 @@ dependencies = [
  "directories",
  "flatbuffers",
  "getrandom 0.3.3",
+ "h3-datagram",
+ "h3-quinn",
  "hex",
  "hmac-sha256",
  "http",

frontend/src/lib/proto/video-update.ts

@@ -0,0 +1,10 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */
+
+export { DecodeUnitBuffer } from './video-update/decode-unit-buffer.js';
+export { DecodeUnitStart } from './video-update/decode-unit-start.js';
+export { FrameType } from './video-update/frame-type.js';
+export { Setup } from './video-update/setup.js';
+export { Update } from './video-update/update.js';
+export { VideoUpdate } from './video-update/video-update.js';

frontend/src/lib/proto/video-update/decode-unit-buffer.ts

@@ -0,0 +1,100 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */
+
+import * as flatbuffers from 'flatbuffers';
+
+export class DecodeUnitBuffer {
+  bb: flatbuffers.ByteBuffer|null = null;
+  bb_pos = 0;
+  __init(i:number, bb:flatbuffers.ByteBuffer):DecodeUnitBuffer {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+}
+
+static getRootAsDecodeUnitBuffer(bb:flatbuffers.ByteBuffer, obj?:DecodeUnitBuffer):DecodeUnitBuffer {
+  return (obj || new DecodeUnitBuffer()).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+}
+
+static getSizePrefixedRootAsDecodeUnitBuffer(bb:flatbuffers.ByteBuffer, obj?:DecodeUnitBuffer):DecodeUnitBuffer {
+  bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH);
+  return (obj || new DecodeUnitBuffer()).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+}
+
+frameNumber():bigint {
+  const offset = this.bb!.__offset(this.bb_pos, 4);
+  return offset ? this.bb!.readUint64(this.bb_pos + offset) : BigInt('0');
+}
+
+bufferIndex():bigint {
+  const offset = this.bb!.__offset(this.bb_pos, 6);
+  return offset ? this.bb!.readUint64(this.bb_pos + offset) : BigInt('0');
+}
+
+bufferOffset():bigint {
+  const offset = this.bb!.__offset(this.bb_pos, 8);
+  return offset ? this.bb!.readUint64(this.bb_pos + offset) : BigInt('0');
+}
+
+data(index: number):number|null {
+  const offset = this.bb!.__offset(this.bb_pos, 10);
+  return offset ? this.bb!.readUint8(this.bb!.__vector(this.bb_pos + offset) + index) : 0;
+}
+
+dataLength():number {
+  const offset = this.bb!.__offset(this.bb_pos, 10);
+  return offset ? this.bb!.__vector_len(this.bb_pos + offset) : 0;
+}
+
+dataArray():Uint8Array|null {
+  const offset = this.bb!.__offset(this.bb_pos, 10);
+  return offset ? new Uint8Array(this.bb!.bytes().buffer, this.bb!.bytes().byteOffset + this.bb!.__vector(this.bb_pos + offset), this.bb!.__vector_len(this.bb_pos + offset)) : null;
+}
+
+static startDecodeUnitBuffer(builder:flatbuffers.Builder) {
+  builder.startObject(4);
+}
+
+static addFrameNumber(builder:flatbuffers.Builder, frameNumber:bigint) {
+  builder.addFieldInt64(0, frameNumber, BigInt('0'));
+}
+
+static addBufferIndex(builder:flatbuffers.Builder, bufferIndex:bigint) {
+  builder.addFieldInt64(1, bufferIndex, BigInt('0'));
+}
+
+static addBufferOffset(builder:flatbuffers.Builder, bufferOffset:bigint) {
+  builder.addFieldInt64(2, bufferOffset, BigInt('0'));
+}
+
+static addData(builder:flatbuffers.Builder, dataOffset:flatbuffers.Offset) {
+  builder.addFieldOffset(3, dataOffset, 0);
+}
+
+static createDataVector(builder:flatbuffers.Builder, data:number[]|Uint8Array):flatbuffers.Offset {
+  builder.startVector(1, data.length, 1);
+  for (let i = data.length - 1; i >= 0; i--) {
+    builder.addInt8(data[i]!);
+  }
+  return builder.endVector();
+}
+
+static startDataVector(builder:flatbuffers.Builder, numElems:number) {
+  builder.startVector(1, numElems, 1);
+}
+
+static endDecodeUnitBuffer(builder:flatbuffers.Builder):flatbuffers.Offset {
+  const offset = builder.endObject();
+  return offset;
+}
+
+static createDecodeUnitBuffer(builder:flatbuffers.Builder, frameNumber:bigint, bufferIndex:bigint, bufferOffset:bigint, dataOffset:flatbuffers.Offset):flatbuffers.Offset {
+  DecodeUnitBuffer.startDecodeUnitBuffer(builder);
+  DecodeUnitBuffer.addFrameNumber(builder, frameNumber);
+  DecodeUnitBuffer.addBufferIndex(builder, bufferIndex);
+  DecodeUnitBuffer.addBufferOffset(builder, bufferOffset);
+  DecodeUnitBuffer.addData(builder, dataOffset);
+  return DecodeUnitBuffer.endDecodeUnitBuffer(builder);
+}
+}

frontend/src/lib/proto/video-update/decode-unit-start.ts

@@ -0,0 +1,91 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */
+
+import * as flatbuffers from 'flatbuffers';
+
+import { FrameType } from '../video-update/frame-type.js';
+
+
+export class DecodeUnitStart {
+  bb: flatbuffers.ByteBuffer|null = null;
+  bb_pos = 0;
+  __init(i:number, bb:flatbuffers.ByteBuffer):DecodeUnitStart {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+}
+
+static getRootAsDecodeUnitStart(bb:flatbuffers.ByteBuffer, obj?:DecodeUnitStart):DecodeUnitStart {
+  return (obj || new DecodeUnitStart()).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+}
+
+static getSizePrefixedRootAsDecodeUnitStart(bb:flatbuffers.ByteBuffer, obj?:DecodeUnitStart):DecodeUnitStart {
+  bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH);
+  return (obj || new DecodeUnitStart()).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+}
+
+frameNumber():bigint {
+  const offset = this.bb!.__offset(this.bb_pos, 4);
+  return offset ? this.bb!.readUint64(this.bb_pos + offset) : BigInt('0');
+}
+
+frameType():FrameType {
+  const offset = this.bb!.__offset(this.bb_pos, 6);
+  return offset ? this.bb!.readInt8(this.bb_pos + offset) : FrameType.PFRAME;
+}
+
+numBuffers():bigint {
+  const offset = this.bb!.__offset(this.bb_pos, 8);
+  return offset ? this.bb!.readUint64(this.bb_pos + offset) : BigInt('0');
+}
+
+receiveTimeMs():number {
+  const offset = this.bb!.__offset(this.bb_pos, 10);
+  return offset ? this.bb!.readUint16(this.bb_pos + offset) : 0;
+}
+
+fullLength():bigint {
+  const offset = this.bb!.__offset(this.bb_pos, 12);
+  return offset ? this.bb!.readUint64(this.bb_pos + offset) : BigInt('0');
+}
+
+static startDecodeUnitStart(builder:flatbuffers.Builder) {
+  builder.startObject(5);
+}
+
+static addFrameNumber(builder:flatbuffers.Builder, frameNumber:bigint) {
+  builder.addFieldInt64(0, frameNumber, BigInt('0'));
+}
+
+static addFrameType(builder:flatbuffers.Builder, frameType:FrameType) {
+  builder.addFieldInt8(1, frameType, FrameType.PFRAME);
+}
+
+static addNumBuffers(builder:flatbuffers.Builder, numBuffers:bigint) {
+  builder.addFieldInt64(2, numBuffers, BigInt('0'));
+}
+
+static addReceiveTimeMs(builder:flatbuffers.Builder, receiveTimeMs:number) {
+  builder.addFieldInt16(3, receiveTimeMs, 0);
+}
+
+static addFullLength(builder:flatbuffers.Builder, fullLength:bigint) {
+  builder.addFieldInt64(4, fullLength, BigInt('0'));
+}
+
+static endDecodeUnitStart(builder:flatbuffers.Builder):flatbuffers.Offset {
+  const offset = builder.endObject();
+  return offset;
+}
+
+static createDecodeUnitStart(builder:flatbuffers.Builder, frameNumber:bigint, frameType:FrameType, numBuffers:bigint, receiveTimeMs:number, fullLength:bigint):flatbuffers.Offset {
+  DecodeUnitStart.startDecodeUnitStart(builder);
+  DecodeUnitStart.addFrameNumber(builder, frameNumber);
+  DecodeUnitStart.addFrameType(builder, frameType);
+  DecodeUnitStart.addNumBuffers(builder, numBuffers);
+  DecodeUnitStart.addReceiveTimeMs(builder, receiveTimeMs);
+  DecodeUnitStart.addFullLength(builder, fullLength);
+  return DecodeUnitStart.endDecodeUnitStart(builder);
+}
+}

frontend/src/lib/proto/video-update/decode-unit.ts

@@ -0,0 +1,103 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */
+
+import * as flatbuffers from 'flatbuffers';
+
+import { FrameType } from '../video-update/frame-type.js';
+
+
+export class DecodeUnit {
+  bb: flatbuffers.ByteBuffer|null = null;
+  bb_pos = 0;
+  __init(i:number, bb:flatbuffers.ByteBuffer):DecodeUnit {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+}
+
+static getRootAsDecodeUnit(bb:flatbuffers.ByteBuffer, obj?:DecodeUnit):DecodeUnit {
+  return (obj || new DecodeUnit()).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+}
+
+static getSizePrefixedRootAsDecodeUnit(bb:flatbuffers.ByteBuffer, obj?:DecodeUnit):DecodeUnit {
+  bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH);
+  return (obj || new DecodeUnit()).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+}
+
+frameNumber():number {
+  const offset = this.bb!.__offset(this.bb_pos, 4);
+  return offset ? this.bb!.readUint16(this.bb_pos + offset) : 0;
+}
+
+frameType():FrameType {
+  const offset = this.bb!.__offset(this.bb_pos, 6);
+  return offset ? this.bb!.readInt8(this.bb_pos + offset) : FrameType.PFRAME;
+}
+
+receiveTimeMs():number {
+  const offset = this.bb!.__offset(this.bb_pos, 8);
+  return offset ? this.bb!.readUint16(this.bb_pos + offset) : 0;
+}
+
+data(index: number):number|null {
+  const offset = this.bb!.__offset(this.bb_pos, 10);
+  return offset ? this.bb!.readUint8(this.bb!.__vector(this.bb_pos + offset) + index) : 0;
+}
+
+dataLength():number {
+  const offset = this.bb!.__offset(this.bb_pos, 10);
+  return offset ? this.bb!.__vector_len(this.bb_pos + offset) : 0;
+}
+
+dataArray():Uint8Array|null {
+  const offset = this.bb!.__offset(this.bb_pos, 10);
+  return offset ? new Uint8Array(this.bb!.bytes().buffer, this.bb!.bytes().byteOffset + this.bb!.__vector(this.bb_pos + offset), this.bb!.__vector_len(this.bb_pos + offset)) : null;
+}
+
+static startDecodeUnit(builder:flatbuffers.Builder) {
+  builder.startObject(4);
+}
+
+static addFrameNumber(builder:flatbuffers.Builder, frameNumber:number) {
+  builder.addFieldInt16(0, frameNumber, 0);
+}
+
+static addFrameType(builder:flatbuffers.Builder, frameType:FrameType) {
+  builder.addFieldInt8(1, frameType, FrameType.PFRAME);
+}
+
+static addReceiveTimeMs(builder:flatbuffers.Builder, receiveTimeMs:number) {
+  builder.addFieldInt16(2, receiveTimeMs, 0);
+}
+
+static addData(builder:flatbuffers.Builder, dataOffset:flatbuffers.Offset) {
+  builder.addFieldOffset(3, dataOffset, 0);
+}
+
+static createDataVector(builder:flatbuffers.Builder, data:number[]|Uint8Array):flatbuffers.Offset {
+  builder.startVector(1, data.length, 1);
+  for (let i = data.length - 1; i >= 0; i--) {
+    builder.addInt8(data[i]!);
+  }
+  return builder.endVector();
+}
+
+static startDataVector(builder:flatbuffers.Builder, numElems:number) {
+  builder.startVector(1, numElems, 1);
+}
+
+static endDecodeUnit(builder:flatbuffers.Builder):flatbuffers.Offset {
+  const offset = builder.endObject();
+  return offset;
+}
+
+static createDecodeUnit(builder:flatbuffers.Builder, frameNumber:number, frameType:FrameType, receiveTimeMs:number, dataOffset:flatbuffers.Offset):flatbuffers.Offset {
+  DecodeUnit.startDecodeUnit(builder);
+  DecodeUnit.addFrameNumber(builder, frameNumber);
+  DecodeUnit.addFrameType(builder, frameType);
+  DecodeUnit.addReceiveTimeMs(builder, receiveTimeMs);
+  DecodeUnit.addData(builder, dataOffset);
+  return DecodeUnit.endDecodeUnit(builder);
+}
+}

frontend/src/lib/proto/video-update/frame-type.ts

@@ -0,0 +1,8 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */
+
+export enum FrameType {
+  PFRAME = 0,
+  IDR = 1
+}

frontend/src/lib/proto/video-update/setup.ts

@@ -0,0 +1,80 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */
+
+import * as flatbuffers from 'flatbuffers';
+
+export class Setup {
+  bb: flatbuffers.ByteBuffer|null = null;
+  bb_pos = 0;
+  __init(i:number, bb:flatbuffers.ByteBuffer):Setup {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+}
+
+static getRootAsSetup(bb:flatbuffers.ByteBuffer, obj?:Setup):Setup {
+  return (obj || new Setup()).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+}
+
+static getSizePrefixedRootAsSetup(bb:flatbuffers.ByteBuffer, obj?:Setup):Setup {
+  bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH);
+  return (obj || new Setup()).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+}
+
+videoFormat():string|null
+videoFormat(optionalEncoding:flatbuffers.Encoding):string|Uint8Array|null
+videoFormat(optionalEncoding?:any):string|Uint8Array|null {
+  const offset = this.bb!.__offset(this.bb_pos, 4);
+  return offset ? this.bb!.__string(this.bb_pos + offset, optionalEncoding) : null;
+}
+
+width():number {
+  const offset = this.bb!.__offset(this.bb_pos, 6);
+  return offset ? this.bb!.readUint16(this.bb_pos + offset) : 0;
+}
+
+height():number {
+  const offset = this.bb!.__offset(this.bb_pos, 8);
+  return offset ? this.bb!.readUint16(this.bb_pos + offset) : 0;
+}
+
+redrawRate():number {
+  const offset = this.bb!.__offset(this.bb_pos, 10);
+  return offset ? this.bb!.readUint16(this.bb_pos + offset) : 0;
+}
+
+static startSetup(builder:flatbuffers.Builder) {
+  builder.startObject(4);
+}
+
+static addVideoFormat(builder:flatbuffers.Builder, videoFormatOffset:flatbuffers.Offset) {
+  builder.addFieldOffset(0, videoFormatOffset, 0);
+}
+
+static addWidth(builder:flatbuffers.Builder, width:number) {
+  builder.addFieldInt16(1, width, 0);
+}
+
+static addHeight(builder:flatbuffers.Builder, height:number) {
+  builder.addFieldInt16(2, height, 0);
+}
+
+static addRedrawRate(builder:flatbuffers.Builder, redrawRate:number) {
+  builder.addFieldInt16(3, redrawRate, 0);
+}
+
+static endSetup(builder:flatbuffers.Builder):flatbuffers.Offset {
+  const offset = builder.endObject();
+  return offset;
+}
+
+static createSetup(builder:flatbuffers.Builder, videoFormatOffset:flatbuffers.Offset, width:number, height:number, redrawRate:number):flatbuffers.Offset {
+  Setup.startSetup(builder);
+  Setup.addVideoFormat(builder, videoFormatOffset);
+  Setup.addWidth(builder, width);
+  Setup.addHeight(builder, height);
+  Setup.addRedrawRate(builder, redrawRate);
+  return Setup.endSetup(builder);
+}
+}

frontend/src/lib/proto/video-update/update.ts

@@ -0,0 +1,42 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */
+
+import { DecodeUnitBuffer } from '../video-update/decode-unit-buffer.js';
+import { DecodeUnitStart } from '../video-update/decode-unit-start.js';
+import { Setup } from '../video-update/setup.js';
+
+
+export enum Update {
+  NONE = 0,
+  Setup = 1,
+  DecodeUnitStart = 2,
+  DecodeUnitBuffer = 3
+}
+
+export function unionToUpdate(
+  type: Update,
+  accessor: (obj:DecodeUnitBuffer|DecodeUnitStart|Setup) => DecodeUnitBuffer|DecodeUnitStart|Setup|null
+): DecodeUnitBuffer|DecodeUnitStart|Setup|null {
+  switch(Update[type]) {
+    case 'NONE': return null; 
+    case 'Setup': return accessor(new Setup())! as Setup;
+    case 'DecodeUnitStart': return accessor(new DecodeUnitStart())! as DecodeUnitStart;
+    case 'DecodeUnitBuffer': return accessor(new DecodeUnitBuffer())! as DecodeUnitBuffer;
+    default: return null;
+  }
+}
+
+export function unionListToUpdate(
+  type: Update, 
+  accessor: (index: number, obj:DecodeUnitBuffer|DecodeUnitStart|Setup) => DecodeUnitBuffer|DecodeUnitStart|Setup|null, 
+  index: number
+): DecodeUnitBuffer|DecodeUnitStart|Setup|null {
+  switch(Update[type]) {
+    case 'NONE': return null; 
+    case 'Setup': return accessor(index, new Setup())! as Setup;
+    case 'DecodeUnitStart': return accessor(index, new DecodeUnitStart())! as DecodeUnitStart;
+    case 'DecodeUnitBuffer': return accessor(index, new DecodeUnitBuffer())! as DecodeUnitBuffer;
+    default: return null;
+  }
+}

frontend/src/lib/proto/video-update/video-update.ts

@@ -0,0 +1,69 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */
+
+import * as flatbuffers from 'flatbuffers';
+
+import { Update, unionToUpdate, unionListToUpdate } from '../video-update/update.js';
+
+
+export class VideoUpdate {
+  bb: flatbuffers.ByteBuffer|null = null;
+  bb_pos = 0;
+  __init(i:number, bb:flatbuffers.ByteBuffer):VideoUpdate {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+}
+
+static getRootAsVideoUpdate(bb:flatbuffers.ByteBuffer, obj?:VideoUpdate):VideoUpdate {
+  return (obj || new VideoUpdate()).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+}
+
+static getSizePrefixedRootAsVideoUpdate(bb:flatbuffers.ByteBuffer, obj?:VideoUpdate):VideoUpdate {
+  bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH);
+  return (obj || new VideoUpdate()).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+}
+
+updateType():Update {
+  const offset = this.bb!.__offset(this.bb_pos, 4);
+  return offset ? this.bb!.readUint8(this.bb_pos + offset) : Update.NONE;
+}
+
+update<T extends flatbuffers.Table>(obj:any):any|null {
+  const offset = this.bb!.__offset(this.bb_pos, 6);
+  return offset ? this.bb!.__union(obj, this.bb_pos + offset) : null;
+}
+
+static startVideoUpdate(builder:flatbuffers.Builder) {
+  builder.startObject(2);
+}
+
+static addUpdateType(builder:flatbuffers.Builder, updateType:Update) {
+  builder.addFieldInt8(0, updateType, Update.NONE);
+}
+
+static addUpdate(builder:flatbuffers.Builder, updateOffset:flatbuffers.Offset) {
+  builder.addFieldOffset(1, updateOffset, 0);
+}
+
+static endVideoUpdate(builder:flatbuffers.Builder):flatbuffers.Offset {
+  const offset = builder.endObject();
+  return offset;
+}
+
+static finishVideoUpdateBuffer(builder:flatbuffers.Builder, offset:flatbuffers.Offset) {
+  builder.finish(offset);
+}
+
+static finishSizePrefixedVideoUpdateBuffer(builder:flatbuffers.Builder, offset:flatbuffers.Offset) {
+  builder.finish(offset, undefined, true);
+}
+
+static createVideoUpdate(builder:flatbuffers.Builder, updateType:Update, updateOffset:flatbuffers.Offset):flatbuffers.Offset {
+  VideoUpdate.startVideoUpdate(builder);
+  VideoUpdate.addUpdateType(builder, updateType);
+  VideoUpdate.addUpdate(builder, updateOffset);
+  return VideoUpdate.endVideoUpdate(builder);
+}
+}

frontend/src/lib/proto/video.ts

@@ -0,0 +1,5 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */
+
+export * as VideoUpdate from './video-update.js';

frontend/src/lib/video.ts

@@ -1,143 +1,184 @@
-type Setup = {
+import { VideoUpdate } from "$lib/proto/video";
-  video_format: string,
+import { ByteBuffer } from "flatbuffers";
-  width: number,
+import { DecodeUnitBuffer } from "./proto/video-update";
-  height: number,
+import { DoorClosed, Video } from "lucide-svelte";
-  redraw_rate: number,
-  dr_flags: number,
-}
 
-type SetupPacket = {
+function getVideoDecoder(canvasElement: OffscreenCanvas): VideoDecoder {
-  Setup: Setup
-}
-
-type DecodeBuffer = {
-  buffer_bype: string,
-  data: Array<number>,
-}
-
-type DecodeUnit = {
-  frame_number: number,
-  frame_type: string,
-  buffer: DecodeBuffer,
-  receieve_time_ms: number,
-}
-
-type DecodeUnitPacket = {
-  DecodeUnit: DecodeUnit
-}
-
-function parseData(newBuffer: Uint8Array, oldBuffer: Uint8Array): [Array<Object>, Uint8Array<ArrayBuffer>] {
-  let packets = new Array<Object>();
-  let unparsedData = new Uint8Array();
-
-  let data = new Uint8Array([...oldBuffer, ...newBuffer]);
-  let index = 0;
-  while (true) {
-    if (index >= data.length) {
-      break
-    }
-    const view = new DataView(data.buffer.slice(index, index + 4));
-    const dataLength = view.getUint32(0, true);
-
-    const slice_start_index = index + 4;
-    const slice_end_index = index + 4 + dataLength;
-
-    if (data.length < slice_end_index) {
-      unparsedData = new Uint8Array(data.buffer.slice(index, data.length));
-      break;
-    }
-
-    const dataToParse = data.buffer.slice(slice_start_index, slice_end_index);
-    const decoder = new TextDecoder('utf-8');
-    const jsonString = decoder.decode(dataToParse);
-
-    packets.push(JSON.parse(jsonString));
-
-    index += 4 + dataLength;
-  }
-  return [packets, unparsedData];
-}
-
-
-export async function streamVideoFromReader(reader: ReadableStreamDefaultReader, canvasElement: OffscreenCanvas) {
   const canvasCtx: OffscreenCanvasRenderingContext2D | null = canvasElement.getContext('2d');
   if (canvasCtx == null) {
     throw new Error(`Could not get 2d canvas context`);
   }
 
-  try {
-    let unparsedData = new Uint8Array();
-
   const videoDecoder = new VideoDecoder({
     output: (frame) => {
-        // Set canvas dimensions to match the frame
+      //console.log(`rendering frame start: ${performance.now()}`);
-        canvasElement.width = frame.displayWidth;
+      //canvasElement.width = frame.displayWidth;
-        canvasElement.height = frame.displayHeight;
+      //canvasElement.height = frame.displayHeight;
 
-        // Draw the decoded frame to canvas
+      //console.log(`rendering frame drawImage: ${performance.now()}`);
       canvasCtx.drawImage(frame, 0, 0);
 
-        // Important: close the frame to free memory
+      //console.log(`rendering frame end: ${performance.now()}`);
       frame.close();
+      //console.log(`rendering frame close: ${performance.now()}`);
     },
     error: (e) => {
       console.error('Decode error:', e);
     }
   });
-
+  return videoDecoder;
-    while (true) {
-      const { value, done } = await reader.read();
-      if (done) break;
-
-      let [packets, remainingData] = parseData(value, unparsedData);
-      unparsedData = remainingData;
-
-      for (let i = 0; i < packets.length; i++) {
-        if (Object.hasOwn(packets[i], "Setup")) {
-          let packet = packets[i] as SetupPacket;
-
-          let config: VideoDecoderConfig | undefined = undefined;
-          if (packet.Setup.video_format == "H264") {
-            config = {
-              //codec: 'avc1.42E01E', // H.264 codec
-              codec: 'avc1.4D002A', // H.264 codec
-              codedWidth: packet.Setup.width,
-              codedHeight: packet.Setup.height,
-            };
-          } else {
-            throw new Error(`Unsupported video codec ${packet.Setup.video_format}`);
 }
 
+async function configureDecoder(videoDecoder: VideoDecoder, videoFormat: string, width: number, height: number) {
+  let config: VideoDecoderConfig = {
+    codec: videoFormat,
+    codedWidth: width,
+    codedHeight: height,
+    optimizeForLatency: true,
+    //hardwareAcceleration: "prefer-hardware",
+  };
+
   const codecSupport = await VideoDecoder.isConfigSupported(config);
+  console.log(codecSupport);
   if (codecSupport.supported) {
     videoDecoder.configure(config);
   } else {
     throw new Error(`Could not configure decoder`);
   }
-
-        } else if (Object.hasOwn(packets[i], "DecodeUnit")) {
-          let packet = packets[i] as DecodeUnitPacket;
-
-
-          let frame_type: EncodedAudioChunkType = "delta";
-          if (packet.DecodeUnit.frame_type == "IDR") {
-            frame_type = "key";
 }
+
+class Decoder {
+  videoDecoder: VideoDecoder;
+
+  frameNumber: bigint | undefined;
+  frameType: EncodedAudioChunkType = "delta";
+  fullLength: bigint = 0n;
+  receiveTimeMs: number = 0;
+
+  //frameBroken: boolean = false;
+  //lastBufferIndex: bigint = 0n;
+  frameCollector: boolean[] = new Array<boolean>();
+  dataOffset: number = 0;
+  data: Uint8Array = new Uint8Array();
+
+  constructor(videoDecoder: VideoDecoder) {
+    this.videoDecoder = videoDecoder;
+  }
+
+  print() {
+    console.log(
+      `
+    frameNumber: ${this.frameNumber}
+    frameType: ${this.frameType}
+    fullLength: ${this.fullLength}
+    receiveTimeMs: ${this.receiveTimeMs}
+    frameCollector: ${this.frameCollector}
+    ts: ${performance.now()}
+    `
+    );
+  }
+
+  processStart(decodeUnitStart: VideoUpdate.DecodeUnitStart) {
+    //this.print();
+    const frameCompleted = !this.frameCollector.includes(false);
+    if (!frameCompleted) {
+      console.log(`Got setup packet for frame ${decodeUnitStart.frameNumber()} but the last frame has not been completed`);
+    }
+
+    this.frameNumber = decodeUnitStart.frameNumber();
+    this.frameType = "delta";
+    if (decodeUnitStart.frameType() == VideoUpdate.FrameType.IDR) {
+      this.frameType = "key";
+    }
+    this.fullLength = decodeUnitStart.fullLength();
+    this.receiveTimeMs = decodeUnitStart.receiveTimeMs();
+
+    //this.frameBroken = false;
+    //this.lastBufferIndex = -1n;
+    //this.dataOffset = 0;
+    this.frameCollector = new Array(Number(decodeUnitStart.numBuffers())).fill(false);
+
+    this.data = new Uint8Array(Number(this.fullLength));
+
+
+    //this.print();
+    //console.log(`start: `, this);
+    //console.log(performance.now());
+  }
+
+  processBuffer(decodeUnitBuffer: VideoUpdate.DecodeUnitBuffer) {
+    //console.log(`buffer: `, this);
+    //console.log(performance.now());
+    //this.print();
+
+
+    const frameNumber = decodeUnitBuffer.frameNumber();
+    if (this.frameNumber === undefined) {
+      console.log("frameNumber is undefined but we got a buffer, ignoring...");
+      return;
+    }
+    if (this.frameNumber != frameNumber) {
+      console.log(`Got buffer for frame ${frameNumber} but we are processing frame ${this.frameNumber}, ignoring...`);
+      return;
+    }
+
+    let offset = decodeUnitBuffer.bufferOffset();
+
+    for (var i = 0; i < decodeUnitBuffer.dataLength(); i++) {
+      this.data[Number(offset) + i] = decodeUnitBuffer.data(i)!;
+    }
+
+    this.frameCollector[Number(decodeUnitBuffer.bufferIndex())] = true;
+
+
+    const gotAllframes = !this.frameCollector.includes(false);
+
+    if (gotAllframes) {
       const chunk = new EncodedVideoChunk({
-            timestamp: packet.DecodeUnit.receieve_time_ms,
+        //timestamp: this.receiveTimeMs,
-            type: frame_type,
+        timestamp: 0,
-            data: new Uint8Array(packet.DecodeUnit.buffer.data),
+        type: this.frameType,
+        data: this.data,
       });
 
-          videoDecoder.decode(chunk);
+      //console.log(`${ performance.now() }: Enqueing a new decode request, current queue size ${ this.videoDecoder.decodeQueueSize } `);
+      this.videoDecoder.decode(chunk);
+    }
+    //this.print();
+  }
+}
+
+export async function streamVideoFromReader(reader: ReadableStreamDefaultReader, canvasElement: OffscreenCanvas) {
+  const videoDecoder = getVideoDecoder(canvasElement);
+  try {
+    let decodeUnitBuffer: VideoUpdate.DecodeUnitBuffer = new VideoUpdate.DecodeUnitBuffer();
+    let decoder = new Decoder(videoDecoder);
+
+    while (true) {
+      const { value, done } = await reader.read();
+      if (done) break;
+
+      const dataToParse = new ByteBuffer(value);
+      const videoUpdate = VideoUpdate.VideoUpdate.getRootAsVideoUpdate(dataToParse);
+
+      if (videoUpdate.updateType() == VideoUpdate.Update.Setup) {
+        let setup = videoUpdate.update(new VideoUpdate.Setup());
+        await configureDecoder(videoDecoder, setup.videoFormat(), setup.width(), setup.height());
+
+      } else if (videoUpdate.updateType() == VideoUpdate.Update.DecodeUnitStart) {
+        let decodeUnitStart: VideoUpdate.DecodeUnitStart = new VideoUpdate.DecodeUnitStart();
+        videoUpdate.update(decodeUnitStart);
+        decoder.processStart(decodeUnitStart);
+
+      } else if (videoUpdate.updateType() == VideoUpdate.Update.DecodeUnitBuffer) {
+        videoUpdate.update(decodeUnitBuffer);
+        decoder.processBuffer(decodeUnitBuffer);
 
       } else {
         throw new Error(`Got packet of unknown type`);
       }
     }
 
-    }
-
   } catch (e) {
     var error = <Error>e;
     console.error('Error connecting to stream:', error);

frontend/src/routes/Cover.svelte

@@ -23,6 +23,8 @@
 		let streamData = await getStreamData(app.id, server_name);
 		streamStore.Url = streamData.Url;
 		streamStore.CertHash = streamData.CertHash;
+		streamStore.Width = streamData.Width;
+		streamStore.Height = streamData.Height;
 
 		console.log(`Stream data retrieved. Navigating to /stream.`);
 		await goto('/stream');

frontend/src/routes/getStreamData.ts

@@ -1,51 +1,30 @@
-//Setup {
-//    video_format: VideoFormat,
-//    width: u64,
-//    height: u64,
-//    redraw_rate: u64,
-//    dr_flags: i32,
-//},
-//DecodeUnit {
-//    frame_number: u64,
-//    frame_type: FrameType,
-
-//    host_processing_latency: u16,
-//    receieve_time_ms: u64,
-//    enqueue_time_ms: u64,
-//    presentation_time: u64,
-
-//    full_length: usize,
-//    //buffers: Vec<Buffer>,
-//    buffer: Buffer,
-//    index: u64,
-
-//    hdr_active: bool,
-//    colorspace: u8,
-//},
-
-
 type StreamData = {
   Url: string,
   CertHash: Array<number>,
+  Width: number,
+  Height: number,
 }
 
 export async function getStreamData(appId: number, server_name: string): Promise<StreamData> {
   try {
     // Create the POST request payload
+    const width = 1920;
+    const height = 1080;
+
     const payload = {
       id: appId,
       server: server_name,
       server_mode: {
         fps: 60,
-        width: 1920,
+        width: width,
-        height: 1080,
+        height: height,
       },
       stream_config: {
-        bitrate_kbps: 1024 * 10,
+        bitrate_kbps: 1024 * 10 * 5,
         mode: {
           fps: 60,
-          width: 1920,
+          width: width,
-          height: 1080,
+          height: height,
         }
       }
     };
@@ -66,7 +45,7 @@ export async function getStreamData(appId: number, server_name: string): Promise
     const streamDataResp = await response.json();
     console.log('Stream started:', streamDataResp);
 
-    let streamData: StreamData = { Url: streamDataResp.url, CertHash: streamDataResp.cert_hash };
+    let streamData: StreamData = { Url: streamDataResp.url, CertHash: streamDataResp.cert_hash, Width: width, Height: height };
     return streamData;
 
 

frontend/src/routes/stores/streamStore.svelte.ts

@@ -1,4 +1,6 @@
 export const streamStore = $state({
   Url: '',
   CertHash: [0],
+  Width: 0,
+  Height: 0,
 });

frontend/src/routes/stream/+page.svelte

@@ -4,6 +4,8 @@
 
 	$: url = streamStore.Url;
 	$: certHash = streamStore.CertHash;
+	$: width = streamStore.Width;
+	$: height = streamStore.Height;
 </script>
 
 <svelte:head>
@@ -13,7 +15,7 @@
 
 <!--<section>
 </section>-->
-<Stream {url} {certHash} />
+<Stream {url} {certHash} {width} {height} />
 
 <style>
 	section {

frontend/src/routes/stream/Stream.svelte

@@ -6,16 +6,26 @@
 	interface Props {
 		url: string;
 		certHash: Array<number>;
+		width: number;
+		height: number;
 	}
 
-	let { url, certHash }: Props = $props();
+	let { url, certHash, width, height }: Props = $props();
 	let loading = $state(true);
 	let fullscreen = $state(false);
 	let gameplayView: HTMLDivElement;
 	let gameplayCanvas: HTMLCanvasElement;
 
 	async function startStream() {
-		await startWebtransportStream(url, certHash, gameplayCanvas, gameplayCanvas, gameplayCanvas);
+		await startWebtransportStream(
+			url,
+			certHash,
+			width,
+			height,
+			gameplayCanvas,
+			gameplayCanvas,
+			gameplayCanvas
+		);
 	}
 
 	async function requestFullscreen() {

frontend/src/routes/stream/stream.ts

@@ -4,12 +4,12 @@ import CanvasWorker from "$lib/canvas.worker?worker";
 export async function getStreamTransport(url: string, certHash: Array<number>): Promise<WebTransport> {
   let certHashArray = new Uint8Array(certHash);
 
-  // Check if WebTransport is supported
   if (!window.WebTransport) {
     throw new Error('WebTransport is not supported in this browser');
   }
 
   const transport = new WebTransport(url, {
+    //congestionControl: "low-latency",
     serverCertificateHashes: [
       {
         algorithm: "sha-256",
@@ -19,9 +19,8 @@ export async function getStreamTransport(url: string, certHash: Array<number>):
   });
 
   console.log('Connecting to WebTransport at ', url);
-  // Wait for the connection to be ready
   await transport.ready;
-  console.log('WebTransport connection established');
+  console.log(`WebTransport connection established`);
 
   return transport;
 }
@@ -43,19 +42,29 @@ export async function spawnWorker(gameplayCanvas: HTMLCanvasElement, reader: Rea
 export async function startWebtransportStream(
   url: string,
   certHash: Array<number>,
+  width: number,
+  height: number,
   gameplayCanvas: HTMLCanvasElement,
   keyEventElement: HTMLElement,
   mouseElement: HTMLElement,
 ) {
+  console.log(width, height);
+  gameplayCanvas.width = width;
+  gameplayCanvas.height = height;
+
+
   console.log(`Connecting to stream at ${url} with cert_hash ${certHash}`);
   const transport = await getStreamTransport(url, certHash);
 
+  const datagrams = transport.datagrams;
+  datagrams.incomingHighWaterMark = 20000;
   const stream = await transport.createBidirectionalStream();
 
-  const reader = stream.readable
+  //const reader = stream.readable
+  const datagramReader = datagrams.readable
   const writer = stream.writable.getWriter();
 
-  spawnWorker(gameplayCanvas, reader);
+  spawnWorker(gameplayCanvas, datagramReader);
 
   keyEventElement.addEventListener("keydown", (event: KeyboardEvent) => { sendKeyboardEvent(writer, event, KeyAction.DOWN) });
   keyEventElement.addEventListener("keyup", (event: KeyboardEvent) => { sendKeyboardEvent(writer, event, KeyAction.UP) });
@@ -67,7 +76,7 @@ export async function startWebtransportStream(
   mouseElement.addEventListener("click", async () => {
     console.log("Requesting pointer lock");
     await mouseElement.requestPointerLock({
-      unadjustedMovement: true,
+      //unadjustedMovement: true,
     });
     console.log("Pointer lock aquired");
   })

gamestream-webtransport-proxy/Cargo.toml

@@ -8,6 +8,8 @@ anyhow = "1.0.98"
 directories = "6.0.0"
 flatbuffers = "25.2.10"
 getrandom = { version = "0.3.3", features = ["std"] }
+h3-datagram = "0.0.2"
+h3-quinn = "0.0.10"
 hex = "0.4.3"
 hmac-sha256 = "1.1.12"
 http = "1.3.1"

gamestream-webtransport-proxy/src/certs.rs

@@ -120,15 +120,41 @@ fn generate_http_cert_and_key(
 
     cert_builder.set_version(2)?;
 
+    let serial = openssl::bn::BigNum::from_u32(1)?;
+    let asn_serial = openssl::asn1::Asn1Integer::from_bn(&serial)?;
+    cert_builder.set_serial_number(&asn_serial)?;
+
     // Set subject (Distinguished Name)
     let mut name_builder = X509NameBuilder::new()?;
-    name_builder.append_entry_by_text("CN", "mumble-web self-signed")?;
+    name_builder.append_entry_by_text("CN", "localhost")?;
     let subject_name = name_builder.build();
     cert_builder.set_subject_name(&subject_name)?;
-
-    // Set issuer (same as subject for self-signed)
     cert_builder.set_issuer_name(&subject_name)?;
 
+    let context = cert_builder.x509v3_context(None, None);
+
+    let mut san = openssl::x509::extension::SubjectAlternativeName::new();
+    san.dns("localhost");
+
+    let san_extension = san.build(&context)?;
+
+    let key_usage = openssl::x509::extension::KeyUsage::new()
+        .digital_signature()
+        .key_encipherment()
+        .build()?;
+
+    let ext_key_usage = openssl::x509::extension::ExtendedKeyUsage::new()
+        .server_auth()
+        .build()?;
+
+    // Add Subject Key Identifier
+    let subject_key_id = openssl::x509::extension::SubjectKeyIdentifier::new().build(&context)?;
+
+    cert_builder.append_extension(san_extension)?;
+    cert_builder.append_extension(key_usage)?;
+    cert_builder.append_extension(ext_key_usage)?;
+    cert_builder.append_extension(subject_key_id)?;
+
     cert_builder.set_not_before(&now)?;
     cert_builder.set_not_after(&expiration_time)?;
     cert_builder.set_pubkey(&key)?;

gamestream-webtransport-proxy/src/gamestream/config.rs

@@ -40,7 +40,7 @@ pub fn stream_config(stream: &crate::backend::Stream) -> _STREAM_CONFIGURATION {
         height: stream.stream_config.mode.height,
         fps: stream.stream_config.mode.fps,
         bitrate: stream.stream_config.bitrate_kbps,
-        packetSize: 512,
+        packetSize: 1024,
         streamingRemotely: STREAM_CFG_AUTO,
         audioConfiguration: (0x3 << 16) | (2 << 8) | 0xCA,
         supportedVideoFormats: VIDEO_FORMAT_H264,

gamestream-webtransport-proxy/src/gamestream/decoder.rs

@@ -12,7 +12,7 @@ use tokio::sync::mpsc;
 use tracing::{debug, error};
 
 #[derive(Serialize)]
-enum FrameType {
+pub enum FrameType {
     PFRAME,
     IDR,
 }
@@ -32,7 +32,7 @@ impl TryFrom<i32> for FrameType {
 }
 
 #[derive(Serialize)]
-enum VideoFormat {
+pub enum VideoFormat {
     H264,
     H264_HIGH8_444,
     H265,
@@ -92,8 +92,8 @@ impl TryFrom<i32> for BufferType {
 }
 
 #[derive(Serialize)]
-struct Buffer {
+pub struct Buffer {
-    data: Vec<u8>,
+    pub data: Vec<u8>,
     buffer_type: BufferType,
 }
 
@@ -119,23 +119,27 @@ pub enum RendererMessage {
         redraw_rate: u64,
         dr_flags: i32,
     },
-    DecodeUnit {
+    DecodeUnitStart {
         frame_number: u64,
         frame_type: FrameType,
+        num_buffers: u64,
 
         host_processing_latency: u16,
-        receieve_time_ms: u64,
+        receive_time_ms: u64,
         enqueue_time_ms: u64,
         presentation_time: u64,
 
-        full_length: usize,
+        full_length: u64,
-        //buffers: Vec<Buffer>,
-        buffer: Buffer,
-        index: u64,
 
         hdr_active: bool,
         colorspace: u8,
     },
+    DecodeUnitBuffer {
+        frame_number: u64,
+        buffer_index: u64,
+        buffer_offset: u64,
+        buffer: Buffer,
+    },
 }
 
 impl RendererMessage {
@@ -155,23 +159,43 @@ impl RendererMessage {
         })
     }
 
-    fn from_decode_unit(decode_unit: _DECODE_UNIT) -> Result<Self> {
+    fn from_decode_unit(decode_unit: _DECODE_UNIT) -> Result<Vec<Self>> {
-        //fn from_decode_unit(decode_unit: _DECODE_UNIT) -> Result<Vec<Self>> {
+        let mut messages = Vec::new();
-        let mut buffer = Vec::new();
-        //let mut buffers = Vec::new();
 
         if decode_unit.bufferList.is_null() {
             return Err(anyhow!("DecodeUnit bufferList is null"));
         }
+        let frame_number = <u64>::try_from(decode_unit.frameNumber)?;
+
+        messages.push(RendererMessage::DecodeUnitStart {
+            frame_number,
+            frame_type: FrameType::try_from(decode_unit.frameType)?,
+            num_buffers: 0,
+            host_processing_latency: decode_unit.frameHostProcessingLatency,
+            receive_time_ms: decode_unit.receiveTimeMs,
+            enqueue_time_ms: decode_unit.enqueueTimeMs,
+            presentation_time: decode_unit.presentationTimeMs as u64,
+            full_length: <u64>::try_from(decode_unit.fullLength)?,
+            hdr_active: decode_unit.hdrActive,
+            colorspace: decode_unit.colorspace,
+        });
+
         let mut next = unsafe { *decode_unit.bufferList };
 
         let mut index = 0;
+        let mut offset = 0;
         loop {
-            let mut b = Buffer::try_from(next)?;
+            let b = Buffer::try_from(next)?;
-            buffer.append(&mut b.data);
+            let buffer_len = b.data.len() as u64;
 
-            //buffers.push(msg);
+            messages.push(RendererMessage::DecodeUnitBuffer {
+                frame_number,
+                buffer_index: index,
+                buffer_offset: offset,
+                buffer: b,
+            });
 
+            offset = offset + buffer_len;
             index = index + 1;
             if next.next.is_null() {
                 break;
@@ -180,22 +204,15 @@ impl RendererMessage {
             next = unsafe { *next.next };
         }
 
-        Ok(RendererMessage::DecodeUnit {
+        if let RendererMessage::DecodeUnitStart {
-            frame_number: <u64>::try_from(decode_unit.frameNumber)?,
+            ref mut num_buffers,
-            frame_type: FrameType::try_from(decode_unit.frameType)?,
+            ..
-            host_processing_latency: decode_unit.frameHostProcessingLatency,
+        } = messages[0]
-            receieve_time_ms: decode_unit.receiveTimeMs,
+        {
-            enqueue_time_ms: decode_unit.enqueueTimeMs,
+            *num_buffers = index;
-            presentation_time: decode_unit.presentationTimeMs as u64,
+        }
-            full_length: <usize>::try_from(decode_unit.fullLength)?,
+
-            buffer: Buffer {
+        Ok(messages)
-                data: buffer,
-                buffer_type: BufferType::PICDATA,
-            },
-            index,
-            hdr_active: decode_unit.hdrActive,
-            colorspace: decode_unit.colorspace,
-        })
     }
 }
 
@@ -261,23 +278,29 @@ extern "C" fn submit_decode_unit_cb(decode_unit: PDECODE_UNIT) -> std::os::raw::
         return -1;
     }
     let decode_unit = unsafe { *decode_unit };
+    //debug!("decode unit bytes: {}", decode_unit.fullLength);
 
-    let message = match RendererMessage::from_decode_unit(decode_unit) {
+    let messages = match RendererMessage::from_decode_unit(decode_unit) {
         Ok(m) => m,
         Err(e) => {
             error!("Cannot construct RendererMessage: {e}");
             return -1;
         }
     };
+    debug!(
+        "got decode unit with {} buffers: {:?}",
+        messages.len() - 1,
+        std::time::Instant::now()
+    );
 
-    send_message(message)
+    for msg in messages {
-    //for msg in messages {
+        let ret = send_message(msg);
-    //    let ret = send_message(msg);
+        if ret != 0 {
-    //    if ret != 0 {
+            return ret;
-    //        return ret;
+        }
-    //    }
+    }
-    //}
+    //debug!("dispatched decode unit: {:?}", std::time::Instant::now());
-    //0
+    0
 }
 
 pub fn decoder_callbacks() -> Result<(DECODER_RENDERER_CALLBACKS, mpsc::Receiver<RendererMessage>)>

gamestream-webtransport-proxy/src/gamestream/mod.rs

@@ -2,7 +2,7 @@ use anyhow::{Result, anyhow};
 use tokio::sync::mpsc;
 
 mod config;
-mod decoder;
+pub mod decoder;
 
 #[derive(Debug)]
 pub struct GamestreamChannels {

gamestream-webtransport-proxy/src/proxy/handler.rs

@@ -29,6 +29,12 @@ async fn setup_webtransport(
 ) -> Result<(
     impl tokio::io::AsyncWrite + Send + Sync + 'static,
     impl tokio::io::AsyncRead + Send + Sync + 'static,
+    h3_datagram::datagram_handler::DatagramSender<
+        <h3_quinn::Connection as h3_datagram::quic_traits::DatagramConnectionExt<
+            salvo::hyper::body::Bytes,
+        >>::SendDatagramHandler,
+        salvo::hyper::body::Bytes,
+    >,
     //salvo::webtransport::stream::SendStream<
     //    impl salvo::proto::quic::SendStream<salvo::hyper::body::Bytes>,
     //    salvo::hyper::body::Bytes,
@@ -39,13 +45,17 @@ async fn setup_webtransport(
     //>,
 )> {
     let session = req.web_transport_mut().await?;
+
+    let datagram_send = session.datagram_sender();
+
     let bidirectional_stream = session
         .accept_bi()
         .await?
         .ok_or(anyhow!("No bidirectional stream"))?;
 
     if let webtransport::server::AcceptedBi::BidiStream(_, stream) = bidirectional_stream {
-        Ok(stream.split())
+        let (stream_send, stream_recv) = stream.split();
+        Ok((stream_send, stream_recv, datagram_send))
     } else {
         Err(anyhow!("bidirectional stream was of the wrong type"))
     }
@@ -76,7 +86,8 @@ impl crate::proxy::Proxy {
         });
 
         info!("WebTransport connection initiated");
-        let (wt_send, wt_recv) = match setup_webtransport(req).await {
+        let (wt_stream_send, wt_stream_recv, wt_datagram_send) = match setup_webtransport(req).await
+        {
             Ok(w) => w,
             Err(e) => {
                 error!("Could not upgrade connection to WebTransport: {e}");
@@ -95,7 +106,7 @@ impl crate::proxy::Proxy {
             }
         };
 
-        match super::proxy_main(stream, wt_send, wt_recv).await {
+        match super::proxy_main(stream, wt_stream_send, wt_stream_recv, wt_datagram_send).await {
             Ok(()) => Ok(()),
             Err(e) => {
                 error!("Proxy main loop failed: {e}");

gamestream-webtransport-proxy/src/proxy/mod.rs

@@ -7,6 +7,7 @@ use crate::{backend, gamestream};
 pub mod handler;
 mod input;
 mod packet_parser;
+mod video;
 
 pub struct Proxy {
     pub cert_hash: [u8; 32],
@@ -33,6 +34,12 @@ async fn proxy_main(
     stream: backend::Stream,
     mut wt_send: impl tokio::io::AsyncWrite + Send + Sync + 'static + std::marker::Unpin,
     mut wt_recv: impl tokio::io::AsyncRead + Send + Sync + 'static + std::marker::Unpin,
+    mut wt_datagram_send: h3_datagram::datagram_handler::DatagramSender<
+        <h3_quinn::Connection as h3_datagram::quic_traits::DatagramConnectionExt<
+            salvo::hyper::body::Bytes,
+        >>::SendDatagramHandler,
+        salvo::hyper::body::Bytes,
+    >,
 ) -> Result<()> {
     debug!(
         "Connecting to stream at address {} with stream config {:?}",
@@ -42,20 +49,14 @@ async fn proxy_main(
     let mut channels = spawn_gamestream(stream).await?;
 
     let mut packet_buffer = packet_parser::PacketBuffer::new();
-    //let mut buffer = vec![0; 65536].into_boxed_slice();
     let mut buffer = [0u8; 65536];
-    //let mut buffer = vec![0; 65536].into_boxed_slice();
 
     loop {
         select! {
             gamestream_packet = channels.gamestream_channels.decoder_rx.recv() => {
                 match gamestream_packet {
                     Some(frame) => {
-                        let frame_json = serde_json::to_vec(&frame)?;
+                        video::send_video_update(&frame, &mut wt_datagram_send).await?;
-                        let frame_json_len: u32 = <u32>::try_from(frame_json.len())?;
-
-                        wt_send.write_all(&frame_json_len.to_le_bytes()).await?;
-                        wt_send.write_all(&frame_json).await?;
                     }
                     None => {
                         error!("Decoder channel is None");

gamestream-webtransport-proxy/src/proxy/video/mod.rs

@@ -0,0 +1,185 @@
+use anyhow::Result;
+use tracing::debug;
+
+use crate::gamestream;
+use video_generated::video_update;
+
+mod video_generated;
+
+fn create_setup_videoupdate(
+    video_format: &gamestream::decoder::VideoFormat,
+    width: u64,
+    height: u64,
+    redraw_rate: u64,
+) -> Vec<u8> {
+    let mut builder = flatbuffers::FlatBufferBuilder::with_capacity(1024);
+
+    //TODO: this is hardcoded to h264 main profile
+    let video_format = Some(builder.create_string("avc1.4D401E"));
+
+    let setup = video_update::Setup::create(
+        &mut builder,
+        &video_update::SetupArgs {
+            video_format,
+            width: width as u16,
+            height: height as u16,
+            redraw_rate: redraw_rate as u16,
+        },
+    );
+
+    let video_update = video_update::VideoUpdate::create(
+        &mut builder,
+        &video_update::VideoUpdateArgs {
+            update_type: video_update::Update::Setup,
+            update: Some(setup.as_union_value()),
+        },
+    );
+
+    builder.finish(video_update, None);
+
+    builder.finished_data().to_vec()
+}
+
+fn create_decodeunitstart_videoupdate(
+    frame_number: u64,
+    frame_type: &gamestream::decoder::FrameType,
+    num_buffers: u64,
+    receive_time_ms: u64,
+    full_length: u64,
+) -> Vec<u8> {
+    let mut builder = flatbuffers::FlatBufferBuilder::with_capacity(1024);
+
+    let frame_type_fb = match frame_type {
+        gamestream::decoder::FrameType::IDR => video_update::FrameType::IDR,
+        gamestream::decoder::FrameType::PFRAME => video_update::FrameType::PFRAME,
+    };
+
+    let decode_unit_start = video_update::DecodeUnitStart::create(
+        &mut builder,
+        &video_update::DecodeUnitStartArgs {
+            frame_number,
+            frame_type: frame_type_fb,
+            num_buffers,
+            receive_time_ms: receive_time_ms as u16,
+            full_length,
+        },
+    );
+
+    let video_update = video_update::VideoUpdate::create(
+        &mut builder,
+        &video_update::VideoUpdateArgs {
+            update_type: video_update::Update::DecodeUnitStart,
+            update: Some(decode_unit_start.as_union_value()),
+        },
+    );
+
+    builder.finish(video_update, None);
+
+    builder.finished_data().to_vec()
+}
+
+fn create_decodeunitbuffer_videoupdate(
+    frame_number: u64,
+    buffer_index: u64,
+    buffer_offset: u64,
+    buffer: &gamestream::decoder::Buffer,
+) -> Vec<u8> {
+    let mut builder = flatbuffers::FlatBufferBuilder::with_capacity(1024);
+
+    let data_vector = builder.create_vector(&buffer.data);
+
+    let decode_unit_buffer = video_update::DecodeUnitBuffer::create(
+        &mut builder,
+        &video_update::DecodeUnitBufferArgs {
+            frame_number,
+            buffer_index,
+            buffer_offset,
+            data: Some(data_vector),
+        },
+    );
+
+    let video_update = video_update::VideoUpdate::create(
+        &mut builder,
+        &video_update::VideoUpdateArgs {
+            update_type: video_update::Update::DecodeUnitBuffer,
+            update: Some(decode_unit_buffer.as_union_value()),
+        },
+    );
+
+    builder.finish(video_update, None);
+
+    builder.finished_data().to_vec()
+}
+
+pub async fn send_video_update(
+    frame: &gamestream::decoder::RendererMessage,
+    wt_datagram_send: &mut h3_datagram::datagram_handler::DatagramSender<
+        <h3_quinn::Connection as h3_datagram::quic_traits::DatagramConnectionExt<
+            salvo::hyper::body::Bytes,
+        >>::SendDatagramHandler,
+        salvo::hyper::body::Bytes,
+    >,
+) -> Result<()> {
+    let mut print_time = false;
+    let buffer = match frame {
+        gamestream::decoder::RendererMessage::Setup {
+            video_format,
+            width,
+            height,
+            redraw_rate,
+            dr_flags,
+        } => create_setup_videoupdate(video_format, *width, *height, *redraw_rate),
+        gamestream::decoder::RendererMessage::DecodeUnitStart {
+            frame_number,
+            frame_type,
+            num_buffers,
+            host_processing_latency,
+            receive_time_ms,
+            enqueue_time_ms,
+            presentation_time,
+            full_length,
+            hdr_active,
+            colorspace,
+        } => {
+            //debug!(
+            //    "sending decodeunitstart {}: {:?}",
+            //    *frame_number,
+            //    std::time::Instant::now()
+            //);
+            create_decodeunitstart_videoupdate(
+                *frame_number,
+                frame_type,
+                *num_buffers,
+                *receive_time_ms,
+                *full_length,
+            )
+        }
+        gamestream::decoder::RendererMessage::DecodeUnitBuffer {
+            frame_number,
+            buffer_index,
+            buffer_offset,
+            buffer,
+        } => {
+            //debug!(
+            //    "sending decodeunitbuffer {}/{}: {:?}",
+            //    *frame_number,
+            //    *buffer_index,
+            //    std::time::Instant::now()
+            //);
+            create_decodeunitbuffer_videoupdate(
+                *frame_number,
+                *buffer_index,
+                *buffer_offset,
+                buffer,
+            )
+        }
+    };
+
+    let bytes = salvo::hyper::body::Bytes::copy_from_slice(&buffer);
+
+    wt_datagram_send.send_datagram(bytes)?;
+    debug!("sent start: {:?}", std::time::Instant::now());
+    //if (print_time) {
+    //}
+    Ok(())
+}

gamestream-webtransport-proxy/src/proxy/video/video_generated.rs

@@ -0,0 +1,925 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+
+// @generated
+
+use core::mem;
+use core::cmp::Ordering;
+
+extern crate flatbuffers;
+use self::flatbuffers::{EndianScalar, Follow};
+
+#[allow(unused_imports, dead_code)]
+pub mod video_update {
+
+  use core::mem;
+  use core::cmp::Ordering;
+
+  extern crate flatbuffers;
+  use self::flatbuffers::{EndianScalar, Follow};
+
+#[deprecated(since = "2.0.0", note = "Use associated constants instead. This will no longer be generated in 2021.")]
+pub const ENUM_MIN_FRAME_TYPE: i8 = 0;
+#[deprecated(since = "2.0.0", note = "Use associated constants instead. This will no longer be generated in 2021.")]
+pub const ENUM_MAX_FRAME_TYPE: i8 = 1;
+#[deprecated(since = "2.0.0", note = "Use associated constants instead. This will no longer be generated in 2021.")]
+#[allow(non_camel_case_types)]
+pub const ENUM_VALUES_FRAME_TYPE: [FrameType; 2] = [
+  FrameType::PFRAME,
+  FrameType::IDR,
+];
+
+#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Default)]
+#[repr(transparent)]
+pub struct FrameType(pub i8);
+#[allow(non_upper_case_globals)]
+impl FrameType {
+  pub const PFRAME: Self = Self(0);
+  pub const IDR: Self = Self(1);
+
+  pub const ENUM_MIN: i8 = 0;
+  pub const ENUM_MAX: i8 = 1;
+  pub const ENUM_VALUES: &'static [Self] = &[
+    Self::PFRAME,
+    Self::IDR,
+  ];
+  /// Returns the variant's name or "" if unknown.
+  pub fn variant_name(self) -> Option<&'static str> {
+    match self {
+      Self::PFRAME => Some("PFRAME"),
+      Self::IDR => Some("IDR"),
+      _ => None,
+    }
+  }
+}
+impl core::fmt::Debug for FrameType {
+  fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result {
+    if let Some(name) = self.variant_name() {
+      f.write_str(name)
+    } else {
+      f.write_fmt(format_args!("<UNKNOWN {:?}>", self.0))
+    }
+  }
+}
+impl<'a> flatbuffers::Follow<'a> for FrameType {
+  type Inner = Self;
+  #[inline]
+  unsafe fn follow(buf: &'a [u8], loc: usize) -> Self::Inner {
+    let b = flatbuffers::read_scalar_at::<i8>(buf, loc);
+    Self(b)
+  }
+}
+
+impl flatbuffers::Push for FrameType {
+    type Output = FrameType;
+    #[inline]
+    unsafe fn push(&self, dst: &mut [u8], _written_len: usize) {
+        flatbuffers::emplace_scalar::<i8>(dst, self.0);
+    }
+}
+
+impl flatbuffers::EndianScalar for FrameType {
+  type Scalar = i8;
+  #[inline]
+  fn to_little_endian(self) -> i8 {
+    self.0.to_le()
+  }
+  #[inline]
+  #[allow(clippy::wrong_self_convention)]
+  fn from_little_endian(v: i8) -> Self {
+    let b = i8::from_le(v);
+    Self(b)
+  }
+}
+
+impl<'a> flatbuffers::Verifiable for FrameType {
+  #[inline]
+  fn run_verifier(
+    v: &mut flatbuffers::Verifier, pos: usize
+  ) -> Result<(), flatbuffers::InvalidFlatbuffer> {
+    use self::flatbuffers::Verifiable;
+    i8::run_verifier(v, pos)
+  }
+}
+
+impl flatbuffers::SimpleToVerifyInSlice for FrameType {}
+#[deprecated(since = "2.0.0", note = "Use associated constants instead. This will no longer be generated in 2021.")]
+pub const ENUM_MIN_UPDATE: u8 = 0;
+#[deprecated(since = "2.0.0", note = "Use associated constants instead. This will no longer be generated in 2021.")]
+pub const ENUM_MAX_UPDATE: u8 = 3;
+#[deprecated(since = "2.0.0", note = "Use associated constants instead. This will no longer be generated in 2021.")]
+#[allow(non_camel_case_types)]
+pub const ENUM_VALUES_UPDATE: [Update; 4] = [
+  Update::NONE,
+  Update::Setup,
+  Update::DecodeUnitStart,
+  Update::DecodeUnitBuffer,
+];
+
+#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Default)]
+#[repr(transparent)]
+pub struct Update(pub u8);
+#[allow(non_upper_case_globals)]
+impl Update {
+  pub const NONE: Self = Self(0);
+  pub const Setup: Self = Self(1);
+  pub const DecodeUnitStart: Self = Self(2);
+  pub const DecodeUnitBuffer: Self = Self(3);
+
+  pub const ENUM_MIN: u8 = 0;
+  pub const ENUM_MAX: u8 = 3;
+  pub const ENUM_VALUES: &'static [Self] = &[
+    Self::NONE,
+    Self::Setup,
+    Self::DecodeUnitStart,
+    Self::DecodeUnitBuffer,
+  ];
+  /// Returns the variant's name or "" if unknown.
+  pub fn variant_name(self) -> Option<&'static str> {
+    match self {
+      Self::NONE => Some("NONE"),
+      Self::Setup => Some("Setup"),
+      Self::DecodeUnitStart => Some("DecodeUnitStart"),
+      Self::DecodeUnitBuffer => Some("DecodeUnitBuffer"),
+      _ => None,
+    }
+  }
+}
+impl core::fmt::Debug for Update {
+  fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result {
+    if let Some(name) = self.variant_name() {
+      f.write_str(name)
+    } else {
+      f.write_fmt(format_args!("<UNKNOWN {:?}>", self.0))
+    }
+  }
+}
+impl<'a> flatbuffers::Follow<'a> for Update {
+  type Inner = Self;
+  #[inline]
+  unsafe fn follow(buf: &'a [u8], loc: usize) -> Self::Inner {
+    let b = flatbuffers::read_scalar_at::<u8>(buf, loc);
+    Self(b)
+  }
+}
+
+impl flatbuffers::Push for Update {
+    type Output = Update;
+    #[inline]
+    unsafe fn push(&self, dst: &mut [u8], _written_len: usize) {
+        flatbuffers::emplace_scalar::<u8>(dst, self.0);
+    }
+}
+
+impl flatbuffers::EndianScalar for Update {
+  type Scalar = u8;
+  #[inline]
+  fn to_little_endian(self) -> u8 {
+    self.0.to_le()
+  }
+  #[inline]
+  #[allow(clippy::wrong_self_convention)]
+  fn from_little_endian(v: u8) -> Self {
+    let b = u8::from_le(v);
+    Self(b)
+  }
+}
+
+impl<'a> flatbuffers::Verifiable for Update {
+  #[inline]
+  fn run_verifier(
+    v: &mut flatbuffers::Verifier, pos: usize
+  ) -> Result<(), flatbuffers::InvalidFlatbuffer> {
+    use self::flatbuffers::Verifiable;
+    u8::run_verifier(v, pos)
+  }
+}
+
+impl flatbuffers::SimpleToVerifyInSlice for Update {}
+pub struct UpdateUnionTableOffset {}
+
+pub enum SetupOffset {}
+#[derive(Copy, Clone, PartialEq)]
+
+pub struct Setup<'a> {
+  pub _tab: flatbuffers::Table<'a>,
+}
+
+impl<'a> flatbuffers::Follow<'a> for Setup<'a> {
+  type Inner = Setup<'a>;
+  #[inline]
+  unsafe fn follow(buf: &'a [u8], loc: usize) -> Self::Inner {
+    Self { _tab: flatbuffers::Table::new(buf, loc) }
+  }
+}
+
+impl<'a> Setup<'a> {
+  pub const VT_VIDEO_FORMAT: flatbuffers::VOffsetT = 4;
+  pub const VT_WIDTH: flatbuffers::VOffsetT = 6;
+  pub const VT_HEIGHT: flatbuffers::VOffsetT = 8;
+  pub const VT_REDRAW_RATE: flatbuffers::VOffsetT = 10;
+
+  #[inline]
+  pub unsafe fn init_from_table(table: flatbuffers::Table<'a>) -> Self {
+    Setup { _tab: table }
+  }
+  #[allow(unused_mut)]
+  pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr, A: flatbuffers::Allocator + 'bldr>(
+    _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr, A>,
+    args: &'args SetupArgs<'args>
+  ) -> flatbuffers::WIPOffset<Setup<'bldr>> {
+    let mut builder = SetupBuilder::new(_fbb);
+    if let Some(x) = args.video_format { builder.add_video_format(x); }
+    builder.add_redraw_rate(args.redraw_rate);
+    builder.add_height(args.height);
+    builder.add_width(args.width);
+    builder.finish()
+  }
+
+
+  #[inline]
+  pub fn video_format(&self) -> Option<&'a str> {
+    // Safety:
+    // Created from valid Table for this object
+    // which contains a valid value in this slot
+    unsafe { self._tab.get::<flatbuffers::ForwardsUOffset<&str>>(Setup::VT_VIDEO_FORMAT, None)}
+  }
+  #[inline]
+  pub fn width(&self) -> u16 {
+    // Safety:
+    // Created from valid Table for this object
+    // which contains a valid value in this slot
+    unsafe { self._tab.get::<u16>(Setup::VT_WIDTH, Some(0)).unwrap()}
+  }
+  #[inline]
+  pub fn height(&self) -> u16 {
+    // Safety:
+    // Created from valid Table for this object
+    // which contains a valid value in this slot
+    unsafe { self._tab.get::<u16>(Setup::VT_HEIGHT, Some(0)).unwrap()}
+  }
+  #[inline]
+  pub fn redraw_rate(&self) -> u16 {
+    // Safety:
+    // Created from valid Table for this object
+    // which contains a valid value in this slot
+    unsafe { self._tab.get::<u16>(Setup::VT_REDRAW_RATE, Some(0)).unwrap()}
+  }
+}
+
+impl flatbuffers::Verifiable for Setup<'_> {
+  #[inline]
+  fn run_verifier(
+    v: &mut flatbuffers::Verifier, pos: usize
+  ) -> Result<(), flatbuffers::InvalidFlatbuffer> {
+    use self::flatbuffers::Verifiable;
+    v.visit_table(pos)?
+     .visit_field::<flatbuffers::ForwardsUOffset<&str>>("video_format", Self::VT_VIDEO_FORMAT, false)?
+     .visit_field::<u16>("width", Self::VT_WIDTH, false)?
+     .visit_field::<u16>("height", Self::VT_HEIGHT, false)?
+     .visit_field::<u16>("redraw_rate", Self::VT_REDRAW_RATE, false)?
+     .finish();
+    Ok(())
+  }
+}
+pub struct SetupArgs<'a> {
+    pub video_format: Option<flatbuffers::WIPOffset<&'a str>>,
+    pub width: u16,
+    pub height: u16,
+    pub redraw_rate: u16,
+}
+impl<'a> Default for SetupArgs<'a> {
+  #[inline]
+  fn default() -> Self {
+    SetupArgs {
+      video_format: None,
+      width: 0,
+      height: 0,
+      redraw_rate: 0,
+    }
+  }
+}
+
+pub struct SetupBuilder<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> {
+  fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a, A>,
+  start_: flatbuffers::WIPOffset<flatbuffers::TableUnfinishedWIPOffset>,
+}
+impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> SetupBuilder<'a, 'b, A> {
+  #[inline]
+  pub fn add_video_format(&mut self, video_format: flatbuffers::WIPOffset<&'b  str>) {
+    self.fbb_.push_slot_always::<flatbuffers::WIPOffset<_>>(Setup::VT_VIDEO_FORMAT, video_format);
+  }
+  #[inline]
+  pub fn add_width(&mut self, width: u16) {
+    self.fbb_.push_slot::<u16>(Setup::VT_WIDTH, width, 0);
+  }
+  #[inline]
+  pub fn add_height(&mut self, height: u16) {
+    self.fbb_.push_slot::<u16>(Setup::VT_HEIGHT, height, 0);
+  }
+  #[inline]
+  pub fn add_redraw_rate(&mut self, redraw_rate: u16) {
+    self.fbb_.push_slot::<u16>(Setup::VT_REDRAW_RATE, redraw_rate, 0);
+  }
+  #[inline]
+  pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>) -> SetupBuilder<'a, 'b, A> {
+    let start = _fbb.start_table();
+    SetupBuilder {
+      fbb_: _fbb,
+      start_: start,
+    }
+  }
+  #[inline]
+  pub fn finish(self) -> flatbuffers::WIPOffset<Setup<'a>> {
+    let o = self.fbb_.end_table(self.start_);
+    flatbuffers::WIPOffset::new(o.value())
+  }
+}
+
+impl core::fmt::Debug for Setup<'_> {
+  fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
+    let mut ds = f.debug_struct("Setup");
+      ds.field("video_format", &self.video_format());
+      ds.field("width", &self.width());
+      ds.field("height", &self.height());
+      ds.field("redraw_rate", &self.redraw_rate());
+      ds.finish()
+  }
+}
+pub enum DecodeUnitStartOffset {}
+#[derive(Copy, Clone, PartialEq)]
+
+pub struct DecodeUnitStart<'a> {
+  pub _tab: flatbuffers::Table<'a>,
+}
+
+impl<'a> flatbuffers::Follow<'a> for DecodeUnitStart<'a> {
+  type Inner = DecodeUnitStart<'a>;
+  #[inline]
+  unsafe fn follow(buf: &'a [u8], loc: usize) -> Self::Inner {
+    Self { _tab: flatbuffers::Table::new(buf, loc) }
+  }
+}
+
+impl<'a> DecodeUnitStart<'a> {
+  pub const VT_FRAME_NUMBER: flatbuffers::VOffsetT = 4;
+  pub const VT_FRAME_TYPE: flatbuffers::VOffsetT = 6;
+  pub const VT_NUM_BUFFERS: flatbuffers::VOffsetT = 8;
+  pub const VT_RECEIVE_TIME_MS: flatbuffers::VOffsetT = 10;
+  pub const VT_FULL_LENGTH: flatbuffers::VOffsetT = 12;
+
+  #[inline]
+  pub unsafe fn init_from_table(table: flatbuffers::Table<'a>) -> Self {
+    DecodeUnitStart { _tab: table }
+  }
+  #[allow(unused_mut)]
+  pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr, A: flatbuffers::Allocator + 'bldr>(
+    _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr, A>,
+    args: &'args DecodeUnitStartArgs
+  ) -> flatbuffers::WIPOffset<DecodeUnitStart<'bldr>> {
+    let mut builder = DecodeUnitStartBuilder::new(_fbb);
+    builder.add_full_length(args.full_length);
+    builder.add_num_buffers(args.num_buffers);
+    builder.add_frame_number(args.frame_number);
+    builder.add_receive_time_ms(args.receive_time_ms);
+    builder.add_frame_type(args.frame_type);
+    builder.finish()
+  }
+
+
+  #[inline]
+  pub fn frame_number(&self) -> u64 {
+    // Safety:
+    // Created from valid Table for this object
+    // which contains a valid value in this slot
+    unsafe { self._tab.get::<u64>(DecodeUnitStart::VT_FRAME_NUMBER, Some(0)).unwrap()}
+  }
+  #[inline]
+  pub fn frame_type(&self) -> FrameType {
+    // Safety:
+    // Created from valid Table for this object
+    // which contains a valid value in this slot
+    unsafe { self._tab.get::<FrameType>(DecodeUnitStart::VT_FRAME_TYPE, Some(FrameType::PFRAME)).unwrap()}
+  }
+  #[inline]
+  pub fn num_buffers(&self) -> u64 {
+    // Safety:
+    // Created from valid Table for this object
+    // which contains a valid value in this slot
+    unsafe { self._tab.get::<u64>(DecodeUnitStart::VT_NUM_BUFFERS, Some(0)).unwrap()}
+  }
+  #[inline]
+  pub fn receive_time_ms(&self) -> u16 {
+    // Safety:
+    // Created from valid Table for this object
+    // which contains a valid value in this slot
+    unsafe { self._tab.get::<u16>(DecodeUnitStart::VT_RECEIVE_TIME_MS, Some(0)).unwrap()}
+  }
+  #[inline]
+  pub fn full_length(&self) -> u64 {
+    // Safety:
+    // Created from valid Table for this object
+    // which contains a valid value in this slot
+    unsafe { self._tab.get::<u64>(DecodeUnitStart::VT_FULL_LENGTH, Some(0)).unwrap()}
+  }
+}
+
+impl flatbuffers::Verifiable for DecodeUnitStart<'_> {
+  #[inline]
+  fn run_verifier(
+    v: &mut flatbuffers::Verifier, pos: usize
+  ) -> Result<(), flatbuffers::InvalidFlatbuffer> {
+    use self::flatbuffers::Verifiable;
+    v.visit_table(pos)?
+     .visit_field::<u64>("frame_number", Self::VT_FRAME_NUMBER, false)?
+     .visit_field::<FrameType>("frame_type", Self::VT_FRAME_TYPE, false)?
+     .visit_field::<u64>("num_buffers", Self::VT_NUM_BUFFERS, false)?
+     .visit_field::<u16>("receive_time_ms", Self::VT_RECEIVE_TIME_MS, false)?
+     .visit_field::<u64>("full_length", Self::VT_FULL_LENGTH, false)?
+     .finish();
+    Ok(())
+  }
+}
+pub struct DecodeUnitStartArgs {
+    pub frame_number: u64,
+    pub frame_type: FrameType,
+    pub num_buffers: u64,
+    pub receive_time_ms: u16,
+    pub full_length: u64,
+}
+impl<'a> Default for DecodeUnitStartArgs {
+  #[inline]
+  fn default() -> Self {
+    DecodeUnitStartArgs {
+      frame_number: 0,
+      frame_type: FrameType::PFRAME,
+      num_buffers: 0,
+      receive_time_ms: 0,
+      full_length: 0,
+    }
+  }
+}
+
+pub struct DecodeUnitStartBuilder<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> {
+  fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a, A>,
+  start_: flatbuffers::WIPOffset<flatbuffers::TableUnfinishedWIPOffset>,
+}
+impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> DecodeUnitStartBuilder<'a, 'b, A> {
+  #[inline]
+  pub fn add_frame_number(&mut self, frame_number: u64) {
+    self.fbb_.push_slot::<u64>(DecodeUnitStart::VT_FRAME_NUMBER, frame_number, 0);
+  }
+  #[inline]
+  pub fn add_frame_type(&mut self, frame_type: FrameType) {
+    self.fbb_.push_slot::<FrameType>(DecodeUnitStart::VT_FRAME_TYPE, frame_type, FrameType::PFRAME);
+  }
+  #[inline]
+  pub fn add_num_buffers(&mut self, num_buffers: u64) {
+    self.fbb_.push_slot::<u64>(DecodeUnitStart::VT_NUM_BUFFERS, num_buffers, 0);
+  }
+  #[inline]
+  pub fn add_receive_time_ms(&mut self, receive_time_ms: u16) {
+    self.fbb_.push_slot::<u16>(DecodeUnitStart::VT_RECEIVE_TIME_MS, receive_time_ms, 0);
+  }
+  #[inline]
+  pub fn add_full_length(&mut self, full_length: u64) {
+    self.fbb_.push_slot::<u64>(DecodeUnitStart::VT_FULL_LENGTH, full_length, 0);
+  }
+  #[inline]
+  pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>) -> DecodeUnitStartBuilder<'a, 'b, A> {
+    let start = _fbb.start_table();
+    DecodeUnitStartBuilder {
+      fbb_: _fbb,
+      start_: start,
+    }
+  }
+  #[inline]
+  pub fn finish(self) -> flatbuffers::WIPOffset<DecodeUnitStart<'a>> {
+    let o = self.fbb_.end_table(self.start_);
+    flatbuffers::WIPOffset::new(o.value())
+  }
+}
+
+impl core::fmt::Debug for DecodeUnitStart<'_> {
+  fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
+    let mut ds = f.debug_struct("DecodeUnitStart");
+      ds.field("frame_number", &self.frame_number());
+      ds.field("frame_type", &self.frame_type());
+      ds.field("num_buffers", &self.num_buffers());
+      ds.field("receive_time_ms", &self.receive_time_ms());
+      ds.field("full_length", &self.full_length());
+      ds.finish()
+  }
+}
+pub enum DecodeUnitBufferOffset {}
+#[derive(Copy, Clone, PartialEq)]
+
+pub struct DecodeUnitBuffer<'a> {
+  pub _tab: flatbuffers::Table<'a>,
+}
+
+impl<'a> flatbuffers::Follow<'a> for DecodeUnitBuffer<'a> {
+  type Inner = DecodeUnitBuffer<'a>;
+  #[inline]
+  unsafe fn follow(buf: &'a [u8], loc: usize) -> Self::Inner {
+    Self { _tab: flatbuffers::Table::new(buf, loc) }
+  }
+}
+
+impl<'a> DecodeUnitBuffer<'a> {
+  pub const VT_FRAME_NUMBER: flatbuffers::VOffsetT = 4;
+  pub const VT_BUFFER_INDEX: flatbuffers::VOffsetT = 6;
+  pub const VT_BUFFER_OFFSET: flatbuffers::VOffsetT = 8;
+  pub const VT_DATA: flatbuffers::VOffsetT = 10;
+
+  #[inline]
+  pub unsafe fn init_from_table(table: flatbuffers::Table<'a>) -> Self {
+    DecodeUnitBuffer { _tab: table }
+  }
+  #[allow(unused_mut)]
+  pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr, A: flatbuffers::Allocator + 'bldr>(
+    _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr, A>,
+    args: &'args DecodeUnitBufferArgs<'args>
+  ) -> flatbuffers::WIPOffset<DecodeUnitBuffer<'bldr>> {
+    let mut builder = DecodeUnitBufferBuilder::new(_fbb);
+    builder.add_buffer_offset(args.buffer_offset);
+    builder.add_buffer_index(args.buffer_index);
+    builder.add_frame_number(args.frame_number);
+    if let Some(x) = args.data { builder.add_data(x); }
+    builder.finish()
+  }
+
+
+  #[inline]
+  pub fn frame_number(&self) -> u64 {
+    // Safety:
+    // Created from valid Table for this object
+    // which contains a valid value in this slot
+    unsafe { self._tab.get::<u64>(DecodeUnitBuffer::VT_FRAME_NUMBER, Some(0)).unwrap()}
+  }
+  #[inline]
+  pub fn buffer_index(&self) -> u64 {
+    // Safety:
+    // Created from valid Table for this object
+    // which contains a valid value in this slot
+    unsafe { self._tab.get::<u64>(DecodeUnitBuffer::VT_BUFFER_INDEX, Some(0)).unwrap()}
+  }
+  #[inline]
+  pub fn buffer_offset(&self) -> u64 {
+    // Safety:
+    // Created from valid Table for this object
+    // which contains a valid value in this slot
+    unsafe { self._tab.get::<u64>(DecodeUnitBuffer::VT_BUFFER_OFFSET, Some(0)).unwrap()}
+  }
+  #[inline]
+  pub fn data(&self) -> Option<flatbuffers::Vector<'a, u8>> {
+    // Safety:
+    // Created from valid Table for this object
+    // which contains a valid value in this slot
+    unsafe { self._tab.get::<flatbuffers::ForwardsUOffset<flatbuffers::Vector<'a, u8>>>(DecodeUnitBuffer::VT_DATA, None)}
+  }
+}
+
+impl flatbuffers::Verifiable for DecodeUnitBuffer<'_> {
+  #[inline]
+  fn run_verifier(
+    v: &mut flatbuffers::Verifier, pos: usize
+  ) -> Result<(), flatbuffers::InvalidFlatbuffer> {
+    use self::flatbuffers::Verifiable;
+    v.visit_table(pos)?
+     .visit_field::<u64>("frame_number", Self::VT_FRAME_NUMBER, false)?
+     .visit_field::<u64>("buffer_index", Self::VT_BUFFER_INDEX, false)?
+     .visit_field::<u64>("buffer_offset", Self::VT_BUFFER_OFFSET, false)?
+     .visit_field::<flatbuffers::ForwardsUOffset<flatbuffers::Vector<'_, u8>>>("data", Self::VT_DATA, false)?
+     .finish();
+    Ok(())
+  }
+}
+pub struct DecodeUnitBufferArgs<'a> {
+    pub frame_number: u64,
+    pub buffer_index: u64,
+    pub buffer_offset: u64,
+    pub data: Option<flatbuffers::WIPOffset<flatbuffers::Vector<'a, u8>>>,
+}
+impl<'a> Default for DecodeUnitBufferArgs<'a> {
+  #[inline]
+  fn default() -> Self {
+    DecodeUnitBufferArgs {
+      frame_number: 0,
+      buffer_index: 0,
+      buffer_offset: 0,
+      data: None,
+    }
+  }
+}
+
+pub struct DecodeUnitBufferBuilder<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> {
+  fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a, A>,
+  start_: flatbuffers::WIPOffset<flatbuffers::TableUnfinishedWIPOffset>,
+}
+impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> DecodeUnitBufferBuilder<'a, 'b, A> {
+  #[inline]
+  pub fn add_frame_number(&mut self, frame_number: u64) {
+    self.fbb_.push_slot::<u64>(DecodeUnitBuffer::VT_FRAME_NUMBER, frame_number, 0);
+  }
+  #[inline]
+  pub fn add_buffer_index(&mut self, buffer_index: u64) {
+    self.fbb_.push_slot::<u64>(DecodeUnitBuffer::VT_BUFFER_INDEX, buffer_index, 0);
+  }
+  #[inline]
+  pub fn add_buffer_offset(&mut self, buffer_offset: u64) {
+    self.fbb_.push_slot::<u64>(DecodeUnitBuffer::VT_BUFFER_OFFSET, buffer_offset, 0);
+  }
+  #[inline]
+  pub fn add_data(&mut self, data: flatbuffers::WIPOffset<flatbuffers::Vector<'b , u8>>) {
+    self.fbb_.push_slot_always::<flatbuffers::WIPOffset<_>>(DecodeUnitBuffer::VT_DATA, data);
+  }
+  #[inline]
+  pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>) -> DecodeUnitBufferBuilder<'a, 'b, A> {
+    let start = _fbb.start_table();
+    DecodeUnitBufferBuilder {
+      fbb_: _fbb,
+      start_: start,
+    }
+  }
+  #[inline]
+  pub fn finish(self) -> flatbuffers::WIPOffset<DecodeUnitBuffer<'a>> {
+    let o = self.fbb_.end_table(self.start_);
+    flatbuffers::WIPOffset::new(o.value())
+  }
+}
+
+impl core::fmt::Debug for DecodeUnitBuffer<'_> {
+  fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
+    let mut ds = f.debug_struct("DecodeUnitBuffer");
+      ds.field("frame_number", &self.frame_number());
+      ds.field("buffer_index", &self.buffer_index());
+      ds.field("buffer_offset", &self.buffer_offset());
+      ds.field("data", &self.data());
+      ds.finish()
+  }
+}
+pub enum VideoUpdateOffset {}
+#[derive(Copy, Clone, PartialEq)]
+
+pub struct VideoUpdate<'a> {
+  pub _tab: flatbuffers::Table<'a>,
+}
+
+impl<'a> flatbuffers::Follow<'a> for VideoUpdate<'a> {
+  type Inner = VideoUpdate<'a>;
+  #[inline]
+  unsafe fn follow(buf: &'a [u8], loc: usize) -> Self::Inner {
+    Self { _tab: flatbuffers::Table::new(buf, loc) }
+  }
+}
+
+impl<'a> VideoUpdate<'a> {
+  pub const VT_UPDATE_TYPE: flatbuffers::VOffsetT = 4;
+  pub const VT_UPDATE: flatbuffers::VOffsetT = 6;
+
+  #[inline]
+  pub unsafe fn init_from_table(table: flatbuffers::Table<'a>) -> Self {
+    VideoUpdate { _tab: table }
+  }
+  #[allow(unused_mut)]
+  pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr, A: flatbuffers::Allocator + 'bldr>(
+    _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr, A>,
+    args: &'args VideoUpdateArgs
+  ) -> flatbuffers::WIPOffset<VideoUpdate<'bldr>> {
+    let mut builder = VideoUpdateBuilder::new(_fbb);
+    if let Some(x) = args.update { builder.add_update(x); }
+    builder.add_update_type(args.update_type);
+    builder.finish()
+  }
+
+
+  #[inline]
+  pub fn update_type(&self) -> Update {
+    // Safety:
+    // Created from valid Table for this object
+    // which contains a valid value in this slot
+    unsafe { self._tab.get::<Update>(VideoUpdate::VT_UPDATE_TYPE, Some(Update::NONE)).unwrap()}
+  }
+  #[inline]
+  pub fn update(&self) -> Option<flatbuffers::Table<'a>> {
+    // Safety:
+    // Created from valid Table for this object
+    // which contains a valid value in this slot
+    unsafe { self._tab.get::<flatbuffers::ForwardsUOffset<flatbuffers::Table<'a>>>(VideoUpdate::VT_UPDATE, None)}
+  }
+  #[inline]
+  #[allow(non_snake_case)]
+  pub fn update_as_setup(&self) -> Option<Setup<'a>> {
+    if self.update_type() == Update::Setup {
+      self.update().map(|t| {
+       // Safety:
+       // Created from a valid Table for this object
+       // Which contains a valid union in this slot
+       unsafe { Setup::init_from_table(t) }
+     })
+    } else {
+      None
+    }
+  }
+
+  #[inline]
+  #[allow(non_snake_case)]
+  pub fn update_as_decode_unit_start(&self) -> Option<DecodeUnitStart<'a>> {
+    if self.update_type() == Update::DecodeUnitStart {
+      self.update().map(|t| {
+       // Safety:
+       // Created from a valid Table for this object
+       // Which contains a valid union in this slot
+       unsafe { DecodeUnitStart::init_from_table(t) }
+     })
+    } else {
+      None
+    }
+  }
+
+  #[inline]
+  #[allow(non_snake_case)]
+  pub fn update_as_decode_unit_buffer(&self) -> Option<DecodeUnitBuffer<'a>> {
+    if self.update_type() == Update::DecodeUnitBuffer {
+      self.update().map(|t| {
+       // Safety:
+       // Created from a valid Table for this object
+       // Which contains a valid union in this slot
+       unsafe { DecodeUnitBuffer::init_from_table(t) }
+     })
+    } else {
+      None
+    }
+  }
+
+}
+
+impl flatbuffers::Verifiable for VideoUpdate<'_> {
+  #[inline]
+  fn run_verifier(
+    v: &mut flatbuffers::Verifier, pos: usize
+  ) -> Result<(), flatbuffers::InvalidFlatbuffer> {
+    use self::flatbuffers::Verifiable;
+    v.visit_table(pos)?
+     .visit_union::<Update, _>("update_type", Self::VT_UPDATE_TYPE, "update", Self::VT_UPDATE, false, |key, v, pos| {
+        match key {
+          Update::Setup => v.verify_union_variant::<flatbuffers::ForwardsUOffset<Setup>>("Update::Setup", pos),
+          Update::DecodeUnitStart => v.verify_union_variant::<flatbuffers::ForwardsUOffset<DecodeUnitStart>>("Update::DecodeUnitStart", pos),
+          Update::DecodeUnitBuffer => v.verify_union_variant::<flatbuffers::ForwardsUOffset<DecodeUnitBuffer>>("Update::DecodeUnitBuffer", pos),
+          _ => Ok(()),
+        }
+     })?
+     .finish();
+    Ok(())
+  }
+}
+pub struct VideoUpdateArgs {
+    pub update_type: Update,
+    pub update: Option<flatbuffers::WIPOffset<flatbuffers::UnionWIPOffset>>,
+}
+impl<'a> Default for VideoUpdateArgs {
+  #[inline]
+  fn default() -> Self {
+    VideoUpdateArgs {
+      update_type: Update::NONE,
+      update: None,
+    }
+  }
+}
+
+pub struct VideoUpdateBuilder<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> {
+  fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a, A>,
+  start_: flatbuffers::WIPOffset<flatbuffers::TableUnfinishedWIPOffset>,
+}
+impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> VideoUpdateBuilder<'a, 'b, A> {
+  #[inline]
+  pub fn add_update_type(&mut self, update_type: Update) {
+    self.fbb_.push_slot::<Update>(VideoUpdate::VT_UPDATE_TYPE, update_type, Update::NONE);
+  }
+  #[inline]
+  pub fn add_update(&mut self, update: flatbuffers::WIPOffset<flatbuffers::UnionWIPOffset>) {
+    self.fbb_.push_slot_always::<flatbuffers::WIPOffset<_>>(VideoUpdate::VT_UPDATE, update);
+  }
+  #[inline]
+  pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>) -> VideoUpdateBuilder<'a, 'b, A> {
+    let start = _fbb.start_table();
+    VideoUpdateBuilder {
+      fbb_: _fbb,
+      start_: start,
+    }
+  }
+  #[inline]
+  pub fn finish(self) -> flatbuffers::WIPOffset<VideoUpdate<'a>> {
+    let o = self.fbb_.end_table(self.start_);
+    flatbuffers::WIPOffset::new(o.value())
+  }
+}
+
+impl core::fmt::Debug for VideoUpdate<'_> {
+  fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
+    let mut ds = f.debug_struct("VideoUpdate");
+      ds.field("update_type", &self.update_type());
+      match self.update_type() {
+        Update::Setup => {
+          if let Some(x) = self.update_as_setup() {
+            ds.field("update", &x)
+          } else {
+            ds.field("update", &"InvalidFlatbuffer: Union discriminant does not match value.")
+          }
+        },
+        Update::DecodeUnitStart => {
+          if let Some(x) = self.update_as_decode_unit_start() {
+            ds.field("update", &x)
+          } else {
+            ds.field("update", &"InvalidFlatbuffer: Union discriminant does not match value.")
+          }
+        },
+        Update::DecodeUnitBuffer => {
+          if let Some(x) = self.update_as_decode_unit_buffer() {
+            ds.field("update", &x)
+          } else {
+            ds.field("update", &"InvalidFlatbuffer: Union discriminant does not match value.")
+          }
+        },
+        _ => {
+          let x: Option<()> = None;
+          ds.field("update", &x)
+        },
+      };
+      ds.finish()
+  }
+}
+#[inline]
+/// Verifies that a buffer of bytes contains a `VideoUpdate`
+/// and returns it.
+/// Note that verification is still experimental and may not
+/// catch every error, or be maximally performant. For the
+/// previous, unchecked, behavior use
+/// `root_as_video_update_unchecked`.
+pub fn root_as_video_update(buf: &[u8]) -> Result<VideoUpdate, flatbuffers::InvalidFlatbuffer> {
+  flatbuffers::root::<VideoUpdate>(buf)
+}
+#[inline]
+/// Verifies that a buffer of bytes contains a size prefixed
+/// `VideoUpdate` and returns it.
+/// Note that verification is still experimental and may not
+/// catch every error, or be maximally performant. For the
+/// previous, unchecked, behavior use
+/// `size_prefixed_root_as_video_update_unchecked`.
+pub fn size_prefixed_root_as_video_update(buf: &[u8]) -> Result<VideoUpdate, flatbuffers::InvalidFlatbuffer> {
+  flatbuffers::size_prefixed_root::<VideoUpdate>(buf)
+}
+#[inline]
+/// Verifies, with the given options, that a buffer of bytes
+/// contains a `VideoUpdate` and returns it.
+/// Note that verification is still experimental and may not
+/// catch every error, or be maximally performant. For the
+/// previous, unchecked, behavior use
+/// `root_as_video_update_unchecked`.
+pub fn root_as_video_update_with_opts<'b, 'o>(
+  opts: &'o flatbuffers::VerifierOptions,
+  buf: &'b [u8],
+) -> Result<VideoUpdate<'b>, flatbuffers::InvalidFlatbuffer> {
+  flatbuffers::root_with_opts::<VideoUpdate<'b>>(opts, buf)
+}
+#[inline]
+/// Verifies, with the given verifier options, that a buffer of
+/// bytes contains a size prefixed `VideoUpdate` and returns
+/// it. Note that verification is still experimental and may not
+/// catch every error, or be maximally performant. For the
+/// previous, unchecked, behavior use
+/// `root_as_video_update_unchecked`.
+pub fn size_prefixed_root_as_video_update_with_opts<'b, 'o>(
+  opts: &'o flatbuffers::VerifierOptions,
+  buf: &'b [u8],
+) -> Result<VideoUpdate<'b>, flatbuffers::InvalidFlatbuffer> {
+  flatbuffers::size_prefixed_root_with_opts::<VideoUpdate<'b>>(opts, buf)
+}
+#[inline]
+/// Assumes, without verification, that a buffer of bytes contains a VideoUpdate and returns it.
+/// # Safety
+/// Callers must trust the given bytes do indeed contain a valid `VideoUpdate`.
+pub unsafe fn root_as_video_update_unchecked(buf: &[u8]) -> VideoUpdate {
+  flatbuffers::root_unchecked::<VideoUpdate>(buf)
+}
+#[inline]
+/// Assumes, without verification, that a buffer of bytes contains a size prefixed VideoUpdate and returns it.
+/// # Safety
+/// Callers must trust the given bytes do indeed contain a valid size prefixed `VideoUpdate`.
+pub unsafe fn size_prefixed_root_as_video_update_unchecked(buf: &[u8]) -> VideoUpdate {
+  flatbuffers::size_prefixed_root_unchecked::<VideoUpdate>(buf)
+}
+#[inline]
+pub fn finish_video_update_buffer<'a, 'b, A: flatbuffers::Allocator + 'a>(
+    fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>,
+    root: flatbuffers::WIPOffset<VideoUpdate<'a>>) {
+  fbb.finish(root, None);
+}
+
+#[inline]
+pub fn finish_size_prefixed_video_update_buffer<'a, 'b, A: flatbuffers::Allocator + 'a>(fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>, root: flatbuffers::WIPOffset<VideoUpdate<'a>>) {
+  fbb.finish_size_prefixed(root, None);
+}
+}  // pub mod VideoUpdate
+

gamestream-webtransport-proxy/src/stream.rs

@@ -315,7 +315,10 @@ impl crate::backend::Backend {
 
         let webtransport_url = url_constructor::UrlConstructor::new()
             .scheme("https")
-            .host(host)
+            // TODO: this is hardcoded to 127.0.0.1 to fix problems with
+            // tls certificates and IPv6 in chrome. This needs to eventually be fixed
+            // but I don't actually know what the fix is
+            .host("127.0.0.1")
             .port(port)
             .subdir("api/stream/connect")
             .build();

schema/video.fbs

@@ -0,0 +1,43 @@
+namespace VideoUpdate;
+
+
+table Setup {
+  video_format: string;
+  width: uint16;
+  height: uint16;
+  redraw_rate: uint16;
+}
+
+enum FrameType: byte {
+  PFRAME,
+  IDR,
+}
+
+table DecodeUnitStart {
+  frame_number: uint64;
+  frame_type: FrameType;
+  num_buffers: uint64;
+
+  receive_time_ms: uint16;
+  full_length: uint64;
+
+}
+
+table DecodeUnitBuffer {
+  frame_number: uint64;
+  buffer_index: uint64;
+  buffer_offset: uint64;
+  data: [ubyte]; 
+}
+
+union Update {
+    Setup:Setup,
+    DecodeUnitStart:DecodeUnitStart,
+    DecodeUnitBuffer:DecodeUnitBuffer,
+}
+
+table VideoUpdate {
+    update: Update; 
+}
+
+root_type VideoUpdate;