backend and frontend: port frame sending to flatbuffers

frontend/src/lib/proto/video-update.ts

@@ -0,0 +1,9 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */
+
+export { DecodeUnit } from './video-update/decode-unit.js';
+export { FrameType } from './video-update/frame-type.js';
+export { Setup } from './video-update/setup.js';
+export { Update } from './video-update/update.js';
+export { VideoUpdate } from './video-update/video-update.js';

frontend/src/lib/proto/video-update/decode-unit.ts

@@ -0,0 +1,103 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */
+
+import * as flatbuffers from 'flatbuffers';
+
+import { FrameType } from '../video-update/frame-type.js';
+
+
+export class DecodeUnit {
+  bb: flatbuffers.ByteBuffer|null = null;
+  bb_pos = 0;
+  __init(i:number, bb:flatbuffers.ByteBuffer):DecodeUnit {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+}
+
+static getRootAsDecodeUnit(bb:flatbuffers.ByteBuffer, obj?:DecodeUnit):DecodeUnit {
+  return (obj || new DecodeUnit()).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+}
+
+static getSizePrefixedRootAsDecodeUnit(bb:flatbuffers.ByteBuffer, obj?:DecodeUnit):DecodeUnit {
+  bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH);
+  return (obj || new DecodeUnit()).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+}
+
+frameNumber():number {
+  const offset = this.bb!.__offset(this.bb_pos, 4);
+  return offset ? this.bb!.readUint16(this.bb_pos + offset) : 0;
+}
+
+frameType():FrameType {
+  const offset = this.bb!.__offset(this.bb_pos, 6);
+  return offset ? this.bb!.readInt8(this.bb_pos + offset) : FrameType.PFRAME;
+}
+
+receiveTimeMs():number {
+  const offset = this.bb!.__offset(this.bb_pos, 8);
+  return offset ? this.bb!.readUint16(this.bb_pos + offset) : 0;
+}
+
+data(index: number):number|null {
+  const offset = this.bb!.__offset(this.bb_pos, 10);
+  return offset ? this.bb!.readUint8(this.bb!.__vector(this.bb_pos + offset) + index) : 0;
+}
+
+dataLength():number {
+  const offset = this.bb!.__offset(this.bb_pos, 10);
+  return offset ? this.bb!.__vector_len(this.bb_pos + offset) : 0;
+}
+
+dataArray():Uint8Array|null {
+  const offset = this.bb!.__offset(this.bb_pos, 10);
+  return offset ? new Uint8Array(this.bb!.bytes().buffer, this.bb!.bytes().byteOffset + this.bb!.__vector(this.bb_pos + offset), this.bb!.__vector_len(this.bb_pos + offset)) : null;
+}
+
+static startDecodeUnit(builder:flatbuffers.Builder) {
+  builder.startObject(4);
+}
+
+static addFrameNumber(builder:flatbuffers.Builder, frameNumber:number) {
+  builder.addFieldInt16(0, frameNumber, 0);
+}
+
+static addFrameType(builder:flatbuffers.Builder, frameType:FrameType) {
+  builder.addFieldInt8(1, frameType, FrameType.PFRAME);
+}
+
+static addReceiveTimeMs(builder:flatbuffers.Builder, receiveTimeMs:number) {
+  builder.addFieldInt16(2, receiveTimeMs, 0);
+}
+
+static addData(builder:flatbuffers.Builder, dataOffset:flatbuffers.Offset) {
+  builder.addFieldOffset(3, dataOffset, 0);
+}
+
+static createDataVector(builder:flatbuffers.Builder, data:number[]|Uint8Array):flatbuffers.Offset {
+  builder.startVector(1, data.length, 1);
+  for (let i = data.length - 1; i >= 0; i--) {
+    builder.addInt8(data[i]!);
+  }
+  return builder.endVector();
+}
+
+static startDataVector(builder:flatbuffers.Builder, numElems:number) {
+  builder.startVector(1, numElems, 1);
+}
+
+static endDecodeUnit(builder:flatbuffers.Builder):flatbuffers.Offset {
+  const offset = builder.endObject();
+  return offset;
+}
+
+static createDecodeUnit(builder:flatbuffers.Builder, frameNumber:number, frameType:FrameType, receiveTimeMs:number, dataOffset:flatbuffers.Offset):flatbuffers.Offset {
+  DecodeUnit.startDecodeUnit(builder);
+  DecodeUnit.addFrameNumber(builder, frameNumber);
+  DecodeUnit.addFrameType(builder, frameType);
+  DecodeUnit.addReceiveTimeMs(builder, receiveTimeMs);
+  DecodeUnit.addData(builder, dataOffset);
+  return DecodeUnit.endDecodeUnit(builder);
+}
+}

frontend/src/lib/proto/video-update/frame-type.ts

@@ -0,0 +1,8 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */
+
+export enum FrameType {
+  PFRAME = 0,
+  IDR = 1
+}

frontend/src/lib/proto/video-update/setup.ts

@@ -0,0 +1,80 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */
+
+import * as flatbuffers from 'flatbuffers';
+
+export class Setup {
+  bb: flatbuffers.ByteBuffer|null = null;
+  bb_pos = 0;
+  __init(i:number, bb:flatbuffers.ByteBuffer):Setup {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+}
+
+static getRootAsSetup(bb:flatbuffers.ByteBuffer, obj?:Setup):Setup {
+  return (obj || new Setup()).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+}
+
+static getSizePrefixedRootAsSetup(bb:flatbuffers.ByteBuffer, obj?:Setup):Setup {
+  bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH);
+  return (obj || new Setup()).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+}
+
+videoFormat():string|null
+videoFormat(optionalEncoding:flatbuffers.Encoding):string|Uint8Array|null
+videoFormat(optionalEncoding?:any):string|Uint8Array|null {
+  const offset = this.bb!.__offset(this.bb_pos, 4);
+  return offset ? this.bb!.__string(this.bb_pos + offset, optionalEncoding) : null;
+}
+
+width():number {
+  const offset = this.bb!.__offset(this.bb_pos, 6);
+  return offset ? this.bb!.readUint16(this.bb_pos + offset) : 0;
+}
+
+height():number {
+  const offset = this.bb!.__offset(this.bb_pos, 8);
+  return offset ? this.bb!.readUint16(this.bb_pos + offset) : 0;
+}
+
+redrawRate():number {
+  const offset = this.bb!.__offset(this.bb_pos, 10);
+  return offset ? this.bb!.readUint16(this.bb_pos + offset) : 0;
+}
+
+static startSetup(builder:flatbuffers.Builder) {
+  builder.startObject(4);
+}
+
+static addVideoFormat(builder:flatbuffers.Builder, videoFormatOffset:flatbuffers.Offset) {
+  builder.addFieldOffset(0, videoFormatOffset, 0);
+}
+
+static addWidth(builder:flatbuffers.Builder, width:number) {
+  builder.addFieldInt16(1, width, 0);
+}
+
+static addHeight(builder:flatbuffers.Builder, height:number) {
+  builder.addFieldInt16(2, height, 0);
+}
+
+static addRedrawRate(builder:flatbuffers.Builder, redrawRate:number) {
+  builder.addFieldInt16(3, redrawRate, 0);
+}
+
+static endSetup(builder:flatbuffers.Builder):flatbuffers.Offset {
+  const offset = builder.endObject();
+  return offset;
+}
+
+static createSetup(builder:flatbuffers.Builder, videoFormatOffset:flatbuffers.Offset, width:number, height:number, redrawRate:number):flatbuffers.Offset {
+  Setup.startSetup(builder);
+  Setup.addVideoFormat(builder, videoFormatOffset);
+  Setup.addWidth(builder, width);
+  Setup.addHeight(builder, height);
+  Setup.addRedrawRate(builder, redrawRate);
+  return Setup.endSetup(builder);
+}
+}

frontend/src/lib/proto/video-update/update.ts

@@ -0,0 +1,38 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */
+
+import { DecodeUnit } from '../video-update/decode-unit.js';
+import { Setup } from '../video-update/setup.js';
+
+
+export enum Update {
+  NONE = 0,
+  Setup = 1,
+  DecodeUnit = 2
+}
+
+export function unionToUpdate(
+  type: Update,
+  accessor: (obj:DecodeUnit|Setup) => DecodeUnit|Setup|null
+): DecodeUnit|Setup|null {
+  switch(Update[type]) {
+    case 'NONE': return null; 
+    case 'Setup': return accessor(new Setup())! as Setup;
+    case 'DecodeUnit': return accessor(new DecodeUnit())! as DecodeUnit;
+    default: return null;
+  }
+}
+
+export function unionListToUpdate(
+  type: Update, 
+  accessor: (index: number, obj:DecodeUnit|Setup) => DecodeUnit|Setup|null, 
+  index: number
+): DecodeUnit|Setup|null {
+  switch(Update[type]) {
+    case 'NONE': return null; 
+    case 'Setup': return accessor(index, new Setup())! as Setup;
+    case 'DecodeUnit': return accessor(index, new DecodeUnit())! as DecodeUnit;
+    default: return null;
+  }
+}

frontend/src/lib/proto/video-update/video-update.ts

@@ -0,0 +1,69 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */
+
+import * as flatbuffers from 'flatbuffers';
+
+import { Update, unionToUpdate, unionListToUpdate } from '../video-update/update.js';
+
+
+export class VideoUpdate {
+  bb: flatbuffers.ByteBuffer|null = null;
+  bb_pos = 0;
+  __init(i:number, bb:flatbuffers.ByteBuffer):VideoUpdate {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+}
+
+static getRootAsVideoUpdate(bb:flatbuffers.ByteBuffer, obj?:VideoUpdate):VideoUpdate {
+  return (obj || new VideoUpdate()).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+}
+
+static getSizePrefixedRootAsVideoUpdate(bb:flatbuffers.ByteBuffer, obj?:VideoUpdate):VideoUpdate {
+  bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH);
+  return (obj || new VideoUpdate()).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+}
+
+updateType():Update {
+  const offset = this.bb!.__offset(this.bb_pos, 4);
+  return offset ? this.bb!.readUint8(this.bb_pos + offset) : Update.NONE;
+}
+
+update<T extends flatbuffers.Table>(obj:any):any|null {
+  const offset = this.bb!.__offset(this.bb_pos, 6);
+  return offset ? this.bb!.__union(obj, this.bb_pos + offset) : null;
+}
+
+static startVideoUpdate(builder:flatbuffers.Builder) {
+  builder.startObject(2);
+}
+
+static addUpdateType(builder:flatbuffers.Builder, updateType:Update) {
+  builder.addFieldInt8(0, updateType, Update.NONE);
+}
+
+static addUpdate(builder:flatbuffers.Builder, updateOffset:flatbuffers.Offset) {
+  builder.addFieldOffset(1, updateOffset, 0);
+}
+
+static endVideoUpdate(builder:flatbuffers.Builder):flatbuffers.Offset {
+  const offset = builder.endObject();
+  return offset;
+}
+
+static finishVideoUpdateBuffer(builder:flatbuffers.Builder, offset:flatbuffers.Offset) {
+  builder.finish(offset);
+}
+
+static finishSizePrefixedVideoUpdateBuffer(builder:flatbuffers.Builder, offset:flatbuffers.Offset) {
+  builder.finish(offset, undefined, true);
+}
+
+static createVideoUpdate(builder:flatbuffers.Builder, updateType:Update, updateOffset:flatbuffers.Offset):flatbuffers.Offset {
+  VideoUpdate.startVideoUpdate(builder);
+  VideoUpdate.addUpdateType(builder, updateType);
+  VideoUpdate.addUpdate(builder, updateOffset);
+  return VideoUpdate.endVideoUpdate(builder);
+}
+}

frontend/src/lib/proto/video.ts

@@ -0,0 +1,5 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */
+
+export * as VideoUpdate from './video-update.js';

frontend/src/lib/video.ts

@@ -1,33 +1,8 @@
-type Setup = {
-  video_format: string,
-  width: number,
-  height: number,
-  redraw_rate: number,
-  dr_flags: number,
-}
+import { VideoUpdate } from "$lib/proto/video";
+import { ByteBuffer } from "flatbuffers";
 
-type SetupPacket = {
-  Setup: Setup
-}
-
-type DecodeBuffer = {
-  buffer_bype: string,
-  data: Array<number>,
-}
-
-type DecodeUnit = {
-  frame_number: number,
-  frame_type: string,
-  buffer: DecodeBuffer,
-  receieve_time_ms: number,
-}
-
-type DecodeUnitPacket = {
-  DecodeUnit: DecodeUnit
-}
-
-function parseData(newBuffer: Uint8Array, oldBuffer: Uint8Array): [Array<Object>, Uint8Array<ArrayBuffer>] {
-  let packets = new Array<Object>();
+function parseData(newBuffer: Uint8Array, oldBuffer: Uint8Array): [Array<VideoUpdate.VideoUpdate>, Uint8Array<ArrayBuffer>] {
+  let packets = new Array<VideoUpdate.VideoUpdate>();
   let unparsedData = new Uint8Array();
 
   let data = new Uint8Array([...oldBuffer, ...newBuffer]);
@@ -43,15 +18,14 @@ function parseData(newBuffer: Uint8Array, oldBuffer: Uint8Array): [Array<Object>
     const slice_end_index = index + 4 + dataLength;
 
     if (data.length < slice_end_index) {
-      unparsedData = new Uint8Array(data.buffer.slice(index, data.length));
+      unparsedData = data.slice(index, data.length);
       break;
     }
 
-    const dataToParse = data.buffer.slice(slice_start_index, slice_end_index);
-    const decoder = new TextDecoder('utf-8');
-    const jsonString = decoder.decode(dataToParse);
+    const dataToParse = new ByteBuffer(data.slice(slice_start_index, slice_end_index));
 
-    packets.push(JSON.parse(jsonString));
+    const videoUpdate = VideoUpdate.VideoUpdate.getRootAsVideoUpdate(dataToParse);
+    packets.push(videoUpdate);
 
     index += 4 + dataLength;
   }
@@ -70,21 +44,18 @@ export async function streamVideoFromReader(reader: ReadableStreamDefaultReader,
 
     const videoDecoder = new VideoDecoder({
       output: (frame) => {
-        // Set canvas dimensions to match the frame
         canvasElement.width = frame.displayWidth;
         canvasElement.height = frame.displayHeight;
 
-        // Draw the decoded frame to canvas
         canvasCtx.drawImage(frame, 0, 0);
 
-        // Important: close the frame to free memory
         frame.close();
       },
       error: (e) => {
         console.error('Decode error:', e);
       }
     });
-
+    let decodeUnit: VideoUpdate.DecodeUnit = new VideoUpdate.DecodeUnit();
     while (true) {
       const { value, done } = await reader.read();
       if (done) break;
@@ -93,20 +64,13 @@ export async function streamVideoFromReader(reader: ReadableStreamDefaultReader,
       unparsedData = remainingData;
 
       for (let i = 0; i < packets.length; i++) {
-        if (Object.hasOwn(packets[i], "Setup")) {
-          let packet = packets[i] as SetupPacket;
-
-          let config: VideoDecoderConfig | undefined = undefined;
-          if (packet.Setup.video_format == "H264") {
-            config = {
-              //codec: 'avc1.42E01E', // H.264 codec
-              codec: 'avc1.4D002A', // H.264 codec
-              codedWidth: packet.Setup.width,
-              codedHeight: packet.Setup.height,
-            };
-          } else {
-            throw new Error(`Unsupported video codec ${packet.Setup.video_format}`);
-          }
+        if (packets[i].updateType() == VideoUpdate.Update.Setup) {
+          let setup = packets[i].update(new VideoUpdate.Setup());
+          let config: VideoDecoderConfig = {
+            codec: setup.videoFormat(),
+            codedWidth: setup.width(),
+            codedHeight: setup.height(),
+          };
 
           const codecSupport = await VideoDecoder.isConfigSupported(config);
           if (codecSupport.supported) {
@@ -115,18 +79,18 @@ export async function streamVideoFromReader(reader: ReadableStreamDefaultReader,
             throw new Error(`Could not configure decoder`);
           }
 
-        } else if (Object.hasOwn(packets[i], "DecodeUnit")) {
-          let packet = packets[i] as DecodeUnitPacket;
+        } else if (packets[i].updateType() == VideoUpdate.Update.DecodeUnit) {
+          packets[i].update(decodeUnit);
 
-
-          let frame_type: EncodedAudioChunkType = "delta";
-          if (packet.DecodeUnit.frame_type == "IDR") {
-            frame_type = "key";
+          let frameType: EncodedAudioChunkType = "delta";
+          if (decodeUnit.frameType() == VideoUpdate.FrameType.IDR) {
+            console.log("GOT KEYFRAME");
+            frameType = "key";
           }
           const chunk = new EncodedVideoChunk({
-            timestamp: packet.DecodeUnit.receieve_time_ms,
-            type: frame_type,
-            data: new Uint8Array(packet.DecodeUnit.buffer.data),
+            timestamp: decodeUnit.receiveTimeMs(),
+            type: frameType,
+            data: decodeUnit.dataArray()!,
           });
 
           videoDecoder.decode(chunk);

frontend/src/routes/getStreamData.ts

@@ -41,7 +41,7 @@ export async function getStreamData(appId: number, server_name: string): Promise
         height: 1080,
       },
       stream_config: {
-        bitrate_kbps: 1024 * 10,
+        bitrate_kbps: 1024 * 10 * 2,
         mode: {
           fps: 60,
           width: 1920,

frontend/src/routes/stream/stream.ts

@@ -4,12 +4,12 @@ import CanvasWorker from "$lib/canvas.worker?worker";
 export async function getStreamTransport(url: string, certHash: Array<number>): Promise<WebTransport> {
   let certHashArray = new Uint8Array(certHash);
 
-  // Check if WebTransport is supported
   if (!window.WebTransport) {
     throw new Error('WebTransport is not supported in this browser');
   }
 
   const transport = new WebTransport(url, {
+    congestionControl: "low-latency",
     serverCertificateHashes: [
       {
         algorithm: "sha-256",
@@ -19,7 +19,6 @@ export async function getStreamTransport(url: string, certHash: Array<number>):
   });
 
   console.log('Connecting to WebTransport at ', url);
-  // Wait for the connection to be ready
   await transport.ready;
   console.log('WebTransport connection established');
 

gamestream-webtransport-proxy/src/gamestream/config.rs

@@ -40,7 +40,7 @@ pub fn stream_config(stream: &crate::backend::Stream) -> _STREAM_CONFIGURATION {
         height: stream.stream_config.mode.height,
         fps: stream.stream_config.mode.fps,
         bitrate: stream.stream_config.bitrate_kbps,
-        packetSize: 512,
+        packetSize: 1024,
         streamingRemotely: STREAM_CFG_AUTO,
         audioConfiguration: (0x3 << 16) | (2 << 8) | 0xCA,
         supportedVideoFormats: VIDEO_FORMAT_H264,

gamestream-webtransport-proxy/src/gamestream/decoder.rs

@@ -12,7 +12,7 @@ use tokio::sync::mpsc;
 use tracing::{debug, error};
 
 #[derive(Serialize)]
-enum FrameType {
+pub enum FrameType {
     PFRAME,
     IDR,
 }
@@ -32,7 +32,7 @@ impl TryFrom<i32> for FrameType {
 }
 
 #[derive(Serialize)]
-enum VideoFormat {
+pub enum VideoFormat {
     H264,
     H264_HIGH8_444,
     H265,
@@ -92,8 +92,8 @@ impl TryFrom<i32> for BufferType {
 }
 
 #[derive(Serialize)]
-struct Buffer {
-    data: Vec<u8>,
+pub struct Buffer {
+    pub data: Vec<u8>,
     buffer_type: BufferType,
 }
 
@@ -124,7 +124,7 @@ pub enum RendererMessage {
         frame_type: FrameType,
 
         host_processing_latency: u16,
-        receieve_time_ms: u64,
+        receive_time_ms: u64,
         enqueue_time_ms: u64,
         presentation_time: u64,
 
@@ -184,7 +184,7 @@ impl RendererMessage {
             frame_number: <u64>::try_from(decode_unit.frameNumber)?,
             frame_type: FrameType::try_from(decode_unit.frameType)?,
             host_processing_latency: decode_unit.frameHostProcessingLatency,
-            receieve_time_ms: decode_unit.receiveTimeMs,
+            receive_time_ms: decode_unit.receiveTimeMs,
             enqueue_time_ms: decode_unit.enqueueTimeMs,
             presentation_time: decode_unit.presentationTimeMs as u64,
             full_length: <usize>::try_from(decode_unit.fullLength)?,

gamestream-webtransport-proxy/src/gamestream/mod.rs

@@ -2,7 +2,7 @@ use anyhow::{Result, anyhow};
 use tokio::sync::mpsc;
 
 mod config;
-mod decoder;
+pub mod decoder;
 
 #[derive(Debug)]
 pub struct GamestreamChannels {

gamestream-webtransport-proxy/src/proxy/mod.rs

@@ -7,6 +7,7 @@ use crate::{backend, gamestream};
 pub mod handler;
 mod input;
 mod packet_parser;
+mod video;
 
 pub struct Proxy {
     pub cert_hash: [u8; 32],
@@ -42,20 +43,14 @@ async fn proxy_main(
     let mut channels = spawn_gamestream(stream).await?;
 
     let mut packet_buffer = packet_parser::PacketBuffer::new();
-    //let mut buffer = vec![0; 65536].into_boxed_slice();
     let mut buffer = [0u8; 65536];
-    //let mut buffer = vec![0; 65536].into_boxed_slice();
 
     loop {
         select! {
             gamestream_packet = channels.gamestream_channels.decoder_rx.recv() => {
                 match gamestream_packet {
                     Some(frame) => {
-                        let frame_json = serde_json::to_vec(&frame)?;
-                        let frame_json_len: u32 = <u32>::try_from(frame_json.len())?;
-
-                        wt_send.write_all(&frame_json_len.to_le_bytes()).await?;
-                        wt_send.write_all(&frame_json).await?;
+                        video::send_video_update(&frame, &mut wt_send).await?;
                     }
                     None => {
                         error!("Decoder channel is None");

gamestream-webtransport-proxy/src/proxy/video/mod.rs

@@ -0,0 +1,114 @@
+use anyhow::Result;
+use tokio::io::AsyncWriteExt;
+use tracing::debug;
+
+use crate::gamestream;
+use video_generated::video_update;
+
+mod video_generated;
+
+fn create_setup_videoupdate(
+    video_format: &gamestream::decoder::VideoFormat,
+    width: u64,
+    height: u64,
+    redraw_rate: u64,
+) -> Vec<u8> {
+    let mut builder = flatbuffers::FlatBufferBuilder::with_capacity(1024);
+
+    //TODO: this is hardcoded to h264 main profile
+    let video_format = Some(builder.create_string("avc1.4D002A"));
+
+    let setup = video_update::Setup::create(
+        &mut builder,
+        &video_update::SetupArgs {
+            video_format,
+            width: width as u16,
+            height: height as u16,
+            redraw_rate: redraw_rate as u16,
+        },
+    );
+
+    let video_update = video_update::VideoUpdate::create(
+        &mut builder,
+        &video_update::VideoUpdateArgs {
+            update_type: video_update::Update::Setup,
+            update: Some(setup.as_union_value()),
+        },
+    );
+
+    builder.finish(video_update, None);
+
+    builder.finished_data().to_vec()
+}
+
+fn create_decodeunit_videoupdate(
+    frame_number: u64,
+    frame_type: &gamestream::decoder::FrameType,
+    receive_time_ms: u64,
+    buffer: &gamestream::decoder::Buffer,
+) -> Vec<u8> {
+    let mut builder = flatbuffers::FlatBufferBuilder::with_capacity(1024);
+
+    let data_vector = builder.create_vector(&buffer.data);
+
+    let frame_type_fb = match frame_type {
+        gamestream::decoder::FrameType::IDR => video_update::FrameType::IDR,
+        gamestream::decoder::FrameType::PFRAME => video_update::FrameType::PFRAME,
+    };
+
+    let decode_unit = video_update::DecodeUnit::create(
+        &mut builder,
+        &video_update::DecodeUnitArgs {
+            frame_number: frame_number as u16,
+            frame_type: frame_type_fb,
+            receive_time_ms: receive_time_ms as u16,
+            data: Some(data_vector),
+        },
+    );
+
+    let video_update = video_update::VideoUpdate::create(
+        &mut builder,
+        &video_update::VideoUpdateArgs {
+            update_type: video_update::Update::DecodeUnit,
+            update: Some(decode_unit.as_union_value()),
+        },
+    );
+
+    builder.finish(video_update, None);
+
+    builder.finished_data().to_vec()
+}
+
+pub async fn send_video_update(
+    frame: &gamestream::decoder::RendererMessage,
+    mut wt_send: impl tokio::io::AsyncWrite + Send + Sync + std::marker::Unpin,
+) -> Result<()> {
+    let buffer = match frame {
+        gamestream::decoder::RendererMessage::Setup {
+            video_format,
+            width,
+            height,
+            redraw_rate,
+            dr_flags,
+        } => create_setup_videoupdate(video_format, *width, *height, *redraw_rate),
+        gamestream::decoder::RendererMessage::DecodeUnit {
+            frame_number,
+            frame_type,
+            host_processing_latency,
+            receive_time_ms,
+            enqueue_time_ms,
+            presentation_time,
+            full_length,
+            buffer,
+            index,
+            hdr_active,
+            colorspace,
+        } => create_decodeunit_videoupdate(*frame_number, frame_type, *receive_time_ms, buffer),
+    };
+
+    let buffer_len = buffer.len() as u32;
+
+    wt_send.write_all(&buffer_len.to_le_bytes()).await?;
+    wt_send.write_all(&buffer).await?;
+    Ok(())
+}

gamestream-webtransport-proxy/src/proxy/video/video_generated.rs

@@ -0,0 +1,733 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+
+// @generated
+
+use core::mem;
+use core::cmp::Ordering;
+
+extern crate flatbuffers;
+use self::flatbuffers::{EndianScalar, Follow};
+
+#[allow(unused_imports, dead_code)]
+pub mod video_update {
+
+  use core::mem;
+  use core::cmp::Ordering;
+
+  extern crate flatbuffers;
+  use self::flatbuffers::{EndianScalar, Follow};
+
+#[deprecated(since = "2.0.0", note = "Use associated constants instead. This will no longer be generated in 2021.")]
+pub const ENUM_MIN_FRAME_TYPE: i8 = 0;
+#[deprecated(since = "2.0.0", note = "Use associated constants instead. This will no longer be generated in 2021.")]
+pub const ENUM_MAX_FRAME_TYPE: i8 = 1;
+#[deprecated(since = "2.0.0", note = "Use associated constants instead. This will no longer be generated in 2021.")]
+#[allow(non_camel_case_types)]
+pub const ENUM_VALUES_FRAME_TYPE: [FrameType; 2] = [
+  FrameType::PFRAME,
+  FrameType::IDR,
+];
+
+#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Default)]
+#[repr(transparent)]
+pub struct FrameType(pub i8);
+#[allow(non_upper_case_globals)]
+impl FrameType {
+  pub const PFRAME: Self = Self(0);
+  pub const IDR: Self = Self(1);
+
+  pub const ENUM_MIN: i8 = 0;
+  pub const ENUM_MAX: i8 = 1;
+  pub const ENUM_VALUES: &'static [Self] = &[
+    Self::PFRAME,
+    Self::IDR,
+  ];
+  /// Returns the variant's name or "" if unknown.
+  pub fn variant_name(self) -> Option<&'static str> {
+    match self {
+      Self::PFRAME => Some("PFRAME"),
+      Self::IDR => Some("IDR"),
+      _ => None,
+    }
+  }
+}
+impl core::fmt::Debug for FrameType {
+  fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result {
+    if let Some(name) = self.variant_name() {
+      f.write_str(name)
+    } else {
+      f.write_fmt(format_args!("<UNKNOWN {:?}>", self.0))
+    }
+  }
+}
+impl<'a> flatbuffers::Follow<'a> for FrameType {
+  type Inner = Self;
+  #[inline]
+  unsafe fn follow(buf: &'a [u8], loc: usize) -> Self::Inner {
+    let b = flatbuffers::read_scalar_at::<i8>(buf, loc);
+    Self(b)
+  }
+}
+
+impl flatbuffers::Push for FrameType {
+    type Output = FrameType;
+    #[inline]
+    unsafe fn push(&self, dst: &mut [u8], _written_len: usize) {
+        flatbuffers::emplace_scalar::<i8>(dst, self.0);
+    }
+}
+
+impl flatbuffers::EndianScalar for FrameType {
+  type Scalar = i8;
+  #[inline]
+  fn to_little_endian(self) -> i8 {
+    self.0.to_le()
+  }
+  #[inline]
+  #[allow(clippy::wrong_self_convention)]
+  fn from_little_endian(v: i8) -> Self {
+    let b = i8::from_le(v);
+    Self(b)
+  }
+}
+
+impl<'a> flatbuffers::Verifiable for FrameType {
+  #[inline]
+  fn run_verifier(
+    v: &mut flatbuffers::Verifier, pos: usize
+  ) -> Result<(), flatbuffers::InvalidFlatbuffer> {
+    use self::flatbuffers::Verifiable;
+    i8::run_verifier(v, pos)
+  }
+}
+
+impl flatbuffers::SimpleToVerifyInSlice for FrameType {}
+#[deprecated(since = "2.0.0", note = "Use associated constants instead. This will no longer be generated in 2021.")]
+pub const ENUM_MIN_UPDATE: u8 = 0;
+#[deprecated(since = "2.0.0", note = "Use associated constants instead. This will no longer be generated in 2021.")]
+pub const ENUM_MAX_UPDATE: u8 = 2;
+#[deprecated(since = "2.0.0", note = "Use associated constants instead. This will no longer be generated in 2021.")]
+#[allow(non_camel_case_types)]
+pub const ENUM_VALUES_UPDATE: [Update; 3] = [
+  Update::NONE,
+  Update::Setup,
+  Update::DecodeUnit,
+];
+
+#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Default)]
+#[repr(transparent)]
+pub struct Update(pub u8);
+#[allow(non_upper_case_globals)]
+impl Update {
+  pub const NONE: Self = Self(0);
+  pub const Setup: Self = Self(1);
+  pub const DecodeUnit: Self = Self(2);
+
+  pub const ENUM_MIN: u8 = 0;
+  pub const ENUM_MAX: u8 = 2;
+  pub const ENUM_VALUES: &'static [Self] = &[
+    Self::NONE,
+    Self::Setup,
+    Self::DecodeUnit,
+  ];
+  /// Returns the variant's name or "" if unknown.
+  pub fn variant_name(self) -> Option<&'static str> {
+    match self {
+      Self::NONE => Some("NONE"),
+      Self::Setup => Some("Setup"),
+      Self::DecodeUnit => Some("DecodeUnit"),
+      _ => None,
+    }
+  }
+}
+impl core::fmt::Debug for Update {
+  fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result {
+    if let Some(name) = self.variant_name() {
+      f.write_str(name)
+    } else {
+      f.write_fmt(format_args!("<UNKNOWN {:?}>", self.0))
+    }
+  }
+}
+impl<'a> flatbuffers::Follow<'a> for Update {
+  type Inner = Self;
+  #[inline]
+  unsafe fn follow(buf: &'a [u8], loc: usize) -> Self::Inner {
+    let b = flatbuffers::read_scalar_at::<u8>(buf, loc);
+    Self(b)
+  }
+}
+
+impl flatbuffers::Push for Update {
+    type Output = Update;
+    #[inline]
+    unsafe fn push(&self, dst: &mut [u8], _written_len: usize) {
+        flatbuffers::emplace_scalar::<u8>(dst, self.0);
+    }
+}
+
+impl flatbuffers::EndianScalar for Update {
+  type Scalar = u8;
+  #[inline]
+  fn to_little_endian(self) -> u8 {
+    self.0.to_le()
+  }
+  #[inline]
+  #[allow(clippy::wrong_self_convention)]
+  fn from_little_endian(v: u8) -> Self {
+    let b = u8::from_le(v);
+    Self(b)
+  }
+}
+
+impl<'a> flatbuffers::Verifiable for Update {
+  #[inline]
+  fn run_verifier(
+    v: &mut flatbuffers::Verifier, pos: usize
+  ) -> Result<(), flatbuffers::InvalidFlatbuffer> {
+    use self::flatbuffers::Verifiable;
+    u8::run_verifier(v, pos)
+  }
+}
+
+impl flatbuffers::SimpleToVerifyInSlice for Update {}
+pub struct UpdateUnionTableOffset {}
+
+pub enum SetupOffset {}
+#[derive(Copy, Clone, PartialEq)]
+
+pub struct Setup<'a> {
+  pub _tab: flatbuffers::Table<'a>,
+}
+
+impl<'a> flatbuffers::Follow<'a> for Setup<'a> {
+  type Inner = Setup<'a>;
+  #[inline]
+  unsafe fn follow(buf: &'a [u8], loc: usize) -> Self::Inner {
+    Self { _tab: flatbuffers::Table::new(buf, loc) }
+  }
+}
+
+impl<'a> Setup<'a> {
+  pub const VT_VIDEO_FORMAT: flatbuffers::VOffsetT = 4;
+  pub const VT_WIDTH: flatbuffers::VOffsetT = 6;
+  pub const VT_HEIGHT: flatbuffers::VOffsetT = 8;
+  pub const VT_REDRAW_RATE: flatbuffers::VOffsetT = 10;
+
+  #[inline]
+  pub unsafe fn init_from_table(table: flatbuffers::Table<'a>) -> Self {
+    Setup { _tab: table }
+  }
+  #[allow(unused_mut)]
+  pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr, A: flatbuffers::Allocator + 'bldr>(
+    _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr, A>,
+    args: &'args SetupArgs<'args>
+  ) -> flatbuffers::WIPOffset<Setup<'bldr>> {
+    let mut builder = SetupBuilder::new(_fbb);
+    if let Some(x) = args.video_format { builder.add_video_format(x); }
+    builder.add_redraw_rate(args.redraw_rate);
+    builder.add_height(args.height);
+    builder.add_width(args.width);
+    builder.finish()
+  }
+
+
+  #[inline]
+  pub fn video_format(&self) -> Option<&'a str> {
+    // Safety:
+    // Created from valid Table for this object
+    // which contains a valid value in this slot
+    unsafe { self._tab.get::<flatbuffers::ForwardsUOffset<&str>>(Setup::VT_VIDEO_FORMAT, None)}
+  }
+  #[inline]
+  pub fn width(&self) -> u16 {
+    // Safety:
+    // Created from valid Table for this object
+    // which contains a valid value in this slot
+    unsafe { self._tab.get::<u16>(Setup::VT_WIDTH, Some(0)).unwrap()}
+  }
+  #[inline]
+  pub fn height(&self) -> u16 {
+    // Safety:
+    // Created from valid Table for this object
+    // which contains a valid value in this slot
+    unsafe { self._tab.get::<u16>(Setup::VT_HEIGHT, Some(0)).unwrap()}
+  }
+  #[inline]
+  pub fn redraw_rate(&self) -> u16 {
+    // Safety:
+    // Created from valid Table for this object
+    // which contains a valid value in this slot
+    unsafe { self._tab.get::<u16>(Setup::VT_REDRAW_RATE, Some(0)).unwrap()}
+  }
+}
+
+impl flatbuffers::Verifiable for Setup<'_> {
+  #[inline]
+  fn run_verifier(
+    v: &mut flatbuffers::Verifier, pos: usize
+  ) -> Result<(), flatbuffers::InvalidFlatbuffer> {
+    use self::flatbuffers::Verifiable;
+    v.visit_table(pos)?
+     .visit_field::<flatbuffers::ForwardsUOffset<&str>>("video_format", Self::VT_VIDEO_FORMAT, false)?
+     .visit_field::<u16>("width", Self::VT_WIDTH, false)?
+     .visit_field::<u16>("height", Self::VT_HEIGHT, false)?
+     .visit_field::<u16>("redraw_rate", Self::VT_REDRAW_RATE, false)?
+     .finish();
+    Ok(())
+  }
+}
+pub struct SetupArgs<'a> {
+    pub video_format: Option<flatbuffers::WIPOffset<&'a str>>,
+    pub width: u16,
+    pub height: u16,
+    pub redraw_rate: u16,
+}
+impl<'a> Default for SetupArgs<'a> {
+  #[inline]
+  fn default() -> Self {
+    SetupArgs {
+      video_format: None,
+      width: 0,
+      height: 0,
+      redraw_rate: 0,
+    }
+  }
+}
+
+pub struct SetupBuilder<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> {
+  fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a, A>,
+  start_: flatbuffers::WIPOffset<flatbuffers::TableUnfinishedWIPOffset>,
+}
+impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> SetupBuilder<'a, 'b, A> {
+  #[inline]
+  pub fn add_video_format(&mut self, video_format: flatbuffers::WIPOffset<&'b  str>) {
+    self.fbb_.push_slot_always::<flatbuffers::WIPOffset<_>>(Setup::VT_VIDEO_FORMAT, video_format);
+  }
+  #[inline]
+  pub fn add_width(&mut self, width: u16) {
+    self.fbb_.push_slot::<u16>(Setup::VT_WIDTH, width, 0);
+  }
+  #[inline]
+  pub fn add_height(&mut self, height: u16) {
+    self.fbb_.push_slot::<u16>(Setup::VT_HEIGHT, height, 0);
+  }
+  #[inline]
+  pub fn add_redraw_rate(&mut self, redraw_rate: u16) {
+    self.fbb_.push_slot::<u16>(Setup::VT_REDRAW_RATE, redraw_rate, 0);
+  }
+  #[inline]
+  pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>) -> SetupBuilder<'a, 'b, A> {
+    let start = _fbb.start_table();
+    SetupBuilder {
+      fbb_: _fbb,
+      start_: start,
+    }
+  }
+  #[inline]
+  pub fn finish(self) -> flatbuffers::WIPOffset<Setup<'a>> {
+    let o = self.fbb_.end_table(self.start_);
+    flatbuffers::WIPOffset::new(o.value())
+  }
+}
+
+impl core::fmt::Debug for Setup<'_> {
+  fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
+    let mut ds = f.debug_struct("Setup");
+      ds.field("video_format", &self.video_format());
+      ds.field("width", &self.width());
+      ds.field("height", &self.height());
+      ds.field("redraw_rate", &self.redraw_rate());
+      ds.finish()
+  }
+}
+pub enum DecodeUnitOffset {}
+#[derive(Copy, Clone, PartialEq)]
+
+pub struct DecodeUnit<'a> {
+  pub _tab: flatbuffers::Table<'a>,
+}
+
+impl<'a> flatbuffers::Follow<'a> for DecodeUnit<'a> {
+  type Inner = DecodeUnit<'a>;
+  #[inline]
+  unsafe fn follow(buf: &'a [u8], loc: usize) -> Self::Inner {
+    Self { _tab: flatbuffers::Table::new(buf, loc) }
+  }
+}
+
+impl<'a> DecodeUnit<'a> {
+  pub const VT_FRAME_NUMBER: flatbuffers::VOffsetT = 4;
+  pub const VT_FRAME_TYPE: flatbuffers::VOffsetT = 6;
+  pub const VT_RECEIVE_TIME_MS: flatbuffers::VOffsetT = 8;
+  pub const VT_DATA: flatbuffers::VOffsetT = 10;
+
+  #[inline]
+  pub unsafe fn init_from_table(table: flatbuffers::Table<'a>) -> Self {
+    DecodeUnit { _tab: table }
+  }
+  #[allow(unused_mut)]
+  pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr, A: flatbuffers::Allocator + 'bldr>(
+    _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr, A>,
+    args: &'args DecodeUnitArgs<'args>
+  ) -> flatbuffers::WIPOffset<DecodeUnit<'bldr>> {
+    let mut builder = DecodeUnitBuilder::new(_fbb);
+    if let Some(x) = args.data { builder.add_data(x); }
+    builder.add_receive_time_ms(args.receive_time_ms);
+    builder.add_frame_number(args.frame_number);
+    builder.add_frame_type(args.frame_type);
+    builder.finish()
+  }
+
+
+  #[inline]
+  pub fn frame_number(&self) -> u16 {
+    // Safety:
+    // Created from valid Table for this object
+    // which contains a valid value in this slot
+    unsafe { self._tab.get::<u16>(DecodeUnit::VT_FRAME_NUMBER, Some(0)).unwrap()}
+  }
+  #[inline]
+  pub fn frame_type(&self) -> FrameType {
+    // Safety:
+    // Created from valid Table for this object
+    // which contains a valid value in this slot
+    unsafe { self._tab.get::<FrameType>(DecodeUnit::VT_FRAME_TYPE, Some(FrameType::PFRAME)).unwrap()}
+  }
+  #[inline]
+  pub fn receive_time_ms(&self) -> u16 {
+    // Safety:
+    // Created from valid Table for this object
+    // which contains a valid value in this slot
+    unsafe { self._tab.get::<u16>(DecodeUnit::VT_RECEIVE_TIME_MS, Some(0)).unwrap()}
+  }
+  #[inline]
+  pub fn data(&self) -> Option<flatbuffers::Vector<'a, u8>> {
+    // Safety:
+    // Created from valid Table for this object
+    // which contains a valid value in this slot
+    unsafe { self._tab.get::<flatbuffers::ForwardsUOffset<flatbuffers::Vector<'a, u8>>>(DecodeUnit::VT_DATA, None)}
+  }
+}
+
+impl flatbuffers::Verifiable for DecodeUnit<'_> {
+  #[inline]
+  fn run_verifier(
+    v: &mut flatbuffers::Verifier, pos: usize
+  ) -> Result<(), flatbuffers::InvalidFlatbuffer> {
+    use self::flatbuffers::Verifiable;
+    v.visit_table(pos)?
+     .visit_field::<u16>("frame_number", Self::VT_FRAME_NUMBER, false)?
+     .visit_field::<FrameType>("frame_type", Self::VT_FRAME_TYPE, false)?
+     .visit_field::<u16>("receive_time_ms", Self::VT_RECEIVE_TIME_MS, false)?
+     .visit_field::<flatbuffers::ForwardsUOffset<flatbuffers::Vector<'_, u8>>>("data", Self::VT_DATA, false)?
+     .finish();
+    Ok(())
+  }
+}
+pub struct DecodeUnitArgs<'a> {
+    pub frame_number: u16,
+    pub frame_type: FrameType,
+    pub receive_time_ms: u16,
+    pub data: Option<flatbuffers::WIPOffset<flatbuffers::Vector<'a, u8>>>,
+}
+impl<'a> Default for DecodeUnitArgs<'a> {
+  #[inline]
+  fn default() -> Self {
+    DecodeUnitArgs {
+      frame_number: 0,
+      frame_type: FrameType::PFRAME,
+      receive_time_ms: 0,
+      data: None,
+    }
+  }
+}
+
+pub struct DecodeUnitBuilder<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> {
+  fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a, A>,
+  start_: flatbuffers::WIPOffset<flatbuffers::TableUnfinishedWIPOffset>,
+}
+impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> DecodeUnitBuilder<'a, 'b, A> {
+  #[inline]
+  pub fn add_frame_number(&mut self, frame_number: u16) {
+    self.fbb_.push_slot::<u16>(DecodeUnit::VT_FRAME_NUMBER, frame_number, 0);
+  }
+  #[inline]
+  pub fn add_frame_type(&mut self, frame_type: FrameType) {
+    self.fbb_.push_slot::<FrameType>(DecodeUnit::VT_FRAME_TYPE, frame_type, FrameType::PFRAME);
+  }
+  #[inline]
+  pub fn add_receive_time_ms(&mut self, receive_time_ms: u16) {
+    self.fbb_.push_slot::<u16>(DecodeUnit::VT_RECEIVE_TIME_MS, receive_time_ms, 0);
+  }
+  #[inline]
+  pub fn add_data(&mut self, data: flatbuffers::WIPOffset<flatbuffers::Vector<'b , u8>>) {
+    self.fbb_.push_slot_always::<flatbuffers::WIPOffset<_>>(DecodeUnit::VT_DATA, data);
+  }
+  #[inline]
+  pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>) -> DecodeUnitBuilder<'a, 'b, A> {
+    let start = _fbb.start_table();
+    DecodeUnitBuilder {
+      fbb_: _fbb,
+      start_: start,
+    }
+  }
+  #[inline]
+  pub fn finish(self) -> flatbuffers::WIPOffset<DecodeUnit<'a>> {
+    let o = self.fbb_.end_table(self.start_);
+    flatbuffers::WIPOffset::new(o.value())
+  }
+}
+
+impl core::fmt::Debug for DecodeUnit<'_> {
+  fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
+    let mut ds = f.debug_struct("DecodeUnit");
+      ds.field("frame_number", &self.frame_number());
+      ds.field("frame_type", &self.frame_type());
+      ds.field("receive_time_ms", &self.receive_time_ms());
+      ds.field("data", &self.data());
+      ds.finish()
+  }
+}
+pub enum VideoUpdateOffset {}
+#[derive(Copy, Clone, PartialEq)]
+
+pub struct VideoUpdate<'a> {
+  pub _tab: flatbuffers::Table<'a>,
+}
+
+impl<'a> flatbuffers::Follow<'a> for VideoUpdate<'a> {
+  type Inner = VideoUpdate<'a>;
+  #[inline]
+  unsafe fn follow(buf: &'a [u8], loc: usize) -> Self::Inner {
+    Self { _tab: flatbuffers::Table::new(buf, loc) }
+  }
+}
+
+impl<'a> VideoUpdate<'a> {
+  pub const VT_UPDATE_TYPE: flatbuffers::VOffsetT = 4;
+  pub const VT_UPDATE: flatbuffers::VOffsetT = 6;
+
+  #[inline]
+  pub unsafe fn init_from_table(table: flatbuffers::Table<'a>) -> Self {
+    VideoUpdate { _tab: table }
+  }
+  #[allow(unused_mut)]
+  pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr, A: flatbuffers::Allocator + 'bldr>(
+    _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr, A>,
+    args: &'args VideoUpdateArgs
+  ) -> flatbuffers::WIPOffset<VideoUpdate<'bldr>> {
+    let mut builder = VideoUpdateBuilder::new(_fbb);
+    if let Some(x) = args.update { builder.add_update(x); }
+    builder.add_update_type(args.update_type);
+    builder.finish()
+  }
+
+
+  #[inline]
+  pub fn update_type(&self) -> Update {
+    // Safety:
+    // Created from valid Table for this object
+    // which contains a valid value in this slot
+    unsafe { self._tab.get::<Update>(VideoUpdate::VT_UPDATE_TYPE, Some(Update::NONE)).unwrap()}
+  }
+  #[inline]
+  pub fn update(&self) -> Option<flatbuffers::Table<'a>> {
+    // Safety:
+    // Created from valid Table for this object
+    // which contains a valid value in this slot
+    unsafe { self._tab.get::<flatbuffers::ForwardsUOffset<flatbuffers::Table<'a>>>(VideoUpdate::VT_UPDATE, None)}
+  }
+  #[inline]
+  #[allow(non_snake_case)]
+  pub fn update_as_setup(&self) -> Option<Setup<'a>> {
+    if self.update_type() == Update::Setup {
+      self.update().map(|t| {
+       // Safety:
+       // Created from a valid Table for this object
+       // Which contains a valid union in this slot
+       unsafe { Setup::init_from_table(t) }
+     })
+    } else {
+      None
+    }
+  }
+
+  #[inline]
+  #[allow(non_snake_case)]
+  pub fn update_as_decode_unit(&self) -> Option<DecodeUnit<'a>> {
+    if self.update_type() == Update::DecodeUnit {
+      self.update().map(|t| {
+       // Safety:
+       // Created from a valid Table for this object
+       // Which contains a valid union in this slot
+       unsafe { DecodeUnit::init_from_table(t) }
+     })
+    } else {
+      None
+    }
+  }
+
+}
+
+impl flatbuffers::Verifiable for VideoUpdate<'_> {
+  #[inline]
+  fn run_verifier(
+    v: &mut flatbuffers::Verifier, pos: usize
+  ) -> Result<(), flatbuffers::InvalidFlatbuffer> {
+    use self::flatbuffers::Verifiable;
+    v.visit_table(pos)?
+     .visit_union::<Update, _>("update_type", Self::VT_UPDATE_TYPE, "update", Self::VT_UPDATE, false, |key, v, pos| {
+        match key {
+          Update::Setup => v.verify_union_variant::<flatbuffers::ForwardsUOffset<Setup>>("Update::Setup", pos),
+          Update::DecodeUnit => v.verify_union_variant::<flatbuffers::ForwardsUOffset<DecodeUnit>>("Update::DecodeUnit", pos),
+          _ => Ok(()),
+        }
+     })?
+     .finish();
+    Ok(())
+  }
+}
+pub struct VideoUpdateArgs {
+    pub update_type: Update,
+    pub update: Option<flatbuffers::WIPOffset<flatbuffers::UnionWIPOffset>>,
+}
+impl<'a> Default for VideoUpdateArgs {
+  #[inline]
+  fn default() -> Self {
+    VideoUpdateArgs {
+      update_type: Update::NONE,
+      update: None,
+    }
+  }
+}
+
+pub struct VideoUpdateBuilder<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> {
+  fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a, A>,
+  start_: flatbuffers::WIPOffset<flatbuffers::TableUnfinishedWIPOffset>,
+}
+impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> VideoUpdateBuilder<'a, 'b, A> {
+  #[inline]
+  pub fn add_update_type(&mut self, update_type: Update) {
+    self.fbb_.push_slot::<Update>(VideoUpdate::VT_UPDATE_TYPE, update_type, Update::NONE);
+  }
+  #[inline]
+  pub fn add_update(&mut self, update: flatbuffers::WIPOffset<flatbuffers::UnionWIPOffset>) {
+    self.fbb_.push_slot_always::<flatbuffers::WIPOffset<_>>(VideoUpdate::VT_UPDATE, update);
+  }
+  #[inline]
+  pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>) -> VideoUpdateBuilder<'a, 'b, A> {
+    let start = _fbb.start_table();
+    VideoUpdateBuilder {
+      fbb_: _fbb,
+      start_: start,
+    }
+  }
+  #[inline]
+  pub fn finish(self) -> flatbuffers::WIPOffset<VideoUpdate<'a>> {
+    let o = self.fbb_.end_table(self.start_);
+    flatbuffers::WIPOffset::new(o.value())
+  }
+}
+
+impl core::fmt::Debug for VideoUpdate<'_> {
+  fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
+    let mut ds = f.debug_struct("VideoUpdate");
+      ds.field("update_type", &self.update_type());
+      match self.update_type() {
+        Update::Setup => {
+          if let Some(x) = self.update_as_setup() {
+            ds.field("update", &x)
+          } else {
+            ds.field("update", &"InvalidFlatbuffer: Union discriminant does not match value.")
+          }
+        },
+        Update::DecodeUnit => {
+          if let Some(x) = self.update_as_decode_unit() {
+            ds.field("update", &x)
+          } else {
+            ds.field("update", &"InvalidFlatbuffer: Union discriminant does not match value.")
+          }
+        },
+        _ => {
+          let x: Option<()> = None;
+          ds.field("update", &x)
+        },
+      };
+      ds.finish()
+  }
+}
+#[inline]
+/// Verifies that a buffer of bytes contains a `VideoUpdate`
+/// and returns it.
+/// Note that verification is still experimental and may not
+/// catch every error, or be maximally performant. For the
+/// previous, unchecked, behavior use
+/// `root_as_video_update_unchecked`.
+pub fn root_as_video_update(buf: &[u8]) -> Result<VideoUpdate, flatbuffers::InvalidFlatbuffer> {
+  flatbuffers::root::<VideoUpdate>(buf)
+}
+#[inline]
+/// Verifies that a buffer of bytes contains a size prefixed
+/// `VideoUpdate` and returns it.
+/// Note that verification is still experimental and may not
+/// catch every error, or be maximally performant. For the
+/// previous, unchecked, behavior use
+/// `size_prefixed_root_as_video_update_unchecked`.
+pub fn size_prefixed_root_as_video_update(buf: &[u8]) -> Result<VideoUpdate, flatbuffers::InvalidFlatbuffer> {
+  flatbuffers::size_prefixed_root::<VideoUpdate>(buf)
+}
+#[inline]
+/// Verifies, with the given options, that a buffer of bytes
+/// contains a `VideoUpdate` and returns it.
+/// Note that verification is still experimental and may not
+/// catch every error, or be maximally performant. For the
+/// previous, unchecked, behavior use
+/// `root_as_video_update_unchecked`.
+pub fn root_as_video_update_with_opts<'b, 'o>(
+  opts: &'o flatbuffers::VerifierOptions,
+  buf: &'b [u8],
+) -> Result<VideoUpdate<'b>, flatbuffers::InvalidFlatbuffer> {
+  flatbuffers::root_with_opts::<VideoUpdate<'b>>(opts, buf)
+}
+#[inline]
+/// Verifies, with the given verifier options, that a buffer of
+/// bytes contains a size prefixed `VideoUpdate` and returns
+/// it. Note that verification is still experimental and may not
+/// catch every error, or be maximally performant. For the
+/// previous, unchecked, behavior use
+/// `root_as_video_update_unchecked`.
+pub fn size_prefixed_root_as_video_update_with_opts<'b, 'o>(
+  opts: &'o flatbuffers::VerifierOptions,
+  buf: &'b [u8],
+) -> Result<VideoUpdate<'b>, flatbuffers::InvalidFlatbuffer> {
+  flatbuffers::size_prefixed_root_with_opts::<VideoUpdate<'b>>(opts, buf)
+}
+#[inline]
+/// Assumes, without verification, that a buffer of bytes contains a VideoUpdate and returns it.
+/// # Safety
+/// Callers must trust the given bytes do indeed contain a valid `VideoUpdate`.
+pub unsafe fn root_as_video_update_unchecked(buf: &[u8]) -> VideoUpdate {
+  flatbuffers::root_unchecked::<VideoUpdate>(buf)
+}
+#[inline]
+/// Assumes, without verification, that a buffer of bytes contains a size prefixed VideoUpdate and returns it.
+/// # Safety
+/// Callers must trust the given bytes do indeed contain a valid size prefixed `VideoUpdate`.
+pub unsafe fn size_prefixed_root_as_video_update_unchecked(buf: &[u8]) -> VideoUpdate {
+  flatbuffers::size_prefixed_root_unchecked::<VideoUpdate>(buf)
+}
+#[inline]
+pub fn finish_video_update_buffer<'a, 'b, A: flatbuffers::Allocator + 'a>(
+    fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>,
+    root: flatbuffers::WIPOffset<VideoUpdate<'a>>) {
+  fbb.finish(root, None);
+}
+
+#[inline]
+pub fn finish_size_prefixed_video_update_buffer<'a, 'b, A: flatbuffers::Allocator + 'a>(fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>, root: flatbuffers::WIPOffset<VideoUpdate<'a>>) {
+  fbb.finish_size_prefixed(root, None);
+}
+}  // pub mod VideoUpdate
+

schema/video.fbs

@@ -0,0 +1,34 @@
+namespace VideoUpdate;
+
+
+table Setup {
+  video_format: string;
+  width: uint16;
+  height: uint16;
+  redraw_rate: uint16;
+}
+
+enum FrameType: byte {
+  PFRAME,
+  IDR,
+}
+
+table DecodeUnit {
+  frame_number: uint16;
+  frame_type: FrameType;
+
+  receive_time_ms: uint16;
+
+  data: [ubyte]; 
+}
+
+union Update {
+    Setup:Setup,
+    DecodeUnit:DecodeUnit,
+}
+
+table VideoUpdate {
+    update: Update; 
+}
+
+root_type VideoUpdate;