From a11a12828c6988a9930b960b991f13e273a43943 Mon Sep 17 00:00:00 2001
From: restitux <restitux@ohea.xyz>
Date: Mon, 21 Jul 2025 02:06:55 -0600
Subject: [PATCH] frontend: video streaming now works!!

- changed the backend to append the buffers together and dispatch them
  as such (this is what moonlight-embedded does)
- fixed the frontend trying to playback an empty buffer because types
  are hard (this is why I should have used protobuf...)
---
 frontend/src/routes/connect.ts                | 18 ++++--
 .../src/gamestream/decoder.rs                 | 64 ++++++++++---------
 2 files changed, 49 insertions(+), 33 deletions(-)

diff --git a/frontend/src/routes/connect.ts b/frontend/src/routes/connect.ts
index 3811b1b..e76d667 100644
--- a/frontend/src/routes/connect.ts
+++ b/frontend/src/routes/connect.ts
@@ -39,10 +39,16 @@ type SetupPacket = {
   Setup: Setup
 }
 
+type DecodeBuffer = {
+  buffer_bype: string,
+  data: Array<number>,
+}
+
 type DecodeUnit = {
   frame_number: number,
   frame_type: string,
-  buffer: Array<number>,
+  buffer: DecodeBuffer,
+  receieve_time_ms: number,
 }
 
 type DecodeUnitPacket = {
@@ -209,7 +215,8 @@ export async function connectToStream(url: string, cert_hash: Array<number>) {
           let config: VideoDecoderConfig | undefined = undefined;
           if (packet.Setup.video_format == "H264") {
             config = {
-              codec: 'avc1.42E01E', // H.264 codec
+              //codec: 'avc1.42E01E', // H.264 codec
+              codec: 'avc1.4D002A', // H.264 codec
               codedWidth: packet.Setup.width,
               codedHeight: packet.Setup.height,
             };
@@ -220,6 +227,8 @@ export async function connectToStream(url: string, cert_hash: Array<number>) {
           const codecSupport = await VideoDecoder.isConfigSupported(config);
           if (codecSupport.supported) {
             videoDecoder.configure(config);
+          } else {
+            throw new Error(`Could not configure decoder`);
           }
 
         } else if (Object.hasOwn(packets[i], "DecodeUnit")) {
@@ -231,10 +240,11 @@ export async function connectToStream(url: string, cert_hash: Array<number>) {
             frame_type = "key";
           }
           const chunk = new EncodedVideoChunk({
-            timestamp: 0,
+            timestamp: packet.DecodeUnit.receieve_time_ms,
             type: frame_type,
-            data: new Uint8Array(packet.DecodeUnit.buffer),
+            data: new Uint8Array(packet.DecodeUnit.buffer.data),
           });
+          console.log(chunk);
 
           videoDecoder.decode(chunk);
 
diff --git a/gamestream-webtransport-proxy/src/gamestream/decoder.rs b/gamestream-webtransport-proxy/src/gamestream/decoder.rs
index 17f8b96..1a81f99 100644
--- a/gamestream-webtransport-proxy/src/gamestream/decoder.rs
+++ b/gamestream-webtransport-proxy/src/gamestream/decoder.rs
@@ -155,8 +155,10 @@ impl RendererMessage {
         })
     }
 
-    fn from_decode_unit(decode_unit: _DECODE_UNIT) -> Result<Vec<Self>> {
-        let mut buffers = Vec::new();
+    fn from_decode_unit(decode_unit: _DECODE_UNIT) -> Result<Self> {
+        //fn from_decode_unit(decode_unit: _DECODE_UNIT) -> Result<Vec<Self>> {
+        let mut buffer = Vec::new();
+        //let mut buffers = Vec::new();
 
         if decode_unit.bufferList.is_null() {
             return Err(anyhow!("DecodeUnit bufferList is null"));
@@ -165,32 +167,35 @@ impl RendererMessage {
 
         let mut index = 0;
         loop {
+            let mut b = Buffer::try_from(next)?;
+            buffer.append(&mut b.data);
+
+            //buffers.push(msg);
+
+            index = index + 1;
             if next.next.is_null() {
                 break;
             }
-            let buffer = Buffer::try_from(next)?;
 
-            let msg = RendererMessage::DecodeUnit {
-                frame_number: <u64>::try_from(decode_unit.frameNumber)?,
-                frame_type: FrameType::try_from(decode_unit.frameType)?,
-                host_processing_latency: decode_unit.frameHostProcessingLatency,
-                receieve_time_ms: decode_unit.receiveTimeMs,
-                enqueue_time_ms: decode_unit.enqueueTimeMs,
-                presentation_time: decode_unit.presentationTimeMs as u64,
-                full_length: <usize>::try_from(decode_unit.fullLength)?,
-                buffer,
-                index,
-                hdr_active: decode_unit.hdrActive,
-                colorspace: decode_unit.colorspace,
-            };
-
-            buffers.push(msg);
-
-            index = index + 1;
             next = unsafe { *next.next };
         }
 
-        Ok(buffers)
+        Ok(RendererMessage::DecodeUnit {
+            frame_number: <u64>::try_from(decode_unit.frameNumber)?,
+            frame_type: FrameType::try_from(decode_unit.frameType)?,
+            host_processing_latency: decode_unit.frameHostProcessingLatency,
+            receieve_time_ms: decode_unit.receiveTimeMs,
+            enqueue_time_ms: decode_unit.enqueueTimeMs,
+            presentation_time: decode_unit.presentationTimeMs as u64,
+            full_length: <usize>::try_from(decode_unit.fullLength)?,
+            buffer: Buffer {
+                data: buffer,
+                buffer_type: BufferType::PICDATA,
+            },
+            index,
+            hdr_active: decode_unit.hdrActive,
+            colorspace: decode_unit.colorspace,
+        })
     }
 }
 
@@ -258,7 +263,7 @@ extern "C" fn submit_decode_unit_cb(decode_unit: PDECODE_UNIT) -> std::os::raw::
     }
     let decode_unit = unsafe { *decode_unit };
 
-    let messages = match RendererMessage::from_decode_unit(decode_unit) {
+    let message = match RendererMessage::from_decode_unit(decode_unit) {
         Ok(m) => m,
         Err(e) => {
             error!("Cannot construct RendererMessage: {e}");
@@ -266,13 +271,14 @@ extern "C" fn submit_decode_unit_cb(decode_unit: PDECODE_UNIT) -> std::os::raw::
         }
     };
 
-    for msg in messages {
-        let ret = send_message(msg);
-        if ret != 0 {
-            return ret;
-        }
-    }
-    0
+    send_message(message)
+    //for msg in messages {
+    //    let ret = send_message(msg);
+    //    if ret != 0 {
+    //        return ret;
+    //    }
+    //}
+    //0
 }
 
 pub fn decoder_callbacks() -> Result<(DECODER_RENDERER_CALLBACKS, mpsc::Receiver<RendererMessage>)>