frontend: port video decoding to web worker

frontend/src/lib/canvas.worker.ts

@@ -0,0 +1,24 @@
+import { streamVideoFromReader } from "./video";
+
+let canvas: OffscreenCanvas | undefined = undefined;
+let reader: ReadableStream | undefined = undefined;
+let resolver: any = undefined;
+const messagePromise = new Promise((resolve) => resolver = resolve);
+
+
+self.onmessage = async ({ data: payload }: { data: any }) => {
+  canvas = payload["canvas"];
+  reader = payload["reader"];
+  self.onmessage = null;
+  resolver?.();
+};
+
+console.log(`Worker spawned successfully`)
+await messagePromise;
+console.log("Worker received required objects")
+console.log("Starting video streaming")
+
+await streamVideoFromReader(reader!.getReader(), canvas!);
+
+
+export { }; // This makes TypeScript happy

frontend/src/lib/video.ts

@@ -0,0 +1,148 @@
+type Setup = {
+  video_format: string,
+  width: number,
+  height: number,
+  redraw_rate: number,
+  dr_flags: number,
+}
+
+type SetupPacket = {
+  Setup: Setup
+}
+
+type DecodeBuffer = {
+  buffer_bype: string,
+  data: Array<number>,
+}
+
+type DecodeUnit = {
+  frame_number: number,
+  frame_type: string,
+  buffer: DecodeBuffer,
+  receieve_time_ms: number,
+}
+
+type DecodeUnitPacket = {
+  DecodeUnit: DecodeUnit
+}
+
+function parseData(newBuffer: Uint8Array, oldBuffer: Uint8Array): [Array<Object>, Uint8Array<ArrayBuffer>] {
+  let packets = new Array<Object>();
+  let unparsedData = new Uint8Array();
+
+  let data = new Uint8Array([...oldBuffer, ...newBuffer]);
+  let index = 0;
+  while (true) {
+    if (index >= data.length) {
+      break
+    }
+    const view = new DataView(data.buffer.slice(index, index + 4));
+    const dataLength = view.getUint32(0, true);
+
+    const slice_start_index = index + 4;
+    const slice_end_index = index + 4 + dataLength;
+
+    if (data.length < slice_end_index) {
+      unparsedData = new Uint8Array(data.buffer.slice(index, data.length));
+      break;
+    }
+
+    const dataToParse = data.buffer.slice(slice_start_index, slice_end_index);
+    const decoder = new TextDecoder('utf-8');
+    const jsonString = decoder.decode(dataToParse);
+
+    packets.push(JSON.parse(jsonString));
+
+    index += 4 + dataLength;
+  }
+  return [packets, unparsedData];
+}
+
+
+export async function streamVideoFromReader(reader: ReadableStreamDefaultReader, canvasElement: OffscreenCanvas) {
+  const canvasCtx: OffscreenCanvasRenderingContext2D | null = canvasElement.getContext('2d');
+  if (canvasCtx == null) {
+    throw new Error(`Could not get 2d canvas context`);
+  }
+
+  try {
+    let unparsedData = new Uint8Array();
+
+    const videoDecoder = new VideoDecoder({
+      output: (frame) => {
+        // Set canvas dimensions to match the frame
+        canvasElement.width = frame.displayWidth;
+        canvasElement.height = frame.displayHeight;
+
+        // Draw the decoded frame to canvas
+        canvasCtx.drawImage(frame, 0, 0);
+
+        // Important: close the frame to free memory
+        frame.close();
+      },
+      error: (e) => {
+        console.error('Decode error:', e);
+      }
+    });
+
+    while (true) {
+      const { value, done } = await reader.read();
+      if (done) break;
+
+      let [packets, remainingData] = parseData(value, unparsedData);
+      unparsedData = remainingData;
+
+      for (let i = 0; i < packets.length; i++) {
+        if (Object.hasOwn(packets[i], "Setup")) {
+          let packet = packets[i] as SetupPacket;
+
+          let config: VideoDecoderConfig | undefined = undefined;
+          if (packet.Setup.video_format == "H264") {
+            config = {
+              //codec: 'avc1.42E01E', // H.264 codec
+              codec: 'avc1.4D002A', // H.264 codec
+              codedWidth: packet.Setup.width,
+              codedHeight: packet.Setup.height,
+            };
+          } else {
+            throw new Error(`Unsupported video codec ${packet.Setup.video_format}`);
+          }
+
+          const codecSupport = await VideoDecoder.isConfigSupported(config);
+          if (codecSupport.supported) {
+            videoDecoder.configure(config);
+          } else {
+            throw new Error(`Could not configure decoder`);
+          }
+
+        } else if (Object.hasOwn(packets[i], "DecodeUnit")) {
+          let packet = packets[i] as DecodeUnitPacket;
+
+
+          let frame_type: EncodedAudioChunkType = "delta";
+          if (packet.DecodeUnit.frame_type == "IDR") {
+            frame_type = "key";
+          }
+          const chunk = new EncodedVideoChunk({
+            timestamp: packet.DecodeUnit.receieve_time_ms,
+            type: frame_type,
+            data: new Uint8Array(packet.DecodeUnit.buffer.data),
+          });
+
+          videoDecoder.decode(chunk);
+
+        } else {
+          throw new Error(`Got packet of unknown type`);
+        }
+      }
+
+    }
+
+  } catch (e) {
+    var error = <Error>e;
+    console.error('Error connecting to stream:', error);
+    alert('Failed to connect to stream: ' + error.message);
+  }
+}
+
+

frontend/src/routes/getStreamData.ts

@@ -41,7 +41,7 @@ export async function getStreamData(appId: number, server_name: string): Promise
         height: 1080,
       },
       stream_config: {
-        bitrate_kbps: 10240,
+        bitrate_kbps: 1024 * 10,
         mode: {
           fps: 60,
           width: 1920,

frontend/src/routes/stream/stream.ts

@@ -1,32 +1,5 @@
 import { sendKeyboardEvent, sendMouseInputEvent, sendMouseMoveEvent, KeyAction } from "$lib/input"
-
+import CanvasWorker from "$lib/canvas.worker?worker";
-type Setup = {
-  video_format: string,
-  width: number,
-  height: number,
-  redraw_rate: number,
-  dr_flags: number,
-}
-
-type SetupPacket = {
-  Setup: Setup
-}
-
-type DecodeBuffer = {
-  buffer_bype: string,
-  data: Array<number>,
-}
-
-type DecodeUnit = {
-  frame_number: number,
-  frame_type: string,
-  buffer: DecodeBuffer,
-  receieve_time_ms: number,
-}
-
-type DecodeUnitPacket = {
-  DecodeUnit: DecodeUnit
-}
 
 export async function getStreamTransport(url: string, certHash: Array<number>): Promise<WebTransport> {
   let certHashArray = new Uint8Array(certHash);
@@ -53,127 +26,19 @@ export async function getStreamTransport(url: string, certHash: Array<number>):
   return transport;
 }
 
-function parseData(newBuffer: Uint8Array, oldBuffer: Uint8Array): [Array<Object>, Uint8Array<ArrayBuffer>] {
-  let packets = new Array<Object>();
-  let unparsedData = new Uint8Array();
 
-  let data = new Uint8Array([...oldBuffer, ...newBuffer]);
+export async function spawnWorker(gameplayCanvas: HTMLCanvasElement, reader: ReadableStream) {
-  let index = 0;
+  const offscreenCanvas = gameplayCanvas.transferControlToOffscreen();
-  while (true) {
-    if (index >= data.length) {
-      break
-    }
-    const view = new DataView(data.buffer.slice(index, index + 4));
-    const dataLength = view.getUint32(0, true);
 
-    const slice_start_index = index + 4;
+  const worker = new CanvasWorker();
-    const slice_end_index = index + 4 + dataLength;
 
-    if (data.length < slice_end_index) {
+  worker.postMessage({
-      unparsedData = new Uint8Array(data.buffer.slice(index, data.length));
+    canvas: offscreenCanvas,
-      break;
+    reader: reader,
-    }
+  }, [offscreenCanvas, reader]);
-
-    const dataToParse = data.buffer.slice(slice_start_index, slice_end_index);
-    const decoder = new TextDecoder('utf-8');
-    const jsonString = decoder.decode(dataToParse);
-
-    packets.push(JSON.parse(jsonString));
-
-    index += 4 + dataLength;
-  }
-  return [packets, unparsedData];
-}
-
-export async function streamVideoFromReader(reader: ReadableStreamDefaultReader, canvasElement: HTMLCanvasElement) {
-  const canvasCtx: CanvasRenderingContext2D | null = canvasElement.getContext('2d');
-  if (canvasCtx == null) {
-    throw new Error(`Could not get 2d canvas context`);
-  }
-
-  try {
-    let unparsedData = new Uint8Array();
-
-    const videoDecoder = new VideoDecoder({
-      output: (frame) => {
-        // Set canvas dimensions to match the frame
-        canvasElement.width = frame.displayWidth;
-        canvasElement.height = frame.displayHeight;
-
-        // Draw the decoded frame to canvas
-        canvasCtx.drawImage(frame, 0, 0);
-
-        // Important: close the frame to free memory
-        frame.close();
-      },
-      error: (e) => {
-        console.error('Decode error:', e);
-      }
-    });
-
-    while (true) {
-      const { value, done } = await reader.read();
-      if (done) break;
-
-      let [packets, remainingData] = parseData(value, unparsedData);
-      unparsedData = remainingData;
-
-      for (let i = 0; i < packets.length; i++) {
-        if (Object.hasOwn(packets[i], "Setup")) {
-          let packet = packets[i] as SetupPacket;
-
-          let config: VideoDecoderConfig | undefined = undefined;
-          if (packet.Setup.video_format == "H264") {
-            config = {
-              //codec: 'avc1.42E01E', // H.264 codec
-              codec: 'avc1.4D002A', // H.264 codec
-              codedWidth: packet.Setup.width,
-              codedHeight: packet.Setup.height,
-            };
-          } else {
-            throw new Error(`Unsupported video codec ${packet.Setup.video_format}`);
-          }
-
-          const codecSupport = await VideoDecoder.isConfigSupported(config);
-          if (codecSupport.supported) {
-            videoDecoder.configure(config);
-          } else {
-            throw new Error(`Could not configure decoder`);
-          }
-
-        } else if (Object.hasOwn(packets[i], "DecodeUnit")) {
-          let packet = packets[i] as DecodeUnitPacket;
-
-
-          let frame_type: EncodedAudioChunkType = "delta";
-          if (packet.DecodeUnit.frame_type == "IDR") {
-            frame_type = "key";
-          }
-          const chunk = new EncodedVideoChunk({
-            timestamp: packet.DecodeUnit.receieve_time_ms,
-            type: frame_type,
-            data: new Uint8Array(packet.DecodeUnit.buffer.data),
-          });
-
-          videoDecoder.decode(chunk);
-
-        } else {
-          throw new Error(`Got packet of unknown type`);
-        }
-      }
 
 }
 
-  } catch (e) {
-    var error = <Error>e;
-    console.error('Error connecting to stream:', error);
-    alert('Failed to connect to stream: ' + error.message);
-  }
-}
-
-
-
-
 
 export async function startWebtransportStream(
   url: string,
@@ -187,9 +52,11 @@ export async function startWebtransportStream(
 
   const stream = await transport.createBidirectionalStream();
 
-  const reader = stream.readable.getReader();
+  const reader = stream.readable
   const writer = stream.writable.getWriter();
 
+  spawnWorker(gameplayCanvas, reader);
+
   keyEventElement.addEventListener("keydown", (event: KeyboardEvent) => { sendKeyboardEvent(writer, event, KeyAction.DOWN) });
   keyEventElement.addEventListener("keyup", (event: KeyboardEvent) => { sendKeyboardEvent(writer, event, KeyAction.UP) });
 
@@ -212,8 +79,6 @@ export async function startWebtransportStream(
   });
 
 
-  await streamVideoFromReader(reader, gameplayCanvas);
-
   // Handle connection close
   transport.closed
     .then(() => {