restitux/gamestream-webtransport-proxy
1
0
Fork 0
Code Issues Pull Requests 6 Actions Packages Projects Releases Wiki Activity

frontend: port video decoding to web worker

Browse Source
This commit is contained in:
restitux
2025-08-10 02:53:16 -06:00
parent a78bb2460e
commit 209c1cffc4
4 changed files with 184 additions and 147 deletions
Download Patch File Download Diff File Expand all files Collapse all files
frontend/src/lib/canvas.worker.ts
+24
View File
@@ -0,0 +1,24 @@
import { streamVideoFromReader } from "./video";
let canvas: OffscreenCanvas | undefined = undefined;
let reader: ReadableStream | undefined = undefined;
let resolver: any = undefined;
const messagePromise = new Promise((resolve) => resolver = resolve);
self.onmessage = async ({ data: payload }: { data: any }) => {
canvas = payload["canvas"];
reader = payload["reader"];
self.onmessage = null;
resolver?.();
};
console.log(`Worker spawned successfully`)
await messagePromise;
console.log("Worker received required objects")
console.log("Starting video streaming")
await streamVideoFromReader(reader!.getReader(), canvas!);
export { }; // This makes TypeScript happy
frontend/src/lib/video.ts
+148
View File
@@ -0,0 +1,148 @@
type Setup = {
video_format: string,
width: number,
height: number,
redraw_rate: number,
dr_flags: number,
}
type SetupPacket = {
Setup: Setup
}
type DecodeBuffer = {
buffer_bype: string,
data: Array<number>,
}
type DecodeUnit = {
frame_number: number,
frame_type: string,
buffer: DecodeBuffer,
receieve_time_ms: number,
}
type DecodeUnitPacket = {
DecodeUnit: DecodeUnit
}
function parseData(newBuffer: Uint8Array, oldBuffer: Uint8Array): [Array<Object>, Uint8Array<ArrayBuffer>] {
let packets = new Array<Object>();
let unparsedData = new Uint8Array();
let data = new Uint8Array([...oldBuffer, ...newBuffer]);
let index = 0;
while (true) {
if (index >= data.length) {
break
}
const view = new DataView(data.buffer.slice(index, index + 4));
const dataLength = view.getUint32(0, true);
const slice_start_index = index + 4;
const slice_end_index = index + 4 + dataLength;
if (data.length < slice_end_index) {
unparsedData = new Uint8Array(data.buffer.slice(index, data.length));
break;
}
const dataToParse = data.buffer.slice(slice_start_index, slice_end_index);
const decoder = new TextDecoder('utf-8');
const jsonString = decoder.decode(dataToParse);
packets.push(JSON.parse(jsonString));
index += 4 + dataLength;
}
return [packets, unparsedData];
}
export async function streamVideoFromReader(reader: ReadableStreamDefaultReader, canvasElement: OffscreenCanvas) {
const canvasCtx: OffscreenCanvasRenderingContext2D | null = canvasElement.getContext('2d');
if (canvasCtx == null) {
throw new Error(`Could not get 2d canvas context`);
}
try {
let unparsedData = new Uint8Array();
const videoDecoder = new VideoDecoder({
output: (frame) => {
// Set canvas dimensions to match the frame
canvasElement.width = frame.displayWidth;
canvasElement.height = frame.displayHeight;
// Draw the decoded frame to canvas
canvasCtx.drawImage(frame, 0, 0);
// Important: close the frame to free memory
frame.close();
},
error: (e) => {
console.error('Decode error:', e);
}
});
while (true) {
const { value, done } = await reader.read();
if (done) break;
let [packets, remainingData] = parseData(value, unparsedData);
unparsedData = remainingData;
for (let i = 0; i < packets.length; i++) {
if (Object.hasOwn(packets[i], "Setup")) {
let packet = packets[i] as SetupPacket;
let config: VideoDecoderConfig | undefined = undefined;
if (packet.Setup.video_format == "H264") {
config = {
//codec: 'avc1.42E01E', // H.264 codec
codec: 'avc1.4D002A', // H.264 codec
codedWidth: packet.Setup.width,
codedHeight: packet.Setup.height,
};
} else {
throw new Error(`Unsupported video codec ${packet.Setup.video_format}`);
}
const codecSupport = await VideoDecoder.isConfigSupported(config);
if (codecSupport.supported) {
videoDecoder.configure(config);
} else {
throw new Error(`Could not configure decoder`);
}
} else if (Object.hasOwn(packets[i], "DecodeUnit")) {
let packet = packets[i] as DecodeUnitPacket;
let frame_type: EncodedAudioChunkType = "delta";
if (packet.DecodeUnit.frame_type == "IDR") {
frame_type = "key";
}
const chunk = new EncodedVideoChunk({
timestamp: packet.DecodeUnit.receieve_time_ms,
type: frame_type,
data: new Uint8Array(packet.DecodeUnit.buffer.data),
});
videoDecoder.decode(chunk);
} else {
throw new Error(`Got packet of unknown type`);
}
}
}
} catch (e) {
var error = <Error>e;
console.error('Error connecting to stream:', error);
alert('Failed to connect to stream: ' + error.message);
}
}
frontend/src/routes/getStreamData.ts
+1 -1
View File
@@ -41,7 +41,7 @@ export async function getStreamData(appId: number, server_name: string): Promise
height: 1080,
},
stream_config: {
bitrate_kbps: 10240,
bitrate_kbps: 1024 * 10,
mode: {
fps: 60,
width: 1920,
frontend/src/routes/stream/stream.ts
+11 -146
View File
@@ -1,32 +1,5 @@
import { sendKeyboardEvent, sendMouseInputEvent, sendMouseMoveEvent, KeyAction } from "$lib/input"
type Setup = {
video_format: string,
width: number,
height: number,
redraw_rate: number,
dr_flags: number,
}
type SetupPacket = {
Setup: Setup
}
type DecodeBuffer = {
buffer_bype: string,
data: Array<number>,
}
type DecodeUnit = {
frame_number: number,
frame_type: string,
buffer: DecodeBuffer,
receieve_time_ms: number,
}
type DecodeUnitPacket = {
DecodeUnit: DecodeUnit
}
import CanvasWorker from "$lib/canvas.worker?worker";
export async function getStreamTransport(url: string, certHash: Array<number>): Promise<WebTransport> {
let certHashArray = new Uint8Array(certHash);
@@ -53,127 +26,19 @@ export async function getStreamTransport(url: string, certHash: Array<number>):
return transport;
}
function parseData(newBuffer: Uint8Array, oldBuffer: Uint8Array): [Array<Object>, Uint8Array<ArrayBuffer>] {
let packets = new Array<Object>();
let unparsedData = new Uint8Array();
let data = new Uint8Array([...oldBuffer, ...newBuffer]);
let index = 0;
while (true) {
if (index >= data.length) {
break
}
const view = new DataView(data.buffer.slice(index, index + 4));
const dataLength = view.getUint32(0, true);
export async function spawnWorker(gameplayCanvas: HTMLCanvasElement, reader: ReadableStream) {
const offscreenCanvas = gameplayCanvas.transferControlToOffscreen();
const slice_start_index = index + 4;
const slice_end_index = index + 4 + dataLength;
const worker = new CanvasWorker();
if (data.length < slice_end_index) {
unparsedData = new Uint8Array(data.buffer.slice(index, data.length));
break;
}
worker.postMessage({
canvas: offscreenCanvas,
reader: reader,
}, [offscreenCanvas, reader]);
const dataToParse = data.buffer.slice(slice_start_index, slice_end_index);
const decoder = new TextDecoder('utf-8');
const jsonString = decoder.decode(dataToParse);
packets.push(JSON.parse(jsonString));
index += 4 + dataLength;
}
return [packets, unparsedData];
}
export async function streamVideoFromReader(reader: ReadableStreamDefaultReader, canvasElement: HTMLCanvasElement) {
const canvasCtx: CanvasRenderingContext2D | null = canvasElement.getContext('2d');
if (canvasCtx == null) {
throw new Error(`Could not get 2d canvas context`);
}
try {
let unparsedData = new Uint8Array();
const videoDecoder = new VideoDecoder({
output: (frame) => {
// Set canvas dimensions to match the frame
canvasElement.width = frame.displayWidth;
canvasElement.height = frame.displayHeight;
// Draw the decoded frame to canvas
canvasCtx.drawImage(frame, 0, 0);
// Important: close the frame to free memory
frame.close();
},
error: (e) => {
console.error('Decode error:', e);
}
});
while (true) {
const { value, done } = await reader.read();
if (done) break;
let [packets, remainingData] = parseData(value, unparsedData);
unparsedData = remainingData;
for (let i = 0; i < packets.length; i++) {
if (Object.hasOwn(packets[i], "Setup")) {
let packet = packets[i] as SetupPacket;
let config: VideoDecoderConfig | undefined = undefined;
if (packet.Setup.video_format == "H264") {
config = {
//codec: 'avc1.42E01E', // H.264 codec
codec: 'avc1.4D002A', // H.264 codec
codedWidth: packet.Setup.width,
codedHeight: packet.Setup.height,
};
} else {
throw new Error(`Unsupported video codec ${packet.Setup.video_format}`);
}
const codecSupport = await VideoDecoder.isConfigSupported(config);
if (codecSupport.supported) {
videoDecoder.configure(config);
} else {
throw new Error(`Could not configure decoder`);
}
} else if (Object.hasOwn(packets[i], "DecodeUnit")) {
let packet = packets[i] as DecodeUnitPacket;
let frame_type: EncodedAudioChunkType = "delta";
if (packet.DecodeUnit.frame_type == "IDR") {
frame_type = "key";
}
const chunk = new EncodedVideoChunk({
timestamp: packet.DecodeUnit.receieve_time_ms,
type: frame_type,
data: new Uint8Array(packet.DecodeUnit.buffer.data),
});
videoDecoder.decode(chunk);
} else {
throw new Error(`Got packet of unknown type`);
}
}
}
} catch (e) {
var error = <Error>e;
console.error('Error connecting to stream:', error);
alert('Failed to connect to stream: ' + error.message);
}
}
export async function startWebtransportStream(
url: string,
@@ -187,9 +52,11 @@ export async function startWebtransportStream(
const stream = await transport.createBidirectionalStream();
const reader = stream.readable.getReader();
const reader = stream.readable
const writer = stream.writable.getWriter();
spawnWorker(gameplayCanvas, reader);
keyEventElement.addEventListener("keydown", (event: KeyboardEvent) => { sendKeyboardEvent(writer, event, KeyAction.DOWN) });
keyEventElement.addEventListener("keyup", (event: KeyboardEvent) => { sendKeyboardEvent(writer, event, KeyAction.UP) });
@@ -212,8 +79,6 @@ export async function startWebtransportStream(
});
await streamVideoFromReader(reader, gameplayCanvas);
// Handle connection close
transport.closed
.then(() => {