diff --git a/frontend/src/lib/canvas.worker.ts b/frontend/src/lib/canvas.worker.ts new file mode 100644 index 0000000..1f23c19 --- /dev/null +++ b/frontend/src/lib/canvas.worker.ts @@ -0,0 +1,24 @@ +import { streamVideoFromReader } from "./video"; + +let canvas: OffscreenCanvas | undefined = undefined; +let reader: ReadableStream | undefined = undefined; +let resolver: any = undefined; +const messagePromise = new Promise((resolve) => resolver = resolve); + + +self.onmessage = async ({ data: payload }: { data: any }) => { + canvas = payload["canvas"]; + reader = payload["reader"]; + self.onmessage = null; + resolver?.(); +}; + +console.log(`Worker spawned successfully`) +await messagePromise; +console.log("Worker received required objects") +console.log("Starting video streaming") + +await streamVideoFromReader(reader!.getReader(), canvas!); + + +export { }; // This makes TypeScript happy diff --git a/frontend/src/lib/video.ts b/frontend/src/lib/video.ts new file mode 100644 index 0000000..6ec46d5 --- /dev/null +++ b/frontend/src/lib/video.ts @@ -0,0 +1,148 @@ +type Setup = { + video_format: string, + width: number, + height: number, + redraw_rate: number, + dr_flags: number, +} + +type SetupPacket = { + Setup: Setup +} + +type DecodeBuffer = { + buffer_bype: string, + data: Array, +} + +type DecodeUnit = { + frame_number: number, + frame_type: string, + buffer: DecodeBuffer, + receieve_time_ms: number, +} + +type DecodeUnitPacket = { + DecodeUnit: DecodeUnit +} + +function parseData(newBuffer: Uint8Array, oldBuffer: Uint8Array): [Array, Uint8Array] { + let packets = new Array(); + let unparsedData = new Uint8Array(); + + let data = new Uint8Array([...oldBuffer, ...newBuffer]); + let index = 0; + while (true) { + if (index >= data.length) { + break + } + const view = new DataView(data.buffer.slice(index, index + 4)); + const dataLength = view.getUint32(0, true); + + const slice_start_index = index + 4; + const slice_end_index = index + 4 + dataLength; + + if (data.length < slice_end_index) { + unparsedData = new Uint8Array(data.buffer.slice(index, data.length)); + break; + } + + const dataToParse = data.buffer.slice(slice_start_index, slice_end_index); + const decoder = new TextDecoder('utf-8'); + const jsonString = decoder.decode(dataToParse); + + packets.push(JSON.parse(jsonString)); + + index += 4 + dataLength; + } + return [packets, unparsedData]; +} + + +export async function streamVideoFromReader(reader: ReadableStreamDefaultReader, canvasElement: OffscreenCanvas) { + const canvasCtx: OffscreenCanvasRenderingContext2D | null = canvasElement.getContext('2d'); + if (canvasCtx == null) { + throw new Error(`Could not get 2d canvas context`); + } + + try { + let unparsedData = new Uint8Array(); + + const videoDecoder = new VideoDecoder({ + output: (frame) => { + // Set canvas dimensions to match the frame + canvasElement.width = frame.displayWidth; + canvasElement.height = frame.displayHeight; + + // Draw the decoded frame to canvas + canvasCtx.drawImage(frame, 0, 0); + + // Important: close the frame to free memory + frame.close(); + }, + error: (e) => { + console.error('Decode error:', e); + } + }); + + while (true) { + const { value, done } = await reader.read(); + if (done) break; + + let [packets, remainingData] = parseData(value, unparsedData); + unparsedData = remainingData; + + for (let i = 0; i < packets.length; i++) { + if (Object.hasOwn(packets[i], "Setup")) { + let packet = packets[i] as SetupPacket; + + let config: VideoDecoderConfig | undefined = undefined; + if (packet.Setup.video_format == "H264") { + config = { + //codec: 'avc1.42E01E', // H.264 codec + codec: 'avc1.4D002A', // H.264 codec + codedWidth: packet.Setup.width, + codedHeight: packet.Setup.height, + }; + } else { + throw new Error(`Unsupported video codec ${packet.Setup.video_format}`); + } + + const codecSupport = await VideoDecoder.isConfigSupported(config); + if (codecSupport.supported) { + videoDecoder.configure(config); + } else { + throw new Error(`Could not configure decoder`); + } + + } else if (Object.hasOwn(packets[i], "DecodeUnit")) { + let packet = packets[i] as DecodeUnitPacket; + + + let frame_type: EncodedAudioChunkType = "delta"; + if (packet.DecodeUnit.frame_type == "IDR") { + frame_type = "key"; + } + const chunk = new EncodedVideoChunk({ + timestamp: packet.DecodeUnit.receieve_time_ms, + type: frame_type, + data: new Uint8Array(packet.DecodeUnit.buffer.data), + }); + + videoDecoder.decode(chunk); + + } else { + throw new Error(`Got packet of unknown type`); + } + } + + } + + } catch (e) { + var error = e; + console.error('Error connecting to stream:', error); + alert('Failed to connect to stream: ' + error.message); + } +} + + diff --git a/frontend/src/routes/getStreamData.ts b/frontend/src/routes/getStreamData.ts index e3f652c..03cf257 100644 --- a/frontend/src/routes/getStreamData.ts +++ b/frontend/src/routes/getStreamData.ts @@ -41,7 +41,7 @@ export async function getStreamData(appId: number, server_name: string): Promise height: 1080, }, stream_config: { - bitrate_kbps: 10240, + bitrate_kbps: 1024 * 10, mode: { fps: 60, width: 1920, diff --git a/frontend/src/routes/stream/stream.ts b/frontend/src/routes/stream/stream.ts index 5acd2dc..f0541ad 100644 --- a/frontend/src/routes/stream/stream.ts +++ b/frontend/src/routes/stream/stream.ts @@ -1,32 +1,5 @@ import { sendKeyboardEvent, sendMouseInputEvent, sendMouseMoveEvent, KeyAction } from "$lib/input" - -type Setup = { - video_format: string, - width: number, - height: number, - redraw_rate: number, - dr_flags: number, -} - -type SetupPacket = { - Setup: Setup -} - -type DecodeBuffer = { - buffer_bype: string, - data: Array, -} - -type DecodeUnit = { - frame_number: number, - frame_type: string, - buffer: DecodeBuffer, - receieve_time_ms: number, -} - -type DecodeUnitPacket = { - DecodeUnit: DecodeUnit -} +import CanvasWorker from "$lib/canvas.worker?worker"; export async function getStreamTransport(url: string, certHash: Array): Promise { let certHashArray = new Uint8Array(certHash); @@ -53,127 +26,19 @@ export async function getStreamTransport(url: string, certHash: Array): return transport; } -function parseData(newBuffer: Uint8Array, oldBuffer: Uint8Array): [Array, Uint8Array] { - let packets = new Array(); - let unparsedData = new Uint8Array(); - let data = new Uint8Array([...oldBuffer, ...newBuffer]); - let index = 0; - while (true) { - if (index >= data.length) { - break - } - const view = new DataView(data.buffer.slice(index, index + 4)); - const dataLength = view.getUint32(0, true); +export async function spawnWorker(gameplayCanvas: HTMLCanvasElement, reader: ReadableStream) { + const offscreenCanvas = gameplayCanvas.transferControlToOffscreen(); - const slice_start_index = index + 4; - const slice_end_index = index + 4 + dataLength; + const worker = new CanvasWorker(); - if (data.length < slice_end_index) { - unparsedData = new Uint8Array(data.buffer.slice(index, data.length)); - break; - } + worker.postMessage({ + canvas: offscreenCanvas, + reader: reader, + }, [offscreenCanvas, reader]); - const dataToParse = data.buffer.slice(slice_start_index, slice_end_index); - const decoder = new TextDecoder('utf-8'); - const jsonString = decoder.decode(dataToParse); - - packets.push(JSON.parse(jsonString)); - - index += 4 + dataLength; - } - return [packets, unparsedData]; } -export async function streamVideoFromReader(reader: ReadableStreamDefaultReader, canvasElement: HTMLCanvasElement) { - const canvasCtx: CanvasRenderingContext2D | null = canvasElement.getContext('2d'); - if (canvasCtx == null) { - throw new Error(`Could not get 2d canvas context`); - } - - try { - let unparsedData = new Uint8Array(); - - const videoDecoder = new VideoDecoder({ - output: (frame) => { - // Set canvas dimensions to match the frame - canvasElement.width = frame.displayWidth; - canvasElement.height = frame.displayHeight; - - // Draw the decoded frame to canvas - canvasCtx.drawImage(frame, 0, 0); - - // Important: close the frame to free memory - frame.close(); - }, - error: (e) => { - console.error('Decode error:', e); - } - }); - - while (true) { - const { value, done } = await reader.read(); - if (done) break; - - let [packets, remainingData] = parseData(value, unparsedData); - unparsedData = remainingData; - - for (let i = 0; i < packets.length; i++) { - if (Object.hasOwn(packets[i], "Setup")) { - let packet = packets[i] as SetupPacket; - - let config: VideoDecoderConfig | undefined = undefined; - if (packet.Setup.video_format == "H264") { - config = { - //codec: 'avc1.42E01E', // H.264 codec - codec: 'avc1.4D002A', // H.264 codec - codedWidth: packet.Setup.width, - codedHeight: packet.Setup.height, - }; - } else { - throw new Error(`Unsupported video codec ${packet.Setup.video_format}`); - } - - const codecSupport = await VideoDecoder.isConfigSupported(config); - if (codecSupport.supported) { - videoDecoder.configure(config); - } else { - throw new Error(`Could not configure decoder`); - } - - } else if (Object.hasOwn(packets[i], "DecodeUnit")) { - let packet = packets[i] as DecodeUnitPacket; - - - let frame_type: EncodedAudioChunkType = "delta"; - if (packet.DecodeUnit.frame_type == "IDR") { - frame_type = "key"; - } - const chunk = new EncodedVideoChunk({ - timestamp: packet.DecodeUnit.receieve_time_ms, - type: frame_type, - data: new Uint8Array(packet.DecodeUnit.buffer.data), - }); - - videoDecoder.decode(chunk); - - } else { - throw new Error(`Got packet of unknown type`); - } - } - - } - - } catch (e) { - var error = e; - console.error('Error connecting to stream:', error); - alert('Failed to connect to stream: ' + error.message); - } -} - - - - export async function startWebtransportStream( url: string, @@ -187,9 +52,11 @@ export async function startWebtransportStream( const stream = await transport.createBidirectionalStream(); - const reader = stream.readable.getReader(); + const reader = stream.readable const writer = stream.writable.getWriter(); + spawnWorker(gameplayCanvas, reader); + keyEventElement.addEventListener("keydown", (event: KeyboardEvent) => { sendKeyboardEvent(writer, event, KeyAction.DOWN) }); keyEventElement.addEventListener("keyup", (event: KeyboardEvent) => { sendKeyboardEvent(writer, event, KeyAction.UP) }); @@ -212,8 +79,6 @@ export async function startWebtransportStream( }); - await streamVideoFromReader(reader, gameplayCanvas); - // Handle connection close transport.closed .then(() => {