From b340c5047c0102865575841122c84591f32c2f2e Mon Sep 17 00:00:00 2001 From: dazzatronus Date: Wed, 28 Oct 2026 21:25:21 +1100 Subject: [PATCH 1/2] feat: hold the nearest baked html5 frame while playing --- src/components/canvas/players/html5-player.ts | 19 ++++++- tests/html5-player.test.ts | 50 ++++++++++++++++++- 2 files changed, 66 insertions(+), 3 deletions(-) diff --git a/src/components/canvas/players/html5-player.ts b/src/components/canvas/players/html5-player.ts index 1cc8ba84..8c2f28dd 100644 --- a/src/components/canvas/players/html5-player.ts +++ b/src/components/canvas/players/html5-player.ts @@ -25,6 +25,8 @@ import { createCaptureLoadingGraphic, createPlaceholderGraphic } from "./placeho const IFRAME_OFFSCREEN_X = -10000; const IFRAME_LOAD_TIMEOUT_MS = 10_000; const DECODED_FRAME_LIMIT = 30; +// One first-pass stride: a held frame is never more than 233 ms behind at 30 fps. +const HELD_FRAME_LIMIT = 7; type HarnessWindow = Window & { ["__shotstackSeek"]?: (ms: number) => void; @@ -99,6 +101,12 @@ export function captureFramesInFlight(logicalCores: number | undefined, frameByt return Math.max(1, Math.min(byCores, Math.floor(CAPTURE_MEMORY_BUDGET_BYTES / frameBytes))); } +/** The latest baked frame at most `maxBehind` frames before `idx`, or null. */ +export function heldFrame(frames: ArrayLike, idx: number, maxBehind: number): number | null { + for (let i = idx; i >= Math.max(0, idx - maxBehind); i -= 1) if (frames[i]) return i; + return null; +} + export function bakeScaleFor(onScreenScale: number): number { return BAKE_SCALES.find(scale => scale >= onScreenScale) ?? 1; } @@ -160,6 +168,8 @@ export class Html5Player extends Player { private stillTexture: pixi.Texture | null = null; private liveStill: pixi.Sprite | null = null; private liveStillFrame: number = -1; + // False while the live still is a held frame from before the playhead, so pausing on it renders the exact one. + private liveStillExact: boolean = true; private liveStillInFlight: boolean = false; private stillRequest: number | null = null; // The frame the running bake draws next. Null between bakes. @@ -1092,8 +1102,9 @@ export class Html5Player extends Player { // One still renders at a time. Positions asked for meanwhile replace each other, so only the newest renders next. private showStillAt(seconds: number): void { const frame = Math.floor(seconds * this.captureFps); - if (frame === this.liveStillFrame) return; + if (frame === this.liveStillFrame && (this.liveStillExact || this.edit.isPlaying)) return; this.liveStillFrame = frame; + this.liveStillExact = true; this.stillRequest = seconds; if (!this.liveStillInFlight) this.renderLiveStills().catch(err => console.warn("[Html5Player] still failed:", err)); } @@ -1111,7 +1122,10 @@ export class Html5Player extends Player { if (!this.disposed) console.warn("[Html5Player] still failed:", err); return null; }); - const baked = this.bakingFrames?.[frame]; + // Playing, a clip holds a recent baked frame rather than spend bake time on a still. + const held = this.bakingFrames ? heldFrame(this.bakingFrames, frame, this.edit.isPlaying ? HELD_FRAME_LIMIT : 0) : null; + const baked = held === null ? undefined : this.bakingFrames?.[held]; + if (baked && held !== frame) this.liveStillExact = false; let texture = baked ? await this.bakingTexture(baked) : await attempt(); // A page can load too slowly once, so a failed still gets one more try while nothing newer is waiting. if (!texture && !baked && this.liveStillFrame === frame && this.stillRequest === null) texture = await attempt(); @@ -1143,6 +1157,7 @@ export class Html5Player extends Player { private removeLiveStill(): void { this.stillRequest = null; this.liveStillFrame = -1; + this.liveStillExact = true; if (!this.liveStill) return; this.contentContainer.removeChild(this.liveStill); this.liveStill.texture.destroy(true); diff --git a/tests/html5-player.test.ts b/tests/html5-player.test.ts index 98e815ef..164be236 100644 --- a/tests/html5-player.test.ts +++ b/tests/html5-player.test.ts @@ -1,7 +1,7 @@ /** @jest-environment jsdom */ import { BakeQueue } from "@canvas/players/bake-queue"; -import { Html5Player, bakeScaleFor, captureFramesInFlight } from "@canvas/players/html5-player"; +import { Html5Player, bakeScaleFor, captureFramesInFlight, heldFrame } from "@canvas/players/html5-player"; import type { Edit } from "@core/edit-session"; import { EditEvent, InternalEvent } from "@core/events/edit-events"; import { sec } from "@core/timing/types"; @@ -1600,6 +1600,54 @@ it("shows a first bake's frames as they land", async () => { await capture; }); +it.each([ + [[0, 3], 2, 7, 0], + [[0, 3], 3, 7, 3], + [[0], 8, 7, null], + [[0, 3], 2, 0, null], + [[19], 20, 7, 19] +])("finds the frame to hold among %p at %p within %p frames: %p", (present, idx, maxBehind, expected) => { + const frames = Array.from({ length: 20 }, (_, i) => (present.includes(i) ? new Blob([`${i}`]) : undefined)); + + expect(heldFrame(frames, idx, maxBehind)).toBe(expected); +}); + +const playingAhead = async (...baked: number[]) => { + const created = createStillPlayer(); + const { player, harness, edit } = created; + Object.assign(edit, { isPlaying: true }); + Object.assign(player, { getPlaybackTime: () => 0.4 }); // frame 12 + baking(harness); + harness.beginCapture(); + Object.assign(harness, { bakingFrames: bakedAt(...baked), bakingScale: 1 }); + player.update(0, 0); + await until(stillShown(harness)); + return created; +}; + +it("holds the nearest baked frame while playing ahead of the bake", async () => { + const { harness, seeks } = await playingAhead(8, 16); + + expect(await frameMarkup(layer(harness)[0].texture.source.blob)).toBe("baked 8"); + expect(seeks.some(seek => seek.startsWith("still"))).toBe(false); +}); + +it("renders a still while playing when the nearest baked frame is more than 7 frames back", async () => { + const { harness } = await playingAhead(0); + + expect(await frameMarkup(layer(harness)[0].texture.source.blob)).toContain(">t=0.4<"); +}); + +it("renders the exact still once playback pauses on a held frame", async () => { + const { player, harness, edit } = await playingAhead(8, 16); + const held = layer(harness)[0].texture; + Object.assign(edit, { isPlaying: false }); + player.update(0, 0); + await until(() => layer(harness)[0].texture !== held); + + expect(await frameMarkup(layer(harness)[0].texture.source.blob)).toContain(">t=0.4<"); +}); + it("swaps the still for baked frames once the bake finishes", async () => { const { player, harness } = createStillPlayer(); let finish: (frames: Blob[]) => void = () => {}; From c0dc65d91bb9af9d15915a1312a41f055a04e9b7 Mon Sep 17 00:00:00 2001 From: dazzatronus Date: Wed, 28 Oct 2026 21:40:57 +1100 Subject: [PATCH 2/2] fix: keep the frame on screen when a held html5 frame would be older --- src/components/canvas/players/html5-player.ts | 65 ++++++++++++------- tests/html5-player.test.ts | 31 ++++++++- 2 files changed, 70 insertions(+), 26 deletions(-) diff --git a/src/components/canvas/players/html5-player.ts b/src/components/canvas/players/html5-player.ts index 8c2f28dd..ba0a6216 100644 --- a/src/components/canvas/players/html5-player.ts +++ b/src/components/canvas/players/html5-player.ts @@ -25,7 +25,7 @@ import { createCaptureLoadingGraphic, createPlaceholderGraphic } from "./placeho const IFRAME_OFFSCREEN_X = -10000; const IFRAME_LOAD_TIMEOUT_MS = 10_000; const DECODED_FRAME_LIMIT = 30; -// One first-pass stride: a held frame is never more than 233 ms behind at 30 fps. +// How stale a held frame may be when it's chosen: 233 ms at 30 fps. const HELD_FRAME_LIMIT = 7; type HarnessWindow = Window & { @@ -170,6 +170,8 @@ export class Html5Player extends Player { private liveStillFrame: number = -1; // False while the live still is a held frame from before the playhead, so pausing on it renders the exact one. private liveStillExact: boolean = true; + // The frame the live still shows, which a held frame trails. + private liveStillShown: number = -1; private liveStillInFlight: boolean = false; private stillRequest: number | null = null; // The frame the running bake draws next. Null between bakes. @@ -1117,31 +1119,14 @@ export class Html5Player extends Player { const frame = Math.floor(seconds * this.captureFps); const hash = this.contentHash; this.stillRequest = null; - const attempt = () => - this.renderStill(seconds).catch(err => { - if (!this.disposed) console.warn("[Html5Player] still failed:", err); - return null; - }); - // Playing, a clip holds a recent baked frame rather than spend bake time on a still. + // A still takes main-thread time from the bake, so a playing clip holds a recent baked frame. const held = this.bakingFrames ? heldFrame(this.bakingFrames, frame, this.edit.isPlaying ? HELD_FRAME_LIMIT : 0) : null; - const baked = held === null ? undefined : this.bakingFrames?.[held]; - if (baked && held !== frame) this.liveStillExact = false; - let texture = baked ? await this.bakingTexture(baked) : await attempt(); - // A page can load too slowly once, so a failed still gets one more try while nothing newer is waiting. - if (!texture && !baked && this.liveStillFrame === frame && this.stillRequest === null) texture = await attempt(); - const unbaked = this.mode === "editing" || this.mode === "stale" || this.mode === "capturing"; - if (!texture || this.disposed || !unbaked || this.contentHash !== hash) { - texture?.destroy(true); - } else if (this.liveStill) { - this.liveStill.texture.destroy(true); - this.liveStill.texture = texture; + const shown = this.liveStillShown; + // Playback never steps back to an older frame, or decodes the frame it already shows. + if (this.edit.isPlaying && held !== null && this.liveStill && held <= shown && shown <= frame) { + this.liveStillExact = shown === frame; } else { - this.removeLoadingGraphic(); - this.liveStill = new pixi.Sprite(texture); - // A capture in progress shows its own copy of the clip. - this.liveStill.visible = !this.staticSprite; - this.contentContainer.addChild(this.liveStill); - if (this.clipConfiguration.width && this.clipConfiguration.height) this.applyFixedDimensions(); + await this.renderLiveStill(seconds, frame, held, hash); } } } finally { @@ -1149,6 +1134,36 @@ export class Html5Player extends Player { } } + private async renderLiveStill(seconds: number, frame: number, held: number | null, hash: string | null): Promise { + const attempt = () => + this.renderStill(seconds).catch(err => { + if (!this.disposed) console.warn("[Html5Player] still failed:", err); + return null; + }); + const baked = held === null ? undefined : this.bakingFrames?.[held]; + if (baked && held !== frame) this.liveStillExact = false; + let texture = baked ? await this.bakingTexture(baked) : await attempt(); + // A page can load too slowly once, so a failed still gets one more try while nothing newer is waiting. + if (!texture && !baked && this.liveStillFrame === frame && this.stillRequest === null) texture = await attempt(); + const unbaked = this.mode === "editing" || this.mode === "stale" || this.mode === "capturing"; + if (!texture || this.disposed || !unbaked || this.contentHash !== hash) { + texture?.destroy(true); + return; + } + this.liveStillShown = baked && held !== null ? held : frame; + if (this.liveStill) { + this.liveStill.texture.destroy(true); + this.liveStill.texture = texture; + return; + } + this.removeLoadingGraphic(); + this.liveStill = new pixi.Sprite(texture); + // A capture in progress shows its own copy of the clip. + this.liveStill.visible = !this.staticSprite; + this.contentContainer.addChild(this.liveStill); + if (this.clipConfiguration.width && this.clipConfiguration.height) this.applyFixedDimensions(); + } + // The resolution makes a scaled frame's texture report the clip's full size. private async bakingTexture(blob: Blob): Promise { return pixi.Texture.from({ resource: await createImageBitmap(blob), resolution: this.bakingScale }); @@ -1157,7 +1172,7 @@ export class Html5Player extends Player { private removeLiveStill(): void { this.stillRequest = null; this.liveStillFrame = -1; - this.liveStillExact = true; + this.liveStillShown = -1; if (!this.liveStill) return; this.contentContainer.removeChild(this.liveStill); this.liveStill.texture.destroy(true); diff --git a/tests/html5-player.test.ts b/tests/html5-player.test.ts index 164be236..4abb328c 100644 --- a/tests/html5-player.test.ts +++ b/tests/html5-player.test.ts @@ -1604,6 +1604,7 @@ it.each([ [[0, 3], 2, 7, 0], [[0, 3], 3, 7, 3], [[0], 8, 7, null], + [[1], 8, 7, 1], [[0, 3], 2, 0, null], [[19], 20, 7, 19] ])("finds the frame to hold among %p at %p within %p frames: %p", (present, idx, maxBehind, expected) => { @@ -1632,8 +1633,36 @@ it("holds the nearest baked frame while playing ahead of the bake", async () => expect(seeks.some(seek => seek.startsWith("still"))).toBe(false); }); +it("holds a baked frame exactly 7 frames back while playing", async () => { + const { harness } = await playingAhead(5); + + expect(await frameMarkup(layer(harness)[0].texture.source.blob)).toBe("baked 5"); +}); + it("renders a still while playing when the nearest baked frame is more than 7 frames back", async () => { - const { harness } = await playingAhead(0); + const { harness } = await playingAhead(4); + + expect(await frameMarkup(layer(harness)[0].texture.source.blob)).toContain(">t=0.4<"); +}); + +it("decodes a held frame once while playing on through it", async () => { + const { player } = await playingAhead(8); + const decode = jest.spyOn(globalThis, "createImageBitmap"); + [0.4, 0.4, 13 / 30, 13 / 30, 14 / 30].forEach(t => { + Object.assign(player, { getPlaybackTime: () => t }); + player.update(0, 0); + }); + await flush(50); + + expect(decode).not.toHaveBeenCalled(); +}); + +it("never steps back to an older baked frame than the still on screen while playing", async () => { + const { player, harness } = await playingAhead(0); // a still of frame 12 + harness.bakingFrames![11] = new Blob(["baked 11"]); + Object.assign(player, { getPlaybackTime: () => 13 / 30 }); + player.update(0, 0); + await flush(50); expect(await frameMarkup(layer(harness)[0].texture.source.blob)).toContain(">t=0.4<"); });