[2/3] feat: hold the nearest baked html5 frame while playing

src/components/canvas/players/html5-player.ts

@@ -25,6 +25,8 @@ import { createCaptureLoadingGraphic, createPlaceholderGraphic } from "./placeho
 const IFRAME_OFFSCREEN_X = -10000;
 const IFRAME_LOAD_TIMEOUT_MS = 10_000;
 const DECODED_FRAME_LIMIT = 30;
+// How stale a held frame may be when it's chosen: 233 ms at 30 fps.
+const HELD_FRAME_LIMIT = 7;
 
 type HarnessWindow = Window & {
 	["__shotstackSeek"]?: (ms: number) => void;
@@ -99,6 +101,12 @@ export function captureFramesInFlight(logicalCores: number | undefined, frameByt
 	return Math.max(1, Math.min(byCores, Math.floor(CAPTURE_MEMORY_BUDGET_BYTES / frameBytes)));
 }
 
+/** The latest baked frame at most `maxBehind` frames before `idx`, or null. */
+export function heldFrame(frames: ArrayLike<Blob | undefined>, idx: number, maxBehind: number): number | null {
+	for (let i = idx; i >= Math.max(0, idx - maxBehind); i -= 1) if (frames[i]) return i;
+	return null;
+}
+
 export function bakeScaleFor(onScreenScale: number): number {
 	return BAKE_SCALES.find(scale => scale >= onScreenScale) ?? 1;
 }
@@ -160,6 +168,10 @@ export class Html5Player extends Player {
 	private stillTexture: pixi.Texture | null = null;
 	private liveStill: pixi.Sprite | null = null;
 	private liveStillFrame: number = -1;
+	// False while the live still is a held frame from before the playhead, so pausing on it renders the exact one.
+	private liveStillExact: boolean = true;
+	// The frame the live still shows, which a held frame trails.
+	private liveStillShown: number = -1;
 	private liveStillInFlight: boolean = false;
 	private stillRequest: number | null = null;
 	// The frame the running bake draws next. Null between bakes.
@@ -1092,8 +1104,9 @@ export class Html5Player extends Player {
 	// One still renders at a time. Positions asked for meanwhile replace each other, so only the newest renders next.
 	private showStillAt(seconds: number): void {
 		const frame = Math.floor(seconds * this.captureFps);
-		if (frame === this.liveStillFrame) return;
+		if (frame === this.liveStillFrame && (this.liveStillExact || this.edit.isPlaying)) return;
 		this.liveStillFrame = frame;
+		this.liveStillExact = true;
 		this.stillRequest = seconds;
 		if (!this.liveStillInFlight) this.renderLiveStills().catch(err => console.warn("[Html5Player] still failed:", err));
 	}
@@ -1106,35 +1119,51 @@ export class Html5Player extends Player {
 				const frame = Math.floor(seconds * this.captureFps);
 				const hash = this.contentHash;
 				this.stillRequest = null;
-				const attempt = () =>
-					this.renderStill(seconds).catch(err => {
-						if (!this.disposed) console.warn("[Html5Player] still failed:", err);
-						return null;
-					});
-				const baked = this.bakingFrames?.[frame];
-				let texture = baked ? await this.bakingTexture(baked) : await attempt();
-				// A page can load too slowly once, so a failed still gets one more try while nothing newer is waiting.
-				if (!texture && !baked && this.liveStillFrame === frame && this.stillRequest === null) texture = await attempt();
-				const unbaked = this.mode === "editing" || this.mode === "stale" || this.mode === "capturing";
-				if (!texture || this.disposed || !unbaked || this.contentHash !== hash) {
-					texture?.destroy(true);
-				} else if (this.liveStill) {
-					this.liveStill.texture.destroy(true);
-					this.liveStill.texture = texture;
+				// A still takes main-thread time from the bake, so a playing clip holds a recent baked frame.
+				const held = this.bakingFrames ? heldFrame(this.bakingFrames, frame, this.edit.isPlaying ? HELD_FRAME_LIMIT : 0) : null;
+				const shown = this.liveStillShown;
+				// Playback never steps back to an older frame, or decodes the frame it already shows.
+				if (this.edit.isPlaying && held !== null && this.liveStill && held <= shown && shown <= frame) {
+					this.liveStillExact = shown === frame;
 				} else {
-					this.removeLoadingGraphic();
-					this.liveStill = new pixi.Sprite(texture);
-					// A capture in progress shows its own copy of the clip.
-					this.liveStill.visible = !this.staticSprite;
-					this.contentContainer.addChild(this.liveStill);
-					if (this.clipConfiguration.width && this.clipConfiguration.height) this.applyFixedDimensions();
+					await this.renderLiveStill(seconds, frame, held, hash);
 				}
 			}
 		} finally {
 			this.liveStillInFlight = false;
 		}
 	}
 
+	private async renderLiveStill(seconds: number, frame: number, held: number | null, hash: string | null): Promise<void> {
+		const attempt = () =>
+			this.renderStill(seconds).catch(err => {
+				if (!this.disposed) console.warn("[Html5Player] still failed:", err);
+				return null;
+			});
+		const baked = held === null ? undefined : this.bakingFrames?.[held];
+		if (baked && held !== frame) this.liveStillExact = false;
+		let texture = baked ? await this.bakingTexture(baked) : await attempt();
+		// A page can load too slowly once, so a failed still gets one more try while nothing newer is waiting.
+		if (!texture && !baked && this.liveStillFrame === frame && this.stillRequest === null) texture = await attempt();
+		const unbaked = this.mode === "editing" || this.mode === "stale" || this.mode === "capturing";
+		if (!texture || this.disposed || !unbaked || this.contentHash !== hash) {
+			texture?.destroy(true);
+			return;
+		}
+		this.liveStillShown = baked && held !== null ? held : frame;
+		if (this.liveStill) {
+			this.liveStill.texture.destroy(true);
+			this.liveStill.texture = texture;
+			return;
+		}
+		this.removeLoadingGraphic();
+		this.liveStill = new pixi.Sprite(texture);
+		// A capture in progress shows its own copy of the clip.
+		this.liveStill.visible = !this.staticSprite;
+		this.contentContainer.addChild(this.liveStill);
+		if (this.clipConfiguration.width && this.clipConfiguration.height) this.applyFixedDimensions();
+	}
+
 	// The resolution makes a scaled frame's texture report the clip's full size.
 	private async bakingTexture(blob: Blob): Promise<pixi.Texture> {
 		return pixi.Texture.from({ resource: await createImageBitmap(blob), resolution: this.bakingScale });
@@ -1143,6 +1172,7 @@ export class Html5Player extends Player {
 	private removeLiveStill(): void {
 		this.stillRequest = null;
 		this.liveStillFrame = -1;
+		this.liveStillShown = -1;
 		if (!this.liveStill) return;
 		this.contentContainer.removeChild(this.liveStill);
 		this.liveStill.texture.destroy(true);

tests/html5-player.test.ts

@@ -1,7 +1,7 @@
 /** @jest-environment jsdom */
 
 import { BakeQueue } from "@canvas/players/bake-queue";
-import { Html5Player, bakeScaleFor, captureFramesInFlight } from "@canvas/players/html5-player";
+import { Html5Player, bakeScaleFor, captureFramesInFlight, heldFrame } from "@canvas/players/html5-player";
 import type { Edit } from "@core/edit-session";
 import { EditEvent, InternalEvent } from "@core/events/edit-events";
 import { sec } from "@core/timing/types";
@@ -1600,6 +1600,83 @@ it("shows a first bake's frames as they land", async () => {
 	await capture;
 });
 
+it.each([
+	[[0, 3], 2, 7, 0],
+	[[0, 3], 3, 7, 3],
+	[[0], 8, 7, null],
+	[[1], 8, 7, 1],
+	[[0, 3], 2, 0, null],
+	[[19], 20, 7, 19]
+])("finds the frame to hold among %p at %p within %p frames: %p", (present, idx, maxBehind, expected) => {
+	const frames = Array.from({ length: 20 }, (_, i) => (present.includes(i) ? new Blob([`${i}`]) : undefined));
+
+	expect(heldFrame(frames, idx, maxBehind)).toBe(expected);
+});
+
+const playingAhead = async (...baked: number[]) => {
+	const created = createStillPlayer();
+	const { player, harness, edit } = created;
+	Object.assign(edit, { isPlaying: true });
+	Object.assign(player, { getPlaybackTime: () => 0.4 }); // frame 12
+	baking(harness);
+	harness.beginCapture();
+	Object.assign(harness, { bakingFrames: bakedAt(...baked), bakingScale: 1 });
+	player.update(0, 0);
+	await until(stillShown(harness));
+	return created;
+};
+
+it("holds the nearest baked frame while playing ahead of the bake", async () => {
+	const { harness, seeks } = await playingAhead(8, 16);
+
+	expect(await frameMarkup(layer(harness)[0].texture.source.blob)).toBe("baked 8");
+	expect(seeks.some(seek => seek.startsWith("still"))).toBe(false);
+});
+
+it("holds a baked frame exactly 7 frames back while playing", async () => {
+	const { harness } = await playingAhead(5);
+
+	expect(await frameMarkup(layer(harness)[0].texture.source.blob)).toBe("baked 5");
+});
+
+it("renders a still while playing when the nearest baked frame is more than 7 frames back", async () => {
+	const { harness } = await playingAhead(4);
+
+	expect(await frameMarkup(layer(harness)[0].texture.source.blob)).toContain(">t=0.4<");
+});
+
+it("decodes a held frame once while playing on through it", async () => {
+	const { player } = await playingAhead(8);
+	const decode = jest.spyOn(globalThis, "createImageBitmap");
+	[0.4, 0.4, 13 / 30, 13 / 30, 14 / 30].forEach(t => {
+		Object.assign(player, { getPlaybackTime: () => t });
+		player.update(0, 0);
+	});
+	await flush(50);
+
+	expect(decode).not.toHaveBeenCalled();
+});
+
+it("never steps back to an older baked frame than the still on screen while playing", async () => {
+	const { player, harness } = await playingAhead(0); // a still of frame 12
+	harness.bakingFrames![11] = new Blob(["baked 11"]);
+	Object.assign(player, { getPlaybackTime: () => 13 / 30 });
+	player.update(0, 0);
+	await flush(50);
+
+	expect(await frameMarkup(layer(harness)[0].texture.source.blob)).toContain(">t=0.4<");
+});
+
+it("renders the exact still once playback pauses on a held frame", async () => {
+	const { player, harness, edit } = await playingAhead(8, 16);
+	const held = layer(harness)[0].texture;
+	Object.assign(edit, { isPlaying: false });
+	player.update(0, 0);
+	await until(() => layer(harness)[0].texture !== held);
+
+	expect(await frameMarkup(layer(harness)[0].texture.source.blob)).toContain(">t=0.4<");
+});
+
 it("swaps the still for baked frames once the bake finishes", async () => {
 	const { player, harness } = createStillPlayer();
 	let finish: (frames: Blob[]) => void = () => {};