From e12313bd837ad369055fe44a96a51a4b93d1efd2 Mon Sep 17 00:00:00 2001 From: EtienneLescot Date: Wed, 14 Oct 2026 17:18:40 +0200 Subject: [PATCH] feat(effects): the recording's surface in the effect contract fx_to_footage and fx_from_footage map between output uv and the recording's own plane through the warp the compositor draws it with: bilinear under a fixed angle, the corners' homography under the real 3D camera. ctx.footage_aspect gives round distances on that plane. The effect picks its space; the contract does not decide for it. --- crates/compositor/src/compositor_linux.rs | 3 +- crates/compositor/src/compositor_windows.rs | 65 +++++++++- crates/compositor/src/frame_geometry.rs | 7 +- crates/compositor/src/shader_fx.rs | 127 +++++++++++++++++++- docs/shader-effects.md | 8 ++ electron/ai-edition/deep-agent/service.ts | 2 +- electron/mcp/effects-contract.test.ts | 8 +- electron/mcp/effects-contract.ts | 28 +++-- 8 files changed, 228 insertions(+), 20 deletions(-) diff --git a/crates/compositor/src/compositor_linux.rs b/crates/compositor/src/compositor_linux.rs index 541840ae28..59b2a3b7d1 100644 --- a/crates/compositor/src/compositor_linux.rs +++ b/crates/compositor/src/compositor_linux.rs @@ -3749,7 +3749,8 @@ impl Compositor { let fx_inputs = crate::shader_fx::FxFrameInputs { cursor: fx_cursor, clicks: fx_clicks, - footage: crate::shader_fx::bounds(&g.footage_quad([rw, rh]).corners), + footage: g.footage_quad([rw, rh]), + footage_aspect: g.footage_aspect([rw, rh]), camera: lp.has_webcam.then_some(g.w_dst), }; let effects = scene_ref diff --git a/crates/compositor/src/compositor_windows.rs b/crates/compositor/src/compositor_windows.rs index 6014ec98c8..8de571408d 100644 --- a/crates/compositor/src/compositor_windows.rs +++ b/crates/compositor/src/compositor_windows.rs @@ -2811,7 +2811,8 @@ impl Compositor { let fx_inputs = crate::shader_fx::FxFrameInputs { cursor: fx_cursor, clicks: fx_clicks, - footage: crate::shader_fx::bounds(&g.footage_quad([self.rw(), self.rh()]).corners), + footage: g.footage_quad([self.rw(), self.rh()]), + footage_aspect: g.footage_aspect([self.rw(), self.rh()]), camera: lp.has_webcam.then_some(w_dst), }; let effects = scene_ref @@ -3872,7 +3873,11 @@ mod tests { /// Une scène à fond rouge plein, l'écran rétréci par le padding : les coins sont du fond. fn shader_fx_scene(effects: serde_json::Value) -> Scene { - let json = serde_json::json!({ + Scene::from_json(&shader_fx_json(effects).to_string()).expect("scène") + } + + fn shader_fx_json(effects: serde_json::Value) -> serde_json::Value { + serde_json::json!({ "clips": [], "layout": {"preset":"no-webcam","webcamSize":1,"webcamShape":"rectangle","webcamMirror":false,"webcamPosition":null,"webcamReactiveZoom":false}, "effects": {"padding":0.5,"blur":false,"shadow":0,"roundnessFrac":0,"motionBlur":0}, @@ -3882,8 +3887,7 @@ mod tests { "cropByClip": [], "output": {"width":320,"height":180,"fps":null}, "shaderEffects": effects, - }); - Scene::from_json(&json.to_string()).expect("scène") + }) } /// Un fragment d'effet de 1 s à 3 s, tel que l'app l'envoie. @@ -3982,6 +3986,59 @@ mod tests { assert_eq!(corner, [0, 255, 0]); } + /// `fx_to_footage` retrouve le point du métrage que le compositeur a dessiné sous un pixel, + /// à plat, sous un angle fixe (warp bilinéaire) et sous la caméra réelle (homographie), et + /// `fx_from_footage` le ramène sur ce pixel. En 1080p, l'arrondi 8 bits et le demi-pixel + /// tiennent sous 0,005 ; la pose orbit testée écarte l'homographie du bilinéaire de ~0,01. + #[test] + fn the_footage_plane_maps_back_to_what_the_compositor_drew() { + let Ok(gpu) = crate::d3d::Gpu::create_auto(false) else { + eprintln!("pas de device D3D11 — test sauté"); + return; + }; + const PLANE: &str = "fn effect(uv: vec2) -> vec4 {\n let p = fx_to_footage(uv);\n return vec4(p, distance(fx_from_footage(p), uv) * 100.0, 1.0);\n}\n"; + for (rotation, projective) in [(None, false), (Some("left"), false), (Some("orbit"), true)] { + let mut json = shader_fx_json(serde_json::json!([fx(PLANE, "image", 0)])); + json["clips"] = serde_json::json!([{"screenPath": "/s.mp4", "webcamPath": "", "sourceStartSec": 0, + "sourceEndSec": 10, "webcamOffsetSec": 0, "hasAudio": false}]); + json["layout"]["screenRect"] = serde_json::json!({"x": 0.1, "y": 0.1, "width": 0.8, "height": 0.8}); + json["cropByClip"] = serde_json::json!([null]); + if let Some(rotation) = rotation { + json["zoomRegions"] = serde_json::json!([{"clipIndex": 0, "startSec": 0, "endSec": 10, + "scale": 1.5, "focusX": 0.5, "focusY": 0.5, "rotation": rotation}]); + } + let comp = Compositor::new_sized(&gpu, 1920, 1080).expect("compositeur"); + let scene = Scene::from_json(&json.to_string()).expect("scène"); + comp.set_live_params(live_params_from_scene(&scene)); + comp.set_scene(Some(scene)); + let (screen, _tex) = nv12_frame(&gpu, 64, 64); + comp.set_timeline_time(Some(2.0)); + comp.set_programme_time(Some(2.0)); + unsafe { comp.compose_frame(&*screen, &*screen, 0.0, &fx_cfg()) }.expect("compose"); + let quad = comp.footage_quad().expect("le métrage de la frame"); + assert_eq!(quad.projective, projective, "{rotation:?}"); + let (w, h, rgba) = unsafe { comp.readback_direct() }.expect("readback"); + let [tl, tr, br, bl] = quad.corners.map(|[x, y]| (x, y)); + for (s, t) in [(0.25, 0.3), (0.7, 0.4), (0.4, 0.75)] { + let (x, y) = if projective { + crate::regions::square_to_quad(&[tl, tr, br, bl], s, t) + } else { + let top = (tl.0 + (tr.0 - tl.0) * s, tl.1 + (tr.1 - tl.1) * s); + let bottom = (bl.0 + (br.0 - bl.0) * s, bl.1 + (br.1 - bl.1) * s); + (top.0 + (bottom.0 - top.0) * t, top.1 + (bottom.1 - top.1) * t) + }; + let i = ((y * h as f32) as usize * w as usize + (x * w as f32) as usize) * 4; + let got = (rgba[i] as f32 / 255.0, rgba[i + 1] as f32 / 255.0); + assert!( + (got.0 - s).abs() < 0.005 && (got.1 - t).abs() < 0.005, + "{rotation:?} ({s}, {t}) : lu {got:?}, coins {:?}", + quad.corners + ); + assert!(rgba[i + 2] < 3, "{rotation:?} ({s}, {t}) : aller-retour à {}", rgba[i + 2]); + } + } + } + /// Les lanes s'appliquent dans l'ordre : bleu puis inversion donne jaune, l'inverse donne bleu. #[test] fn lanes_apply_in_order() { diff --git a/crates/compositor/src/frame_geometry.rs b/crates/compositor/src/frame_geometry.rs index 24de397595..c68321efe9 100644 --- a/crates/compositor/src/frame_geometry.rs +++ b/crates/compositor/src/frame_geometry.rs @@ -2857,12 +2857,17 @@ impl FrameGeometry { let [x, y, w, h] = self.s_dst; FootageQuad { corners: [[x, y], [x + w, y], [x + w, y + h], [x, y + h]], projective: false } } + + /// Largeur sur hauteur du métrage dans son propre plan : celles du rect qu'incline le tilt. + pub fn footage_aspect(&self, render_px: [f32; 2]) -> f32 { + self.s_dst[2] * render_px[0] / (self.s_dst[3] * render_px[1]).max(1e-6) + } } /// Le métrage dans l'image : ses coins TL, TR, BR, BL en fractions de la sortie, et la /// correspondance d'un point `(u, v)` du métrage à l'image, bilinéaire entre eux ou, sous la /// caméra réelle, l'homographie qu'ils définissent (`TiltedQuad::point_px`). -#[derive(Debug, Clone, Copy, PartialEq)] +#[derive(Debug, Clone, Copy, PartialEq, Default)] pub struct FootageQuad { pub corners: [[f32; 2]; 4], pub projective: bool, diff --git a/crates/compositor/src/shader_fx.rs b/crates/compositor/src/shader_fx.rs index 87e4ad54e0..b43f9e6ad7 100644 --- a/crates/compositor/src/shader_fx.rs +++ b/crates/compositor/src/shader_fx.rs @@ -11,6 +11,7 @@ //! pas d'état d'une frame à l'autre. C'est ce qui garde la preview (qui saute d'une frame à //! l'autre au seek) et l'export (qui les rend dans l'ordre) d'accord. +use crate::frame_geometry::FootageQuad; use crate::scene::{SceneShaderEffect, ShaderEffectTarget, ShaderParamDecl, ShaderParamType, ShaderParamValue}; use std::hash::{Hash, Hasher}; @@ -31,6 +32,9 @@ struct FxContext { clicks: array, 4>, footage: vec4, camera: vec4, + footage_corners: array, 2>, + footage_aspect: f32, + footage_warp: f32, }; @group(0) @binding(0) var fx_frame: texture_2d; @group(0) @binding(1) var fx_sampler: sampler; @@ -44,6 +48,98 @@ fn fx_px(n: f32) -> f32 { return n / 1080.0; } +// The recording's surface, wherever the zoom, the layout and the 3D camera put it, through the +// same warp the compositor draws it with. A point of the surface is (0,0) at the recording's +// top-left corner and (1,1) at its bottom-right, extended past its edges. Mirror of +// `quad_forward` in shaders.hlsl. +fn fx_from_footage(p: vec2) -> vec2 { + let r = ctx.resolution; + let c00 = ctx.footage_corners[0].xy * r; + let c10 = ctx.footage_corners[0].zw * r; + let c11 = ctx.footage_corners[1].xy * r; + let c01 = ctx.footage_corners[1].zw * r; + if (ctx.footage_warp > 0.5) { + let p1 = c10 - c00; + let p2 = c11 - c00; + let p3 = c01 - c00; + let d1 = p1 - p2; + let d2 = p3 - p2; + let d3 = p2 - p1 - p3; + let den = d1.x * d2.y - d2.x * d1.y; + let g = (d3.x * d2.y - d2.x * d3.y) / den; + let h = (d1.x * d3.y - d3.x * d1.y) / den; + return (c00 + (p1 * (1.0 + g) * p.x + p3 * (1.0 + h) * p.y) / (g * p.x + h * p.y + 1.0)) / r; + } + return (c00 + p.x * (c10 - c00) + p.y * (c01 - c00) + p.x * p.y * (c00 - c10 - c01 + c11)) / r; +} + +fn fx_footage_root(t: f32, e: vec2, f: vec2, g: vec2, h: vec2) -> vec3 { + let dx = e.x + g.x * t; + let dy = e.y + g.y * t; + var s = (h.y - f.y * t) / dy; + if (abs(dx) > abs(dy)) { + s = (h.x - f.x * t) / dx; + } + let inside = s >= -0.02 && s <= 1.02 && t >= -0.02 && t <= 1.02; + return vec3(s, t, select(0.0, 1.0, inside)); +} + +// The point of the recording's surface under `uv`: fx_from_footage backwards. Where no point of +// the surface lands (past its horizon), a point far outside 0..1. Mirror of `quad_inverse` in +// shaders.hlsl. +fn fx_to_footage(uv: vec2) -> vec2 { + let none = vec2(-1.0e4); + let r = ctx.resolution; + let c00 = ctx.footage_corners[0].xy * r; + let c10 = ctx.footage_corners[0].zw * r; + let c11 = ctx.footage_corners[1].xy * r; + let c01 = ctx.footage_corners[1].zw * r; + if (ctx.footage_warp > 0.5) { + let p1 = c10 - c00; + let p2 = c11 - c00; + let p3 = c01 - c00; + let d1 = p1 - p2; + let d2 = p3 - p2; + let d3 = p2 - p1 - p3; + let den = d1.x * d2.y - d2.x * d1.y; + let g = (d3.x * d2.y - d2.x * d3.y) / den; + let h = (d1.x * d3.y - d3.x * d1.y) / den; + let q = uv * r - c00; + let m00 = p1.x * (1.0 + g) - g * q.x; + let m01 = p3.x * (1.0 + h) - h * q.x; + let m10 = p1.y * (1.0 + g) - g * q.y; + let m11 = p3.y * (1.0 + h) - h * q.y; + let det = m00 * m11 - m01 * m10; + let st = vec2(q.x * m11 - m01 * q.y, m00 * q.y - q.x * m10) / det; + // Behind the camera: the homography reaches the point, the picture never shows it. + if (g * st.x + h * st.y + 1.0 <= 0.0) { + return none; + } + return st; + } + let e = c10 - c00; + let f = c01 - c00; + let g = c00 - c10 - c01 + c11; + let h = uv * r - c00; + let k2 = g.x * f.y - g.y * f.x; + let k1 = e.x * f.y - e.y * f.x + h.x * g.y - h.y * g.x; + let k0 = h.x * e.y - h.y * e.x; + if (abs(k2) < 1e-5 * abs(k1)) { + if (abs(k1) < 1e-6) { + return none; + } + return fx_footage_root(-k0 / k1, e, f, g, h).xy; + } + let disc = k1 * k1 - 4.0 * k2 * k0; + if (disc < 0.0) { + return none; + } + let q = -0.5 * (k1 + select(-1.0, 1.0, k1 >= 0.0) * sqrt(disc)); + let r0 = fx_footage_root(q / k2, e, f, g, h); + let r1 = fx_footage_root(select(q / k2, k0 / q, abs(q) > 0.0), e, f, g, h); + return select(r1.xy, r0.xy, r0.z > 0.5); +} + fn fx_pcg(v: u32) -> u32 { let state = v * 747796405u + 2891336453u; let word = ((state >> ((state >> 28u) + 4u)) ^ state) * 277803737u; @@ -84,7 +180,7 @@ pub const MAX_ITERATIONS_PER_PIXEL: u64 = 4096; /// Taille du constant buffer des paramètres : le plafond technique (32) de `vec4`. pub const PARAMS_BUFFER_BYTES: usize = 32 * 16; -/// Le constant buffer `ctx`, au layout de `FxContext` (144 octets). +/// Le constant buffer `ctx`, au layout de `FxContext` (192 octets). #[repr(C)] #[derive(Debug, Clone, Copy, PartialEq, Default)] pub struct FxContext { @@ -96,6 +192,13 @@ pub struct FxContext { pub clicks: [[f32; 4]; MAX_CLICKS], pub footage: [f32; 4], pub camera: [f32; 4], + /// Les coins du métrage, TL et TR puis BR et BL, en uv de sortie : `fx_to_footage`. + pub footage_corners: [[f32; 4]; 2], + /// Largeur sur hauteur du métrage dans son propre plan, avant toute inclinaison. + pub footage_aspect: f32, + /// 1 : l'homographie des coins (la caméra réelle), 0 : leur warp bilinéaire (`TiltedQuad`). + pub footage_warp: f32, + pub _pad: [f32; 2], } /// Ce que le plan du curseur et la géométrie de la frame donnent aux effets, en uv de sortie. @@ -104,7 +207,8 @@ pub struct FxFrameInputs { pub cursor: Option<[f32; 2]>, /// Les clics récents, du plus récent au plus ancien : point et âge en secondes. pub clicks: Vec<([f32; 2], f32)>, - pub footage: [f32; 4], + pub footage: FootageQuad, + pub footage_aspect: f32, pub camera: Option<[f32; 4]>, } @@ -558,6 +662,7 @@ pub fn context( resolution: [f32; 2], inputs: &FxFrameInputs, ) -> FxContext { + let [tl, tr, br, bl] = inputs.footage.corners; let mut clicks = [[0.0; 4]; MAX_CLICKS]; for (slot, ([x, y], age)) in clicks.iter_mut().zip(&inputs.clicks) { *slot = [*x, *y, *age, 1.0]; @@ -569,8 +674,12 @@ pub fn context( region: region(effect, source_t, programme_t), cursor: inputs.cursor.map_or([0.0; 4], |[x, y]| [x, y, 1.0, 0.0]), clicks, - footage: inputs.footage, + footage: bounds(&inputs.footage.corners), camera: inputs.camera.unwrap_or([0.0; 4]), + footage_corners: [[tl[0], tl[1], tr[0], tr[1]], [br[0], br[1], bl[0], bl[1]]], + footage_aspect: inputs.footage_aspect, + footage_warp: if inputs.footage.projective { 1.0 } else { 0.0 }, + _pad: [0.0; 2], } } @@ -820,6 +929,8 @@ mod tests { assert_eq!(offset("region"), &c.region as *const _ as usize - base); assert_eq!(offset("clicks"), &c.clicks as *const _ as usize - base); assert_eq!(offset("camera"), &c.camera as *const _ as usize - base); + assert_eq!(offset("footage_corners"), &c.footage_corners as *const _ as usize - base); + assert_eq!(offset("footage_warp"), &c.footage_warp as *const _ as usize - base); } #[test] @@ -937,13 +1048,19 @@ mod tests { let inputs = FxFrameInputs { cursor: Some([0.25, 0.5]), clicks: vec![([0.3, 0.6], 0.2), ([0.1, 0.1], 1.5)], - footage: [0.1, 0.1, 0.8, 0.8], + footage: FootageQuad { + corners: [[0.25, 0.5], [0.75, 0.25], [0.625, 0.75], [0.25, 0.625]], + projective: true, + }, + footage_aspect: 16.0 / 9.0, camera: Some([0.7, 0.7, 0.2, 0.2]), }; let ctx = context(&whole, 2.0, 10.0, [1920.0, 1080.0], &inputs); assert_eq!(ctx.cursor, [0.25, 0.5, 1.0, 0.0]); assert_eq!(ctx.clicks[..3], [[0.3, 0.6, 0.2, 1.0], [0.1, 0.1, 1.5, 1.0], [0.0; 4]]); - assert_eq!((ctx.footage, ctx.camera), ([0.1, 0.1, 0.8, 0.8], [0.7, 0.7, 0.2, 0.2])); + assert_eq!((ctx.footage, ctx.camera), ([0.25, 0.25, 0.5, 0.5], [0.7, 0.7, 0.2, 0.2])); + assert_eq!(ctx.footage_corners, [[0.25, 0.5, 0.75, 0.25], [0.625, 0.75, 0.25, 0.625]]); + assert_eq!((ctx.footage_aspect, ctx.footage_warp), (16.0 / 9.0, 1.0)); } #[test] diff --git a/docs/shader-effects.md b/docs/shader-effects.md index b4d1a6ed2d..039445a18b 100644 --- a/docs/shader-effects.md +++ b/docs/shader-effects.md @@ -217,6 +217,10 @@ libre, et les erreurs pointent ainsi les lignes que l'agent a écrites. | `ctx.clicks[i]` | `vec4` | Les 4 derniers clics des 2 dernières secondes, du plus récent au plus ancien. xy : le point du clic en uv de sortie. z : secondes écoulées depuis. w : 1 si l'entrée est occupée. | | `ctx.footage` | `vec4` | Le rect de l'enregistrement en uv de sortie (x, y, w, h), après zoom et layout. Sous un tilt 3D, c'est sa boîte englobante. | | `ctx.camera` | `vec4` | Le rect de la webcam en uv de sortie. Taille nulle sans caméra. | +| `ctx.footage_aspect` | `f32` | Largeur sur hauteur de l'enregistrement dans son propre plan, pour les distances circulaires sur lui. | +| `ctx.footage_corners`, `ctx.footage_warp` | | Ce que lisent les deux fonctions suivantes : les coins TL, TR, BR, BL du métrage en uv de sortie, et son warp (1 homographie, 0 bilinéaire). | +| `fx_to_footage(uv)` | `vec2` | Le point du plan de l'enregistrement sous `uv` : (0, 0) son coin haut-gauche, (1, 1) le bas-droit, prolongé au-delà de ses bords. Loin hors de 0..1 là où aucun point du plan ne tombe. | +| `fx_from_footage(p)` | `vec2` | L'inverse : où tombe en uv de sortie le point `p` du plan. | | `params.` | selon le type | Les paramètres déclarés par l'effet (C.3). | | `fx_px(n)` | `f32` | `n` pixels d'une sortie 1080p, en fraction de la hauteur. | | `fx_hash(p)` | `f32` | Hasard déterministe dans [0, 1) pour un `vec2`, identique sur tous les GPU (PCG). | @@ -224,6 +228,10 @@ libre, et les erreurs pointent ainsi les lignes que l'agent a écrites. - **Tout ce qui est en uv de sortie** est déjà passé par le zoom, le layout, le cadre et le tilt. Ces valeurs viennent de `plan_frame` et `plan_cursor` (`frame_geometry.rs`), que les trois backends partagent. +- **Le plan de l'enregistrement** suit le warp que le compositeur dessine : bilinéaire sous un + angle fixe, homographie sous la caméra réelle (`FootageQuad`, `quad_inverse` des shaders). + L'effet choisit son repère : l'image de sortie, ou ce plan. Le contrat ne décide pas lequel + convient à quel effet. - **Le point d'un clic** est la pointe à l'instant du clic, au même endroit que l'impact intégré (mode 16). - **Le curseur dessiné** passe par le lissage et peut traîner un instant derrière ce point quand diff --git a/electron/ai-edition/deep-agent/service.ts b/electron/ai-edition/deep-agent/service.ts index f97fa9cf6a..b3e2f978b8 100644 --- a/electron/ai-edition/deep-agent/service.ts +++ b/electron/ai-edition/deep-agent/service.ts @@ -263,7 +263,7 @@ export const TOOL_DESCRIPTIONS: Record = { addCameraFullscreen: "Add a camera-fullscreen region over a span of the edited timeline (virtual seconds): the webcam fills the frame for that span. This only does something when the footage under that span comes from an asset with a linked webcam — check assets[].hasCameraTrack (or hasAnyCamera) in getCurrentDocument first. On footage with no camera the call is refused rather than storing a region that would render nothing; say so instead of retrying.", addEffect: - "Write a visual effect as a shader and pose it over a span of the edited timeline (virtual seconds), or pose one of this project's effects again (effectId). Use it for a look the settings do not offer. `effect.code` is WGSL that defines `fn effect(uv: vec2) -> vec4`, called for every pixel; uv is 0..1 with (0,0) top-left. Available inside it: `frame(uv)` samples the composed picture; `params.` reads each declared parameter; `ctx.time` is programme seconds; `ctx.region.x` goes 0 to 1 across the region (y its length in s, z seconds since it began); `ctx.resolution` and `ctx.aspect`; `ctx.cursor` (xy in uv, z = 1 when on screen); `ctx.clicks[0..3]`, the last four clicks, newest first (xy in uv where it happened, z seconds since, w = 1 when the slot holds one); `ctx.footage` and `ctx.camera`, the recording's and the webcam's rects (x, y, w, h in uv; camera zero-sized when absent) — all already placed through zoom and layout; `fx_px(n)`, n pixels of a 1080p frame as a fraction of the height; `fx_hash(vec2)`, a random number in [0,1) identical on every GPU. Rules: the effect is a pure function of those inputs (no state between frames), so the preview and the export show the same pixels; reason in uv or fx_px, never raw pixels; every loop is a `for` with literal bounds (`for (var i = 0; i < 16; i++)`), at most 4096 iterations per pixel — the one hard rule. `params` declares up to 8 (recommended) parameters, each `{type: number|toggle|color|point, name, label, default}` (number also `min`, `max`); declare one only when moving it visibly changes the effect over its whole range. `target`: `image` (default: the picture under the annotations and captions), `text` (the annotations and captions alone, premultiplied alpha), `both` (everything); use text or both only when asked. Effects of one `lane` (0–7) cannot overlap; lanes apply in order, lane 0 first. The result gives the cost tier (frame() reads per pixel: light ≤ 8, about 0.06 ms per 1080p frame on a fast GPU; medium ≤ 32, 0.14 ms; heavy ≤ 128, 0.54 ms; very heavy beyond) and a recommendation when the tier runs long: light anywhere, medium up to a minute, heavy up to 10 s, very heavy a few seconds. It is a recommendation, never a refusal. Code that does not compile is refused with the compiler's error at the line of your code: fix it and call again. Check the result with getFrames.", + "Write a visual effect as a shader and pose it over a span of the edited timeline (virtual seconds), or pose one of this project's effects again (effectId). Use it for a look the settings do not offer. `effect.code` is WGSL that defines `fn effect(uv: vec2) -> vec4`, called for every pixel; uv is 0..1 with (0,0) top-left. Available inside it: `frame(uv)` samples the composed picture; `params.` reads each declared parameter; `ctx.time` is programme seconds; `ctx.region.x` goes 0 to 1 across the region (y its length in s, z seconds since it began); `ctx.resolution` and `ctx.aspect`; `ctx.cursor` (xy in uv, z = 1 when on screen); `ctx.clicks[0..3]`, the last four clicks, newest first (xy in uv where it happened, z seconds since, w = 1 when the slot holds one); `ctx.footage` and `ctx.camera`, the recording's and the webcam's rects (x, y, w, h in uv; camera zero-sized when absent) — all already placed through zoom and layout; `fx_to_footage(uv)` and `fx_from_footage(p)` map between output uv and the recording's own surface (0..1 across the recording) through the warp the compositor draws it with, zoom, layout and 3D camera included, and `ctx.footage_aspect` is its width over height: the effect works in whichever of the two spaces fits it; `fx_px(n)`, n pixels of a 1080p frame as a fraction of the height; `fx_hash(vec2)`, a random number in [0,1) identical on every GPU. Rules: the effect is a pure function of those inputs (no state between frames), so the preview and the export show the same pixels; reason in uv or fx_px, never raw pixels; every loop is a `for` with literal bounds (`for (var i = 0; i < 16; i++)`), at most 4096 iterations per pixel — the one hard rule. `params` declares up to 8 (recommended) parameters, each `{type: number|toggle|color|point, name, label, default}` (number also `min`, `max`); declare one only when moving it visibly changes the effect over its whole range. `target`: `image` (default: the picture under the annotations and captions), `text` (the annotations and captions alone, premultiplied alpha), `both` (everything); use text or both only when asked. Effects of one `lane` (0–7) cannot overlap; lanes apply in order, lane 0 first. The result gives the cost tier (frame() reads per pixel: light ≤ 8, about 0.06 ms per 1080p frame on a fast GPU; medium ≤ 32, 0.14 ms; heavy ≤ 128, 0.54 ms; very heavy beyond) and a recommendation when the tier runs long: light anywhere, medium up to a minute, heavy up to 10 s, very heavy a few seconds. It is a recommendation, never a refusal. Code that does not compile is refused with the compiler's error at the line of your code: fix it and call again. Check the result with getFrames.", setEffect: "Move, resize or re-tune an effect region by id (virtual seconds): its span, its parameter `values`, its `target` or its `lane`. Only the fields you pass change; a value equal to the default is dropped. Refused if the new span runs into another effect of the same lane.", setEffectDefinition: diff --git a/electron/mcp/effects-contract.test.ts b/electron/mcp/effects-contract.test.ts index fce3e8b65f..b8d54cbc87 100644 --- a/electron/mcp/effects-contract.test.ts +++ b/electron/mcp/effects-contract.test.ts @@ -32,7 +32,13 @@ describe("the effect contract resource", () => { }); it("names every helper the prelude gives", () => { - for (const helper of ["frame(uv)", "fx_px(n)", "fx_hash(p)"]) { + for (const helper of [ + "frame(uv)", + "fx_px(n)", + "fx_hash(p)", + "fx_to_footage(uv)", + "fx_from_footage(p)", + ]) { expect(EFFECTS_CONTRACT).toContain(helper); } }); diff --git a/electron/mcp/effects-contract.ts b/electron/mcp/effects-contract.ts index 929de027e9..47c28e37d8 100644 --- a/electron/mcp/effects-contract.ts +++ b/electron/mcp/effects-contract.ts @@ -33,6 +33,10 @@ added after your code, so compiler errors point at your lines. | ctx.clicks[i] | vec4 | The last 4 clicks of the last 2 s, newest first. xy: where it happened, in uv; z: seconds since; w: 1 when the slot holds one. | | ctx.footage | vec4 | The recording's rect in uv (x, y, w, h), after zoom and layout; under a 3D tilt, its bounding box. | | ctx.camera | vec4 | The webcam's rect in uv; zero-sized without a camera. | +| ctx.footage_aspect | f32 | The recording's width over height on its own surface, for round distances on it. | +| ctx.footage_corners, ctx.footage_warp | | What the two functions below read: the recording's corners and its warp. Use the functions. | +| fx_to_footage(uv) | vec2 | The point of the recording's surface under uv: (0, 0) its top-left corner, (1, 1) its bottom-right, extended past its edges. Far outside 0..1 where no point of the surface lands. | +| fx_from_footage(p) | vec2 | Back: where point p of the recording's surface lands, in uv. | | params. | per type | Your declared parameters. | | fx_px(n) | f32 | n pixels of a 1080p output, as a fraction of the height. | | fx_hash(p) | f32 | A random number in [0, 1) for a vec2, identical on every GPU. | @@ -40,6 +44,12 @@ added after your code, so compiler errors point at your lines. Everything in uv is already placed through zoom, layout, frame and tilt: never recompute where the screen, the camera or the pointer sit. +There are two spaces to work in, and the effect picks the one that fits it. Output uv is the +picture as it is framed. The recording's surface is the recording itself, wherever the zoom, the +layout and the 3D camera put it, through the same warp the compositor draws it with: something +computed there moves, scales and tilts with the recording. Map into it with fx_to_footage, compute, +and map back with fx_from_footage before calling frame(). + ## Parameters Declare each one as \`{ type, name, label, default }\`; a number also takes \`min\` and \`max\`. @@ -92,27 +102,31 @@ multiplied by their bound), and its tier. Measured on a fast GPU (RTX 4070 Ti) a An integrated GPU pays several times more. The tiers are recommendations, never refusals. -## Example: a ripple on every click +## Example: a ripple on the recording, on every click Parameters: strength (number, 0–2, default 1), radius (number, 0.05–0.5, default 0.2), -duration (number, 0.3–2, default 0.9). +duration (number, 0.3–2, default 0.9). The wave is computed on the recording's surface, so it +follows the zoom and lies flat on the recording under the 3D camera. - fn ripple(uv: vec2, click: vec4) -> vec2 { + fn ripple(p: vec2, click: vec4) -> vec2 { if (click.w < 0.5 || click.z > params.duration) { return vec2(0.0); } let t = click.z / params.duration; - let d = (uv - click.xy) * vec2(ctx.aspect, 1.0); + let metric = vec2(ctx.footage_aspect, 1.0); + let d = (p - fx_to_footage(click.xy)) * metric; let r = length(d); let x = params.radius * (1.0 - pow(1.0 - t, 3.0)) - r; let env = smoothstep(-0.008, 0.0, x) * exp(-max(x, 0.0) / 0.07); let wave = sin(x / 0.03 * 6.2831853) * env * (1.0 - t) * (1.0 - t) * params.strength; let dir = select(vec2(0.0), d / r, r > 1e-5); - return dir * wave * 0.01 / vec2(ctx.aspect, 1.0); + return dir * wave * 0.01 / metric; } fn effect(uv: vec2) -> vec4 { + let p = fx_to_footage(uv); var offset = vec2(0.0); - for (var i = 0; i < 4; i++) { offset += ripple(uv, ctx.clicks[i]); } - return frame(uv - offset); + for (var i = 0; i < 4; i++) { offset += ripple(p, ctx.clicks[i]); } + if (dot(offset, offset) == 0.0) { return frame(uv); } + return frame(fx_from_footage(p - offset)); } ## Example: a vignette that breathes across the region