diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index a27dd4e..8a94b26 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -2776,6 +2776,38 @@ in-place tests' world, rows given back; two models in a shared pose posed again (tests/render/data/arm.glb, a skinned strip of two materials); `geometry/emitter-model-live` (a model's emitting section that is not its first, the model a unit under software traversal). +**Meshes that deform are skinned on the GPU (2026-11-21).** A new pose of a shared pose's mesh was skinned on the +CPU and every triangle of it sent again: 180 bytes each (its vertices, its shading attributes, the BLAS's input). +Now the CPU sends the pose's matrices (48 bytes a joint) and a compute pass skins the mesh where it lies +(`shaders/skin.wgsl`, `Gpu::skin`): a triangle an invocation, from its model's corners as the file has them +(`skin::SkinCorner`: position, normal, tangent, four matrices and weights; one copy a model, whatever its poses, +`Scene::skin_sources`) and the pose's matrices (`skin::Model::palette`: a skin's joints, and the nodes of primitives +beside it), into the triangles, the attributes' normals, tangent and texel density, and the BLAS's input. The +commit builds the mesh's BLAS after it, as before (a BLAS is built again: wgpu has no update yet). +The CPU keeps no triangles of such a pose: a click or a query skins the mesh when it asks (`LiveCore::mesh_tris`). +A pose that mirrors other primitives than the one the mesh was made for has its triangles in another order: a +rebuild's. An authored tangent of a skinned vertex follows its joints now (it was placed by the object alone). +Still on the CPU: a posed model alone in its pose (world-space triangles: the next step makes it a mesh of its own), +and every pose under software traversal (a rebuild there). +Measured, a crowd of 150 walkers in 8 shared poses, hardware traversal, alternated: a frame's commit 2.9 ms -> 2.1 +(the upload 0.82 -> 0.18, the skin 0.45 -> 0.30); the trace the same. With every walker a mesh of its own (a switch +of the experiment, not in the tree): the CPU's frame 24 ms with the CPU skin (15 ms of upload: 40 fps), 7.4 ms with +the pass (60 fps), against 7.1 with the 8 shared poses. +A fault found on the way, older than this: a posed model the scene did not call movable was moved by skinning it +again where it stands, so its move was recorded as a new pose. One that is an instance of a mesh that deforms was +then not moved at all (no pose had changed), and nothing rebuilt. A posed model's move is a move now +(`World::note_move`): the renderer moves an instance, and skins world-space triangles again. +Tests: the pass's output read back from the GPU and compared word by word with the CPU skin after each new pose, for +two model files in two poses (`tests/render/data/skin-blend.glb`: more triangles than a block of the pass, vertices +two joints hold, weights that do not sum to one, a primitive with tangents, one without normals, a joint turned +about an axis that is neither the normals' nor the tangents', a mesh on a node beside the skin that mirrors it, +materials in an order the mesh does not keep), with world-space triangles before the meshes'; the previous frame's +vertices, the mesh as the CPU has it when asked, the picture against a build's; a model moved alone, posed and +moved at once, posed after the scene's buffers grew; a pose that mirrors other primitives, by a rebuild; the +corners placed by the palette on the CPU against the model's triangles. 27 mutations of the new logic: each fails a +test (two did not at first: the mesh's triangles were in the model's order and first in the GPU's, in every scene +the tests had). + **A posed rigid model is edited in place (2026-11-20).** As a mesh and a still model are. Hidden, its parts are instances no ray sees, their emitters at no weight, and it is posed and moved on; shown, it is placed where it stands (`show_meshes`, `repose_rigid`). Removed, its instances are no object's (`edit_geometry`). One the scene has diff --git a/renderer/shaders/skin.wgsl b/renderer/shaders/skin.wgsl new file mode 100644 index 0000000..d18b17f --- /dev/null +++ b/renderer/shaders/skin.wgsl @@ -0,0 +1,188 @@ +// The skinning pass: the triangles of meshes that deform, from their corners (skin.rs `SkinCorner`, as the file has +// them) and their poses' palettes, into the scene's triangles, shading attributes and BLAS vertices. One invocation a +// triangle; a workgroup is 64 triangles of one mesh (`blocks`: the frame's work, the meshes with a new pose alone). +// What it writes is what the CPU skin writes (skin.rs `Model::triangles`, scene.rs `tri_attr`). + +struct Params { + // x: the frame's blocks. + n: vec4, +} + +// A mesh that deforms: its first triangle in the scene's triangles and how many, its first triangle among the +// corners, its palette's first matrix. +struct Job { + first: u32, + count: u32, + source: u32, + palette: u32, +} + +struct Corner { + // Position; w: the first two palette indices (16 bits each). + p: vec4, + // Normal; w: the other two. + n: vec4, + // Tangent; w: CORNER_NORMAL | CORNER_TANGENT, which of the two the file has. + t: vec4, + // The four weights. + w: vec4, +} + +// (pathtrace.wgsl `TriAttr`.) +struct TriAttr { + n0: vec4, + n1: vec4, + n2: vec4, + uv: vec4, + t: vec4, + x: vec4, +} + +// (pathtrace.wgsl `TRI_DYNAMIC`.) +const TRI_DYNAMIC: u32 = 4u; +const CORNER_NORMAL: u32 = 1u; +const CORNER_TANGENT: u32 = 2u; +// Workgroups a row of the dispatch (a dimension holds 65,535). +const ROW: u32 = 32768u; + +@group(0) @binding(0) var PP: Params; +// (mesh, its first triangle of the block) +@group(0) @binding(1) var blocks: array>; +@group(0) @binding(2) var jobs: array; +@group(0) @binding(3) var corners: array; +// Three rows a matrix. +@group(0) @binding(4) var palette: array>; +@group(0) @binding(5) var verts: array>; +@group(0) @binding(6) var attrs: array; +@group(0) @binding(7) var blas: array; + +struct Placed { + p: vec3, + n: vec3, + t: vec3, +} + +fn unit(v: vec3) -> vec3 { + let l = length(v); + if l > 0.0 && finite(l) { + return v / l; + } + return vec3(0.0); +} + +fn finite(v: f32) -> bool { + return (bitcast(v) & 0x7f800000u) != 0x7f800000u; +} + +// A corner by its four matrices, blended by its weights; its normal by the blend's inverse transpose. +fn place(c: Corner, base: u32) -> Placed { + let a = bitcast(c.p.w); + let b = bitcast(c.n.w); + let j0 = (base + (a & 0xffffu)) * 3u; + let j1 = (base + (a >> 16u)) * 3u; + let j2 = (base + (b & 0xffffu)) * 3u; + let j3 = (base + (b >> 16u)) * 3u; + let r0 = palette[j0] * c.w.x + palette[j1] * c.w.y + palette[j2] * c.w.z + palette[j3] * c.w.w; + let r1 = palette[j0 + 1u] * c.w.x + palette[j1 + 1u] * c.w.y + palette[j2 + 1u] * c.w.z + palette[j3 + 1u] * c.w.w; + let r2 = palette[j0 + 2u] * c.w.x + palette[j1 + 2u] * c.w.y + palette[j2 + 2u] * c.w.z + palette[j3 + 2u] * c.w.w; + let p = vec4(c.p.xyz, 1.0); + let c12 = cross(r1.xyz, r2.xyz); + var n = vec3(dot(c12, c.n.xyz), dot(cross(r2.xyz, r0.xyz), c.n.xyz), dot(cross(r0.xyz, r1.xyz), c.n.xyz)); + if dot(r0.xyz, c12) < 0.0 { + n = -n; + } + return Placed(vec3(dot(r0, p), dot(r1, p), dot(r2, p)), unit(n), unit(vec3(dot(r0.xyz, c.t.xyz), dot(r1.xyz, c.t.xyz), dot(r2.xyz, c.t.xyz)))); +} + +@compute @workgroup_size(64) +fn main(@builtin(workgroup_id) wg: vec3, @builtin(local_invocation_index) li: u32) { + let b = wg.x + wg.y * ROW; + if b >= PP.n.x { + return; + } + let block = blocks[b]; + let job = jobs[block.x]; + let local = block.y + li; + if local >= job.count { + return; + } + let k = job.first + local; + let s = (job.source + local) * 3u; + let flags = bitcast(corners[s].t.w); + let c0 = place(corners[s], job.palette); + let c1 = place(corners[s + 1u], job.palette); + let c2 = place(corners[s + 2u], job.palette); + let e1 = c1.p - c0.p; + let e2 = c2.p - c0.p; + let area2 = cross(e1, e2); + var n0 = c0.n; + var n1 = c1.n; + var n2 = c2.n; + if (flags & CORNER_NORMAL) == 0u { + n0 = unit(area2); + n1 = n0; + n2 = n0; + } + var at = attrs[k]; + let uv0 = vec2(at.n0.w, at.n1.w); + let d1 = vec2(at.n2.w, at.uv.x) - uv0; + let d2 = vec2(at.uv.y, at.uv.z) - uv0; + let det = d1.x * d2.y - d2.x * d1.y; + // The tangent frame: the file's, its first corner's (its hand stays); or the triangle's own from its UVs. + var t = c0.t; + var hand = at.t.w; + if (flags & CORNER_TANGENT) == 0u { + if abs(det) < 1e-12 { + var any = vec3(0.0, 1.0, 0.0); + if abs(n0.x) < 0.9 { + any = vec3(1.0, 0.0, 0.0); + } + t = normalize(cross(n0, any)); + hand = 1.0; + } else { + let r = 1.0 / det; + t = unit((e1 * d2.y - e2 * d1.y) * r); + let bt = (e2 * d1.x - e1 * d2.x) * r; + hand = -1.0; + if dot(cross(n0, t), bt) < 0.0 { + hand = 1.0; + } + } + } + // Texel density for the mip choice: 0.5 log2(uv area / area). + let area = 0.5 * length(area2); + let uv_area = 0.5 * abs(det); + var lod = 0.0; + if area > 1e-12 && uv_area > 1e-14 { + lod = 0.5 * log2(uv_area / area); + } + at.n0 = vec4(n0, at.n0.w); + at.n1 = vec4(n1, at.n1.w); + at.n2 = vec4(n2, at.n2.w); + at.t = vec4(t, hand); + at.x.x = lod; + attrs[k] = at; + // (The flag word: the kernel takes a deforming mesh's motion from the previous frame's vertices.) + verts[k * 3u] = vec4(c0.p, bitcast(TRI_DYNAMIC)); + verts[k * 3u + 1u] = vec4(e1, 0.0); + verts[k * 3u + 2u] = vec4(e2, 0.0); + // The BLAS's vertices: a triangle that is not finite has no size there. + var q0 = c0.p; + var q1 = c1.p; + var q2 = c2.p; + if !(finite(q0.x) && finite(q0.y) && finite(q0.z) && finite(q1.x) && finite(q1.y) && finite(q1.z) && finite(q2.x) && finite(q2.y) && finite(q2.z)) { + q0 = vec3(0.0); + q1 = q0; + q2 = q0; + } + let v = k * 9u; + blas[v] = q0.x; + blas[v + 1u] = q0.y; + blas[v + 2u] = q0.z; + blas[v + 3u] = q1.x; + blas[v + 4u] = q1.y; + blas[v + 5u] = q1.z; + blas[v + 6u] = q2.x; + blas[v + 7u] = q2.y; + blas[v + 8u] = q2.z; +} diff --git a/renderer/src/gpu.rs b/renderer/src/gpu.rs index 46a513a..31502e7 100644 --- a/renderer/src/gpu.rs +++ b/renderer/src/gpu.rs @@ -469,6 +469,8 @@ pub struct Uploaded { /// The particle pass, when the scene has particle systems (hardware RT). pub particles: Option, pub gpu_particles: Option, + /// The skinning pass, when meshes of the scene deform (hardware RT). + pub skin: Option, pub fills: EmitterFills, /// The density grids (the atlas the kernel reads, the grid programs' pass), when the scene has any. pub grids: Option, @@ -730,6 +732,44 @@ fn grid_halves(data: &[f32]) -> Vec { /// Spawns one frame takes at most (gpu_particles.wgsl `spawns`: two vec4 each). pub const MAX_SPAWNS: usize = 1 << 16; +/// Triangles a workgroup of the skinning pass, and workgroups a row of its dispatch (skin.wgsl). +const SKIN_BLOCK: u32 = 64; +const SKIN_ROW: u32 = 32768; + +/// skin.wgsl `Job`: a mesh that deforms: its first triangle in the scene's and how many, its first triangle among the +/// corners, its palette's first matrix. +#[repr(C)] +#[derive(Clone, Copy, Debug, bytemuck::Pod, bytemuck::Zeroable)] +struct SkinJob { + first: u32, + count: u32, + source: u32, + palette: u32, +} + +/// The skinning pass of a scene (shaders/skin.wgsl): the meshes that deform (a job each, `scene::PoseGroup`'s index) +/// are skinned from their models' corners and their poses' palettes into the scene's triangles, attributes and BLAS +/// input. Hardware RT. +pub struct SkinGpu { + pipeline: wgpu::ComputePipeline, + bg: wgpu::BindGroup, + params: wgpu::Buffer, + /// A frame's work: (job, its first triangle of the block), `SKIN_BLOCK` triangles each. + blocks: wgpu::Buffer, + table: wgpu::Buffer, + corners: wgpu::Buffer, + /// Every job's palette, three rows a matrix (`skin::PaletteRows`). + palette: wgpu::Buffer, + /// Per job, and how many matrices its palette is. + jobs: Vec<(SkinJob, u32)>, +} + +fn skin_bind_group(d: &wgpu::Device, pipeline: &wgpu::ComputePipeline, own: [&wgpu::Buffer; 5], verts: &wgpu::Buffer, attrs: &wgpu::Buffer, blas_verts: &wgpu::Buffer) -> wgpu::BindGroup { + let mut entries: Vec = own.iter().enumerate().map(|(k, b)| buffer_entry(k as u32, b)).collect(); + entries.extend([buffer_entry(5, verts), buffer_entry(6, attrs), buffer_entry(7, blas_verts)]); + d.create_bind_group(&wgpu::BindGroupDescriptor { label: Some("skin"), layout: &pipeline.get_bind_group_layout(0), entries: &entries }) +} + /// Stateful GPU particles (gpu_particles.wgsl; ROADMAP ยง 6 step 5a): the pass, its tables and buffers, and per system /// what the CPU keeps between frames: where the next emission starts in its ring of slots, the rate's fraction carried. pub struct GpuParticlesGpu { @@ -2663,6 +2703,11 @@ impl Gpu { } _ => None, }; + // The skinning pass (hardware RT: it writes the BLASes' input), a job each mesh that deforms. + let skin = match &accel { + Some(a) if !scene.retained.pose_groups.is_empty() => Some(self.skin_pass(scene, a.pool_first, &b_verts, &b_attrs, &a.verts)), + _ => None, + }; self.queue.submit([]); self.device.poll(wgpu::PollType::wait_indefinitely())?; timings.push(("upload_ms".into(), t0.elapsed().as_secs_f64() * 1000.0)); @@ -2673,7 +2718,7 @@ impl Gpu { Some(_) => EmitterFills { moved: self.emitter_fill(&emitter_runs.moved, &b_verts, &heap), particles: self.emitter_fill(&emitter_runs.particles, &b_verts, &heap) }, None => EmitterFills::default(), }; - Ok(Uploaded { bg, bg_parts, heap, uniform, accum: b_accum, params: b_params, kernel, verts: b_verts, attrs: b_attrs, prev_verts: b_prev, dynamic_first, accel, units: scene.units.clone(), particles, gpu_particles, fills, grids, translucent, vcolors: !scene.gpu_vcolors.is_empty(), media_nodes: scene.media_nodes.clone(), media_hidden: std::cell::RefCell::new(scene.media.iter().map(|m| m.a[1] == crate::scene::MEDIUM_HIDDEN).collect()), room, words, cdf_words: env_cdf.len(), region_words, env_tex }) + Ok(Uploaded { bg, bg_parts, heap, uniform, accum: b_accum, params: b_params, kernel, verts: b_verts, attrs: b_attrs, prev_verts: b_prev, dynamic_first, accel, units: scene.units.clone(), particles, gpu_particles, skin, fills, grids, translucent, vcolors: !scene.gpu_vcolors.is_empty(), media_nodes: scene.media_nodes.clone(), media_hidden: std::cell::RefCell::new(scene.media.iter().map(|m| m.a[1] == crate::scene::MEDIUM_HIDDEN).collect()), room, words, cdf_words: env_cdf.len(), region_words, env_tex }) } /// Bytes Metal allocates for an AABB BLAS over `count` boxes in `boxes` (its own size query through wgpu-hal; None @@ -3351,6 +3396,71 @@ impl Gpu { if let Some(p) = &mut up.gpu_particles { p.bg = particle_bind_group(&self.device, &p.pipeline, "gpu particles", &[(0, &p.params), (1, &p.table), (7, &p.last), (8, &p.state), (9, &p.spawns), (10, &p.grid)], &written); } + if let Some(k) = &mut up.skin { + k.bg = skin_bind_group(&self.device, &k.pipeline, [&k.params, &k.blocks, &k.table, &k.corners, &k.palette], &up.verts, &up.attrs, &a.verts); + } + } + + /// The skinning pass of `scene`: its pipeline, the models' corners, a job each pose group, and room for a frame's + /// work and every job's palette. + fn skin_pass(&self, scene: &Scene, pool_first: usize, verts: &wgpu::Buffer, attrs: &wgpu::Buffer, blas_verts: &wgpu::Buffer) -> SkinGpu { + let d = &self.device; + let module = d.create_shader_module(wgpu::ShaderModuleDescriptor { label: Some("skin"), source: wgpu::ShaderSource::Wgsl(include_str!("../shaders/skin.wgsl").into()) }); + let pipeline = d.create_compute_pipeline(&wgpu::ComputePipelineDescriptor { label: Some("skin"), layout: None, module: &module, entry_point: Some("main"), compilation_options: Default::default(), cache: self.cache() }); + let mut source_first = vec![]; + let mut at = 0u32; + for source in &scene.skin_sources { + source_first.push(at); + at += source.len() as u32; + } + let corners: Vec = scene.skin_sources.iter().flatten().flatten().copied().collect(); + let (mut jobs, mut matrices, mut blocks) = (vec![], 0u32, 0u32); + for g in &scene.retained.pose_groups { + let m = &scene.meshes[g.mesh as usize]; + jobs.push((SkinJob { first: pool_first as u32 + m.first, count: m.count, source: source_first[g.source as usize], palette: matrices }, g.palette_len)); + matrices += g.palette_len; + blocks += m.count.div_ceil(SKIN_BLOCK); + } + let table: Vec = jobs.iter().map(|j| j.0).collect(); + let storage = wgpu::BufferUsages::STORAGE | wgpu::BufferUsages::COPY_DST; + let init = |label: &str, bytes: &[u8]| d.create_buffer_init(&wgpu::util::BufferInitDescriptor { label: Some(label), contents: bytes, usage: storage }); + let room = |label: &str, size: u64, usage| d.create_buffer(&wgpu::BufferDescriptor { label: Some(label), size: size.max(16), usage, mapped_at_creation: false }); + let (b_corners, b_table) = (init("skin corners", bytemuck::cast_slice(&corners)), init("skin jobs", bytemuck::cast_slice(&table))); + let params = room("skin params", 16, wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST); + let (b_blocks, palette) = (room("skin blocks", blocks as u64 * 8, storage), room("skin palette", matrices as u64 * 48, storage)); + let bg = skin_bind_group(d, &pipeline, [¶ms, &b_blocks, &b_table, &b_corners, &palette], verts, attrs, blas_verts); + SkinGpu { pipeline, bg, params, blocks: b_blocks, table: b_table, corners: b_corners, palette, jobs } + } + + /// Skins the meshes of `work` (a job and its pose's palette each) on the GPU: their triangles, shading attributes + /// and BLAS input, in a submit of its own (before the commit builds their BLASes). False: the scene has no such + /// pass, or a palette is not the job's size (nothing was written). + pub fn skin(&self, up: &Uploaded, work: &[(u32, Vec)]) -> bool { + let Some(s) = &up.skin else { return false }; + if work.iter().any(|(job, rows)| s.jobs.get(*job as usize).is_none_or(|j| j.1 as usize != rows.len())) { + return false; + } + let mut blocks: Vec<[u32; 2]> = vec![]; + for (job, rows) in work { + let j = s.jobs[*job as usize].0; + self.queue.write_buffer(&s.palette, j.palette as u64 * 48, bytemuck::cast_slice(rows)); + blocks.extend((0..j.count.div_ceil(SKIN_BLOCK)).map(|b| [*job, b * SKIN_BLOCK])); + } + if blocks.is_empty() { + return true; + } + let n = blocks.len() as u32; + self.queue.write_buffer(&s.blocks, 0, bytemuck::cast_slice(&blocks)); + self.queue.write_buffer(&s.params, 0, bytemuck::cast_slice(&[n, 0, 0, 0])); + let mut e = self.device.create_command_encoder(&Default::default()); + { + let mut pass = e.begin_compute_pass(&wgpu::ComputePassDescriptor { label: Some("skin"), timestamp_writes: None }); + pass.set_pipeline(&s.pipeline); + pass.set_bind_group(0, &s.bg, &[]); + pass.dispatch_workgroups(n.min(SKIN_ROW), n.div_ceil(SKIN_ROW), 1); + } + self.queue.submit([e.finish()]); + true } /// The loop guard's count as the GPU has it now (None: `kernel` has no guard): the invocations it stopped since @@ -3913,6 +4023,17 @@ mod tests { wesl_wgsl(&[RECORDS, src].concat(), &[], &[]); } wesl_wgsl(include_str!("../shaders/grid_sim.wgsl"), &[], &[]); + wesl_wgsl(include_str!("../shaders/skin.wgsl"), &[], &[]); + } + + /// The skinning pass is valid WGSL, and its block size and dispatch row are the ones the CPU lays its work out by. + #[test] + fn the_skinning_pass_is_valid_and_its_sizes_are_the_cpus() { + let src = include_str!("../shaders/skin.wgsl"); + let module = naga::front::wgsl::parse_str(src).expect("skin.wgsl parses"); + naga::valid::Validator::new(naga::valid::ValidationFlags::all(), naga::valid::Capabilities::all()).validate(&module).expect("skin.wgsl validates"); + assert!(src.contains(&format!("@compute @workgroup_size({SKIN_BLOCK})")) && src.contains(&format!("const ROW: u32 = {SKIN_ROW}u;"))); + assert_eq!((std::mem::size_of::(), std::mem::size_of::(), std::mem::size_of::()), (16, 64, 48)); } /// Each switch changes what it says, under both traversals (that every combination is valid WGSL: diff --git a/renderer/src/live/edits.rs b/renderer/src/live/edits.rs index b9ba70b..446ec44 100644 --- a/renderer/src/live/edits.rs +++ b/renderer/src/live/edits.rs @@ -1824,14 +1824,14 @@ impl LiveCore { Ok(true) } - /// Models in a shared pose (`scene::PoseGroup`): each group whose pose changed is skinned once, in model space (the - /// groups in parallel), into its mesh's pool triangles; the commit builds their BLASes and the TLAS over their - /// new bounds. False when a group's members no longer share one pose, or one is hidden or changed: - /// the caller rebuilds, which groups them again. (Deforming meshes get no motion vectors of their own yet: an - /// instance's are its transform's.) + /// Models in a shared pose (`scene::PoseGroup`): each group whose pose changed is skinned once, in model space, by + /// the skinning pass (gpu.rs `skin`: the CPU sends its pose's matrices); the commit builds their BLASes and the + /// TLAS over their new bounds. False when a group's members no longer share one pose, one is hidden or changed, + /// or its new pose mirrors other primitives than the one its mesh was made for: the caller rebuilds, which groups + /// them again. (The CPU keeps no triangles of such a mesh's pose: `LiveCore::mesh_tris` skins it when asked.) pub(super) fn repose_groups(&mut self, world: &World, members: &[u32], build: &mut gpu::TopBuild) -> Result { use rayon::prelude::*; - if self.sc.up.accel.is_none() { + if self.sc.up.accel.is_none() || self.sc.up.skin.is_none() { return Ok(false); } let r = &self.sc.tables.retained; @@ -1861,35 +1861,29 @@ impl LiveCore { if jobs.is_empty() { return Ok(true); } + // The poses' matrices (the groups in parallel). let t_skin = Instant::now(); let assets = &self.assets; - type Skinned = (Vec<[f32; 4]>, Vec, Vec<[f32; 3]>, Vec<[glam::Vec3A; 3]>); - let skinned: Vec>> = jobs + let palettes: Vec>>> = jobs .par_iter() .map(|(g, a)| { let pg = &r.pose_groups[*g as usize]; - let tris = scene::model_gltf(&pg.uri, assets)?.triangles(glam::Mat4::IDENTITY, pg.nodes.as_deref(), Some(a))?; - if tris.len() != pg.order.len() { + let model = scene::model_gltf(&pg.uri, assets)?; + if model.mirrored(Some(a))? != pg.mirrors { return Ok(None); } - let n = pg.order.len(); - let (mut v, mut at, mut b, mut t) = (Vec::with_capacity(n * 3), Vec::with_capacity(n), Vec::with_capacity(n * 3), Vec::with_capacity(n)); - for &(k, section, rank) in &pg.order { - let q = &tris[k as usize]; - let (e1, e2) = (q.p[1] - q.p[0], q.p[2] - q.p[0]); - // The flag word keeps TRI_DYNAMIC: the kernel takes a deforming mesh's motion from `prev_verts`. - v.extend_from_slice(&[[q.p[0].x, q.p[0].y, q.p[0].z, f32::from_bits(scene::TRI_DYNAMIC)], [e1.x, e1.y, e1.z, 0.0], [e2.x, e2.y, e2.z, 0.0]]); - at.push(scene::section_attr(q.p, q.n, q.uv, q.tangent, section, rank)); - b.extend(scene::blas_tri(q.p.map(|p| p.to_array()))); - t.push(q.p.map(glam::Vec3A::from)); - } - Ok(Some((v, at, b, t))) + Ok(Some(model.palette(Some(a))?)) }) .collect(); + let mut work = Vec::with_capacity(jobs.len()); + for ((g, _), rows) in jobs.iter().zip(palettes) { + let Some(rows) = rows? else { return Ok(false) }; + work.push((*g, rows)); + } self.cpu_skin_ms += t_skin.elapsed().as_secs_f32() * 1000.0; let t_up = Instant::now(); // Motion vectors: the GPU still holds the previous pose of these meshes (and of the last re-skin's, if others): - // keep it in `prev_verts` before uploading the new one. + // keep it in `prev_verts` before the pass writes the new one. let df = self.sc.dynamic.0; let (mut lo, mut hi) = (usize::MAX, 0usize); for (g, _) in &jobs { @@ -1898,25 +1892,16 @@ impl LiveCore { (lo, hi) = (lo.min(k), hi.max(k + pg.order.len())); } self.keep_prev(self.pose_span.map_or((lo, hi), |(a, b)| (a.min(lo), b.max(hi)))); + if !self.g.skin(&self.sc.up, &work) { + return Ok(false); + } self.pose_span = Some((lo, hi)); self.pose_reposed = true; - let (q, up) = (&self.g.queue, &self.sc.up); - let accel = up.accel.as_ref().unwrap(); - let mut meshes = Vec::with_capacity(jobs.len()); - for ((g, a), res) in jobs.into_iter().zip(skinned) { - let Some((v, at, b, t)) = res? else { return Ok(false) }; + for (g, a) in jobs { let pg = &mut self.sc.tables.retained.pose_groups[g as usize]; - // The mesh's first triangle in the GPU order: any member's record has it. - let k = self.sc.gpu_instances[pg.members[0].1 as usize].info[0] as u64; - q.write_buffer(&up.verts, k * 48, bytemuck::cast_slice(&v)); - q.write_buffer(&up.attrs, k * 96, bytemuck::cast_slice(&at)); - q.write_buffer(&accel.verts, k * 36, bytemuck::cast_slice(&b)); - let m = self.sc.meshes[pg.mesh as usize]; - self.sc.pool_tris[m.first as usize..(m.first + m.count) as usize].copy_from_slice(&t); pg.animation = a; - meshes.push(pg.mesh); + build.meshes.push(pg.mesh); } - build.meshes.extend(meshes); self.cpu_upload_ms += t_up.elapsed().as_secs_f32() * 1000.0; Ok(true) } diff --git a/renderer/src/live/inplace_tests.rs b/renderer/src/live/inplace_tests.rs index f3dbaaa..e7adaf7 100644 --- a/renderer/src/live/inplace_tests.rs +++ b/renderer/src/live/inplace_tests.rs @@ -2559,6 +2559,159 @@ fn models_in_a_shared_pose_posed_again_keep_their_sections() { assert!(apart(&a, &b) < 0.5, "the picture is {:.2} steps from a build's", apart(&a, &b)); } +/// The skinning pass writes what the CPU skin writes. Two models (tests/render/data/skin-blend.glb: 72 triangles, +/// vertices two joints hold, weights that do not sum to one, a primitive with tangents and one without normals, a +/// mesh on a node beside the skin that mirrors it, a joint turned about an axis that is neither the normals' nor the +/// tangents', its materials in an order a mesh laid out by material does not keep; and arm.glb), two of each in a +/// pose, in a scene with world-space triangles before the meshes': after each new pose the GPU's triangles, shading attributes and +/// BLAS input of each mesh are its model's triangles for the pose, the previous frame's vertices are the pose +/// before, and the CPU has the mesh as it is now when asked. Also after the scene's buffers were made again larger, +/// and a pose that mirrors other primitives than the mesh was made for is a rebuild's. +#[test] +fn the_skinning_pass_writes_what_the_cpu_skin_writes() { + let presets = crate::default_presets_dir(); + let open = |w: &World| gpu::Gpu::new().and_then(|g| LiveCore::new(g, w, Path::new(".."), &presets, 96, 54)); + let pose = |clip: &str, time| crate::desc::Animation { clip: clip.into(), time, r#loop: true, nodes: Default::default() }; + let bend = |time| pose("bend", time); + let mut w = world(); + for (id, uri, x) in [(30, "skin-blend", -0.5), (31, "skin-blend", 0.9), (32, "arm", -1.5), (33, "arm", 1.6)] { + w.set_object(id, format!("m{id}"), None, Kind::Model { uri: format!("tests/render/data/{uri}.glb"), nodes: None, animation: Some(bend(0.0)), functions: vec![] }, true, false).unwrap(); + w.set_transform(id, Vec3::new(x, -0.6, 0.8), glam::Quat::from_rotation_y(0.3), Vec3::splat(1.1)).unwrap(); + } + // (A rect light's card, far below: world-space triangles, which come before the meshes' in the GPU's order.) + let card = serde_json::from_value(serde_json::json!({ "type": "rect", "color": [1, 1, 1], "castShadow": true, "intensity": 1, "width": 0.2, "height": 0.2, "range": 0 })).unwrap(); + w.set_object(21, "card".into(), None, Kind::Light(card), true, false).unwrap(); + w.set_transform(21, Vec3::new(0.0, -60.0, 0.0), glam::Quat::IDENTITY, Vec3::ONE).unwrap(); + let mut live = match open(&w) { + Ok(c) => c, + Err(e) => return eprintln!("skipped: {e}"), + }; + w.take_changes(); + if !live.g.hwrt { + return eprintln!("skipped: no hardware ray tracing"); + } + assert!(live.sc.edit.pool_first > 0, "a mesh's first triangle on the GPU is not its first in the pool"); + { + let g = live.sc.tables.retained.pose_groups.iter().find(|g| g.uri.ends_with("skin-blend.glb")).expect("the model's mesh"); + assert!(g.order.iter().enumerate().any(|(at, &(k, _, _))| at as u32 != k), "the mesh has the model's triangles in another order (by material)"); + } + // (A mesh, its first instance and its object, for each deforming mesh of the scene.) + let deforming = |c: &LiveCore| c.sc.tables.retained.pose_groups.iter().map(|g| (g.uri.clone(), g.order.clone(), g.members[0].1, g.members[0].0)).collect::>(); + assert_eq!(deforming(&live).len(), 2, "two models, each a pose its two placements share"); + let read = |c: &LiveCore, b: &wgpu::Buffer, at: u64, bytes: u64| -> Vec { + let to = c.g.device.create_buffer(&wgpu::BufferDescriptor { label: None, size: bytes, usage: wgpu::BufferUsages::MAP_READ | wgpu::BufferUsages::COPY_DST, mapped_at_creation: false }); + let mut e = c.g.device.create_command_encoder(&Default::default()); + e.copy_buffer_to_buffer(b, at, &to, 0, bytes); + c.g.queue.submit([e.finish()]); + to.slice(..).map_async(wgpu::MapMode::Read, |_| {}); + c.g.device.poll(wgpu::PollType::wait_indefinitely()).unwrap(); + let words = bytemuck::pod_collect_to_vec(&to.slice(..).get_mapped_range().unwrap()); + words + }; + // What the CPU skin gives a mesh for a pose: vertices (48 B a triangle), attributes (96 B), BLAS input (36 B). + let cpu = |c: &LiveCore, g: usize, a: &crate::desc::Animation| { + let (uri, order, _, _) = &deforming(c)[g]; + let tris = scene::model_gltf(uri, Path::new("..")).unwrap().triangles(glam::Mat4::IDENTITY, None, Some(a)).unwrap(); + let (mut v, mut at, mut b): (Vec, Vec, Vec) = Default::default(); + for &(k, section, rank) in order { + let q = &tris[k as usize]; + let (e1, e2) = (q.p[1] - q.p[0], q.p[2] - q.p[0]); + v.extend([q.p[0].x, q.p[0].y, q.p[0].z, f32::from_bits(scene::TRI_DYNAMIC), e1.x, e1.y, e1.z, 0.0, e2.x, e2.y, e2.z, 0.0]); + at.extend(bytemuck::cast_slice::<_, f32>(&[scene::section_attr(q.p, q.n, q.uv, q.tangent, section, rank)])); + b.extend(q.p.iter().flat_map(|p| p.to_array())); + } + (v, at, b) + }; + // (Numbers within a rounding of each other; the words that are no numbers, a flag, a section, a rank, the same.) + let same = |what: &str, got: &[f32], want: &[f32], words: &[usize], stride: usize| { + assert_eq!(got.len(), want.len(), "{what}: its length"); + for (i, (g, x)) in got.iter().zip(want).enumerate() { + let ok = if words.contains(&(i % stride)) { g.to_bits() == x.to_bits() } else { (g - x).abs() <= 2e-5 * x.abs().max(1.0) }; + assert!(ok, "{what}: word {} of triangle {}: {g} ({:#x}), the CPU's {x} ({:#x})", i % stride, i / stride, g.to_bits(), x.to_bits()); + } + }; + let gpu = |c: &LiveCore, g: usize| { + let (_, order, instance, _) = &deforming(c)[g]; + let (k, n) = (c.sc.gpu_instances[*instance as usize].info[0] as u64, order.len() as u64); + let up = &c.sc.up; + (read(c, &up.verts, k * 48, n * 48), read(c, &up.attrs, k * 96, n * 96), read(c, &up.accel.as_ref().unwrap().verts, k * 36, n * 36), read(c, &up.prev_verts, (k - c.sc.dynamic.0 as u64) * 48, n * 48)) + }; + let check = |what: &str, c: &LiveCore, now: &crate::desc::Animation, before: &crate::desc::Animation| { + for g in 0..deforming(c).len() { + let what = format!("{what}, {}", deforming(c)[g].0); + let ((v, at, b, prev), (cv, cat, cb)) = (gpu(c, g), cpu(c, g, now)); + same(&format!("{what}: the vertices"), &v, &cv, &[3], 12); + // (An attribute's words: the section in `uv.w` (15), the rank in `x.w` (23), the object and a spare beside.) + same(&format!("{what}: the attributes"), &at, &cat, &[15, 21, 22, 23], 24); + same(&format!("{what}: the BLAS input"), &b, &cb, &[], 9); + same(&format!("{what}: the previous frame's vertices"), &prev, &cpu(c, g, before).0, &[3], 12); + // The CPU's own: the build's pose in the pool, the pose as it is now when asked. + let (instance, id) = (deforming(c)[g].2, deforming(c)[g].3); + let inst = &c.sc.instances[instance as usize]; + let asked: Vec = c.node_tris(&format!("m{id}")).iter().flatten().flat_map(|p| p.to_array()).collect(); + let placed: Vec = cb.chunks_exact(3).flat_map(|p| inst.world.transform_point3(Vec3::new(p[0], p[1], p[2])).to_array()).collect(); + same(&format!("{what}: the mesh as the CPU has it when asked"), &asked, &placed, &[], 9); + } + }; + let all = |w: &mut World, a: &crate::desc::Animation| { + for id in [30, 31, 32, 33] { + w.set_animation(id, Some(a.clone())).unwrap(); + } + }; + let (first, second, third) = (bend(0.0), bend(0.75), bend(0.4)); + let loaded: Vec> = (0..2).map(|g| cpu(&live, g, &first).0).collect(); + for (what, now, before) in [("a new pose", &second, &first), ("the next", &third, &second)] { + all(&mut w, now); + assert!(!live.commit(&mut w).unwrap(), "{what}: in place ({:?})", live.fallback); + check(what, &live, now, before); + } + assert!((0..2).all(|g| cpu(&live, g, &third).0.iter().zip(&loaded[g]).any(|(a, b)| (a - b).abs() > 0.05)), "the pose moves both meshes"); + // The picture is a build's. + let pictured = |what: &str, live: &mut LiveCore, w: &World| { + let mut fresh = open(w).unwrap(); + forget(live); + let (a, b) = (picture(live, 64), picture(&mut fresh, 64)); + assert!(apart(&a, &b) < 0.5, "{what}: the picture is {:.2} steps from a build's", apart(&a, &b)); + }; + pictured("posed in place", &mut live, &w); + // Moved alone (a model the scene did not call movable): its instance, in place, and the picture a build's. + w.set_transform(31, Vec3::new(1.1, -0.4, 0.9), glam::Quat::from_rotation_y(0.5), Vec3::splat(1.1)).unwrap(); + assert!(!live.commit(&mut w).unwrap(), "a posed model moved: in place ({:?})", live.fallback); + let stands = |c: &LiveCore, id: u32| c.sc.instances[c.sc.tables.retained.instance(id).unwrap().instance as usize].world; + assert_eq!(stands(&live, 31), w.world_matrix(31), "its instance stands where the model was moved to"); + pictured("a posed model moved", &mut live, &w); + // Posed in the same frame as moved: in place, the mesh skinned, the instance moved. + w.set_transform(30, Vec3::new(-0.9, -0.5, 0.9), glam::Quat::from_rotation_y(-0.2), Vec3::splat(1.1)).unwrap(); + all(&mut w, &bend(0.9)); + assert!(!live.commit(&mut w).unwrap(), "posed and moved at once: in place ({:?})", live.fallback); + check("posed and moved at once", &live, &bend(0.9), &third); + assert_eq!(stands(&live, 30), w.world_matrix(30), "its instance stands where the model was moved to"); + pictured("posed and moved at once", &mut live, &w); + // The scene's buffers made again, larger (a mesh of more triangles than the room for edits has, added in place, + // far below the view): the pass writes the new ones. + let size = live.sc.up.verts.size(); + let n = 200u32; + let positions: Vec = (0..=n).flat_map(|y| (0..=n).flat_map(move |x| [x as f32, y as f32, 0.0])).collect(); + let indices: Vec = (0..n).flat_map(|y| (0..n).flat_map(move |x| { let k = y * (n + 1) + x; [k, k + 1, k + n + 2, k, k + n + 2, k + n + 1] })).collect(); + let corners = ((n + 1) * (n + 1)) as usize; + w.set_geometry(7, Geometry { normals: [0.0, 0.0, 1.0].repeat(corners), uvs: vec![0.0; corners * 2], positions, indices, groups: vec![], colors: vec![] }).unwrap(); + w.set_object(40, "sheet".into(), None, Kind::Mesh { geometry: 7, material: 1, own_blas: false, materials: vec![] }, true, false).unwrap(); + w.set_transform(40, Vec3::new(0.0, -80.0, 0.0), glam::Quat::IDENTITY, Vec3::splat(0.001)).unwrap(); + assert!(!live.commit(&mut w).unwrap(), "a large mesh added: in place ({:?})", live.fallback); + assert!(live.sc.up.verts.size() > size, "the triangles' buffer was made again, larger ({} -> {})", size, live.sc.up.verts.size()); + all(&mut w, &bend(0.25)); + assert!(!live.commit(&mut w).unwrap(), "posed after the buffers grew: in place ({:?})", live.fallback); + check("posed after the buffers grew", &live, &bend(0.25), &bend(0.9)); + pictured("posed after the buffers grew", &mut live, &w); + // A pose that mirrors other primitives than the mesh was made for (the clip `flip` turns the badge's node the + // other way round): its triangles are in another order, a build's to make. + for id in [30, 31] { + w.set_animation(id, Some(pose("flip", 0.9))).unwrap(); + } + assert!(live.commit(&mut w).unwrap(), "a pose that mirrors other primitives: a rebuild"); + pictured("a pose that mirrors other primitives", &mut live, &w); +} + /// Models the game poses node by node (tests/render/data/arm.glb's joint `tip`, no clip): each is a pose of its own, even /// in the same pose as another, so posing them apart is in place; the picture is a build's. #[test] diff --git a/renderer/src/live/load.rs b/renderer/src/live/load.rs index 7b7372c..a9ffd63 100644 --- a/renderer/src/live/load.rs +++ b/renderer/src/live/load.rs @@ -12,6 +12,7 @@ pub(super) struct LoadedScene { pub(super) tri_nodes: Vec<(String, u32, u32)>, /// Flattened instances of instanced objects: (first, end, instance index), for picking. pub(super) tri_instances: Vec<(u32, u32, u32)>, + /// (A mesh the GPU skins has its build's pose here: `LiveCore::mesh_tris` has it as it is now.) pub(super) pool_tris: Vec<[glam::Vec3A; 3]>, /// Software traversal: each pool triangle's section and its index in it (a unit's tree orders its mesh's /// triangles; live/edits.rs `mesh_places`). Empty under hardware traversal, where they lie by section. diff --git a/renderer/src/live/mod.rs b/renderer/src/live/mod.rs index f3141ad..3f5d926 100644 --- a/renderer/src/live/mod.rs +++ b/renderer/src/live/mod.rs @@ -919,6 +919,25 @@ impl LiveCore { /// Closest surface under (u, v): (node id, world-space hit point). CPU ray cast over the scene triangles, and over /// each instance's model-space mesh with the ray in its space (same ray parameter t). + /// A mesh's triangles in its own space, as they are now. One that deforms is skinned by the GPU, and the CPU keeps + /// none of its pose: it is skinned here when asked for (a click, a query), once a call (`posed`, by mesh). + fn mesh_tris<'a>(&'a self, mesh: u32, posed: &'a mut std::collections::HashMap>) -> &'a [[glam::Vec3A; 3]] { + let m = self.sc.meshes[mesh as usize]; + let pool = &self.sc.pool_tris[m.first as usize..(m.first + m.count) as usize]; + if !m.deforms || self.sc.up.skin.is_none() { + return pool; + } + let Some(pg) = self.sc.tables.retained.pose_groups.iter().find(|g| g.mesh == mesh) else { return pool }; + if !posed.contains_key(&mesh) { + let tris = scene::model_gltf(&pg.uri, &self.assets).and_then(|model| model.triangles(glam::Mat4::IDENTITY, pg.nodes.as_deref(), Some(&pg.animation))); + match tris { + Ok(t) if t.len() == pg.order.len() => posed.insert(mesh, pg.order.iter().map(|&(k, _, _)| t[k as usize].p.map(glam::Vec3A::from)).collect()), + _ => return pool, + }; + } + &posed[&mesh] + } + /// The object under view point (u, v): its name, the hit point, and for an instanced object the instance's index. pub fn pick(&self, u: f32, v: f32) -> Option<(String, Vec3, Option)> { let (o, d) = self.ray(u, v); @@ -954,6 +973,7 @@ impl LiveCore { let (mut t, i) = closest(&self.sc.tris, o, d); let mut node = self.sc.tri_nodes.iter().find(|(_, a, b)| (*a as usize..*b as usize).contains(&i)).map(|n| n.0.as_str()); let mut sub = self.sc.tri_instances.iter().find(|(a, b, _)| (*a as usize..*b as usize).contains(&i)).map(|r| r.2); + let mut posed = Default::default(); for inst in self.sc.instances.iter().filter(|i| !i.hidden) { let m = self.sc.meshes[inst.mesh as usize]; let inv = inst.world.inverse(); @@ -964,7 +984,7 @@ impl LiveCore { if far < near.max(0.0) || near > t { continue; } - let (ti, _) = closest(&self.sc.pool_tris[m.first as usize..(m.first + m.count) as usize], lo, ld); + let (ti, _) = closest(&self.mesh_tris(inst.mesh, &mut posed), lo, ld); if ti < t { t = ti; node = Some(inst.name.as_str()); @@ -1001,9 +1021,9 @@ impl LiveCore { // A model's first instance; every drawn instance of an instanced object. let mut of_node = self.sc.instances.iter().filter(|i| i.name == id && !i.hidden); let first = of_node.next(); + let mut posed = Default::default(); for inst in first.into_iter().chain(of_node.filter(|i| i.sub.is_some())) { - let m = self.sc.meshes[inst.mesh as usize]; - out.extend(self.sc.pool_tris[m.first as usize..(m.first + m.count) as usize].iter().map(|t| inst.tri(t))); + out.extend(self.mesh_tris(inst.mesh, &mut posed).iter().map(|t| inst.tri(t))); } out } diff --git a/renderer/src/scene.rs b/renderer/src/scene.rs index 4622d0c..ecd6504 100644 --- a/renderer/src/scene.rs +++ b/renderer/src/scene.rs @@ -672,6 +672,9 @@ pub struct Scene { pub pool_attrs: Vec, pub meshes: Vec, pub instances: Vec, + /// What the skinning pass reads of the models whose meshes deform (`PoseGroup::source`): a model's triangles as + /// its file has them, in its mesh's order. One a model, whatever its poses. + pub skin_sources: Vec>, /// The meshes' sections (`InstMesh::sect_first`) and the instances' rows (`Instance::rows`): as the kernel reads /// the rows once `prepare` has set where each section's triangles and emitters are. pub sections: Vec
, @@ -974,6 +977,12 @@ pub struct PoseGroup { pub order: Vec<(u32, u32, u32)>, /// (object, instance) of each member. pub members: Vec<(u32, u32)>, + /// What the skinning pass skins it from (`Scene::skin_sources`), how many matrices its pose is + /// (`skin::Model::palette`), and the primitives the pose its mesh was made for mirrors (`skin::Model::mirrored`: + /// a pose that mirrors others has its triangles in another order, a build's to make). + pub source: u32, + pub palette_len: u32, + pub mirrors: Vec, } /// What groups posed models into a `PoseGroup`: model, nodes, clip, time (bits), loop; for a model whose pose names @@ -1555,6 +1564,9 @@ pub fn build(w: &World, assets: &Path, presets_dir: &Path, overrides: Option<&se pose_key_of.retain(|_, k| count[k] >= 2 || k.5.is_some()); } let mut pose_group_of: HashMap> = HashMap::new(); + // What the skinning pass reads, by (model, nodes, the primitives mirrored): shared by the model's pose groups. + let mut skin_sources: Vec> = vec![]; + let mut skin_source_of: HashMap<(String, Option>, Vec), u32> = HashMap::new(); // Flattened instances' triangle ranges. let mut tri_instances: Vec<(u32, u32, u32)> = vec![]; // Instance colours anywhere: object entries of 4 vec4s, and the kernel's F_TINT. @@ -1883,10 +1895,19 @@ pub fn build(w: &World, assets: &Path, presets_dir: &Path, overrides: Option<&se classed[if emits { FLAT } else { class as usize }].push((k, mat)); } // Triangles flattened per placement (emissive): each model skinned alone. - let g = (classed[FLAT].is_empty() && !local.is_empty()).then(|| { + let shared = classed[FLAT].is_empty() && !local.is_empty(); + let (order, sects, mine) = model_sections(&kept(&classed)); + // (The model's corners in the mesh's order: the same for every pose group of the model.) + let mirrors = if shared { glb.mirrored(Some(a))? } else { vec![] }; + let skey = (key.0.clone(), nodes.clone(), mirrors.clone()); + if shared && !skin_source_of.contains_key(&skey) { + let corners = glb.skin_source(nodes.as_deref(), Some(a))?; + skin_sources.push(order.iter().map(|&(k, _, _)| corners[k]).collect()); + skin_source_of.insert(skey.clone(), skin_sources.len() as u32 - 1); + } + let g = shared.then(|| { let first = pool_tris.len() as u32; let (mut lo, mut hi) = (Vec3A::splat(f32::MAX), Vec3A::splat(f32::MIN)); - let (order, sects, mine) = model_sections(&kept(&classed)); for &(k, section, rank) in &order { let t = &local[k]; for v in t.p { @@ -1903,7 +1924,7 @@ pub fn build(w: &World, assets: &Path, presets_dir: &Path, overrides: Option<&se meshes.push(InstMesh { lo: lo - pad, hi: hi + pad, deforms: true, sect_first: sections.len() as u32, sect_count: sects.len() as u32, ..own_inst_mesh(&pool_tris[first as usize..], first) }); sections.extend(sects); retained.mesh_rows.insert(meshes.len() as u32 - 1, mine); - retained.pose_groups.push(PoseGroup { mesh: meshes.len() as u32 - 1, uri: uri.clone(), nodes: nodes.clone(), animation: a.clone(), order: tri_order, members: vec![] }); + retained.pose_groups.push(PoseGroup { mesh: meshes.len() as u32 - 1, uri: uri.clone(), nodes: nodes.clone(), animation: a.clone(), order: tri_order, members: vec![], source: skin_source_of[&skey], palette_len: glb.palette_len() as u32, mirrors }); retained.pose_groups.len() as u32 - 1 }); pose_group_of.insert(key.clone(), g); @@ -2286,6 +2307,7 @@ pub fn build(w: &World, assets: &Path, presets_dir: &Path, overrides: Option<&se pool_attrs, meshes, instances, + skin_sources, sections, rows, gpu_instances: vec![], diff --git a/renderer/src/skin.rs b/renderer/src/skin.rs index 2ce72f5..bc309c6 100644 --- a/renderer/src/skin.rs +++ b/renderer/src/skin.rs @@ -1,7 +1,8 @@ //! glTF models, posed: a clip sampled at a time (node translation / rotation / scale channels with step, linear or //! cubic-spline interpolation) and linear blend skinning on the CPU (UE: a skeletal mesh with an animation sequence). //! `Model::triangles` yields a model's triangles in world space for a pose, in the same order every time, so a posed -//! model can be re-skinned in place (`tables.rs`, `LiveCore::commit`). +//! model can be re-skinned in place (`tables.rs`, `LiveCore::commit`). A mesh that deforms is skinned by the GPU +//! (shaders/skin.wgsl) from the same triangles as the file has them (`skin_source`) and a pose's matrices (`palette`). use crate::desc; use anyhow::{anyhow, Result}; @@ -67,6 +68,50 @@ pub struct ModelTri { pub material: Option, } +/// A corner of a model's triangle as the skinning pass reads it (skin.wgsl `Corner`): where it is in the file, and +/// the matrices of the model's palette (`Model::palette`) that place it. +#[repr(C)] +#[derive(Clone, Copy, Debug, Default, PartialEq, bytemuck::Pod, bytemuck::Zeroable)] +pub struct SkinCorner { + /// Position; w: the first two palette indices (16 bits each). + pub p: [f32; 4], + /// Normal; w: the other two. + pub n: [f32; 4], + /// Tangent; w: `CORNER_NORMAL` | `CORNER_TANGENT` (bits): which of the two the file has. + pub t: [f32; 4], + /// The four weights, summing to one. + pub w: [f32; 4], +} + +pub const CORNER_NORMAL: u32 = 1; +pub const CORNER_TANGENT: u32 = 2; + +/// A palette matrix: the three rows of an affine transform. +pub type PaletteRows = [[f32; 4]; 3]; + +fn palette_rows(m: Mat4) -> PaletteRows { + let t = m.transpose().to_cols_array_2d(); + [t[0], t[1], t[2]] +} + +/// A corner placed by a palette, as skin.wgsl places it: its position, and its normal and tangent (zero where the +/// file has none). The four matrices blended by the weights; the normal by the blend's inverse transpose. +pub fn skin_corner(c: &SkinCorner, palette: &[PaletteRows]) -> (Vec3, Vec3, Vec3) { + let (a, b) = (c.p[3].to_bits(), c.n[3].to_bits()); + let joints = [a & 0xffff, a >> 16, b & 0xffff, b >> 16]; + let mut rows = [glam::Vec4::ZERO; 3]; + for (j, w) in joints.iter().zip(c.w) { + for (r, m) in rows.iter_mut().zip(palette[*j as usize]) { + *r += glam::Vec4::from(m) * w; + } + } + let (p, n, t) = (Vec3::new(c.p[0], c.p[1], c.p[2]), Vec3::new(c.n[0], c.n[1], c.n[2]), Vec3::new(c.t[0], c.t[1], c.t[2])); + let r = rows.map(|r| r.truncate()); + let det = r[0].dot(r[1].cross(r[2])); + let normal = (Vec3::new(r[1].cross(r[2]).dot(n), r[2].cross(r[0]).dot(n), r[0].cross(r[1]).dot(n)) * det.signum()).normalize_or_zero(); + (Vec3::new(rows[0].dot(p.extend(1.0)), rows[1].dot(p.extend(1.0)), rows[2].dot(p.extend(1.0))), normal, Vec3::new(r[0].dot(t), r[1].dot(t), r[2].dot(t)).normalize_or_zero()) +} + impl Model { pub fn new(gltf: Gltf) -> Result { let (doc, bufs, _) = &gltf; @@ -253,6 +298,114 @@ impl Model { .collect()) } + /// Where each matrix of the palette is: the first of each skin's joints, each node that holds a primitive without + /// a skin, and the palette's length. Entry 0 is the identity (a vertex no joint holds). + fn palette_layout(&self) -> (Vec, HashMap, usize) { + let mut at = 1; + let mut skins = vec![]; + for skin in self.gltf.0.skins() { + skins.push(at); + at += skin.joints().count(); + } + let mut nodes = HashMap::new(); + for p in self.prims.iter().filter(|p| p.skin.is_none()) { + nodes.entry(p.node).or_insert_with(|| { + at += 1; + at - 1 + }); + } + (skins, nodes, at) + } + + /// The matrices that place the model's vertices in its own space for `pose` (None = rest pose): what + /// `skin_source`'s corners index. A skin's joints (joint x inverse bind), then the nodes of primitives without a + /// skin. + pub fn palette(&self, pose: Option<&desc::Animation>) -> Result> { + let (doc, bufs, _) = &self.gltf; + let globals = self.globals(pose)?; + let (_, nodes, len) = self.palette_layout(); + let mut out = vec![palette_rows(Mat4::IDENTITY); len]; + let mut at = 1; + for skin in doc.skins() { + let ibm: Vec = skin.reader(|b| Some(&bufs[b.index()])).read_inverse_bind_matrices().map(|it| it.map(|m| Mat4::from_cols_array_2d(&m)).collect()).unwrap_or_default(); + for (k, j) in skin.joints().enumerate() { + out[at] = palette_rows(globals[j.index()] * ibm.get(k).copied().unwrap_or(Mat4::IDENTITY)); + at += 1; + } + } + for (node, k) in nodes { + out[k] = palette_rows(globals[node]); + } + Ok(out) + } + + /// The palette's length. + pub fn palette_len(&self) -> usize { + self.palette_layout().2 + } + + /// Per primitive: whether `pose` places it mirrored (a node's transform; a skinned one never is). `triangles` + /// keeps such a primitive's winding front-facing, so the corners of `skin_source` are in a pose's order: they + /// serve the poses that mirror the same primitives. + pub fn mirrored(&self, pose: Option<&desc::Animation>) -> Result> { + let globals = self.globals(pose)?; + let nodes: Vec = self.gltf.0.nodes().collect(); + Ok(self.prims.iter().map(|p| !(p.skin.is_some() && nodes[p.node].skin().is_some()) && globals[p.node].determinant() < 0.0).collect()) + } + + /// The model's triangles as the skinning pass reads them, three corners each, in `triangles`' order for `pose` + /// (restricted to the mesh nodes named in `only`, as there): skinned by `palette`'s matrices they are `triangles` + /// in the model's own space, for every pose that mirrors the primitives `pose` does (`mirrored`). + pub fn skin_source(&self, only: Option<&[String]>, pose: Option<&desc::Animation>) -> Result> { + let (doc, _, _) = &self.gltf; + let mirrors = self.mirrored(pose)?; + let nodes: Vec = doc.nodes().collect(); + let (skins, of_node, _) = self.palette_layout(); + let mut out = vec![]; + for (p, &mirrored) in self.prims.iter().zip(&mirrors) { + let node = &nodes[p.node]; + if only.is_some_and(|names| !node.name().is_some_and(|nm| names.iter().any(|x| x == nm))) { + continue; + } + let skin = p.skin.as_ref().zip(node.skin()); + let corner = |v: usize, tangent: Option<[f32; 4]>| { + let (joints, weights) = match &skin { + Some(((joints, weights), skin)) => { + let (j, w) = (joints[v], weights[v]); + let sum = w[0] + w[1] + w[2] + w[3]; + let count = skin.joints().count(); + if sum > 0.0 { + // (A weight that is not positive holds nothing; a joint the skin has not is the identity.) + let held = |c: usize| w[c] > 0.0; + ([0, 1, 2, 3].map(|c| if held(c) && (j[c] as usize) < count { skins[skin.index()] + j[c] as usize } else { 0 }), [0, 1, 2, 3].map(|c| if held(c) { w[c] / sum } else { 0.0 })) + } else { + ([0; 4], [1.0, 0.0, 0.0, 0.0]) + } + } + None => ([of_node[&p.node], 0, 0, 0], [1.0, 0.0, 0.0, 0.0]), + }; + let pos = p.positions[v]; + let n = p.normals.as_ref().map(|n| n[v]); + let flags = if n.is_some() { CORNER_NORMAL } else { 0 } | if tangent.is_some() { CORNER_TANGENT } else { 0 }; + let (n, t) = (n.unwrap_or(Vec3::ZERO), tangent.unwrap_or([0.0; 4])); + SkinCorner { + p: [pos.x, pos.y, pos.z, f32::from_bits(joints[0] as u32 | (joints[1] as u32) << 16)], + n: [n.x, n.y, n.z, f32::from_bits(joints[2] as u32 | (joints[3] as u32) << 16)], + t: [t[0], t[1], t[2], f32::from_bits(flags)], + w: weights, + } + }; + // (A node that mirrors: the winding stays front-facing, as `triangles` has it.) + for t in p.indices.chunks_exact(3) { + let (a, b, c) = if mirrored { (t[0], t[2], t[1]) } else { (t[0], t[1], t[2]) }; + // (A triangle's tangent is its first corner's.) + let tangent = p.tangents.as_ref().map(|tv| tv[a as usize]); + out.push([corner(a as usize, tangent), corner(b as usize, tangent), corner(c as usize, tangent)]); + } + } + Ok(out) + } + /// Node transforms relative to the glTF scene root for `pose` (None = rest pose). pub fn node_globals(&self, pose: Option<&desc::Animation>) -> Result> { self.globals(pose) @@ -297,11 +450,14 @@ impl Model { let nodes: Vec = doc.nodes().collect(); let mut skins: HashMap> = HashMap::new(); let mut out = vec![]; + // (A skinned primitive's matrix a vertex, where it has tangents: they follow their joints as its normals do.) + let mut blends: Vec = vec![]; for p in &self.prims { let node = &nodes[p.node]; if only.is_some_and(|names| !node.name().is_some_and(|nm| names.iter().any(|x| x == nm))) { continue; } + blends.clear(); let (pos, nrm): (Vec, Option>) = match (&p.skin, node.skin()) { (Some((joints, weights)), Some(skin)) => { // glTF: a skinned mesh ignores its node's transform; joints place it (relative to the scene root). @@ -311,6 +467,7 @@ impl Model { }); let mut pos = Vec::with_capacity(p.positions.len()); let mut nrm = p.normals.as_ref().map(|n| Vec::with_capacity(n.len())); + blends.clear(); for (v, (j, w)) in joints.iter().zip(weights).enumerate() { let sum = w[0] + w[1] + w[2] + w[3]; let k = if sum > 0.0 { 1.0 / sum } else { 0.0 }; @@ -324,6 +481,9 @@ impl Model { m = world; } pos.push(m.transform_point3(p.positions[v])); + if p.tangents.is_some() { + blends.push(m); + } if let (Some(out), Some(src)) = (nrm.as_mut(), &p.normals) { out.push((Mat3::from_mat4(m).inverse().transpose() * src[v]).normalize_or_zero()); } @@ -346,7 +506,8 @@ impl Model { let n = nrm.as_ref().map(|n| [n[a], n[b], n[c]]).unwrap_or([gn; 3]); let tangent = p.tangents.as_ref().map(|tv| { let tt = tv[a]; - (m.transform_vector3(Vec3::new(tt[0], tt[1], tt[2])).normalize_or_zero(), if mirrored { -tt[3] } else { tt[3] }) + let by = blends.get(a).copied().unwrap_or(m); + (by.transform_vector3(Vec3::new(tt[0], tt[1], tt[2])).normalize_or_zero(), if mirrored { -tt[3] } else { tt[3] }) }); out.push(ModelTri { p: tp, n, uv: [p.uvs[a], p.uvs[b], p.uvs[c]], tangent, material: p.material }); } @@ -454,6 +615,69 @@ mod tests { assert!(m.moving_nodes(&pose("raise-both")).is_err(), "no such clip"); } + /// The skinning pass's inputs give the triangles the CPU skin gives: a model's corners placed by its palette are + /// `triangles` in the model's own space, for a skinned strip of two materials (tests/render/data/arm.glb) and for + /// primitives on nodes (`RIGID`), at rest and posed. + #[test] + fn corners_placed_by_the_palette_are_the_models_triangles() { + let arm = Model::new(gltf::import("../tests/render/data/arm.glb").unwrap()).unwrap(); + // (Vertices two joints hold, weights that do not sum to one, a primitive with tangents and one without + // normals, a mesh on a node beside the skin that mirrors it; a joint turned about an axis that is neither + // the normals' nor the tangents'.) + let blend = Model::new(gltf::import("../tests/render/data/skin-blend.glb").unwrap()).unwrap(); + let rigid = Model::new(gltf::import_slice(RIGID.as_bytes()).unwrap()).unwrap(); + let pose = |clip: &str, time| desc::Animation { clip: clip.into(), time, r#loop: true, nodes: Default::default() }; + let mut seen = 0; + let bends = || vec![None, Some(pose("bend", 0.3)), Some(pose("bend", 0.8))]; + for (m, poses) in [(&arm, bends()), (&blend, bends()), (&rigid, vec![None, Some(pose("move", 0.5))])] { + let source = m.skin_source(None, None).unwrap(); + assert!(poses.iter().all(|a| m.mirrored(a.as_ref()).unwrap() == m.mirrored(None).unwrap()), "no pose here mirrors a primitive"); + for a in &poses { + let (tris, palette) = (m.triangles(Mat4::IDENTITY, None, a.as_ref()).unwrap(), m.palette(a.as_ref()).unwrap()); + assert_eq!(source.len(), tris.len()); + for (corners, t) in source.iter().zip(&tris) { + for (k, c) in corners.iter().enumerate() { + let (p, n, tg) = skin_corner(c, &palette); + assert!((p - t.p[k]).length() < 1e-5, "position {p} against {}", t.p[k]); + if c.t[3].to_bits() & CORNER_NORMAL != 0 { + assert!((n - t.n[k]).length() < 1e-5, "normal {n} against {}", t.n[k]); + } + if let (0, Some((tt, _))) = (k, t.tangent) { + assert!((tg - tt).length() < 1e-5, "tangent {tg} against {tt}"); + } + seen += 1; + } + } + } + } + assert_eq!(seen, 36 + 648 + 12, "corners compared"); + assert_eq!((blend.palette_len(), blend.skin_source(None, None).unwrap().len()), (4, 72), "the identity, two joints and the badge's node; 72 triangles"); + // The badge's node mirrors it: its corners are in the order that keeps it front-facing, and a pose that does + // not mirror it has them in another. + // (Primitives in the order the model emits them: the badge's node first.) + assert_eq!(blend.mirrored(None).unwrap(), vec![true, false, false]); + assert_eq!(blend.mirrored(Some(&pose("flip", 0.9))).unwrap(), vec![false, false, false]); + assert!(blend.skin_source(None, None).unwrap() != blend.skin_source(None, Some(&pose("flip", 0.9))).unwrap()); + // A blended vertex's normal is not its matrix's: the matrix of two joints is no rotation. + let turned = blend.palette(Some(&pose("bend", 0.8))).unwrap(); + let by_the_matrix = |c: &SkinCorner| { + let (a, b) = (c.p[3].to_bits(), c.n[3].to_bits()); + let mut rows = [glam::Vec4::ZERO; 3]; + for (j, w) in [a & 0xffff, a >> 16, b & 0xffff, b >> 16].iter().zip(c.w) { + for (r, m) in rows.iter_mut().zip(turned[*j as usize]) { + *r += glam::Vec4::from(m) * w; + } + } + let n = Vec3::new(c.n[0], c.n[1], c.n[2]); + Vec3::new(rows[0].truncate().dot(n), rows[1].truncate().dot(n), rows[2].truncate().dot(n)).normalize_or_zero() + }; + assert!(blend.skin_source(None, None).unwrap().iter().flatten().any(|c| c.t[3].to_bits() & CORNER_NORMAL != 0 && (skin_corner(c, &turned).1 - by_the_matrix(c)).length() > 0.01), "the fixture tells a normal by the inverse transpose from one by the matrix"); + // A pose bends the strip: its corners are not where the file has them. + let (source, bent) = (arm.skin_source(None, None).unwrap(), arm.palette(Some(&pose("bend", 0.8))).unwrap()); + assert_eq!(bent.len(), arm.palette_len()); + assert!(source.iter().flatten().any(|c| (skin_corner(c, &bent).0 - Vec3::new(c.p[0], c.p[1], c.p[2])).length() > 0.05), "the pose moves corners"); + } + #[test] fn nodes_posed_by_name_over_the_clip() { let m = Model::new(gltf::import_slice(RIGID.as_bytes()).unwrap()).unwrap(); diff --git a/renderer/src/world.rs b/renderer/src/world.rs index 06cc26f..1b1caa8 100644 --- a/renderer/src/world.rs +++ b/renderer/src/world.rs @@ -699,7 +699,9 @@ impl World { // So are an instanced object's instances (`LiveCore::update_instanced`). self.changes.instanced.insert(id); } else if posed && self.leaf(id) { - self.changes.poses.insert(id); + // A posed model moves in place however it is drawn: an instance of its mesh by its transform, world-space + // triangles by being skinned again where it stands (the renderer's to tell: live/edits.rs `commit`). + self.changes.moved.insert(id); } else if light && !sun && self.leaf(id) { self.changes.lights.insert(id); } else if sun && self.leaf(id) { diff --git a/tests/render/data/skin-blend.glb b/tests/render/data/skin-blend.glb new file mode 100644 index 0000000..404d87b Binary files /dev/null and b/tests/render/data/skin-blend.glb differ