diff --git a/README.md b/README.md index b17edcd..66201d9 100644 --- a/README.md +++ b/README.md @@ -90,8 +90,8 @@ Both versions are printed at launch, and Inline Studio checks PyPI once a day in says so when either half is behind: ``` -Versions: inline-core 1.3.11, inline-studio-frontend 1.3.10 -UPDATE AVAILABLE: inline-studio-frontend 1.3.10 -> 1.3.11 +Versions: inline-core 1.3.13, inline-studio-frontend 1.3.12 +UPDATE AVAILABLE: inline-studio-frontend 1.3.12 -> 1.3.13 Update with: ./webui.sh --install (or: pip install -U inline-studio-frontend) ``` @@ -195,7 +195,9 @@ Getting the same person across shots normally means training a LoRA for each one same reference photos into every node by hand. Build a character once instead, then pick it from a dropdown. -![Two reference photos compiled into a portable character file, then the same person generated in an office, a cafe, a park, a street and at a lakeside](https://raw.githubusercontent.com/inlineresearch/Inline-Studio/main/screenshots/character_showcase.png) + + +[**Workflow: Minimax H3. Consistent face, body & cloths via reference identity →**](https://inlinestudio.art/workflows/minimax-h3-guided-consistent-characters-via-reference-identity-face-body-cloths) Drop in a photo or two and Inline Studio compiles a **`.char`**: one portable file holding your references and an identity fingerprint. Describe the scene, and the references carry the likeness. diff --git a/TODO b/TODO index 36dad67..5df9db8 100644 --- a/TODO +++ b/TODO @@ -1,26 +1,20 @@ Backlog - Remove frame node entirely - Update asset node to resize based on input -- drag drop from output to spawn asset node -- double click, search node settings - project output share settings - Group based settings -- HF Token handle: -- Clearly mention if gated model download failed New models -- Ltx 2.5 +- Minimax union CN Training: -- Snapshot export on checkpoint +- Improve ui for edit dataset/trigger name Char - Body & cloth cosistency -- Support for minimax & krea2 -- Look into this error: https://www.reddit.com/r/StableDiffusion/comments/1vp0xct/comment/p3vef5w/?utm_source=share&utm_medium=web3x&utm_name=web3xcss&utm_term=1&utm_content=share_button diff --git a/core/pyproject.toml b/core/pyproject.toml index bdbb5ba..5fe8f7c 100644 --- a/core/pyproject.toml +++ b/core/pyproject.toml @@ -1,7 +1,7 @@ [project] # PyPI name; the import package is `inline_core` (src/inline_core). name = "inline-core" -version = "1.3.12" +version = "1.3.13" description = "The generation engine behind Inline Studio." readme = "README.md" license = "GPL-3.0-or-later" diff --git a/core/src/inline_core/characters/apply.py b/core/src/inline_core/characters/apply.py index e0984be..cfc971f 100644 --- a/core/src/inline_core/characters/apply.py +++ b/core/src/inline_core/characters/apply.py @@ -32,22 +32,48 @@ def __init__( description: str, lora: Path | None = None, lora_strength: float = 1.0, + roles: list[str] | None = None, ) -> None: self.name = name self.refs = refs self.description = description + #: One role per ref, in the same order. All face when a character predates roles, which is + #: what keeps an old character's prompt byte-identical to what it used to produce. + self.roles = roles or [cf.ROLE_FACE] * len(refs) #: A trained adapter, which for a model with no reference channel is the only route. self.lora = lora #: What it fuses at. Set on Attach Adapter, because an overfit adapter is only usable #: turned down and the character wire carries no controls of its own. self.lora_strength = lora_strength - def prompt_prefix(self, first_position: int, style: str = "ordinal") -> str: + def _role_lines(self, first_position: int, style: str) -> str: + """Sentences binding each role to the positions it actually landed on. + + Written from the allocation rather than alongside it, so the numbers cannot drift from the + references. Silent when everything is face: an old character keeps its exact prompt. + """ + grouped: dict[str, list[int]] = {} + for offset, role in enumerate(self.roles[: len(self.refs)]): + grouped.setdefault(role, []).append(first_position + offset) + if set(grouped) <= {cf.ROLE_FACE}: + return "" + out = "" + for role in cf.ROLES: + numbers = grouped.get(role) + if numbers: + out += f" {_positions(style, numbers)} show {self.name}'s {_ROLE_BINDINGS[role]}." + return out + + def prompt_prefix( + self, first_position: int, style: str = "ordinal", role_lines: bool = False + ) -> str: """Text naming the positions the character lands on, so positional prompting resolves. ``style`` because a model only resolves the form it was trained on: FLUX.2 reads the ordinal prose below, MiniMax H3 reads ```` tokens (``models/references.py``), and handing either the other one names positions it cannot see. + + ``role_lines`` defaults off: the bindings are unvalidated, see docs/characters.md. """ if not self.refs: # A LoRA carries the likeness, so the description is all the prompt needs. @@ -64,6 +90,8 @@ def prompt_prefix(self, first_position: int, style: str = "ordinal") -> str: ordinals = [str(n) for n in positions] which = f"Images {', '.join(ordinals[:-1])} and {ordinals[-1]} show" line = f"{which} {self.name}, the same character in every image." + if role_lines: + line += self._role_lines(first_position, style) detail = " ".join(self.description.split()) if not detail: return f"{line} " @@ -73,18 +101,42 @@ def prompt_prefix(self, first_position: int, style: str = "ordinal") -> str: return f"{line} {detail} " +def _positions(style: str, numbers: list[int]) -> str: + """How a role line refers *back* to positions, in prose, never re-declaring them.""" + # Never ``: that is H3's reserved label and repeating it replayed the references. + noun = "Picture" if style == "token" else "Image" + if len(numbers) == 1: + return f"{noun} {numbers[0]}" + return f"{noun}s {', '.join(str(n) for n in numbers[:-1])} and {numbers[-1]}" + + +#: What each role binds to. Naming only, never describing: "slim build" or "red jacket" would be +#: text competing with the reference images, which is the caption-overrides-identity failure the +#: LoRA evals already found. +_ROLE_BINDINGS = { + cf.ROLE_FACE: "face", + cf.ROLE_BODY: "full body and build", + cf.ROLE_CLOTH: "outfit", +} + + def _cache_root() -> Path: return data_dir() / "characters" def char_apply( - chosen: str, arch: str = encode.FLUX2_KLEIN_ARCH, prefer: str | None = None + chosen: str, + arch: str = encode.FLUX2_KLEIN_ARCH, + prefer: str | None = None, + limit: int | None = None, + keep_roles: tuple[str, ...] | None = None, ) -> AppliedCharacter | None: """How a character applies on ``arch``, or None when none is picked. An unreadable pick raises rather than silently generating the wrong person. ``arch`` matters because a model without a reference channel can only take the adapter, and its - payloads are keyed separately.""" + payloads are keyed separately. ``keep_roles`` narrows which of them are sent at all, per render, + so testing a character face-only does not mean writing a second character.""" name = str(chosen or "").strip() if not name: return None @@ -113,16 +165,44 @@ def char_apply( # No reference channel on this arch, so the adapter is the only way it can apply at all. if not references: mode = "lora" - refs = [] if mode == "lora" else _extract(doc, digest, arch) + refs, roles = ([], []) if mode == "lora" else _extract(doc, digest, arch) + if keep_roles is not None: + kept = [i for i, role in enumerate(roles) if role in keep_roles] + refs, roles = [refs[i] for i in kept], [roles[i] for i in kept] + if limit is not None: + refs, roles = _fit_roles(refs, roles, limit) return AppliedCharacter( doc.manifest.name or path.stem, refs, description, lora if mode == "lora" else None, strength, + roles=roles, ) +def _fit_roles( + refs: list[AssetRef], roles: list[str], limit: int +) -> tuple[list[AssetRef], list[str]]: + """Cut to what a model takes, dividing the slots by role rather than by arrival. + + Trimming the tail instead would drop whichever role happens to be last, so a character with + face, body and cloth would silently lose its wardrobe on any model that takes fewer than it + holds. Order is preserved, because order is what the prompt numbers. + """ + if limit >= len(refs) or limit <= 0: + return refs[: max(0, limit)], roles[: max(0, limit)] + counts = {role: roles.count(role) for role in cf.ROLES} + share = encode.allocate_roles(counts, limit) + keep: list[int] = [] + taken = dict.fromkeys(cf.ROLES, 0) + for index, role in enumerate(roles): + if taken[role] < share.get(role, 0): + taken[role] += 1 + keep.append(index) + return [refs[i] for i in keep], [roles[i] for i in keep] + + def _extract_lora(doc: cf.CharDoc, digest: str, arch: str) -> Path | None: """The adapter for ``arch``, or None. A stale one is the wrong face, so it is ignored.""" entry = encode.lora_payload(doc.manifest, arch) @@ -172,11 +252,13 @@ def _recompile(doc: cf.CharDoc, path: Path, arch: str = encode.FLUX2_KLEIN_ARCH) return doc -def _extract(doc: cf.CharDoc, digest: str, arch: str) -> list[AssetRef]: +def _extract(doc: cf.CharDoc, digest: str, arch: str) -> tuple[list[AssetRef], list[str]]: payload = doc.manifest.payloads.get(arch) or {} - files = [str(entry.get("path")) for entry in payload.get("files") or []] + entries = list(payload.get("files") or []) + files = [str(entry.get("path")) for entry in entries] + roles = [cf.role_of(entry) for entry in entries] if not files: - return [] + return [], [] root = _cache_root() / digest marker = root / ".complete" @@ -194,7 +276,7 @@ def _extract(doc: cf.CharDoc, digest: str, arch: str) -> list[AssetRef]: staging.replace(root) _prune(_cache_root()) - return [AssetRef(ref="path", path=str(root / Path(member).name)) for member in files] + return [AssetRef(ref="path", path=str(root / Path(member).name)) for member in files], roles def _prune(root: Path) -> None: diff --git a/core/src/inline_core/characters/charfile.py b/core/src/inline_core/characters/charfile.py index 2c0782d..7265087 100644 --- a/core/src/inline_core/characters/charfile.py +++ b/core/src/inline_core/characters/charfile.py @@ -53,6 +53,26 @@ def origin_of(ref: dict[str, Any]) -> str: return str(ref.get("origin") or ORIGIN_ORIGINAL) +#: What a reference is *of*. Sits beside `origin` for the same reason, and absent means face, so +#: every character written before roles existed keeps behaving exactly as it did. +ROLE_FACE = "face" +ROLE_BODY = "body" +ROLE_CLOTH = "cloth" +ROLES = (ROLE_FACE, ROLE_BODY, ROLE_CLOTH) + + +def role_of(ref: dict[str, Any]) -> str: + """A reference's role, defaulting to face. An unknown value reads as face rather than raising: + a manifest is user data, and refusing to open a character over one bad string helps nobody.""" + role = str(ref.get("role") or ROLE_FACE) + return role if role in ROLES else ROLE_FACE + + +def by_role(refs: list[dict[str, Any]], role: str) -> list[dict[str, Any]]: + """The references carrying one role, in manifest order.""" + return [ref for ref in refs if role_of(ref) == role] + + class CharFileError(Exception): """A ``.char`` that cannot be trusted. The message is shown to the user.""" diff --git a/core/src/inline_core/characters/encode.py b/core/src/inline_core/characters/encode.py index 03c8f31..b42faa5 100644 --- a/core/src/inline_core/characters/encode.py +++ b/core/src/inline_core/characters/encode.py @@ -58,20 +58,56 @@ def payload_key(arch: str, kind: str = PAYLOAD_REF) -> str: } +#: How the slots divide when there are more references than a model takes: face gets half, body and +#: cloth a quarter each. Face is weighted because it is what identity is actually carried by; the +#: other two are conditioning on top of it. +ROLE_RATIO: dict[str, int] = {cf.ROLE_FACE: 2, cf.ROLE_BODY: 1, cf.ROLE_CLOTH: 1} + + +def allocate_roles(counts: dict[str, int], cap: int) -> dict[str, int]: + """How many of each role to send, capped at ``cap`` total. + + Under the cap everything goes. Over it, the split is `ROLE_RATIO` by largest remainder, and + then any slot a role cannot fill is handed to the roles that still have references waiting - + a character with no body shots should not lose those slots to nothing. + """ + wanted = {role: max(0, int(counts.get(role, 0))) for role in cf.ROLES} + if sum(wanted.values()) <= cap: + return wanted + + total_weight = sum(ROLE_RATIO.values()) + exact = {role: cap * ROLE_RATIO[role] / total_weight for role in cf.ROLES} + share = {role: min(wanted[role], int(exact[role])) for role in cf.ROLES} + + # Largest remainder first, then whoever still has references left, so no slot goes unused. + spare = cap - sum(share.values()) + order = sorted(cf.ROLES, key=lambda r: (-(exact[r] - int(exact[r])), -ROLE_RATIO[r], r)) + while spare > 0: + moved = False + for role in order: + if spare and share[role] < wanted[role]: + share[role] += 1 + spare -= 1 + moved = True + if not moved: + break + return share + + def reference_policy(arch: str) -> dict[str, Any]: return REFERENCE_POLICIES.get(arch, PAYLOAD_POLICY) -#: What "Resized Reference Resolution" means when left at -1: the model's own policy, uncapped. +#: What "Stored Reference Resolution" means when left at -1: the model's own policy, uncapped. NO_REFERENCE_CAP = -1 def capped_policy(arch: str, resolution: int | None) -> dict[str, Any]: """A model's reference policy, with its target lowered to ``resolution``. - Capping the source image instead would do nothing for a model whose policy scales *up*: H3 - takes a 2048 short edge whatever it is handed, so a 4K reference and a 512 one cost the same - 36,864 vision tokens. The lever has to be the target the policy resizes onto. + This sets what the `.char` stores, not what a render costs. H3's pipeline calls + `resolve_reference_image_size` on the way in and puts every reference back onto a 2048 short + edge, upscaling included, so lowering this saves disk and compile time and no VRAM at all. """ policy = dict(reference_policy(arch)) if resolution is None or int(resolution) <= 0: @@ -239,6 +275,7 @@ def char_encode( refs: list[Path | str], *, name: str, + roles: list[str] | None = None, description: str = "", app_version: str = "", char_id: str | None = None, @@ -251,6 +288,9 @@ def char_encode( downloads ~370MB of encoder weights, which without a signal is indistinguishable from a hang.""" report = on_progress or (lambda _fraction, _status: None) paths = [Path(p) for p in refs] + # Face when unsaid, so a caller that predates roles writes exactly what it always did. + tags = list(roles or [cf.ROLE_FACE] * len(paths)) + tags += [cf.ROLE_FACE] * (len(paths) - len(tags)) if not paths: raise ValueError("A character needs at least one reference image.") missing = [p.name for p in paths if not p.is_file()] @@ -281,6 +321,7 @@ def char_encode( "height": image.height, "source_name": path.name, "origin": cf.ORIGIN_ORIGINAL, + "role": tags[index], } ) @@ -436,7 +477,10 @@ def build_payload( member = f"payloads/{arch}/ref_{slot:03d}.png" data = _png_bytes(normalise_reference(images[index], policy)) members[member] = data - files.append({"path": member, "sha256": cf.sha256_bytes(data)}) + # The role rides with the compiled file: `apply` needs it to number the prompt, and + # recomputing the manifest order there would be the same rule written twice. + role = cf.role_of(manifest.refs[index]) if index < len(manifest.refs) else cf.ROLE_FACE + files.append({"path": member, "sha256": cf.sha256_bytes(data), "role": role}) manifest.payloads[arch] = { "payload_version": 1, "type": PAYLOAD_REF, diff --git a/core/src/inline_core/characters/verify.py b/core/src/inline_core/characters/verify.py index 587ea8e..ce49f7c 100644 --- a/core/src/inline_core/characters/verify.py +++ b/core/src/inline_core/characters/verify.py @@ -31,6 +31,10 @@ class Verdict: flagged: list[int] = field(default_factory=list) duplicates: list[int] = field(default_factory=list) unchecked: list[int] = field(default_factory=list) + #: Body and clothing references. Held out of scoring entirely rather than scored and excused: + #: SFace measures faces, and a body shot that happens to show one would be judged on the wrong + #: thing and could be flagged as an outlier for it. + unscored: list[int] = field(default_factory=list) note: str = "" def to_json(self) -> dict[str, Any]: @@ -41,6 +45,7 @@ def to_json(self) -> dict[str, Any]: "flagged": self.flagged, "duplicates": self.duplicates, "unchecked": self.unchecked, + "unscored": self.unscored, "note": self.note, } @@ -76,13 +81,20 @@ def verify( images = encode.ref_images(doc) total = len(images) + faces = { + i for i, ref in enumerate(doc.manifest.refs) if cf.role_of(ref) == cf.ROLE_FACE + } + verdict.unscored = [i for i in range(len(images)) if i not in faces] slots: list[list[float]] = [] for index, image in enumerate(images): + if index not in faces: + slots.append([]) + continue report(0.1 + 0.6 * index / max(1, total), f"Checking reference {index + 1} of {total}…") slots.append(scoring.embed_face(image) or []) - verdict.unchecked = [i for i, vector in enumerate(slots) if not vector] + verdict.unchecked = [i for i, v in enumerate(slots) if not v and i in faces] - live = [i for i in range(len(slots)) if i not in set(duplicates)] + live = [i for i in range(len(slots)) if i not in set(duplicates) and i in faces] if existing: verdict.agreement = _against_frozen(doc, slots, live) else: @@ -92,6 +104,9 @@ def verify( for index in duplicates: verdict.agreement[index] = None + for index in verdict.unscored: + if index < len(verdict.agreement): + verdict.agreement[index] = None measured = [i for i, value in enumerate(verdict.agreement) if value is not None] if len(measured) < scoring.MIN_REFS_TO_FLAG: verdict.note = ( @@ -100,6 +115,12 @@ def verify( ) return verdict verdict.flagged = [i for i in measured if (verdict.agreement[i] or 0.0) < floor] + if verdict.unscored and not verdict.note: + verdict.note = ( + f"{len(verdict.unscored)} body or clothing reference(s) are used but not scored: " + "the face measure does not apply to them, and the subject measure conflates a body " + "with its clothing and background." + ) return verdict @@ -170,3 +191,4 @@ def _reindex(verdict: Verdict, before: list[str], after: list[str]) -> None: verdict.flagged = sorted(moved[i] for i in verdict.flagged if i in moved) verdict.unchecked = sorted(moved[i] for i in verdict.unchecked if i in moved) verdict.duplicates = sorted(moved[i] for i in verdict.duplicates if i in moved) + verdict.unscored = sorted(moved[i] for i in verdict.unscored if i in moved) diff --git a/core/src/inline_core/models/character/runner.py b/core/src/inline_core/models/character/runner.py index f94bdce..d788151 100644 --- a/core/src/inline_core/models/character/runner.py +++ b/core/src/inline_core/models/character/runner.py @@ -61,6 +61,10 @@ class Payload: output_kind=None, inputs=( Port("images", "References", PortKind.IMAGE_LIST, required=True), + # Body and wardrobe as their own wires, so each reference carries what it is *of*. They + # compete with the face for a model's reference slots rather than adding to them. + Port("body", "Body references", PortKind.IMAGE_LIST, required=False), + Port("cloth", "Clothing references", PortKind.IMAGE_LIST, required=False), Port("description", "Description", PortKind.TEXT, required=False), ), outputs=(Port("character", "Character", PortKind.CHARACTER),), @@ -98,9 +102,11 @@ class Payload: "arch", "Model", Widget.SELECT, encode.FLUX2_KLEIN_ARCH, options=tuple(Option(value=a, label=a) for a in encode.REFERENCE_POLICIES), ), - # On the face: it decides both the file size and, for H3, whether the run fits the card. + # On the face because it decides how big the .char is. It does *not* decide what a render + # costs: H3 re-resizes every reference onto a 2048 short edge on the way in whatever this + # says, so the only lever on the vision tower is how many references are wired. ParamField( - "ref_resolution", "Resized Reference Resolution", Widget.NUMBER, 1024, + "ref_resolution", "Stored Reference Resolution", Widget.NUMBER, 1024, min=encode.NO_REFERENCE_CAP, max=8192, step=64, on_face=True, ), ), @@ -332,9 +338,20 @@ class EncodeCharacterRunner(NodeRunner): def run(self, node: Node, inputs: dict[str, list[Any]], ctx: ExecutionContext) -> NodeResult: _require_encoders() - refs = list(inputs.get("images") or []) - if not refs: - raise ValueError("A character needs at least one reference image.") + # Order is the manifest's order and therefore the prompt's numbering: face, then body, + # then cloth, which is the priority the slot allocation also uses. + grouped = [ + (cf.ROLE_FACE, list(inputs.get("images") or [])), + (cf.ROLE_BODY, list(inputs.get("body") or [])), + (cf.ROLE_CLOTH, list(inputs.get("cloth") or [])), + ] + refs = [ref for _role, wired in grouped for ref in wired] + roles = [role for role, wired in grouped for _ref in wired] + if not inputs.get("images"): + raise ValueError( + "A character needs at least one face reference. Body and clothing references " + "condition on top of an identity; they cannot carry one on their own." + ) name = str(node.params.get("name") or "").strip() or "Character" # A wired description wins over the typed one, so a Prompt node can drive it. description = str(_first(inputs.get("description")) or node.params.get("description") or "") @@ -345,8 +362,14 @@ def run(self, node: Node, inputs: dict[str, list[Any]], ctx: ExecutionContext) - def report(fraction: float, status: str) -> None: ctx.emitter.emit(progress_event(ctx, node, Phase.ENCODE, fraction, status=status)) - doc = encode.char_encode(paths, name=name, description=description, on_progress=report) - logger.info("Encoded character %s from %d reference(s)", name, len(paths)) + doc = encode.char_encode( + paths, name=name, roles=roles, description=description, on_progress=report + ) + counts = {role: roles.count(role) for role in cf.ROLES if roles.count(role)} + logger.info( + "Encoded character %s from %d reference(s): %s", + name, len(paths), ", ".join(f"{n} {role}" for role, n in counts.items()), + ) return NodeResult(outputs={"character": Identity(doc=doc)}) diff --git a/core/src/inline_core/models/minimaxh3/nvfp4.py b/core/src/inline_core/models/minimaxh3/nvfp4.py index 751fcf6..68d5f10 100644 --- a/core/src/inline_core/models/minimaxh3/nvfp4.py +++ b/core/src/inline_core/models/minimaxh3/nvfp4.py @@ -163,9 +163,13 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: rows, cols = self.weight.shape[0], self.weight.shape[1] * 2 scales = from_blocked(self.weight_scale.to(torch.float32), rows, cols // BLOCK) global_scale = float(self.weight_scale_2) - # Row-chunked so the whole weight never exists: the conditioner runs on one short prompt, so - # the output is a few kilobytes while the dequantised weight would be hundreds of megabytes. - parts: list[torch.Tensor] = [] + # Row-chunked so the whole dequantised weight never exists, and written into one output + # rather than concatenated: `torch.cat` holds every chunk *and* the result, which doubles + # the peak. That is invisible on a short prompt and 2 GB once references push the sequence + # past twenty thousand tokens. + out = torch.empty( + (*x.shape[:-1], self.out_features), dtype=x.dtype, device=x.device + ) for start, stop in _row_chunks(rows, cols): if start >= self.out_features: break @@ -173,9 +177,10 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: self.weight[start:stop], scales[start:stop], global_scale, x.dtype ) block = block[: self.out_features - start, : self.in_features] - bias = None if self.bias is None else self.bias[start : start + block.shape[0]] - parts.append(torch.nn.functional.linear(x, block, bias)) - return torch.cat(parts, dim=-1) + end = start + block.shape[0] + bias = None if self.bias is None else self.bias[start:end] + out[..., start:end] = torch.nn.functional.linear(x, block, bias) + return out def extra_repr(self) -> str: return f"in_features={self.in_features}, out_features={self.out_features}, nvfp4" diff --git a/core/src/inline_core/models/minimaxh3/pipeline.py b/core/src/inline_core/models/minimaxh3/pipeline.py index 9dd9929..ee4c8fa 100644 --- a/core/src/inline_core/models/minimaxh3/pipeline.py +++ b/core/src/inline_core/models/minimaxh3/pipeline.py @@ -388,11 +388,7 @@ def _build( # the card is full, and badly wrong when it is not: at 960x544 it turned a 6 minute render into # 32. Staged, the conditioner is on the CPU by decode time, so if the measured free VRAM covers # the VAE plus a working margin it stays resident instead. - vae_resident = staged and _vae_fits( - video_vae, - placement.device, - _denoiser_card_bytes(transformer, recipe, resident_blocks), - ) + vae_resident = staged and _vae_fits(video_vae, placement.device) if vae_resident: recipe = _replace(recipe, vae_offload=None) @@ -406,22 +402,25 @@ def _build( vae=getattr(pipe, "vae", None), device=placement.device, ) - if vae_resident: - pipe.vae.to(rt.torch_device(placement)) + # False means the VAE carries streaming hooks instead, which a `.to()` would fight. + pipe._inline_staged_vae = bool(vae_resident) # noqa: SLF001 pipe._inline_denoiser = denoiser_name # noqa: SLF001 - our own attribute on our own pipeline # Whether the phase swap may hand the denoiser the whole card. False on the offload plan, where # it carries streaming hooks and is far larger than the card: moving it wholesale is an OOM. pipe._inline_resident_denoiser = recipe.denoiser_offload is None # noqa: SLF001 if staged: - # Split once, at build time: the halves share every component object, so this costs nothing - # but the block graph. `render_staged` below does the swapping. + # Split once at build time: the parts share every component, so only the block graph costs. + owners, parts = _staged_phases(blocks) pipe._inline_phases = tuple( # noqa: SLF001 - our own attribute on our own pipeline - pipeline_class(blocks=half) - for half in rt.split_blocks(blocks, through="text_encoder") + pipeline_class(blocks=part) for part in parts ) - for half in pipe._inline_phases: # noqa: SLF001 - half.update_components(**dict(pipe.components)) + pipe._inline_phase_owners = owners # noqa: SLF001 + for part in pipe._inline_phases: # noqa: SLF001 + part.update_components(**dict(pipe.components)) transformer.to("cpu") # the conditioner has the card until the prompt is encoded + if vae_resident: + # `render_staged` places it for the two phases that read it, not for the whole run. + pipe.vae.to("cpu") rt.free_vram() elif not recipe.denoiser_offload: pipe.to(device) @@ -432,51 +431,30 @@ def _build( return pipe -#: Left free beside a resident VAE, to cover the decode's own activations - the largest single -#: allocation of the render. -_VAE_RESIDENT_MARGIN_GB = 12.0 - - -def _denoiser_card_bytes(transformer: Any, recipe: Any, resident_blocks: int) -> int: - """What the denoiser will claim on the card once ``apply_offload`` places it. - - It is loaded but still on the CPU when the VAE decision is taken, so the card reads empty and - every later placement is invisible to anything measuring free VRAM at that moment. - """ - from ..offload import block_stack - - total = sum(t.numel() * t.element_size() for t in transformer.parameters()) - total += sum(t.numel() * t.element_size() for t in transformer.buffers()) - if recipe.denoiser_offload is None: - return total - blocks = len(list(block_stack(transformer))) - if not blocks: - return total - # Streaming leaves only the placed head blocks resident; the rest arrives and leaves per step. - return int(total * min(1.0, max(0, resident_blocks) / blocks)) +def _staged_phases(blocks: Any) -> tuple[tuple[str, ...], tuple[Any, ...]]: + """The staged render in four parts, each labelled with the one large component it reads.""" + head, rest = rt.split_blocks(blocks, through="text_encoder") + # Taken positionally: ref2va calls it `reference_encoder` and fl2va `vae_encoder`. + encode_refs, rest = rt.split_blocks(rest, through=next(iter(rest.sub_blocks))) + denoise, decode = rt.split_blocks(rest, through="denoise") + return ("encoder", "vae", "denoiser", "vae"), (head, encode_refs, denoise, decode) -def _vae_fits(path: Path, device: Any, denoiser_bytes: int = 0) -> bool: - """Whether the video VAE can stay on the card rather than streaming leaf by leaf. +#: Beside a resident VAE, covering its decode: measured at 1.43 GB above weights, see docs. +_VAE_RESIDENT_MARGIN_GB = 4.0 - fp32 doubles what the file weighs, and that is what actually has to fit. - ``denoiser_bytes`` is what the denoiser will take once it is placed, which happens *after* this - runs: measured free VRAM here is an empty card, and reserving against it put a 10.4 GB VAE and - a 33 GB denoiser onto 44 GB. The same correction `_plan_residency` already makes for host RAM. - """ - free = rt.free_vram_bytes(device) - denoiser_bytes - if free <= 0: - logger.info( - "MiniMax H3 video VAE: streamed leaf by leaf (the denoiser claims the card first)" - ) - return False +def _vae_fits(path: Path, device: Any) -> bool: + """Whether the video VAE can stay on the card rather than streaming leaf by leaf.""" + # Nothing subtracted for the denoiser: `render_staged` parks it for both phases that read this. + free = rt.free_vram_bytes(device) + # fp32 doubles what the file weighs, and that is what has to fit. needed = path.stat().st_size * 2 + int(_VAE_RESIDENT_MARGIN_GB * 1e9) fits = free > needed logger.info( - "MiniMax H3 video VAE: %s (%.1f GB free after the denoiser's %.1f GB, needs %.1f GB)", - "resident" if fits else "streamed leaf by leaf", - free / 1e9, denoiser_bytes / 1e9, needed / 1e9, + "MiniMax H3 video VAE: %s (%.1f GB free, needs %.1f GB)", + "resident, placed per phase" if fits else "streamed leaf by leaf", + free / 1e9, needed / 1e9, ) return fits @@ -491,16 +469,7 @@ def _denoiser_name(blocks: Any) -> str: def render_staged(pipe: Any, device: Any, cancel_check: Any = None, **call: Any) -> Any: - """Encode the prompt, get the conditioner off the card, then denoise with the card to itself. - - H3's conditioner is a 32B model that rivals its denoiser, and holding both means the denoiser - streams every block of every step. Encoding is one pass per prompt, so the two never actually - need the card at the same time - but the convenience call runs all eight blocks together, which - is what makes it look as though they do. - - Falls back to the single call when the pipeline was not built staged, so a caller never has to - ask which kind it holds. - """ + """Run the render a phase at a time, each holding only the weights it reads.""" def check() -> None: if cancel_check is not None: cancel_check() @@ -509,40 +478,46 @@ def check() -> None: if phases is None: check() return pipe(**call) - head, tail = phases device = str(device) + owners: tuple[str, ...] = pipe._inline_phase_owners # noqa: SLF001 - # Parked, not released: the next render needs it again, and 19.5 GB across the bus is seconds - # against the minutes of streaming it buys back. - denoiser = getattr(pipe, getattr(pipe, "_inline_denoiser", "transformer")) - resident = getattr(pipe, "_inline_resident_denoiser", True) - - # Each half claims the card *before* it runs rather than the previous one restoring the layout - # on its way out. Same two transfers per render in the steady state, but a cancelled denoise no - # longer pays ~40 GB of restore traffic before the exception surfaces, which read as the cancel - # being ignored for half a minute. A `.to()` onto the device a module already sits on is free. - check() - if resident: - denoiser.to("cpu") - pipe.text_encoder.to(device) - rt.free_vram() + # Parked, never released: the next render needs them, and the bus costs seconds against minutes. + movable: dict[str, Any] = {"encoder": pipe.text_encoder} + # Absent on the offload plan: it carries streaming hooks, so moving it wholesale is an OOM. + if getattr(pipe, "_inline_resident_denoiser", True): + movable["denoiser"] = getattr(pipe, getattr(pipe, "_inline_denoiser", "transformer")) + if getattr(pipe, "_inline_staged_vae", False): + movable["vae"] = pipe.vae - # Every kwarg goes to BOTH halves, not just the first. `set_timesteps` lives in the tail, so a - # head-only handoff left it reading the descriptor default: a render that asked for 8 steps - # silently took 49. The halves ignore what their own blocks do not declare, so this is safe. - state = head(**call) - # The conditioner is a 32B model, so the encode alone runs for a while with no step hook to - # cancel from; without this a cancel there waits for the whole denoise as well. - check() - - # Parking the conditioner is worth doing either way: it frees the card for activations even when - # the denoiser is too big to take it. Moving the denoiser across is only right when it fits. - pipe.text_encoder.to("cpu") - rt.free_vram() - if resident: - denoiser.to(device) + # A no-op `.to()` still walks every parameter, which the VAE would pay four times a render. + placed: dict[str, str] = {} + + state: Any = None try: - return tail(state, **call) + for index, (owner, phase) in enumerate(zip(owners, phases, strict=True)): + # Claimed before the phase runs, not restored after: a cancel then costs no transfers. + check() + before = rt.own_vram_bytes() + for name, module in movable.items(): + if name != owner and placed.get(name) != "cpu": + module.to("cpu") + placed[name] = "cpu" + rt.free_vram() + parked = rt.own_vram_bytes() + held = movable.get(owner) + if held is not None and placed.get(owner) != device: + held.to(device) + placed[owner] = device + rt.free_vram() + # Logged because a `.to("cpu")` that does not actually release is invisible otherwise. + logger.info( + "MiniMax H3 phase %d/%d holds the %s: %.1f GB -> %.1f GB parked -> %.1f GB placed", + index + 1, len(phases), owner, + before / 1e9, parked / 1e9, rt.own_vram_bytes() / 1e9, + ) + # Every kwarg to every phase: `set_timesteps` reading a default took 49 steps for 8. + state = phase(**call) if index == 0 else phase(state, **call) + return state finally: rt.free_vram() diff --git a/core/src/inline_core/models/minimaxh3/runner.py b/core/src/inline_core/models/minimaxh3/runner.py index 9dfad0e..8d2aa64 100644 --- a/core/src/inline_core/models/minimaxh3/runner.py +++ b/core/src/inline_core/models/minimaxh3/runner.py @@ -21,7 +21,7 @@ from ...device.policy import DevicePolicy from ...errors import CancelledError, ComponentError -from ...graph.descriptor import NodeDescriptor, ParamField, Port, Widget +from ...graph.descriptor import NodeDescriptor, Option, ParamField, Port, Widget from ...graph.runners import NodeResult, NodeRunner from ...graph.schema import Node, PortKind from ...media import MediaKind @@ -102,6 +102,27 @@ def _params(variant: Variant) -> tuple[ParamField, ...]: ParamField("num_inference_steps", "Steps", Widget.NUMBER, 50, min=1, max=200, step=1), ParamField("seed", "Seed (-1 = random)", Widget.SEED, -1), ] + if variant.references: + # References are ~99.8% of what the conditioner reads, so their count is the only lever. + fields.append( + ParamField( + "character_references", "Character references", Widget.NUMBER, + REFERENCE_LIMITS.max_images, min=1, max=REFERENCE_LIMITS.max_images, step=1, + ) + ) + fields.append( + ParamField( + "character_reference_roles", "Character reference roles", Widget.SELECT, "all", + options=(Option("all", "Face, body and clothing"), Option("face", "Face only")), + ) + ) + # Off by default: unvalidated, and one render replayed the references as opening frames. + fields.append( + ParamField( + "character_role_lines", "Name character reference roles in the prompt", + Widget.BOOLEAN, False, + ) + ) fields.append( ParamField( "model", "Diffusion model", Widget.SELECT, "", @@ -207,7 +228,7 @@ def build_request( multiple=CANVAS_MULTIPLE, minimum=CANVAS_MULTIPLE, ) - character = _apply_character(inputs, variant) + character = _apply_character(inputs, variant, params) loras: tuple[Any, ...] = () references: tuple[Any, ...] = () if variant.references: @@ -261,18 +282,33 @@ def _character_file(inputs: dict[str, list[Any]]) -> str: return name -def _apply_character(inputs: dict[str, list[Any]], variant: Variant) -> _Character | None: +def _apply_character( + inputs: dict[str, list[Any]], variant: Variant, params: dict[str, Any] +) -> _Character | None: """A wired character as references or as its adapter, or None when none is wired.""" chosen = _character_file(inputs) if not chosen: return None from ...characters import apply as characters + from ...characters import charfile as cf from ...graph.loader_runners import LoraRef - # The reference partition cannot run on an adapter alone, so it asks for references outright - # rather than taking the adapter a character prefers by default. + # Capped inside `char_apply` so the slots divide by role, not by arrival order. + wired = len([v for v in (inputs.get("references") or []) if v is not None]) + budget = int(params.get("character_references") or REFERENCE_LIMITS.max_images) + slots = max(0, min(budget, REFERENCE_LIMITS.max_images - wired)) + keep = None if params.get("character_reference_roles", "all") == "all" else (cf.ROLE_FACE,) + if variant.references and slots == 0: + raise ComponentError( + f"{variant.title} takes {REFERENCE_LIMITS.max_images} images and {wired} are wired, so " + f"{chosen} has no slot left. Unwire one, or raise Character references." + ) applied = characters.char_apply( - chosen, ARCH, prefer="reference" if variant.references else None + chosen, + ARCH, + prefer="reference" if variant.references else None, + limit=slots if variant.references else None, + keep_roles=keep if variant.references else None, ) if applied is None: return None @@ -295,23 +331,19 @@ def _apply_character(inputs: dict[str, list[Any]], variant: Variant) -> _Charact "Wire it through Compile References with Model set to minimax-h3 and write it again, " "or wire images into this node's References input." ) - how = "adapter" if applied.lora is not None else f"{len(applied.refs)} reference(s)" - logger.info("Applying character %s by %s", applied.name, how) - # H3 resolves ``, not FLUX.2's ordinal prose, and the character's images land after - # whatever the user already wired. - wired = len([v for v in (inputs.get("references") or []) if v is not None]) - # Trimmed here rather than by the caller, so the prefix can never name a position that was - # dropped: a character is a library artefact and H3's 9 images is not every model's limit. - keep = list(applied.refs)[: max(0, REFERENCE_LIMITS.max_images - wired)] - if len(keep) < len(applied.refs): - logger.info( - "%s: using %d of %s's %d references, the most it takes beside %d wired", - variant.title, len(keep), chosen, len(applied.refs), wired, - ) - applied.refs = keep + counts = {role: applied.roles.count(role) for role in cf.ROLES if applied.roles.count(role)} + logger.info( + "Applying character %s by %s (%s), beside %d wired", + applied.name, + "adapter" if applied.lora is not None else f"{len(applied.refs)} reference(s)", + ", ".join(f"{n} {role}" for role, n in counts.items()) or "no references", + wired, + ) return _Character( - refs=keep, - prefix=applied.prompt_prefix(wired + 1, style="token"), + refs=applied.refs, + prefix=applied.prompt_prefix( + wired + 1, style="token", role_lines=bool(params.get("character_role_lines")) + ), lora=( LoraRef(file=str(applied.lora), strength=applied.lora_strength) if applied.lora is not None @@ -513,11 +545,10 @@ def _result( def _reference_tokens(request: Request) -> tuple[int, int]: - """Wired image references and what they cost the vision tower, measured from the pixels. + """Wired image references and their vision-tower cost, counted through the 2048 short edge the + pipeline forces: the stored pixels under-report a downscaled reference 16x.""" + from .vendor.packing_ref2va import resolve_reference_image_size - Read off the files rather than a setting, because the size that matters was decided when the - character was compiled and nothing on this node records it. - """ images = [r for r in request.references if getattr(r, "kind", None) == ReferenceKind.IMAGE] tokens = 0 for ref in images: @@ -526,9 +557,10 @@ def _reference_tokens(request: Request) -> tuple[int, int]: with Image.open(getattr(ref.value, "path", ref.value)) as handle: width, height = handle.size + resolved_h, resolved_w = resolve_reference_image_size(width, height) except Exception: # noqa: BLE001 - an error path must not raise a second error continue - tokens += (width // 32) * (height // 32) + tokens += (resolved_w // 32) * (resolved_h // 32) return len(images), tokens @@ -564,12 +596,11 @@ def _oom(request: Request, *, host: bool = False, held: int = 0) -> str: if not images: return canvas cost = f", which is {tokens:,} vision tokens" if tokens else "" - # Named alone because references are encoded before a frame exists: the canvas cannot move this - # step at all, and a hint that leads with it sends the user to resize for nothing. + # Fewer references is the only lever: resolution cannot reduce this, nor can the canvas. return ( - f"{where} ran out encoding {images} reference(s){cost}. The canvas does not affect this " - "step. Lower Resized Reference Resolution on the Compile References node and write the " - "character again - halving it quarters the tokens - or wire fewer references." + f"{where} ran out encoding {images} reference(s){cost}. Each one costs about 4,000 tokens " + "whatever resolution it was stored at, so wire fewer references. Neither the canvas nor " + "Resized Reference Resolution affects this step." ) diff --git a/core/src/inline_core/models/pipeline_runtime.py b/core/src/inline_core/models/pipeline_runtime.py index 98338d7..3b5d24e 100644 --- a/core/src/inline_core/models/pipeline_runtime.py +++ b/core/src/inline_core/models/pipeline_runtime.py @@ -71,11 +71,20 @@ def is_resident(policy: DevicePolicy) -> bool: _ENCODER_RESIDENT_HEADROOM_MB = 8 * 1024 +def tensor_bytes(tensor: Any) -> int: + """What a tensor occupies, following quantised subclasses down to their real storage.""" + # `element_size()` reports the dtype a torchao tensor was quantised *from*: int8 reads 2x. + names = getattr(tensor, "__tensor_flatten__", None) + if names is None: + return tensor.numel() * tensor.element_size() + return sum(tensor_bytes(getattr(tensor, name)) for name in names()[0]) + + def module_bytes(module: Any) -> int: """On-device size of a module's parameters and buffers.""" try: - params = sum(p.numel() * p.element_size() for p in module.parameters()) - buffers = sum(b.numel() * b.element_size() for b in module.buffers()) + params = sum(tensor_bytes(p) for p in module.parameters()) + buffers = sum(tensor_bytes(b) for b in module.buffers()) return params + buffers except Exception: # noqa: BLE001 - a size estimate must never break a run return 0 diff --git a/core/src/inline_core/studio/handlers.py b/core/src/inline_core/studio/handlers.py index 1e8ddd2..680ee4b 100644 --- a/core/src/inline_core/studio/handlers.py +++ b/core/src/inline_core/studio/handlers.py @@ -245,7 +245,7 @@ def _size(core_type: str) -> tuple[int, int]: "moodboard:addGenNode", lambda mid, x, y: mb.add_gen_node(conn(), mid, x, y, kind="image", params={}, title=mid), ) - reg("moodboard:updateItem", lambda iid, patch: mb.update_item(conn(), iid, patch)) + reg("moodboard:updateItem", lambda iid, patch: mb.client_update_item(conn(), iid, patch)) reg("moodboard:deleteItem", lambda iid: mb.delete_item(conn(), iid)) def remove_core_output(item_id: str, take_id: str) -> None: diff --git a/core/src/inline_core/studio/moodboard.py b/core/src/inline_core/studio/moodboard.py index dde1c1d..c97bb06 100644 --- a/core/src/inline_core/studio/moodboard.py +++ b/core/src/inline_core/studio/moodboard.py @@ -306,8 +306,9 @@ def add_trim(conn: sqlite3.Connection, x: float, y: float) -> dict[str, Any]: def add_loader(conn: sqlite3.Connection, x: float, y: float) -> dict[str, Any]: # A "Load Assets" node holds library asset refs in its data (no frame, no frame_inputs) and # feeds its hero (first) asset downstream via graph_build. + # Portrait by default; the renderer refits it to the real aspect once its first asset lands. return _insert_item( - conn, item_type="loader", x=x, y=y, width=220, height=200, data={"assetIds": []} + conn, item_type="loader", x=x, y=y, width=240, height=340, data={"assetIds": []} ) @@ -325,6 +326,34 @@ def add_prompt(conn: sqlite3.Connection, x: float, y: float) -> dict[str, Any]: ) +# How many recent renders a Core node keeps in its on-node take history (newest first). Bounds the +# JSON we carry on the moodboard item; older entries drop off (their files stay in takes/). +_CORE_HISTORY_MAX = 24 + + +def client_update_item( + conn: sqlite3.Connection, item_id: str, patch: dict[str, Any] +) -> dict[str, Any]: + """``update_item`` for a browser write, with take history held back: ``data`` is replaced + wholesale, so a stale patch erased renders that landed after the tab last loaded.""" + data = patch.get("data") + core = data.get("core") if isinstance(data, dict) else None + if isinstance(core, dict): + stored = (get_item(conn, item_id).get("data") or {}).get("core") or {} + history = stored.get("outputs") + if history is not None: + seen = {o.get("takeId") for o in (core.get("outputs") or []) if isinstance(o, dict)} + missed = [o for o in history if o.get("takeId") not in seen] + if missed: + merged = [*missed, *(core.get("outputs") or [])] + merged.sort(key=lambda o: o.get("createdAt") or 0, reverse=True) + patch = { + **patch, + "data": {**data, "core": {**core, "outputs": merged[:_CORE_HISTORY_MAX]}}, + } + return update_item(conn, item_id, patch) + + def update_item(conn: sqlite3.Connection, item_id: str, patch: dict[str, Any]) -> dict[str, Any]: get_item(conn, item_id) # ensure exists sets: list[str] = [] @@ -348,11 +377,6 @@ def update_item(conn: sqlite3.Connection, item_id: str, patch: dict[str, Any]) - return get_item(conn, item_id) -# How many recent renders a Core node keeps in its on-node take history (newest first). Bounds the -# JSON we carry on the moodboard item; older entries drop off (their files stay in takes/). -_CORE_HISTORY_MAX = 24 - - def set_core_node_output(conn: sqlite3.Connection, item_id: str, output: dict[str, Any]) -> None: """Record a render a Core media node produced: make it the node's active ``output`` and prepend it to the node's ``outputs`` take history (newest first, deduped by takeId, capped). A fresh @@ -366,6 +390,8 @@ def set_core_node_output(conn: sqlite3.Connection, item_id: str, output: dict[st if item["type"] != "core" or not core: return take_id = output.get("takeId") + # The node's own params beside the runner's: only these answer "changed since it rendered". + output = {**output, "nodeParams": dict(core.get("params") or {})} prior = [o for o in (core.get("outputs") or []) if o.get("takeId") != take_id] outputs = [output, *prior][:_CORE_HISTORY_MAX] update_item( diff --git a/core/tests/test_character_nodes.py b/core/tests/test_character_nodes.py index e1197f5..e34b1ef 100644 --- a/core/tests/test_character_nodes.py +++ b/core/tests/test_character_nodes.py @@ -496,10 +496,19 @@ def fake_lora(doc: cf.CharDoc) -> None: def test_encoding_refuses_with_no_references(tmp_path: Path, encoders: None) -> None: - with pytest.raises(ValueError, match="at least one reference"): + with pytest.raises(ValueError, match="at least one face reference"): EncodeCharacterRunner().run(_node({"name": "Ada"}), {"images": []}, _ctx()) # type: ignore[arg-type] +def test_body_and_clothing_alone_are_not_a_character(tmp_path: Path, encoders: None) -> None: + """They condition on top of an identity. Encoding from them would produce a character whose + likeness nothing carries, and the face is what every model's identity signal comes from.""" + with pytest.raises(ValueError, match="at least one face reference"): + EncodeCharacterRunner().run( # type: ignore[arg-type] + _node({"name": "Ada"}), {"images": [], "body": ["b"], "cloth": ["c"]}, _ctx() + ) + + def test_the_last_reference_cannot_be_dropped(tmp_path: Path, encoders: None) -> None: """A character with no references is not a character, so the edit is refused, not silently ignored.""" diff --git a/core/tests/test_characters_apply.py b/core/tests/test_characters_apply.py index b4de0d9..ae39b05 100644 --- a/core/tests/test_characters_apply.py +++ b/core/tests/test_characters_apply.py @@ -107,3 +107,25 @@ def test_krea2_applies_a_character_only_as_its_adapter(tmp_path: Path) -> None: # The Flux payload is untouched by any of this: the two archs are keyed separately. assert ax.char_apply("Ada.char").lora is None + + +def test_a_render_can_send_fewer_of_a_characters_references_than_it_holds(tmp_path: Path) -> None: + """References are ~99.8% of what H3's conditioner reads and sit on the video's own rotary + clock, so their count is the lever on how hard they pull. Dialling it must not mean + re-encoding the character.""" + from inline_core.characters import apply as ax + from inline_core.characters import charfile as cf + from inline_core.characters import encode, library + + roles = [cf.ROLE_FACE, cf.ROLE_FACE, cf.ROLE_BODY, cf.ROLE_CLOTH, cf.ROLE_CLOTH] + images = [_image(tmp_path / f"r{i}.png") for i in range(len(roles))] + library.save(encode.char_encode(images, name="Ada", roles=roles)) + arch = encode.FLUX2_KLEIN_ARCH + + assert len(ax.char_apply("Ada.char", arch, prefer="reference", limit=9).refs) == 5 + assert len(ax.char_apply("Ada.char", arch, prefer="reference", limit=3).refs) == 3 + face_only = ax.char_apply( + "Ada.char", arch, prefer="reference", limit=9, keep_roles=(cf.ROLE_FACE,) + ) + assert face_only is not None + assert face_only.roles == [cf.ROLE_FACE] * 2, "body and clothing must not be sent" diff --git a/core/tests/test_encoder_parking.py b/core/tests/test_encoder_parking.py index 78dab90..2f4f396 100644 --- a/core/tests/test_encoder_parking.py +++ b/core/tests/test_encoder_parking.py @@ -75,3 +75,20 @@ def test_module_bytes_counts_parameters() -> None: def test_module_bytes_never_raises_on_an_odd_object() -> None: # It only feeds a heuristic, so a module that does not behave must not break a run. assert module_bytes(object()) == 0 + + +def test_a_quantised_weight_is_sized_by_its_real_storage() -> None: + """Believing `element_size()` read H3's int8 denoiser as 40.2 GB against a real 20, and that + figure is what the video VAE's residency was decided against.""" + import torch + + torchao = pytest.importorskip("torchao.quantization") + linear = torch.nn.Linear(512, 512, bias=False, dtype=torch.bfloat16) + bf16 = module_bytes(linear) + + torchao.quantize_(linear, torchao.Int8WeightOnlyConfig()) + assert type(linear.weight).__name__ != "Parameter", "quantisation did not take" + # The naive sum still reports the bf16 figure, which is the whole bug. + assert sum(p.numel() * p.element_size() for p in linear.parameters()) == bf16 + # Int8 weights plus a small per-row scale: about half, never the same. + assert module_bytes(linear) < bf16 * 0.6 diff --git a/core/tests/test_h3_characters.py b/core/tests/test_h3_characters.py index 3eef5df..eb75360 100644 --- a/core/tests/test_h3_characters.py +++ b/core/tests/test_h3_characters.py @@ -8,6 +8,7 @@ import pytest +from inline_core.characters import charfile as cf from inline_core.characters import encode from inline_core.characters.apply import AppliedCharacter @@ -146,16 +147,23 @@ def test_the_reference_node_asks_for_references_over_an_adapter(tmp_path, monkey seen: dict[str, object] = {} - def fake(chosen: str, arch: str = "", prefer: str | None = None): - seen["arch"], seen["prefer"] = arch, prefer + def fake( + chosen: str, arch: str = "", prefer: str | None = None, + limit: int | None = None, **_: Any, + ): + seen["arch"], seen["prefer"], seen["limit"] = arch, prefer, limit return None monkeypatch.setattr(characters, "char_apply", fake) from inline_core.models.minimaxh3.runner import VARIANTS, _apply_character ref = next(v for v in VARIANTS if v.references) - _apply_character({"character": [type("I", (), {"file": "x.char"})()]}, ref) - assert seen == {"arch": "minimax-h3", "prefer": "reference"} + _apply_character( + {"character": [type("I", (), {"file": "x.char"})()]}, ref, {} + ) + # The cap travels with the call: `char_apply` divides the slots by role, which trimming the + # returned list could not do without knowing what each reference is of. + assert seen == {"arch": "minimax-h3", "prefer": "reference", "limit": 9} def test_a_node_with_no_reference_channel_takes_whatever_the_character_prefers(monkeypatch) -> None: @@ -163,16 +171,23 @@ def test_a_node_with_no_reference_channel_takes_whatever_the_character_prefers(m seen: dict[str, object] = {} - def fake(chosen: str, arch: str = "", prefer: str | None = None): - seen["prefer"] = prefer + def fake( + chosen: str, arch: str = "", prefer: str | None = None, + limit: int | None = None, **_: Any, + ): + seen["prefer"], seen["limit"] = prefer, limit return None monkeypatch.setattr(characters, "char_apply", fake) from inline_core.models.minimaxh3.runner import VARIANTS, _apply_character fl2va = next(v for v in VARIANTS if not v.references) - _apply_character({"character": [type("I", (), {"file": "x.char"})()]}, fl2va) + _apply_character( + {"character": [type("I", (), {"file": "x.char"})()]}, fl2va, {} + ) assert seen["prefer"] is None + # No reference channel, so no cap to state. + assert seen["limit"] is None def test_prefer_overrides_the_adapter_default() -> None: @@ -184,37 +199,40 @@ def test_prefer_overrides_the_adapter_default() -> None: assert "prefer" in inspect.signature(char_apply).parameters -def test_a_character_with_more_references_than_the_model_takes_is_trimmed(monkeypatch) -> None: +def test_a_character_with_more_references_than_the_model_takes_is_trimmed() -> None: """H3 takes 9 images; a character built for another model may carry more. Refusing sent a user to unwire images they had not wired, because every one of them came from the character.""" - from inline_core.characters import apply as characters - from inline_core.characters.apply import AppliedCharacter - from inline_core.models.minimaxh3.runner import VARIANTS, _apply_character + from inline_core.characters.apply import _fit_roles - monkeypatch.setattr( - characters, "char_apply", - lambda *_a, **_k: AppliedCharacter("Ada", [f"r{i}" for i in range(10)], "freckles"), - ) - ref = next(v for v in VARIANTS if v.references) - out = _apply_character({"character": [type("I", (), {"file": "x.char"})()]}, ref) - assert out is not None and len(out.refs) == 9 + refs = [f"r{i}" for i in range(12)] + roles = [cf.ROLE_FACE] * 12 + kept, kept_roles = _fit_roles(refs, roles, 9) + assert len(kept) == len(kept_roles) == 9 + assert kept == refs[:9], "order is the prompt's numbering, so it has to be preserved" + + +def test_trimming_divides_the_slots_by_role() -> None: + """Cutting the tail dropped whichever role happened to be last: a character with wardrobe lost + its cloth references on every model that takes fewer than it holds.""" + from inline_core.characters.apply import _fit_roles + roles = [cf.ROLE_FACE] * 6 + [cf.ROLE_BODY] * 4 + [cf.ROLE_CLOTH] * 3 + refs = [f"{r}{i}" for i, r in enumerate(roles)] + _kept, kept_roles = _fit_roles(refs, roles, 9) + counts = {role: kept_roles.count(role) for role in cf.ROLES} + assert sum(counts.values()) == 9 + assert counts[cf.ROLE_CLOTH] > 0, "the last role must survive the cut" + assert counts[cf.ROLE_FACE] >= counts[cf.ROLE_BODY] >= counts[cf.ROLE_CLOTH] -def test_the_prefix_never_names_a_reference_that_was_trimmed(monkeypatch) -> None: + +def test_the_prefix_never_names_a_reference_that_was_trimmed() -> None: """The prefix is what the prompt resolves; naming when nine were sent addresses a - position the model cannot see.""" - from inline_core.characters import apply as characters - from inline_core.characters.apply import AppliedCharacter - from inline_core.models.minimaxh3.runner import VARIANTS, _apply_character + position the model cannot see. Refs and roles are cut together, so it cannot drift.""" + from inline_core.characters.apply import AppliedCharacter, _fit_roles - monkeypatch.setattr( - characters, "char_apply", - lambda *_a, **_k: AppliedCharacter("Ada", [f"r{i}" for i in range(10)], "freckles"), - ) - ref = next(v for v in VARIANTS if v.references) - out = _apply_character({"character": [type("I", (), {"file": "x.char"})()]}, ref) - assert out is not None - assert "" in out.prefix and "" not in out.prefix + refs, roles = _fit_roles([f"r{i}" for i in range(12)], [cf.ROLE_FACE] * 12, 9) + prefix = AppliedCharacter("Ada", refs, "freckles", roles=roles).prompt_prefix(1, style="token") + assert "" in prefix and "" not in prefix def test_wired_images_keep_priority_over_the_character(monkeypatch) -> None: @@ -223,28 +241,35 @@ def test_wired_images_keep_priority_over_the_character(monkeypatch) -> None: from inline_core.characters.apply import AppliedCharacter from inline_core.models.minimaxh3.runner import VARIANTS, _apply_character - monkeypatch.setattr( - characters, "char_apply", - lambda *_a, **_k: AppliedCharacter("Ada", [f"r{i}" for i in range(10)], "freckles"), - ) + seen: dict[str, Any] = {} + + def fake( + chosen: str, arch: str = "", prefer: str | None = None, + limit: int | None = None, **_: Any, + ): + seen["limit"] = limit + return AppliedCharacter("Ada", [f"r{i}" for i in range(limit or 0)], "freckles") + + monkeypatch.setattr(characters, "char_apply", fake) ref = next(v for v in VARIANTS if v.references) inputs = { "character": [type("I", (), {"file": "x.char"})()], "references": ["mine1", "mine2", "mine3"], } - out = _apply_character(inputs, ref) - assert out is not None and len(out.refs) == 6, "3 wired + 6 from the character is the 9 cap" + out = _apply_character(inputs, ref, {}) + assert seen["limit"] == 6, "3 wired leaves 6 of H3's 9 for the character" + assert out is not None and len(out.refs) == 6 assert out.prefix.startswith(""), "and it is numbered after the wired ones" def test_the_resolution_param_is_on_the_node_face_and_defaults_to_capping() -> None: - """Default 1024, not uncapped: H3's own policy is 2048, and a character compiled there is what - put 36,864 vision tokens on the card.""" + """Default 1024, not uncapped, and named for what it does. It sets what the `.char` stores; + H3 re-resizes every reference onto 2048 on the way in, so it buys disk and never VRAM.""" from inline_core.models.character.runner import COMPILE_REFS field = next(p for p in COMPILE_REFS.params if p.key == "ref_resolution") - assert field.label == "Resized Reference Resolution" + assert field.label == "Stored Reference Resolution" assert field.default == 1024 assert field.on_face is True assert field.min == encode.NO_REFERENCE_CAP @@ -305,51 +330,48 @@ def tokens(cap: int) -> int: -def test_an_encoder_oom_points_at_the_character_not_the_canvas(monkeypatch) -> None: - """The canvas hint sent a user to resize twice for nothing: references are encoded before any - frame exists, so a 1344x768 -> 544x768 drop left the failing allocation byte-identical. The - size that matters was fixed when the character was compiled, so that is what the error names. - """ +def test_an_encoder_oom_counts_what_the_model_sees_not_what_was_stored(monkeypatch) -> None: + """The pipeline calls `resolve_reference_image_size` and puts every reference back onto a 2048 + short edge. Counting the stored pixels instead reported 1,280 tokens for a set that really cost + 20,480, and told the user to lower a setting that could not have helped.""" import tempfile from PIL import Image from inline_core.models import pipeline_runtime as rt - from inline_core.models.minimaxh3.runner import Request, _oom + from inline_core.models.minimaxh3.runner import Request, _oom, _reference_tokens from inline_core.models.references import ReferenceKind - # Stubbed because it reads the live card otherwise, so this asserted on whatever else happened - # to be running: it passed on an idle box and failed beside a training run. monkeypatch.setattr(rt, "foreign_vram_bytes", lambda *a, **k: 0) with tempfile.TemporaryDirectory() as tmp: - paths = [] - for index in range(9): - path = f"{tmp}/ref{index}.png" - Image.new("RGB", (2048, 2048)).save(path) - paths.append(path) - refs = tuple( - type("R", (), {"kind": ReferenceKind.IMAGE, "value": type("V", (), {"path": p})()})() - for p in paths - ) - request = Request( - prompt="", num_frames=144, width=544, height=768, num_inference_steps=50, - seed=1, partition="ref2va", references=refs, - ) - message = _oom(request) - - # Measured off the pixels, not off a setting this node no longer carries. - assert "36,864 vision tokens" in message - assert "Resized Reference Resolution" in message - assert "does not affect this step" in message - assert "960x544" not in message + counts = {} + for stored in (512, 2048): + paths = [] + for index in range(5): + path = f"{tmp}/{stored}_{index}.png" + Image.new("RGB", (stored, stored)).save(path) + paths.append(path) + refs = tuple( + type("R", (), {"kind": ReferenceKind.IMAGE, + "value": type("V", (), {"path": p})()})() + for p in paths + ) + request = Request( + prompt="", num_frames=124, width=544, height=768, num_inference_steps=20, + seed=1, partition="ref2va", references=refs, + ) + counts[stored] = _reference_tokens(request)[1] + + # The stored size is irrelevant: both are five references at an enforced 2048. + assert counts[512] == counts[2048] == 5 * 64 * 64 - # With no references the canvas really is the lever, so that hint has to survive untouched. - plain = Request( - prompt="", num_frames=144, width=1344, height=768, - num_inference_steps=50, seed=1, partition="fl2va", - ) - assert "960x544" in _oom(plain) + message = _oom(request) + assert "20,480 vision tokens" in message + assert "wire fewer references" in message + # The two levers that cannot move this must not be offered as though they can. + assert "960x544" not in message + assert "Lower Resized Reference Resolution" not in message def test_a_card_held_by_another_process_is_named_before_anything_on_this_node(monkeypatch) -> None: @@ -422,3 +444,143 @@ def test_the_runner_clears_the_pipeline_cache_on_a_vram_failure() -> None: # the card still held 43.5 GB after a failed run that did call clear(). assert "pipe = None" in handler assert handler.index("pipe = None") < handler.index("PIPELINES.clear()") + + +def test_body_and_clothing_references_are_never_scored_against_the_face() -> None: + """SFace measures faces. A body shot that happens to show one would be judged on the wrong + thing and could be flagged as an outlier for it, so it is held out of scoring entirely.""" + from inline_core.characters import verify + + manifest = cf.Manifest(char_id="c", name="Ada", created_at=0, modified_at=0) + manifest.refs = [ + {"path": f"refs/{i:03d}.png", "sha256": f"h{i}", "role": role} + for i, role in enumerate([cf.ROLE_FACE] * 3 + [cf.ROLE_BODY, cf.ROLE_CLOTH]) + ] + doc = cf.CharDoc(manifest=manifest, members={}) + + scored: list[int] = [] + + def fake_images(_doc): + return [object()] * len(manifest.refs) + + def fake_embed(image): + scored.append(id(image)) + return [1.0, 0.0] + + import inline_core.characters.encode as enc + import inline_core.characters.scoring as sc + + original_images, original_embed = enc.ref_images, sc.embed_face + enc.ref_images, sc.embed_face = fake_images, fake_embed + try: + verdict = verify.verify(doc) + finally: + enc.ref_images, sc.embed_face = original_images, original_embed + + assert verdict.unscored == [3, 4], "the body and cloth refs, by position" + assert len(scored) == 3, "only the three face refs reached the face encoder" + assert 3 not in verdict.flagged and 4 not in verdict.flagged + assert "not scored" in verdict.note + + +def test_the_verdict_says_body_references_are_unscored() -> None: + """The UI has to be able to say it: a reference that is used but not measured is not the same + as one that passed, and showing it as passing would be a claim nothing checked.""" + from inline_core.characters import verify + + verdict = verify.Verdict(mode=verify.MODE_BOOTSTRAP, floor=50.0, unscored=[2, 3]) + assert verdict.to_json()["unscored"] == [2, 3] + + +def test_removing_a_reference_keeps_the_unscored_positions_pointing_at_the_right_images() -> None: + """Every list in a verdict is a position into `manifest.refs`, so a removal shifts them all. + Remapping three of the four would leave `unscored` ringing whatever moved into its slot.""" + from inline_core.characters import verify + + verdict = verify.Verdict( + mode=verify.MODE_BOOTSTRAP, floor=50.0, + agreement=[90.0, 80.0, None, None], + flagged=[1], unchecked=[], duplicates=[], unscored=[2, 3], + ) + before = ["a.png", "b.png", "c.png", "d.png"] + after = ["a.png", "c.png", "d.png"] # "b.png" removed + verify._reindex(verdict, before, after) + assert verdict.unscored == [1, 2], "c and d kept their identity, one slot earlier" + assert verdict.flagged == [], "the flagged reference is the one that went" + + +def test_a_role_line_refers_back_to_a_picture_without_re_declaring_it() -> None: + """`` is H3's reserved label, emitted before each vision block. The role lines used + to repeat it with nothing behind the second, and the model replayed the references as the + opening frames of the video. Each label must be declared exactly once.""" + from inline_core.characters.apply import AppliedCharacter + + roles = [cf.ROLE_FACE] * 4 + [cf.ROLE_BODY] * 2 + [cf.ROLE_CLOTH] * 2 + refs = [f"r{i}" for i in range(len(roles))] + character = AppliedCharacter("Ada", refs, "", roles=roles) + prefix = character.prompt_prefix(1, style="token", role_lines=True) + + for n in range(1, len(roles) + 1): + assert prefix.count(f"") == 1, f" is declared more than once" + # The roles still have to be bound, just in prose that cannot be mistaken for a label. + assert "Pictures 5 and 6 show Ada's full body and build." in prefix + assert "Pictures 7 and 8 show Ada's outfit." in prefix + + +def test_the_role_line_switch_restores_the_prompt_a_character_had_before_roles() -> None: + """Off, the bindings go and nothing else does, so the switch isolates one variable: the roles + still decide which references are sent.""" + from inline_core.characters.apply import AppliedCharacter + + roles = [cf.ROLE_FACE] * 4 + [cf.ROLE_BODY] * 2 + [cf.ROLE_CLOTH] * 2 + refs = [f"r{i}" for i in range(len(roles))] + with_roles = AppliedCharacter("Ada", refs, "freckles", roles=roles) + # The same references with no roles recorded at all: a character written before the feature. + without = AppliedCharacter("Ada", refs, "freckles") + + off = with_roles.prompt_prefix(1, style="token", role_lines=False) + assert off == without.prompt_prefix(1, style="token") + assert "full body and build" not in off + assert "full body and build" in with_roles.prompt_prefix(1, style="token", role_lines=True) + + +def test_the_h3_node_leaves_the_role_lines_off_unless_asked() -> None: + """A param that defaults on would ship the behaviour that replayed the references.""" + from inline_core.models.minimaxh3.runner import DESCRIPTORS, VARIANTS + + ref = next(v for v in VARIANTS if v.references) + field = next( + f for f in DESCRIPTORS[ref.node_type].params if f.key == "character_role_lines" + ) + assert field.default is False + + +def test_the_node_defaults_to_every_reference_the_model_takes() -> None: + """A smaller default would quietly change what every existing graph renders.""" + from inline_core.models.minimaxh3.runner import DESCRIPTORS, REFERENCE_LIMITS, VARIANTS + + ref = next(v for v in VARIANTS if v.references) + params = {f.key: f for f in DESCRIPTORS[ref.node_type].params} + assert params["character_references"].default == REFERENCE_LIMITS.max_images + assert params["character_references"].max == REFERENCE_LIMITS.max_images + assert params["character_reference_roles"].default == "all" + + +def test_a_full_reference_input_names_the_wiring_not_the_character(monkeypatch) -> None: + """Nine wired images leave the character no slot, which is not the same fact as an empty .char. + + It used to raise "has no minimax-h3 references ... write it again", sending the user off to + rebuild a character that was never the problem. + """ + from inline_core.errors import ComponentError + from inline_core.models.minimaxh3.runner import REFERENCE_LIMITS, VARIANTS, _apply_character + + ref = next(v for v in VARIANTS if v.references) + inputs = { + "character": [type("I", (), {"file": "x.char"})()], + "references": ["img"] * REFERENCE_LIMITS.max_images, + } + with pytest.raises(ComponentError) as raised: + _apply_character(inputs, ref, {}) + assert "no slot left" in str(raised.value) + assert "Compile References" not in str(raised.value), "the character is not at fault" diff --git a/core/tests/test_minimaxh3_nvfp4.py b/core/tests/test_minimaxh3_nvfp4.py index 9362ba4..283dede 100644 --- a/core/tests/test_minimaxh3_nvfp4.py +++ b/core/tests/test_minimaxh3_nvfp4.py @@ -149,10 +149,12 @@ def test_the_build_satisfies_the_vendored_depth_guard() -> None: assert isinstance(model.lm_head, torch.nn.Identity), "this build ships no head" -def test_the_vae_budget_counts_the_denoiser_that_lands_after_it(monkeypatch) -> None: - """`_vae_fits` runs three lines before `apply_offload`, so the card it measures is empty and - every later placement is invisible. Reserving against that put a 10.4 GB fp32 VAE beside a - 33 GB denoiser on a 44 GB card, and the render peaked at 43.4 GB and died.""" +def test_the_vae_budget_does_not_reserve_for_a_denoiser_it_never_meets(monkeypatch) -> None: + """`render_staged` parks the denoiser for both phases that read the VAE, so they never contend. + + This used to subtract the denoiser's ~21 GB, which on a 24 GB card refused a VAE that fits + in 14.4 and sent it back to streaming - the path that turned a 6 minute render into 32. + """ import pathlib from inline_core.models import pipeline_runtime as rt @@ -161,30 +163,58 @@ def test_the_vae_budget_counts_the_denoiser_that_lands_after_it(monkeypatch) -> vae = pathlib.Path("models/vae/minimax_h3_video_vae_fp16.safetensors") if not vae.is_file(): pytest.skip("video VAE not present") + needed = vae.stat().st_size * 2 + pl._VAE_RESIDENT_MARGIN_GB * 1e9 + + monkeypatch.setattr(rt, "free_vram_bytes", lambda *a, **k: int(needed * 1.5)) + assert pl._vae_fits(vae, None) + # The card that the old denoiser reservation talked itself out of. + monkeypatch.setattr(rt, "free_vram_bytes", lambda *a, **k: int(24e9)) + assert pl._vae_fits(vae, None), "24 GB holds a 14.4 GB VAE once the denoiser is parked" + # Still a decision and not a crash when the card genuinely cannot hold it. + monkeypatch.setattr(rt, "free_vram_bytes", lambda *a, **k: int(needed * 0.5)) + assert not pl._vae_fits(vae, None) + + +def test_the_forward_is_correct_at_a_reference_length_sequence() -> None: + """The chunked matmul writes into a preallocated output, so a wrong slice bound would corrupt + part of it rather than raising. Five references put ~20k tokens through this, and the short + prompt the earlier tests use would not catch an off-by-one in the row bounds.""" + weight = _weight(96, 64) + packed, block_scale, global_scale = nvfp4.quantize_reference(weight) - monkeypatch.setattr(rt, "free_vram_bytes", lambda *a, **k: int(46.6e9)) - assert pl._vae_fits(vae, None, 0), "an empty card looks like room, which is the old bug" - assert not pl._vae_fits(vae, None, int(33e9)), "counting the denoiser has to flip it" - - # The fast path survives where the card genuinely has room: leaf offload turned a 6 minute - # render into 32, so this must not become "always stream". - monkeypatch.setattr(rt, "free_vram_bytes", lambda *a, **k: int(80e9)) - assert pl._vae_fits(vae, None, int(33e9)) - - # A denoiser larger than the whole card is a decision, not a crash. - monkeypatch.setattr(rt, "free_vram_bytes", lambda *a, **k: int(40e9)) - assert not pl._vae_fits(vae, None, int(80e9)) - - -def test_a_resident_denoiser_is_sized_whole() -> None: - """With no offload every byte of it lands on the card, so that is what the VAE must budget.""" - from dataclasses import dataclass + layer = nvfp4.NVFP4Linear(64, 96, bias=True, dtype=torch.float32) + layer.weight.copy_(packed) + layer.weight_scale.copy_(block_scale) + layer.weight_scale_2.copy_(global_scale) + bias = torch.randn(96, dtype=torch.float32) * 0.01 + with torch.no_grad(): + layer.bias.copy_(bias) - from inline_core.models.minimaxh3.pipeline import _denoiser_card_bytes + x = torch.randn(1, 512, 64, dtype=torch.float32) + got = layer(x) + want = torch.nn.functional.linear(x, weight, bias) + assert got.shape == want.shape == (1, 512, 96) + # Every column must be right, not just the aggregate: a bad chunk bound leaves a correct-looking + # tensor with one band of zeros, which a whole-tensor cosine hides. + per_column = torch.nn.functional.cosine_similarity(got, want, dim=1) + assert float(per_column.min()) > 0.99, float(per_column.min()) + + +def test_the_output_is_allocated_once_not_concatenated(monkeypatch) -> None: + """`torch.cat` holds every chunk and the result at the same time. At 20k tokens that is an + extra gigabyte, which is what it ran out of. Asserted by watching for the call rather than + grepping the source, which matched the comment explaining why it is gone.""" + weight = _weight(96, 64) + packed, block_scale, global_scale = nvfp4.quantize_reference(weight) + layer = nvfp4.NVFP4Linear(64, 96, bias=False, dtype=torch.float32) + layer.weight.copy_(packed) + layer.weight_scale.copy_(block_scale) + layer.weight_scale_2.copy_(global_scale) - @dataclass - class _Recipe: - denoiser_offload: object = None + calls: list[int] = [] + real_cat = torch.cat + monkeypatch.setattr(torch, "cat", lambda *a, **k: (calls.append(1), real_cat(*a, **k))[1]) - model = torch.nn.Linear(64, 32, bias=False, dtype=torch.float32) - assert _denoiser_card_bytes(model, _Recipe(), 0) == 64 * 32 * 4 + out = layer(torch.randn(1, 256, 64, dtype=torch.float32)) + assert out.shape == (1, 256, 96) + assert not calls, "the chunks are written into one output, never concatenated" diff --git a/core/tests/test_staged_residency.py b/core/tests/test_staged_residency.py index d52f3e7..64ec13b 100644 --- a/core/tests/test_staged_residency.py +++ b/core/tests/test_staged_residency.py @@ -131,6 +131,7 @@ def __call__(self, *_args: Any, **_kw: Any) -> Any: class _StagedPipe: def __init__(self, head: _Phase, tail: _Phase) -> None: self._inline_phases = (head, tail) + self._inline_phase_owners = ("encoder", "denoiser") self.text_encoder = _Module() self.transformer = _Module() @@ -191,3 +192,65 @@ def cancel_check() -> None: with pytest.raises(CancelledError): _render(_StagedPipe(head, tail), cancel_check=cancel_check) assert head.calls == 1 and tail.calls == 0 + + +# --- the phase split ------------------------------------------------------------------------------ + + +def test_no_staged_phase_holds_a_component_it_never_reads() -> None: + """Held as one tail, the vision tower encode carried a 10.4 GB VAE it never touches, which on a + 45 GB card was the difference between rendering and an OOM. Derived from `expected_components` + so it keeps holding if the blockset is reordered.""" + pytest.importorskip("diffusers") + from inline_core.models.minimaxh3.pipeline import _denoiser_name, _staged_phases + from inline_core.models.minimaxh3.vendor import MiniMaxH3Blocks, MiniMaxH3Ref2VABlocks + + for blocks in (MiniMaxH3Ref2VABlocks(), MiniMaxH3Blocks()): + # `transformer_ref` on ref2va, `transformer` on fl2va. + big = {"encoder": "text_encoder", "vae": "vae", "denoiser": _denoiser_name(blocks)} + owners, parts = _staged_phases(blocks) + assert len(parts) == 4 + for owner, part in zip(owners, parts, strict=True): + declared: set[str] = set() + for block in part.sub_blocks.values(): + declared |= { + getattr(spec, "name", spec) for spec in (block.expected_components or []) + } + for role, component in big.items(): + held = component in declared + if role == owner: + assert held, f"the {owner} phase does not read {component}" + else: + assert not held, f"the {owner} phase would hold {component} unused" + + +class _StagedPipeVAE: + """A four phase pipe whose three large components each record where they are moved.""" + + def __init__(self, phases: tuple[_Phase, ...]) -> None: + self._inline_phases = phases + self._inline_phase_owners = ("encoder", "vae", "denoiser", "vae") + self._inline_staged_vae = True + self.text_encoder = _Module() + self.transformer = _Module() + self.vae = _Module() + + +def test_the_vae_is_off_the_card_for_the_encode_and_the_denoise() -> None: + """Exactly the two phases that do not read it, and the two that do get it back.""" + pipe = _StagedPipeVAE(tuple(_Phase(result=f"s{i}") for i in range(4))) + assert _render(pipe) == "s3" + + # Parked for phase 1, placed for 2, parked for 3, placed again for 4. + assert pipe.vae.moves == ["cpu", "cuda:0", "cpu", "cuda:0"] + # The conditioner goes up once and never comes back; the denoiser only for its own phase. + assert pipe.text_encoder.moves == ["cuda:0", "cpu"] + assert pipe.transformer.moves == ["cpu", "cuda:0", "cpu"] + + +def test_a_streamed_vae_is_never_moved_by_the_phase_loop() -> None: + """Without `_inline_staged_vae` it carries offload hooks, and a `.to()` fights them.""" + pipe = _StagedPipeVAE(tuple(_Phase() for _ in range(4))) + pipe._inline_staged_vae = False + _render(pipe) + assert pipe.vae.moves == [] diff --git a/core/tests/test_studio_moodboard.py b/core/tests/test_studio_moodboard.py index 543f833..2b8b11d 100644 --- a/core/tests/test_studio_moodboard.py +++ b/core/tests/test_studio_moodboard.py @@ -99,3 +99,40 @@ def test_prompt_text_for_frame(conn) -> None: assert mb.prompt_text_for_frame(conn, gen["frameId"]) is None # not wired yet mb.create_connector(conn, prompt["id"], gen["id"], "out", "prompt") assert mb.prompt_text_for_frame(conn, gen["frameId"]) == "a neon city" + + +def test_a_client_write_cannot_drop_a_take_it_had_not_loaded(conn) -> None: + """`data` is one JSON blob replaced wholesale, so a browser patch carries whatever that tab last + loaded. A render landing between its load and its write used to be erased by it: the file stayed + on disk and `core.outputs`, its only record, was gone. Cost two finished videos.""" + iid = mb.add_core_node(conn, "minimax/h3-reference-to-video", 0, 0)["id"] + stale = mb.get_item(conn, iid)["data"] # what a tab loaded before the render + + mb.set_core_node_output( + conn, iid, {"takeId": "landed", "filePath": "takes/x.mp4", "kind": "video", "createdAt": 20} + ) + + # The tab now writes back its own copy, which has never seen "landed". + patch = {"data": {**stale, "core": {**stale["core"], "params": {"steps": 4}}}} + mb.client_update_item(conn, iid, patch) + + core = mb.get_item(conn, iid)["data"]["core"] + assert [o["takeId"] for o in core["outputs"]] == ["landed"], "the take must survive the write" + assert core["params"] == {"steps": 4}, "the client's own edit must still land" + + +def test_a_client_can_still_choose_which_take_is_active(conn) -> None: + """Only entries the client could not see are restored, so selecting a take keeps working.""" + iid = mb.add_core_node(conn, "minimax/h3-reference-to-video", 0, 0)["id"] + for n, at in (("a", 10), ("b", 20)): + mb.set_core_node_output( + conn, iid, {"takeId": n, "filePath": f"takes/{n}.mp4", "kind": "video", "createdAt": at} + ) + data = mb.get_item(conn, iid)["data"] + older = next(o for o in data["core"]["outputs"] if o["takeId"] == "a") + + mb.client_update_item(conn, iid, {"data": {**data, "core": {**data["core"], "output": older}}}) + + core = mb.get_item(conn, iid)["data"]["core"] + assert core["output"]["takeId"] == "a" + assert {o["takeId"] for o in core["outputs"]} == {"a", "b"} diff --git a/core/uv.lock b/core/uv.lock index c06bfea..f4f258f 100644 --- a/core/uv.lock +++ b/core/uv.lock @@ -610,7 +610,7 @@ wheels = [ [[package]] name = "inline-core" -version = "1.3.12" +version = "1.3.13" source = { editable = "." } dependencies = [ { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, diff --git a/package.json b/package.json index 2431649..5113d3d 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "inline-studio", - "version": "1.3.12", + "version": "1.3.13", "description": "AI filmmaking on a node canvas. Generate locally on your own GPU and train your own LoRAs on the same canvas, with the built-in Inline Core engine and hosted models. Every render is kept as a versioned take.", "keywords": [ "ai-filmmaking", diff --git a/packages/frontend/pyproject.toml b/packages/frontend/pyproject.toml index 7ef1635..3a75abf 100644 --- a/packages/frontend/pyproject.toml +++ b/packages/frontend/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "inline-studio-frontend" -version = "1.3.12" +version = "1.3.13" description = "Prebuilt Inline Studio web UI (SPA), served by Inline Core. Mirrors comfyui-frontend-package." requires-python = ">=3.9" readme = "README.md" diff --git a/screenshots/char-mm.mp4 b/screenshots/char-mm.mp4 new file mode 100644 index 0000000..f4a7ebc Binary files /dev/null and b/screenshots/char-mm.mp4 differ diff --git a/src/renderer/components/VideoPreview.tsx b/src/renderer/components/VideoPreview.tsx index dcd272b..069cce8 100644 --- a/src/renderer/components/VideoPreview.tsx +++ b/src/renderer/components/VideoPreview.tsx @@ -13,6 +13,7 @@ export function VideoPreview({ poster, className, onLoadedMetadata, + onLoadedData, onContextMenu, onDoubleClick, }: { @@ -20,6 +21,7 @@ export function VideoPreview({ poster?: string className?: string onLoadedMetadata?: React.ReactEventHandler + onLoadedData?: React.ReactEventHandler onContextMenu?: React.MouseEventHandler onDoubleClick?: React.MouseEventHandler }): React.JSX.Element { @@ -58,6 +60,7 @@ export function VideoPreview({ onEnded={onEnded} onMouseEnter={onMouseEnter} onLoadedMetadata={onLoadedMetadata} + onLoadedData={onLoadedData} onContextMenu={onContextMenu} onDoubleClick={onDoubleClick} className={className} diff --git a/src/renderer/index.css b/src/renderer/index.css index d9feb48..52df974 100644 --- a/src/renderer/index.css +++ b/src/renderer/index.css @@ -49,3 +49,38 @@ body { border-radius: 0; background: transparent; } + +/* The Current slot's working tile: three dots lighting 1-2-3 on a loop. A percentage was worse than + nothing there - a run inside a model load reports none for minutes, so the tile sat at 0% and read + as stuck. */ +@keyframes run-dot-cycle { + 0%, + 100% { + opacity: 0.2; + } + 20% { + opacity: 1; + } + 55% { + opacity: 0.2; + } +} + +.run-dot { + animation: run-dot-cycle 1.2s linear infinite; +} + +.run-dot:nth-child(2) { + animation-delay: 0.4s; +} + +.run-dot:nth-child(3) { + animation-delay: 0.8s; +} + +@media (prefers-reduced-motion: reduce) { + .run-dot { + animation: none; + opacity: 0.7; + } +} diff --git a/src/renderer/lib/loaderFit.test.ts b/src/renderer/lib/loaderFit.test.ts new file mode 100644 index 0000000..5c5ceac --- /dev/null +++ b/src/renderer/lib/loaderFit.test.ts @@ -0,0 +1,90 @@ +import { describe, expect, it } from 'vitest' +import { + LOADER_CHROME_H, + LOADER_LONG_EDGE, + LOADER_MAX_BODY, + LOADER_MAX_W, + LOADER_MIN_BODY, + LOADER_MIN_W, + fitLoaderSize, + needsBlurFill, +} from './loaderFit' + +const body = (h: number): number => h - LOADER_CHROME_H + +describe('fitLoaderSize', () => { + it('matches the long edge on both orientations', () => { + expect(fitLoaderSize(2 / 3)).toEqual({ width: 227, height: 341 + LOADER_CHROME_H }) + expect(fitLoaderSize(16 / 9)).toEqual({ width: 340, height: 191 + LOADER_CHROME_H }) + expect(fitLoaderSize(1)).toEqual({ width: 340, height: 340 + LOADER_CHROME_H }) + }) + + it('reproduces the media aspect within a pixel when nothing clamps', () => { + for (const aspect of [0.75, 1, 1.25, 1.5, 16 / 9]) { + const { width, height } = fitLoaderSize(aspect) + expect(Math.abs(width / body(height) - aspect)).toBeLessThan(0.01) + } + }) + + it('scales a narrow portrait up to the minimum width without introducing bars', () => { + const { width, height } = fitLoaderSize(9 / 16) + expect(width).toBe(LOADER_MIN_W) + // The body is derived from the clamped width, so the aspect survives the floor. + expect(Math.abs(width / body(height) - 9 / 16)).toBeLessThan(0.01) + }) + + it('leaves bars only where the body itself clamps', () => { + const wide = fitLoaderSize(3) + expect(body(wide.height)).toBe(LOADER_MIN_BODY) + expect(wide.width / body(wide.height)).toBeLessThan(3) + + const tall = fitLoaderSize(0.3) + expect(body(tall.height)).toBe(LOADER_MAX_BODY) + expect(tall.width / body(tall.height)).toBeGreaterThan(0.3) + }) + + it('stays inside every bound for any aspect', () => { + for (let a = 0.05; a <= 20; a += 0.05) { + const { width, height } = fitLoaderSize(a) + expect(width).toBeGreaterThanOrEqual(LOADER_MIN_W) + expect(width).toBeLessThanOrEqual(LOADER_MAX_W) + expect(body(height)).toBeGreaterThanOrEqual(LOADER_MIN_BODY) + expect(body(height)).toBeLessThanOrEqual(LOADER_MAX_BODY) + } + }) + + it('keeps an unusable aspect out of the stored size', () => { + for (const bad of [0, -1, Number.NaN, Number.POSITIVE_INFINITY]) { + const { width, height } = fitLoaderSize(bad) + expect(Number.isFinite(width) && Number.isFinite(height)).toBe(true) + expect(width).toBe(LOADER_LONG_EDGE) + expect(body(height)).toBe(LOADER_LONG_EDGE) + } + }) +}) + +describe('needsBlurFill', () => { + it('is quiet on a node still at its fitted size', () => { + for (const aspect of [2 / 3, 1, 16 / 9, 9 / 16]) { + const { width, height } = fitLoaderSize(aspect) + expect(needsBlurFill(aspect, width / body(height))).toBe(false) + } + }) + + it('fires once the box no longer matches the media', () => { + expect(needsBlurFill(2 / 3, 16 / 9)).toBe(true) + expect(needsBlurFill(1, 1.5)).toBe(true) + }) + + it('fires on the aspects whose body clamps, since those keep bars by design', () => { + for (const aspect of [3, 0.3]) { + const { width, height } = fitLoaderSize(aspect) + expect(needsBlurFill(aspect, width / body(height))).toBe(true) + } + }) + + it('is quiet on a degenerate box', () => { + expect(needsBlurFill(1, 0)).toBe(false) + expect(needsBlurFill(Number.NaN, 1)).toBe(false) + }) +}) diff --git a/src/renderer/lib/loaderFit.ts b/src/renderer/lib/loaderFit.ts new file mode 100644 index 0000000..53a4a6f --- /dev/null +++ b/src/renderer/lib/loaderFit.ts @@ -0,0 +1,29 @@ +// Sizes a "Load Assets" node to its media once, when its first asset lands. + +/** The node's footer bar; the fit runs before mount, so it cannot be measured off the DOM. */ +export const LOADER_CHROME_H = 26 +/** Both orientations get the same visual weight by matching on their long edge. */ +export const LOADER_LONG_EDGE = 340 +export const LOADER_MIN_W = 200 +export const LOADER_MAX_W = 460 +export const LOADER_MIN_BODY = 150 +export const LOADER_MAX_BODY = 460 + +const clamp = (n: number, lo: number, hi: number): number => Math.max(lo, Math.min(hi, n)) + +/** Node size for a media aspect; extreme ratios clamp and keep a blurred sliver. */ +export function fitLoaderSize(aspect: number): { width: number; height: number } { + // Callers measure first, so this only keeps NaN out of the stored width/height. + const safe = Number.isFinite(aspect) && aspect > 0 ? aspect : 1 + const long = safe >= 1 ? LOADER_LONG_EDGE : LOADER_LONG_EDGE * safe + const width = clamp(Math.round(long), LOADER_MIN_W, LOADER_MAX_W) + const body = clamp(Math.round(width / safe), LOADER_MIN_BODY, LOADER_MAX_BODY) + return { width, height: body + LOADER_CHROME_H } +} + +/** Whether a box of this aspect leaves visible bars around media of that aspect. */ +export function needsBlurFill(mediaAspect: number, boxAspect: number): boolean { + if (!Number.isFinite(mediaAspect) || !Number.isFinite(boxAspect)) return false + if (mediaAspect <= 0 || boxAspect <= 0) return false + return Math.abs(mediaAspect - boxAspect) / boxAspect > 0.02 +} diff --git a/src/renderer/lib/mediaSize.ts b/src/renderer/lib/mediaSize.ts new file mode 100644 index 0000000..c0216d7 --- /dev/null +++ b/src/renderer/lib/mediaSize.ts @@ -0,0 +1,35 @@ +// Core stores no dimensions on an asset, so an aspect ratio has to be decoded in the browser. + +/** Give up rather than hang a caller on a file whose metadata never arrives. */ +const MEASURE_TIMEOUT_MS = 8000 + +export function imageSize(url: string): Promise<{ w: number; h: number } | null> { + return new Promise((resolve) => { + const img = new Image() + img.onload = () => resolve({ w: img.naturalWidth, h: img.naturalHeight }) + img.onerror = () => resolve(null) + img.src = url + }) +} + +export function videoSize(url: string): Promise<{ w: number; h: number } | null> { + return new Promise((resolve) => { + const v = document.createElement('video') + v.preload = 'metadata' + v.muted = true + v.onloadedmetadata = () => + resolve(v.videoWidth && v.videoHeight ? { w: v.videoWidth, h: v.videoHeight } : null) + v.onerror = () => resolve(null) + v.src = url + }) +} + +/** The media's width/height ratio, or null when it can't be decoded (audio always). */ +export async function mediaAspect(url: string, kind: string): Promise { + if (kind !== 'image' && kind !== 'video') return null + const timeout = new Promise((resolve) => + setTimeout(() => resolve(null), MEASURE_TIMEOUT_MS), + ) + const size = await Promise.race([kind === 'video' ? videoSize(url) : imageSize(url), timeout]) + return size && size.w > 0 && size.h > 0 ? size.w / size.h : null +} diff --git a/src/renderer/lib/recipeGraph.ts b/src/renderer/lib/recipeGraph.ts index e758561..7f02355 100644 --- a/src/renderer/lib/recipeGraph.ts +++ b/src/renderer/lib/recipeGraph.ts @@ -76,6 +76,7 @@ export async function buildGraphFromRecipe(recipe: Recipe, drop: Point): Promise const assetIds = ((data.assetIds as string[] | undefined) ?? []).filter((id) => known.has(id)) if (created && assetIds.length) { await store.updateItem(created.id, { data: { ...created.data, assetIds } }, false) + await store.fitLoaderToAsset(created.id, assetIds[0]) } } else if (it.type === 'frame' && (data.fal as { modelId?: string } | undefined)?.modelId) { // A fal gen node - Core authored it with the model + params. diff --git a/src/renderer/store/generationStore.ts b/src/renderer/store/generationStore.ts index 94ceb0a..23cdcca 100644 --- a/src/renderer/store/generationStore.ts +++ b/src/renderer/store/generationStore.ts @@ -146,6 +146,18 @@ export const useGenerationStore = create((set) => ({ busyByFrame: { ...s.busyByFrame, [itemId]: true }, progressByFrame: { ...s.progressByFrame, [itemId]: 0 }, })) + // Snapshot before the call: the node's params can be edited while it renders, and until a take + // lands there is nothing else holding the recipe this run is using. + const board = useMoodboardStore.getState() + const core = board.items.find((i) => i.id === itemId)?.data.core + if (core) { + void board.setPendingRun(itemId, { + params: { ...core.params }, + prompt: board.connectedPromptText(itemId), + startedAt: Date.now(), + status: 'running', + }) + } try { const res = await studio().generation.runWorkflow(itemId) if (!res.ok) { @@ -312,6 +324,15 @@ function openMissingModels(itemId: string, error: string): boolean { export function subscribeGenerationEvents(): () => void { const gen = useGenerationStore.getState() + // A stopped one keeps its snapshot, and so its slot. Browsing history overwrites the node's + // params, so after a comparison this is the only copy of the settings that were submitted; + // clearing it on stop lost them for good. The next run on this node replaces it. + const markPending = async (nodeId: string, status: 'cancelled' | 'failed'): Promise => { + const board = useMoodboardStore.getState() + const pending = board.items.find((i) => i.id === nodeId)?.data.core?.pending + if (pending) await board.setPendingRun(nodeId, { ...pending, status }) + } + // Finish any generations still running when the app last closed; their events arrive below. void gen.resumePending() // A page refresh throws away this tab's copy of the queue while Core keeps working, and a run @@ -341,6 +362,7 @@ export function subscribeGenerationEvents(): () => void { }), studio().events.onGenerationError((e) => { gen.finishRun(e.frameId ?? e.targetFrameId) + void markPending(e.frameId ?? e.targetFrameId, 'failed') // The node that stopped, which for a chain is rarely the one the run was started from. gen.setFailedNode(e.frameId ?? e.targetFrameId) // A missing weight file is a thing to fix, not a sentence to read: open the popup that can @@ -350,6 +372,7 @@ export function subscribeGenerationEvents(): () => void { }), studio().events.onGenerationCancelled((e) => { gen.finishRun(e.targetFrameId) + void markPending(e.targetFrameId, 'cancelled') }), ] return () => unsubs.forEach((u) => u()) diff --git a/src/renderer/store/moodboardStore.ts b/src/renderer/store/moodboardStore.ts index 6178a6c..db92d2f 100644 --- a/src/renderer/store/moodboardStore.ts +++ b/src/renderer/store/moodboardStore.ts @@ -5,14 +5,18 @@ * to main via studio().moodboard. */ import { create } from 'zustand' -import type { MoodboardItem, MoodboardConnector } from '@shared/types' +import type { CorePendingRun, MoodboardItem, MoodboardConnector } from '@shared/types' import type { MoodboardItemPatch } from '@shared/ipc' import { ipcErrorMessage } from '../lib/ipcError' +import { fitLoaderSize } from '../lib/loaderFit' +import { resolveMedia } from '@/lib/media' +import { mediaAspect } from '../lib/mediaSize' import { studio } from '@/lib/studio' /** Training node kinds this canvas can add; they share the Trainer graph's channels. */ export type TrainingNodeKind = 'train/dataset' | 'train/caption' | 'train/lora' | 'train/loss' +import { useAssetStore } from './assetStore' import { useFrameStore } from './frameStore' /** A board snapshot for the undo/redo stacks. */ @@ -54,6 +58,15 @@ interface MoodboardState { addControlSpace: (x: number, y: number) => Promise /** Append library assets to a loader's ordered asset list (deduped). */ addLoaderAssets: (itemId: string, assetIds: string[]) => Promise + /** Place a library asset on the canvas as its own Load Assets node, sized to its media. */ + addLoaderFromAssetInLayer: ( + assetId: string, + x: number, + y: number, + parentId: string | null, + ) => Promise + /** Size a loader to its media's aspect. Fires once, when the node's first assets land. */ + fitLoaderToAsset: (itemId: string, assetId: string) => Promise /** Remove one asset from a loader. */ removeLoaderAsset: (itemId: string, assetId: string) => Promise /** Move an asset to the front of a loader's list (its hero, fed downstream). */ @@ -96,9 +109,14 @@ interface MoodboardState { updateItem: (id: string, patch: MoodboardItemPatch, recordHistory?: boolean) => Promise /** Merge into an item's `data`. Node selections (dataset, run, hyperparams) live there. */ patchItemData: (id: string, data: Record) => Promise - /** Restore the text of the prompt node wired into `nodeId`'s `prompt` input (no-op if none). Used - * when switching a gen node's take history so the shown image's prompt is restored non-destructively. */ + /** Restore the text of the prompt node wired into `nodeId`'s `prompt` input (no-op if none). + * Only ever from an explicit "use these settings", never from browsing take history. */ setConnectedPromptText: (nodeId: string, text: string) => Promise + /** The text of the prompt node wired into `nodeId`'s `prompt` input, or undefined. */ + connectedPromptText: (nodeId: string) => string | undefined + /** Record (or clear) a Core node's submitted-but-unlanded run. Never an undo step: the user did + * not edit anything, and a snapshot on the stack would make one ⌘Z look like it did nothing. */ + setPendingRun: (nodeId: string, pending: CorePendingRun | null) => Promise deleteItem: (id: string) => Promise /** Delete one render from a Core node's output history (and its file). */ removeCoreOutput: (itemId: string, takeId: string) => Promise @@ -420,6 +438,26 @@ export const useMoodboardStore = create((set, get) => ({ const next = [...current, ...assetIds.filter((id) => !current.includes(id))] if (next.length === current.length) return await get().updateItem(itemId, { data: { ...item.data, assetIds: next } }) + if (current.length === 0) await get().fitLoaderToAsset(itemId, next[0]) + }, + + addLoaderFromAssetInLayer: async (assetId, x, y, parentId) => { + const created = await get().addLoader(x, y) + if (!created) return + // One patch, and no second snapshot: `addLoader` already recorded the whole drop as one step. + const patch: MoodboardItemPatch = { data: { ...created.data, assetIds: [assetId] } } + if (parentId) patch.parentId = parentId + await get().updateItem(created.id, patch, false) + await get().fitLoaderToAsset(created.id, assetId) + }, + + fitLoaderToAsset: async (itemId, assetId) => { + const asset = useAssetStore.getState().assets.find((a) => a.id === assetId) + if (!asset?.filePath) return + const aspect = await mediaAspect(resolveMedia(asset.filePath), asset.kind) + if (aspect == null) return + // Programmatic layout fit, part of the drop - it must not land on the undo stack of its own. + await get().updateItem(itemId, fitLoaderSize(aspect), false) }, removeLoaderAsset: async (itemId, assetId) => { @@ -774,6 +812,32 @@ export const useMoodboardStore = create((set, get) => ({ } }, + connectedPromptText: (nodeId) => { + const { items, connectors } = get() + const conn = connectors.find( + (c) => + c.toItemId === nodeId && (c.data as { targetHandle?: string }).targetHandle === 'prompt', + ) + if (!conn) return undefined + const node = items.find((i) => i.id === conn.fromItemId && i.type === 'prompt') + const text = node?.data.promptText + return typeof text === 'string' ? text : undefined + }, + + setPendingRun: async (nodeId, pending) => { + // Reloaded first because `updateItem` PUTs the whole `data` blob: writing it from a client copy + // that Core has since added a take to silently drops that take, which is how a finished render + // went missing with only its file left behind. + await get().load() + const item = get().items.find((i) => i.id === nodeId) + const core = item?.data.core + if (!item || !core) return + const next: NonNullable = { ...core } + if (pending) next.pending = pending + else delete next.pending + await get().updateItem(nodeId, { data: { ...item.data, core: next } }, false) + }, + setConnectedPromptText: async (nodeId, text) => { const { items, connectors } = get() const conn = connectors.find( diff --git a/src/renderer/views/Moodboard/AddNodeMenu.tsx b/src/renderer/views/Moodboard/AddNodeMenu.tsx index e608820..b75faa2 100644 --- a/src/renderer/views/Moodboard/AddNodeMenu.tsx +++ b/src/renderer/views/Moodboard/AddNodeMenu.tsx @@ -17,7 +17,7 @@ import { useMenuPlacement } from './useMenuPlacement' import { addableCoreNodes, type NodeDescriptor } from '@shared/coreNodes' import { isExtensionNode, extensionOf } from '@shared/extensions' import { listNodeDefs, groupByOwner } from '@shared/nodes/registry' -import { CaptionGlyph, ChartIcon, CpuIcon, LayersIcon, WandIcon } from './nodes/NodeBadge' +import { CaptionGlyph, ChartIcon, CpuIcon, FalIcon, LayersIcon, WandIcon } from './nodes/NodeBadge' import type { AddNodeKind } from './nodeKinds' @@ -126,7 +126,12 @@ export function AddNodeMenu({ (d.title.toLowerCase().includes(needle) || d.id.toLowerCase().includes(needle)), ) .map((d) => ( - } accent onClick={() => onPickGen?.(d.id)}> + } + accent + onClick={() => onPickGen?.(d.id)} + > {d.title} )), @@ -200,7 +205,7 @@ export function AddNodeMenu({ {group.defs.map((def) => ( } + icon={} accent onClick={() => onPickGen?.(def.id)} > @@ -426,15 +431,6 @@ function ScissorsIcon(): React.JSX.Element { ) } -function SparklesIcon(): React.JSX.Element { - return ( - - - - - ) -} - function PromptIcon(): React.JSX.Element { return ( diff --git a/src/renderer/views/Moodboard/MoodboardPanel.tsx b/src/renderer/views/Moodboard/MoodboardPanel.tsx index 1be16a3..0bd71da 100644 --- a/src/renderer/views/Moodboard/MoodboardPanel.tsx +++ b/src/renderer/views/Moodboard/MoodboardPanel.tsx @@ -305,7 +305,7 @@ function Board(): React.JSX.Element { const { items, connectors, error, load, updateItem, deleteItem, connect, disconnect } = useMoodboardStore() const addTextAt = useMoodboardStore((s) => s.addTextAt) - const addFrameFromAssetInLayer = useMoodboardStore((s) => s.addFrameFromAssetInLayer) + const addLoaderFromAssetInLayer = useMoodboardStore((s) => s.addLoaderFromAssetInLayer) const addFrameItemInLayer = useMoodboardStore((s) => s.addFrameItemInLayer) const addPreview = useMoodboardStore((s) => s.addPreview) const addLayer = useMoodboardStore((s) => s.addLayer) @@ -931,7 +931,7 @@ function Board(): React.JSX.Element { const ids = getAssetDragIds(e.dataTransfer) if (ids.length === 0) { - // Files dropped from the OS → import into the library, then place as frames. A shared Inline + // Files dropped from the OS → import into the library, then place them. A shared Inline // PNG carries its recipe, so offer to rebuild the graph instead of just importing it. const files = Array.from(e.dataTransfer.files ?? []) if (files.length === 0) return @@ -992,7 +992,7 @@ function Board(): React.JSX.Element { else onAsset() } - /** Place existing library assets as frames at/near a drop point (cascaded). */ + /** Place existing library assets as Load Assets nodes at/near a drop point (cascaded). */ const placeAssetsAt = (assetIds: string[], drop: { x: number; y: number }): void => { assetIds.forEach((assetId, i) => { const abs = { x: drop.x + i * 24, y: drop.y + i * 24 } @@ -1000,11 +1000,11 @@ function Board(): React.JSX.Element { // Children store positions relative to their layer. const x = layer ? abs.x - layer.x : abs.x const y = layer ? abs.y - layer.y : abs.y - void addFrameFromAssetInLayer(assetId, x, y, layer?.id ?? null) + void addLoaderFromAssetInLayer(assetId, x, y, layer?.id ?? null) }) } - /** Import dropped OS files (paths under Electron, upload in the browser), then place as frames. */ + /** Import dropped OS files (paths under Electron, upload in the browser), then place them. */ const placeDroppedFiles = async ( files: File[], drop: { x: number; y: number }, diff --git a/src/renderer/views/Moodboard/nodes/FrameNode.tsx b/src/renderer/views/Moodboard/nodes/FrameNode.tsx index 0f60f36..072f072 100644 --- a/src/renderer/views/Moodboard/nodes/FrameNode.tsx +++ b/src/renderer/views/Moodboard/nodes/FrameNode.tsx @@ -20,8 +20,8 @@ import { pickFilesViaInput, } from '../../../lib/importFiles' import { useLightboxStore } from '../../../store/lightboxStore' -import { VideoPreview } from '../../../components/VideoPreview' import { Waveform } from '../../../components/Waveform' +import { MediaBody } from './MediaBody' import { NodeFrame } from './NodeFrame' import { FilmIcon, NodeBadge, NodeBadgeRow, StarIcon, UploadIcon } from './NodeBadge' import { ThumbStrip } from './ThumbStrip' @@ -79,7 +79,9 @@ export function FrameNode({ id, data, selected }: NodeProps): React.JSX.Element // A "Load Assets" loader: a pure viewer with no generation. It's freely resizable (the aspect // auto-fit below is skipped) and passes its loaded asset straight through as its output. const isLoader = !!item?.data.loader - const mediaFit = isLoader ? 'object-contain' : 'object-cover' + const mediaFit = isLoader ? 'contain' : 'cover' + // Audio renders as a waveform, so the media body only ever sees the other two kinds. + const mediaKind: 'image' | 'video' = cur?.kind === 'video' ? 'video' : 'image' // Fit the node height to the media's aspect ratio at the current width, so the // body shows the image edge-to-edge with no black bars. The `lastFit` guard makes @@ -228,60 +230,33 @@ export function FrameNode({ id, data, selected }: NodeProps): React.JSX.Element className="relative flex flex-1 items-center justify-center overflow-hidden bg-black" > {cur ? ( - cur.kind === 'video' ? ( - // `cur.url` is the playable source (transcoded preview when needed); - // the poster shows while that transcode is still in progress. - { - const v = e.currentTarget - if (v.videoWidth && v.videoHeight) setAspect(v.videoWidth / v.videoHeight) - }} - onContextMenu={(e) => - onMediaContextMenu(e, { - src: cur.saveSrc, - name: frame ? `Frame ${frame.name}` : 'input', - kind: 'video', - }) - } - onDoubleClick={() => - openLightbox({ - src: cur.saveSrc, - kind: 'video', - name: frame ? `Frame ${frame.name}` : 'input', - }) - } - className={`h-full w-full ${mediaFit}`} - /> - ) : cur.kind === 'audio' ? ( + cur.kind === 'audio' ? (
) : ( - { - const img = e.currentTarget - if (img.naturalWidth && img.naturalHeight) - setAspect(img.naturalWidth / img.naturalHeight) - }} + kind={mediaKind} + poster={cur.poster} + fit={mediaFit} + onAspect={setAspect} onContextMenu={(e) => onMediaContextMenu(e, { src: cur.saveSrc, name: frame ? `Frame ${frame.name}` : 'input', - kind: 'image', + kind: mediaKind, }) } onDoubleClick={() => openLightbox({ src: cur.saveSrc, - kind: 'image', + kind: mediaKind, name: frame ? `Frame ${frame.name}` : 'input', }) } - className={`h-full w-full ${mediaFit}`} /> ) ) : isLoader ? ( diff --git a/src/renderer/views/Moodboard/nodes/GenNode.tsx b/src/renderer/views/Moodboard/nodes/GenNode.tsx index 25a6671..ef24def 100644 --- a/src/renderer/views/Moodboard/nodes/GenNode.tsx +++ b/src/renderer/views/Moodboard/nodes/GenNode.tsx @@ -24,10 +24,12 @@ import { NodeFrame } from './NodeFrame' import { AdjustIcon, AudioGlyph, + FalIcon, ImageGlyph, NodeBadge, NodeBadgeRow, VideoGlyph, + RunningDots, } from './NodeBadge' import { ThumbStrip } from './ThumbStrip' import { NodeRunToolbar } from './NodeRunToolbar' @@ -40,24 +42,6 @@ interface GenNodeData extends Record { frameId: string } -/** Two-sparkle mark flagging an AI/API-backed node. Matches the toolbar's create button. */ -function SparkleIcon(): React.JSX.Element { - return ( - - - - - ) -} - /** * A left-edge input dot that reveals a small hint chip on hover - an icon + label naming what to * connect (a "T" for the text prompt, an image/video icon for the media input). @@ -262,7 +246,9 @@ export function GenNode({ id, data, selected }: NodeProps): React.JSX.Element { /> {/* Title + live price-estimate badges - float above the node. */} - }>Generate + }> + Generate + {price && ( )} - {/* Multiple takes → a thumbnail strip; click one to make it this node's chosen output - (its hero), so the shown image is what flows to anything wired downstream. */} - {!busy && ( - ({ - id: t.id, - url: resolveMedia(t.filePath), - kind: t.kind, - }))} - selected={safeTakeIdx} - onSelect={(i) => { - void setHero(frameId, ordered[i].id) - setTakeIdx(0) - }} - /> - )} + {/* Takes behind a Current slot; click a take to make it this node's chosen output (its + hero), so the shown image is what flows downstream. The strip used to vanish while + busy, which hid the history exactly when there was a render to compare against. */} + ({ + id: t.id, + url: resolveMedia(t.filePath), + kind: t.kind, + }))} + selected={busy ? -1 : safeTakeIdx} + onSelect={(i) => { + void setHero(frameId, ordered[i].id) + setTakeIdx(0) + }} + leading={ + busy ? ( +
+ +
+ ) : undefined + } + /> {/* Footer: model picker + settings (adjust). Run lives on the graph's output node. */} diff --git a/src/renderer/views/Moodboard/nodes/GraphNode.tsx b/src/renderer/views/Moodboard/nodes/GraphNode.tsx index beee996..a780615 100644 --- a/src/renderer/views/Moodboard/nodes/GraphNode.tsx +++ b/src/renderer/views/Moodboard/nodes/GraphNode.tsx @@ -1,4 +1,4 @@ -import { useEffect } from 'react' +import { useEffect, useState } from 'react' import { type NodeProps } from '@xyflow/react' import { isModelPort } from '@shared/coreNodes' import { isExtensionNode, extensionOf } from '@shared/extensions' @@ -14,6 +14,18 @@ import { useLightboxStore } from '../../../store/lightboxStore' import { matchControlAspect } from '../../../lib/matchControlAspect' import { resolveCoreInputThumbs } from './coreInputThumbs' import { CoreOutputPreview, CoreOutputThumb } from './CoreOutputPreview' +import type { SlotId } from './takeSlots' +import { + applyableParams, + buildSlots, + activePending, + hasEdits, + restorableKeys, + slotMedia, + slotPrompt, + slotRecipe, +} from './takeSlots' +import type { CorePendingRun, CoreTakeRef } from '@shared/types' import { NodeFrame } from './NodeFrame' import { bottomStyle, compactNodeMinHeight, topStyle } from './nodeSize' import { PortHandle } from './PortHandle' @@ -34,6 +46,9 @@ import { SquareIcon, TypeIcon, WandIcon, + RunningDots, + StopIcon, + PencilIcon, } from './NodeBadge' import { resolveMedia } from '@/lib/media' @@ -82,6 +97,15 @@ export function GraphNode({ id, data, selected }: NodeProps): React.JSX.Element const frames = useFrameStore((s) => s.frames) const takesByFrame = useFrameStore((s) => s.takesByFrame) const openLightbox = useLightboxStore((s) => s.open) + // Not persisted: a saved browse position reopens a project showing history, not the present. + const [slot, setSlot] = useState('current') + // Follow each render as it lands: Core promotes a finished take to the node's active output. + const activeTakeId = item?.data.core?.output?.takeId + useEffect(() => { + if (activeTakeId) setSlot(activeTakeId) + }, [activeTakeId]) + // What Generate will actually send, so the front slot never shows an older take's prompt. + const livePrompt = useMoodboardStore((s) => s.connectedPromptText(itemId)) const coreType = item?.type === 'core' ? item.data.core?.type : undefined const descriptor = useCoreNodesStore((s) => coreType ? s.descriptors.find((d) => d.type === coreType) : undefined, @@ -175,22 +199,51 @@ export function GraphNode({ id, data, selected }: NodeProps): React.JSX.Element // Take history for the on-node output strip (newest first). Older items predate history and only // carry a single `output` - treat that as a one-entry history. `output` marks the active take. - const outputs = core.outputs ?? (core.output ? [core.output] : []) - const activeTakeId = core.output?.takeId - // Switching a take restores that image's recipe non-destructively: its settings onto this node's - // params, and its prompt onto the wired prompt node (if one still exists). No nodes are created. - // The take's *seed* is deliberately NOT restored - keep the node's current seed so browsing history - // never pins a fixed seed (which would make every re-generation identical and turn on the node - // cache, so connection/control changes would stop taking effect until the seed was reset). - const setActiveOutput = (o: NonNullable): void => { - let params = core.params - if (o.params) { - const restored: Record = { ...o.params } - delete restored.seed - params = { ...core.params, ...restored } - } - void updateItem(itemId, { data: { ...item.data, core: { ...core, output: o, params } } }) - if (typeof o.prompt === 'string') void setConnectedPromptText(itemId, o.prompt) + // Minus the installed-file dropdowns: a take records the runner's `model`, not the filename. + const restorable = restorableKeys(descriptor?.params) + const edited = hasEdits(core, livePrompt) + const slots = buildSlots(core, busy || executing, edited) + // A slot that has gone falls back to the active output rather than highlighting nothing. + const shown = slots.some((e) => e.id === slot) ? slot : (core.output?.takeId ?? 'current') + + const shownPrompt = slotPrompt(core, shown, livePrompt) + const shownMedia = slotMedia(core, shown) + const rendering = (busy || executing) && shown === 'current' + // Selecting a slot restores the graph that produced it, seed excluded: a pinned seed turns on + // the node cache and freezes re-generation. + const restore = ( + recipe: { params?: Record; prompt?: string } | undefined, + output?: CoreTakeRef, + pending?: CorePendingRun, + ): void => { + const params = recipe ? { ...core.params, ...applyableParams(recipe, restorable) } : core.params + const next = { ...core, params } + if (output) next.output = output + if (pending) next.pending = pending + void updateItem(itemId, { data: { ...item.data, core: next } }) + if (typeof recipe?.prompt === 'string') void setConnectedPromptText(itemId, recipe.prompt) + } + + const selectTake = (o: CoreTakeRef): void => { + // Settings edited but never generated have no take to come back to, and this restore is about + // to overwrite them. Captured here rather than on every keystroke: this is the only moment they + // can be lost, so it is the only moment worth a write. It replaces a stopped run's snapshot, + // which the edit has superseded; it never replaces a running one, whose settings are the only + // record of what is on the GPU right now. + // A snapshot must exist before this overwrites the node's params. Never over a draft or a + // running run: see "the draft survives browsing" in docs/generation-recipe.md. A stopped run's snapshot is replaced only by a real edit. + const held = activePending(core)?.status + const draft: CorePendingRun | undefined = + held === 'running' || held === 'draft' || (held !== undefined && !edited) + ? undefined + : { params: { ...core.params }, prompt: livePrompt, startedAt: Date.now(), status: 'draft' } + setSlot(o.takeId) + restore(o, o, draft) + } + + const selectCurrent = (): void => { + setSlot('current') + restore(slotRecipe(core, 'current')) } // Real "models missing" signal from the requirements check (replaces the old options heuristic, @@ -541,20 +594,27 @@ export function GraphNode({ id, data, selected }: NodeProps): React.JSX.Element {/* Edge-to-edge output preview. */}
{references.length > 0 && } - {core.output ? ( - - core.output && - openLightbox({ - src: resolveMedia(core.output.filePath), - kind, - name: core.output.prompt || descriptor.title, - }) + {shownMedia ? ( + // Dimmed while rendering rather than blanked: an empty card through a long render + // reads as broken, and the previous take is the most useful thing to look at. +
+ > + + openLightbox({ + src: resolveMedia(shownMedia.filePath), + kind, + name: shownMedia.prompt || descriptor.title, + }) + } + /> +
) : (
@@ -605,46 +665,86 @@ export function GraphNode({ id, data, selected }: NodeProps): React.JSX.Element )}
- {/* Take history: every render this node produced, newest first. Click one to make it the - active output (shown large + flowed downstream). Only shown once there's more than one. */} - {outputs.length > 1 && ( -
- {outputs.map((o) => ( + {/* Take history behind a permanent Current slot. Current holds the node's live settings, + which is what makes browsing reversible: selecting it again is how you get back. */} +
+ {slots.map((entry) => { + const take = entry.take + const active = entry.id === shown + const ring = active + ? 'border-emerald-400 ring-1 ring-emerald-400/40' + : 'border-border hover:border-zinc-500' + if (!take) { + // One slot for everything that is not a finished render: edited, running, stopped. + const face = { + draft: { + tone: 'border-dashed border-zinc-500 hover:border-zinc-400', + title: 'Edited since the last render - click to restore these settings', + glyph: , + }, + failed: { + tone: 'border-red-500/70 hover:border-red-400', + title: 'Failed - click to restore the settings it ran with', + glyph: , + }, + cancelled: { + tone: 'border-amber-500/70 hover:border-amber-400', + title: 'Cancelled - click to restore the settings it ran with', + glyph: , + }, + running: { + tone: ring, + title: `Rendering${pct === null ? '' : ` ${pct}%`} - click to restore its settings`, + // Dots mean working, so no other state may wear them. + glyph: , + }, + }[entry.state === 'take' ? 'draft' : entry.state] + return ( + + ) + } + return ( - ))} -
- )} + ) + })} +
- {/* The active take's prompt (restored from its recipe on switch), so the shown image's - prompt is visible without opening Adjust. */} - {core.output?.prompt && ( + {/* The selected slot's prompt: the browsed take's, or the live one on Current. It used to + always show the active take's, so an edited prompt left the node advertising the old. */} + {shownPrompt && (
- {core.output.prompt} + {shownPrompt}
)} diff --git a/src/renderer/views/Moodboard/nodes/LoaderNode.tsx b/src/renderer/views/Moodboard/nodes/LoaderNode.tsx index 77df654..7668c27 100644 --- a/src/renderer/views/Moodboard/nodes/LoaderNode.tsx +++ b/src/renderer/views/Moodboard/nodes/LoaderNode.tsx @@ -10,6 +10,7 @@ import { ASSET_DND_TYPE, MEDIA_FILE_DND_TYPE, } from '../../../lib/dnd' +import { LOADER_CHROME_H, LOADER_MIN_BODY, LOADER_MIN_W } from '../../../lib/loaderFit' import { useMediaContextMenu } from '../../../lib/mediaContextMenu' import { resolveMedia } from '@/lib/media' import { @@ -18,8 +19,8 @@ import { pickFilesViaInput, } from '../../../lib/importFiles' import { useLightboxStore } from '../../../store/lightboxStore' -import { VideoPreview } from '../../../components/VideoPreview' import { Waveform } from '../../../components/Waveform' +import { MediaBody } from './MediaBody' import { NodeFrame } from './NodeFrame' import { ImageGlyph, NodeBadge, NodeBadgeRow, StarIcon, UploadIcon } from './NodeBadge' import { ThumbStrip } from './ThumbStrip' @@ -119,6 +120,8 @@ export function LoaderNode({ id, selected }: NodeProps): React.JSX.Element { } const heroName = cur ? (assets.find((a) => a.id === cur.assetId)?.name ?? 'asset') : 'asset' + // Audio renders as a waveform, so the media body only ever sees the other two kinds. + const mediaKind: 'image' | 'video' = cur?.kind === 'video' ? 'video' : 'image' return ( <> @@ -131,8 +134,8 @@ export function LoaderNode({ id, selected }: NodeProps): React.JSX.Element { @@ -144,33 +147,21 @@ export function LoaderNode({ id, selected }: NodeProps): React.JSX.Element { >
{cur ? ( - cur.kind === 'video' ? ( - - onMediaContextMenu(e, { src: cur.saveSrc, name: heroName, kind: 'video' }) - } - onDoubleClick={() => - openLightbox({ src: cur.saveSrc, kind: 'video', name: heroName }) - } - className="h-full w-full object-contain" - /> - ) : cur.kind === 'audio' ? ( + cur.kind === 'audio' ? (
) : ( - - onMediaContextMenu(e, { src: cur.saveSrc, name: heroName, kind: 'image' }) + onMediaContextMenu(e, { src: cur.saveSrc, name: heroName, kind: mediaKind }) } onDoubleClick={() => - openLightbox({ src: cur.saveSrc, kind: 'image', name: heroName }) + openLightbox({ src: cur.saveSrc, kind: mediaKind, name: heroName }) } - className="h-full w-full object-contain" /> ) ) : ( diff --git a/src/renderer/views/Moodboard/nodes/MediaBody.tsx b/src/renderer/views/Moodboard/nodes/MediaBody.tsx new file mode 100644 index 0000000..c699ccc --- /dev/null +++ b/src/renderer/views/Moodboard/nodes/MediaBody.tsx @@ -0,0 +1,113 @@ +import { useEffect, useRef, useState } from 'react' +import { VideoPreview } from '../../../components/VideoPreview' +import { needsBlurFill } from '../../../lib/loaderFit' + +/** Downscaled hard so the blurred backdrop costs a thumbnail, not a second full-res decode. */ +const FRAME_GRAB_W = 64 + +// A node's media, plus a blurred copy of itself filling whatever a contain fit leaves over. +export function MediaBody({ + src, + kind, + poster, + fit = 'contain', + onContextMenu, + onDoubleClick, + onAspect, +}: { + src: string + kind: 'image' | 'video' + poster?: string + fit?: 'contain' | 'cover' + onContextMenu?: (e: React.MouseEvent) => void + onDoubleClick?: () => void + /** Fires with the media's intrinsic width/height ratio once it decodes. */ + onAspect?: (aspect: number) => void +}): React.JSX.Element { + const boxRef = useRef(null) + const [aspect, setAspect] = useState(null) + const [boxAspect, setBoxAspect] = useState(null) + // A video frame can't be a CSS background, so one grab stands in as the backdrop's still. + const [videoStill, setVideoStill] = useState(null) + + useEffect(() => { + setAspect(null) + setVideoStill(null) + }, [src]) + + useEffect(() => { + const box = boxRef.current + if (!box || fit !== 'contain') return + const ro = new ResizeObserver(([entry]) => { + const { width, height } = entry.contentRect + setBoxAspect(width > 0 && height > 0 ? width / height : null) + }) + ro.observe(box) + return () => ro.disconnect() + }, [fit]) + + const takeAspect = (w: number, h: number): void => { + if (!w || !h) return + setAspect(w / h) + onAspect?.(w / h) + } + + const grabStill = (v: HTMLVideoElement): void => { + if (fit !== 'contain' || videoStill || !v.videoWidth || !v.videoHeight) return + const canvas = document.createElement('canvas') + canvas.width = FRAME_GRAB_W + canvas.height = Math.max(1, Math.round((FRAME_GRAB_W * v.videoHeight) / v.videoWidth)) + const ctx = canvas.getContext('2d') + if (!ctx) return + try { + ctx.drawImage(v, 0, 0, canvas.width, canvas.height) + setVideoStill(canvas.toDataURL('image/jpeg', 0.6)) + } catch { + // A frame we can't read just means no backdrop; the plain black body still works. + } + } + + const fitClass = fit === 'cover' ? 'object-cover' : 'object-contain' + const backdrop = kind === 'video' ? (videoStill ?? poster) : src + const showBlur = + fit === 'contain' && !!backdrop && aspect != null && boxAspect != null + ? needsBlurFill(aspect, boxAspect) + : false + + return ( +
+ {showBlur && ( + <> +
+
+ + )} + {kind === 'video' ? ( + + takeAspect(e.currentTarget.videoWidth, e.currentTarget.videoHeight) + } + onLoadedData={(e) => grabStill(e.currentTarget)} + onContextMenu={onContextMenu} + onDoubleClick={onDoubleClick} + className={`relative h-full w-full ${fitClass}`} + /> + ) : ( + takeAspect(e.currentTarget.naturalWidth, e.currentTarget.naturalHeight)} + onContextMenu={onContextMenu} + onDoubleClick={onDoubleClick} + className={`relative h-full w-full ${fitClass}`} + /> + )} +
+ ) +} diff --git a/src/renderer/views/Moodboard/nodes/NodeBadge.tsx b/src/renderer/views/Moodboard/nodes/NodeBadge.tsx index 6a3d9ae..43577c2 100644 --- a/src/renderer/views/Moodboard/nodes/NodeBadge.tsx +++ b/src/renderer/views/Moodboard/nodes/NodeBadge.tsx @@ -365,6 +365,32 @@ export function SparkleIcon({ className }: { className?: string }): React.JSX.El ) } +/** The fal wordmark in a bordered chip - marks a node that runs on fal, not locally. */ +export function FalIcon({ className }: { className?: string }): React.JSX.Element { + return ( + + ) +} + /** Box - a loader-style node (`icon:"box"`). */ export function BoxIcon({ className }: { className?: string }): React.JSX.Element { return ( @@ -460,3 +486,32 @@ export function UploadIcon({ className }: { className?: string }): React.JSX.Ele ) } + +/** The working indicator on a running node's Current slot: three dots lighting in sequence. */ +export function RunningDots({ className }: { className?: string }): React.JSX.Element { + return ( + + + + + + ) +} + +/** Pencil, marking the strip slot whose settings have been edited but not yet rendered. */ +export function PencilIcon({ className }: { className?: string }): React.JSX.Element { + return ( + + + + + ) +} diff --git a/src/renderer/views/Moodboard/nodes/ThumbStrip.tsx b/src/renderer/views/Moodboard/nodes/ThumbStrip.tsx index 27910d3..6cdf2a6 100644 --- a/src/renderer/views/Moodboard/nodes/ThumbStrip.tsx +++ b/src/renderer/views/Moodboard/nodes/ThumbStrip.tsx @@ -32,6 +32,7 @@ export function ThumbStrip({ onSelect, onRemove, edge = 'bottom', + leading, }: { items: StripItem[] selected?: number @@ -39,10 +40,13 @@ export function ThumbStrip({ /** When provided, each thumbnail gets a hover × that calls this with its index. */ onRemove?: (index: number) => void edge?: 'top' | 'bottom' + /** A tile pinned before the items, owning no media of its own. Used for the Current slot, whose + * presence is also what forces the strip open on a node with fewer than two takes. */ + leading?: React.ReactNode }): React.JSX.Element | null { const manage = !!onRemove - if (items.length === 0) return null - if (!manage && items.length <= 1) return null + if (items.length === 0 && !leading) return null + if (!manage && !leading && items.length <= 1) return null const scrim = edge === 'top' ? 'top-0 bg-gradient-to-b from-black/80 via-black/40 to-transparent pt-1.5 pb-5' @@ -54,6 +58,7 @@ export function ThumbStrip({
+ {leading} {items.map((it, i) => (