Skip to content
Draft
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -1247,18 +1247,31 @@ private struct EngineImpl: ~Copyable {
}
}

continuation.yield(lastToken)
// NOTE: `lastToken` is deliberately NOT yielded here. A sampled token is
// only emitted after the grammar has accepted it (at the top of the loop)
// and we've confirmed the acceptance did not terminate the grammar.
// Otherwise the grammar's terminal token (e.g. <|endoftext|>, 151643) would
// be streamed to the consumer before termination is detected and would leak
// into the structured output. This mirrors the CPU ConstrainedDecodingStrategy
// fix in #117, which returns nil instead of emitting the terminal token's text.

// Constrained decode loop
var generated = 1
var generated = 0
while generated < maxTokens {
guard !isCancelled.load(ordering: .relaxed) else { break }
try Task.checkCancellation()

// Accept previous token in grammar
// Accept the most recently sampled token in the grammar.
if !session.acceptToken(lastToken) { break }
// If accepting it terminated the grammar, `lastToken` is the terminal
// (stop) token — break WITHOUT emitting it.
if session.isTerminated { break }

// `lastToken` is confirmed to be valid structured content — emit it now.
continuation.yield(lastToken)
generated += 1
if generated >= maxTokens { break }

// Check for jump-forward: deterministic grammar segments that can be
// batch-encoded without per-token sampling (saves N-1 round-trips)
if let jumpString = session.findJumpForwardString(),
Expand Down Expand Up @@ -1288,12 +1301,13 @@ private struct EngineImpl: ~Copyable {
}
}

// Yield the jump-forward tokens (deterministic) + the sampled token
// Emit the deterministic jump-forward tokens now. The freshly sampled
// `lastToken` is deferred to the next iteration's accept/termination
// check so a terminal token never leaks into the output.
for jt in jumpTokens {
continuation.yield(jt)
}
continuation.yield(lastToken)
generated += jumpTokens.count + 1
generated += jumpTokens.count
continue
}

Expand Down Expand Up @@ -1322,8 +1336,7 @@ private struct EngineImpl: ~Copyable {
}
}

continuation.yield(lastToken)
generated += 1
// `lastToken` is deferred to the next iteration's accept/termination check.
}

// Drain: sentinel command buffer to ensure all GPU work completes
Expand Down