From 71c08f1a58751d0e00e01cff28cd8644c9a8b3b0 Mon Sep 17 00:00:00 2001 From: Brian Carbone Date: Mon, 27 Jul 2026 18:29:50 -0400 Subject: [PATCH 01/11] fix: cache processDoc results across requests --- .../src/service/script/index.ts | 38 ++++++++----- .../src/service/style/index.ts | 54 ++++++++++--------- packages/language-server/src/utils/file.ts | 21 +++----- 3 files changed, 60 insertions(+), 53 deletions(-) diff --git a/packages/language-server/src/service/script/index.ts b/packages/language-server/src/service/script/index.ts index ee2304ca..a2eba4a9 100644 --- a/packages/language-server/src/service/script/index.ts +++ b/packages/language-server/src/service/script/index.ts @@ -647,21 +647,31 @@ const ScriptService: Partial = { }, }; +const processScriptCallbacks = new WeakMap< + TSProject, + (file: MarkoFile) => Extracted +>(); + function processScript(doc: TextDocument, tsProject: TSProject) { - return processDoc(doc, ({ filename, parsed, lookup, dirname }) => { - const { host, markoScriptLang } = tsProject; - return extractScript({ - ts, - parsed, - lookup, - translator: Project.getConfig(dirname).translator, - scriptLang: filename - ? Project.getScriptLang(filename, markoScriptLang, ts, host) - : markoScriptLang, - runtimeTypesCode: Project.getTypeLibs(tsProject.rootDir, ts, host) - ?.markoTypesCode, - }); - }); + let process = processScriptCallbacks.get(tsProject); + if (!process) { + process = ({ filename, parsed, lookup, dirname }) => { + const { host, markoScriptLang } = tsProject; + return extractScript({ + ts, + parsed, + lookup, + translator: Project.getConfig(dirname).translator, + scriptLang: filename + ? Project.getScriptLang(filename, markoScriptLang, ts, host) + : markoScriptLang, + runtimeTypesCode: Project.getTypeLibs(tsProject.rootDir, ts, host) + ?.markoTypesCode, + }); + }; + processScriptCallbacks.set(tsProject, process); + } + return processDoc(doc, process); } /** diff --git a/packages/language-server/src/service/style/index.ts b/packages/language-server/src/service/style/index.ts index 0fe83a81..17e83090 100644 --- a/packages/language-server/src/service/style/index.ts +++ b/packages/language-server/src/service/style/index.ts @@ -23,7 +23,7 @@ import { } from "vscode-languageserver"; import { TextDocument } from "vscode-languageserver-textdocument"; -import { processDoc } from "../../utils/file"; +import { type MarkoFile, processDoc } from "../../utils/file"; import fileSystemProvider from "../../utils/file-system"; import resolveReference from "../../utils/resolve-url"; import type { Plugin } from "../types"; @@ -405,34 +405,36 @@ const StyleSheetService: Partial = { export { StyleSheetService as default }; function processStyle(doc: TextDocument) { - return processDoc(doc, ({ uri, version, parsed, lookup }) => { - const result: ProcessedStyle[] = []; - for (const [ext, extracted] of extractStyle({ - parsed, - lookup, - })) { - const service = services[ext]?.({ - fileSystemProvider, - clientCapabilities, + return processDoc(doc, extractStyles); +} + +function extractStyles({ uri, version, parsed, lookup }: MarkoFile) { + const result: ProcessedStyle[] = []; + for (const [ext, extracted] of extractStyle({ + parsed, + lookup, + })) { + const service = services[ext]?.({ + fileSystemProvider, + clientCapabilities, + }); + if (service) { + const virtualDoc = TextDocument.create( + uri, + "css", + version, + extracted.toString(), + ); + result.push({ + service, + extracted, + virtualDoc, + parsed: service.parseStylesheet(virtualDoc), }); - if (service) { - const virtualDoc = TextDocument.create( - uri, - "css", - version, - extracted.toString(), - ); - result.push({ - service, - extracted, - virtualDoc, - parsed: service.parseStylesheet(virtualDoc), - }); - } } + } - return result; - }); + return result; } function getSourceEdits( diff --git a/packages/language-server/src/utils/file.ts b/packages/language-server/src/utils/file.ts index f0e34049..72e76f12 100644 --- a/packages/language-server/src/utils/file.ts +++ b/packages/language-server/src/utils/file.ts @@ -70,21 +70,16 @@ export function processDoc( process: (file: MarkoFile) => T, ): T { const file = getMarkoFile(doc); - const cache = processorCaches.get(file.parsed) as - | Map - | undefined; - let result: T | undefined; + let cache = processorCaches.get(file.parsed); + if (!cache) { + processorCaches.set(file.parsed, (cache = new Map())); + } - if (cache) { - result = cache.get(process); - if (!result) { - result = process(file); - cache.set(process, result); - } - } else { - result = process(file); - processorCaches.set(file.parsed, new Map([[process, result]])); + if (cache.has(process)) { + return cache.get(process) as T; } + const result = process(file); + cache.set(process, result); return result; } From 41616e112d6272e84ea1b496edd9cf448b450caf Mon Sep 17 00:00:00 2001 From: Brian Carbone Date: Tue, 28 Jul 2026 09:32:56 -0400 Subject: [PATCH 02/11] feat: semantic tokens --- .changeset/semantic-tokens.md | 6 + .../__snapshots__/concise.expected/index.html | 1 + .../__snapshots__/concise.expected/index.md | 0 .../concise.expected/index.tokens.md | 3 + .../__snapshots__/concise.expected/index.ts | 83 +++++++ .../semantic-tokens/concise/index.marko | 3 + .../core-tags.expected/index.html | 10 + .../__snapshots__/core-tags.expected/index.md | 0 .../core-tags.expected/index.tokens.md | 16 ++ .../__snapshots__/core-tags.expected/index.ts | 109 ++++++++ .../semantic-tokens/core-tags/index.marko | 13 + .../components/my-button.html | 1 + .../components/my-button.md | 0 .../components/my-button.tokens.md | 4 + .../components/my-button.ts | 64 +++++ .../custom-tag.expected/index.html | 4 + .../custom-tag.expected/index.md | 0 .../custom-tag.expected/index.tokens.md | 2 + .../custom-tag.expected/index.ts | 71 ++++++ .../custom-tag/components/my-button.marko | 5 + .../semantic-tokens/custom-tag/index.marko | 6 + .../edge-tags.expected/index.html | 3 + .../__snapshots__/edge-tags.expected/index.md | 0 .../edge-tags.expected/index.tokens.md | 4 + .../__snapshots__/edge-tags.expected/index.ts | 71 ++++++ .../semantic-tokens/edge-tags/index.marko | 4 + .../expressions.expected/index.html | 3 + .../expressions.expected/index.md | 0 .../expressions.expected/index.tokens.md | 6 + .../expressions.expected/index.ts | 79 ++++++ .../semantic-tokens/expressions/index.marko | 6 + .../script-blocks.expected/index.html | 1 + .../script-blocks.expected/index.md | 0 .../script-blocks.expected/index.tokens.md | 9 + .../script-blocks.expected/index.ts | 67 +++++ .../semantic-tokens/script-blocks/index.marko | 9 + .../semantic-tokens/script-blocks/util.ts | 3 + .../style-block.expected/index.html | 1 + .../style-block.expected/index.md | 0 .../style-block.expected/index.tokens.md | 1 + .../style-block.expected/index.ts | 56 +++++ .../semantic-tokens/style-block/index.marko | 7 + .../tag-params.expected/index.html | 5 + .../tag-params.expected/index.md | 0 .../tag-params.expected/index.tokens.md | 7 + .../tag-params.expected/index.ts | 77 ++++++ .../semantic-tokens/tag-params/index.marko | 5 + .../src/__tests__/index.test.ts | 24 ++ .../src/__tests__/semantic-tokens.test.ts | 235 ++++++++++++++++++ .../src/__tests__/util/semantic-tokens.ts | 68 +++++ packages/language-server/src/browser.ts | 3 +- packages/language-server/src/index.ts | 64 ++++- .../src/service/create-service.ts | 96 ++++++- .../src/service/marko/index.ts | 2 + .../src/service/marko/semantic-tokens.ts | 118 +++++++++ .../marko/util/get-tag-name-completion.ts | 7 +- .../src/service/marko/util/is-core-tag.ts | 18 ++ .../src/service/script/index.ts | 153 +++++++++++- .../src/service/semantic-tokens.ts | 57 +++++ packages/language-server/src/service/types.ts | 13 + 60 files changed, 1674 insertions(+), 9 deletions(-) create mode 100644 .changeset/semantic-tokens.md create mode 100644 packages/language-server/src/__tests__/fixtures/semantic-tokens/concise/__snapshots__/concise.expected/index.html create mode 100644 packages/language-server/src/__tests__/fixtures/semantic-tokens/concise/__snapshots__/concise.expected/index.md create mode 100644 packages/language-server/src/__tests__/fixtures/semantic-tokens/concise/__snapshots__/concise.expected/index.tokens.md create mode 100644 packages/language-server/src/__tests__/fixtures/semantic-tokens/concise/__snapshots__/concise.expected/index.ts create mode 100644 packages/language-server/src/__tests__/fixtures/semantic-tokens/concise/index.marko create mode 100644 packages/language-server/src/__tests__/fixtures/semantic-tokens/core-tags/__snapshots__/core-tags.expected/index.html create mode 100644 packages/language-server/src/__tests__/fixtures/semantic-tokens/core-tags/__snapshots__/core-tags.expected/index.md create mode 100644 packages/language-server/src/__tests__/fixtures/semantic-tokens/core-tags/__snapshots__/core-tags.expected/index.tokens.md create mode 100644 packages/language-server/src/__tests__/fixtures/semantic-tokens/core-tags/__snapshots__/core-tags.expected/index.ts create mode 100644 packages/language-server/src/__tests__/fixtures/semantic-tokens/core-tags/index.marko create mode 100644 packages/language-server/src/__tests__/fixtures/semantic-tokens/custom-tag/__snapshots__/custom-tag.expected/components/my-button.html create mode 100644 packages/language-server/src/__tests__/fixtures/semantic-tokens/custom-tag/__snapshots__/custom-tag.expected/components/my-button.md create mode 100644 packages/language-server/src/__tests__/fixtures/semantic-tokens/custom-tag/__snapshots__/custom-tag.expected/components/my-button.tokens.md create mode 100644 packages/language-server/src/__tests__/fixtures/semantic-tokens/custom-tag/__snapshots__/custom-tag.expected/components/my-button.ts create mode 100644 packages/language-server/src/__tests__/fixtures/semantic-tokens/custom-tag/__snapshots__/custom-tag.expected/index.html create mode 100644 packages/language-server/src/__tests__/fixtures/semantic-tokens/custom-tag/__snapshots__/custom-tag.expected/index.md create mode 100644 packages/language-server/src/__tests__/fixtures/semantic-tokens/custom-tag/__snapshots__/custom-tag.expected/index.tokens.md create mode 100644 packages/language-server/src/__tests__/fixtures/semantic-tokens/custom-tag/__snapshots__/custom-tag.expected/index.ts create mode 100644 packages/language-server/src/__tests__/fixtures/semantic-tokens/custom-tag/components/my-button.marko create mode 100644 packages/language-server/src/__tests__/fixtures/semantic-tokens/custom-tag/index.marko create mode 100644 packages/language-server/src/__tests__/fixtures/semantic-tokens/edge-tags/__snapshots__/edge-tags.expected/index.html create mode 100644 packages/language-server/src/__tests__/fixtures/semantic-tokens/edge-tags/__snapshots__/edge-tags.expected/index.md create mode 100644 packages/language-server/src/__tests__/fixtures/semantic-tokens/edge-tags/__snapshots__/edge-tags.expected/index.tokens.md create mode 100644 packages/language-server/src/__tests__/fixtures/semantic-tokens/edge-tags/__snapshots__/edge-tags.expected/index.ts create mode 100644 packages/language-server/src/__tests__/fixtures/semantic-tokens/edge-tags/index.marko create mode 100644 packages/language-server/src/__tests__/fixtures/semantic-tokens/expressions/__snapshots__/expressions.expected/index.html create mode 100644 packages/language-server/src/__tests__/fixtures/semantic-tokens/expressions/__snapshots__/expressions.expected/index.md create mode 100644 packages/language-server/src/__tests__/fixtures/semantic-tokens/expressions/__snapshots__/expressions.expected/index.tokens.md create mode 100644 packages/language-server/src/__tests__/fixtures/semantic-tokens/expressions/__snapshots__/expressions.expected/index.ts create mode 100644 packages/language-server/src/__tests__/fixtures/semantic-tokens/expressions/index.marko create mode 100644 packages/language-server/src/__tests__/fixtures/semantic-tokens/script-blocks/__snapshots__/script-blocks.expected/index.html create mode 100644 packages/language-server/src/__tests__/fixtures/semantic-tokens/script-blocks/__snapshots__/script-blocks.expected/index.md create mode 100644 packages/language-server/src/__tests__/fixtures/semantic-tokens/script-blocks/__snapshots__/script-blocks.expected/index.tokens.md create mode 100644 packages/language-server/src/__tests__/fixtures/semantic-tokens/script-blocks/__snapshots__/script-blocks.expected/index.ts create mode 100644 packages/language-server/src/__tests__/fixtures/semantic-tokens/script-blocks/index.marko create mode 100644 packages/language-server/src/__tests__/fixtures/semantic-tokens/script-blocks/util.ts create mode 100644 packages/language-server/src/__tests__/fixtures/semantic-tokens/style-block/__snapshots__/style-block.expected/index.html create mode 100644 packages/language-server/src/__tests__/fixtures/semantic-tokens/style-block/__snapshots__/style-block.expected/index.md create mode 100644 packages/language-server/src/__tests__/fixtures/semantic-tokens/style-block/__snapshots__/style-block.expected/index.tokens.md create mode 100644 packages/language-server/src/__tests__/fixtures/semantic-tokens/style-block/__snapshots__/style-block.expected/index.ts create mode 100644 packages/language-server/src/__tests__/fixtures/semantic-tokens/style-block/index.marko create mode 100644 packages/language-server/src/__tests__/fixtures/semantic-tokens/tag-params/__snapshots__/tag-params.expected/index.html create mode 100644 packages/language-server/src/__tests__/fixtures/semantic-tokens/tag-params/__snapshots__/tag-params.expected/index.md create mode 100644 packages/language-server/src/__tests__/fixtures/semantic-tokens/tag-params/__snapshots__/tag-params.expected/index.tokens.md create mode 100644 packages/language-server/src/__tests__/fixtures/semantic-tokens/tag-params/__snapshots__/tag-params.expected/index.ts create mode 100644 packages/language-server/src/__tests__/fixtures/semantic-tokens/tag-params/index.marko create mode 100644 packages/language-server/src/__tests__/semantic-tokens.test.ts create mode 100644 packages/language-server/src/__tests__/util/semantic-tokens.ts create mode 100644 packages/language-server/src/service/marko/semantic-tokens.ts create mode 100644 packages/language-server/src/service/marko/util/is-core-tag.ts create mode 100644 packages/language-server/src/service/semantic-tokens.ts diff --git a/.changeset/semantic-tokens.md b/.changeset/semantic-tokens.md new file mode 100644 index 00000000..ba839fa8 --- /dev/null +++ b/.changeset/semantic-tokens.md @@ -0,0 +1,6 @@ +--- +"@marko/language-server": minor +"marko-vscode": minor +--- + +Add semantic token support (`textDocument/semanticTokens` full and range). Custom component tags stop looking like native HTML — `` and `` both render in the component/type color, matching how `.tsx` files color ``, while `
` keeps its tag color. Core tags the grammar doesn't recognize (``, ``, ``) pick up core-tag coloring, and identifiers in placeholders, attribute values, tag params/vars, and `static`/`server`/`class` blocks are colored from TypeScript's view of the code. diff --git a/packages/language-server/src/__tests__/fixtures/semantic-tokens/concise/__snapshots__/concise.expected/index.html b/packages/language-server/src/__tests__/fixtures/semantic-tokens/concise/__snapshots__/concise.expected/index.html new file mode 100644 index 00000000..029cfa59 --- /dev/null +++ b/packages/language-server/src/__tests__/fixtures/semantic-tokens/concise/__snapshots__/concise.expected/index.html @@ -0,0 +1 @@ +
total placeholder
\ No newline at end of file diff --git a/packages/language-server/src/__tests__/fixtures/semantic-tokens/concise/__snapshots__/concise.expected/index.md b/packages/language-server/src/__tests__/fixtures/semantic-tokens/concise/__snapshots__/concise.expected/index.md new file mode 100644 index 00000000..e69de29b diff --git a/packages/language-server/src/__tests__/fixtures/semantic-tokens/concise/__snapshots__/concise.expected/index.tokens.md b/packages/language-server/src/__tests__/fixtures/semantic-tokens/concise/__snapshots__/concise.expected/index.tokens.md new file mode 100644 index 00000000..9b3a023d --- /dev/null +++ b/packages/language-server/src/__tests__/fixtures/semantic-tokens/concise/__snapshots__/concise.expected/index.tokens.md @@ -0,0 +1,3 @@ +Ln 2, Col 3 (len 3) macro `let` +Ln 2, Col 7 (len 5) variable [declaration local] `count` +Ln 3, Col 19 (len 5) variable [local] `count` diff --git a/packages/language-server/src/__tests__/fixtures/semantic-tokens/concise/__snapshots__/concise.expected/index.ts b/packages/language-server/src/__tests__/fixtures/semantic-tokens/concise/__snapshots__/concise.expected/index.ts new file mode 100644 index 00000000..bf388ed9 --- /dev/null +++ b/packages/language-server/src/__tests__/fixtures/semantic-tokens/concise/__snapshots__/concise.expected/index.ts @@ -0,0 +1,83 @@ +export interface Input {} +(function (this: void) { + const input = Marko._.any as Input; + const $signal = Marko._.any as AbortSignal; + const $global = Marko._.getGlobal( + // @ts-expect-error We expect the compiler to error because we are checking if the MarkoRun.Context is defined. + (Marko._.error, Marko._.any as MarkoRun.Context), + ); + const count = Marko._.hoist(() => __marko_internal_hoist__count); + const __marko_internal_rendered_1 = Marko._.renderNativeTag("div")()()({ + class: Marko._.interpolated`layout`, + [Marko._.content]: (() => { + const __marko_internal_tag_2 = Marko._.resolveTemplate( + import("@marko/runtime-tags/tags/let.d.marko"), + ); + { + const count = Marko._.returned(() => __marko_internal_rendered_2); + const __marko_internal_rendered_2 = Marko._.renderTemplate( + __marko_internal_tag_2, + )()()({ + value: 7, + }); + Marko._.renderNativeTag("span")()()({ + [Marko._.content]: (() => { + count; + return () => { + return Marko._.voidReturn; + }; + })(), + }); + return () => { + return new (class MarkoReturn { + readonly [Marko._.scope] = { count }; + declare return: Return; + constructor(_?: Return) {} + })(); + }; + } + })(), + }); + var { count: __marko_internal_hoist__count } = Marko._.readScope( + __marko_internal_rendered_1, + ); + Marko._.noop({ count, input, $global, $signal }); + return; +})(); +const __marko_internal_api = "tags"; +export { __marko_internal_api as "~api" }; +export default new (class Template extends Marko._.Template<{ + render( + input: Marko.TemplateInput, + stream?: { + write: (chunk: string) => void; + end: (chunk?: string) => void; + }, + ): Marko.Out; + + render( + input: Marko.TemplateInput, + cb?: (err: Error | null, result: Marko.RenderResult) => void, + ): Marko.Out; + + renderSync(input: Marko.TemplateInput): Marko.RenderResult; + + renderToString(input: Marko.TemplateInput): string; + + stream( + input: Marko.TemplateInput, + ): ReadableStream & NodeJS.ReadableStream; + + mount( + input: Marko.TemplateInput, + reference: Node, + position?: "afterbegin" | "afterend" | "beforebegin" | "beforeend", + ): Marko.MountedTemplate; + + api: typeof __marko_internal_api; + _(): () => <__marko_internal_input extends unknown>( + input: Marko.Directives & + Input & + Marko._.Relate<__marko_internal_input, Marko.Directives & Input>, + ) => Marko._.ReturnWithScope<__marko_internal_input, void>; +}> {})(); diff --git a/packages/language-server/src/__tests__/fixtures/semantic-tokens/concise/index.marko b/packages/language-server/src/__tests__/fixtures/semantic-tokens/concise/index.marko new file mode 100644 index 00000000..3dc1babe --- /dev/null +++ b/packages/language-server/src/__tests__/fixtures/semantic-tokens/concise/index.marko @@ -0,0 +1,3 @@ +div.layout + let/count=7 + span -- total ${count} diff --git a/packages/language-server/src/__tests__/fixtures/semantic-tokens/core-tags/__snapshots__/core-tags.expected/index.html b/packages/language-server/src/__tests__/fixtures/semantic-tokens/core-tags/__snapshots__/core-tags.expected/index.html new file mode 100644 index 00000000..bd3d25e3 --- /dev/null +++ b/packages/language-server/src/__tests__/fixtures/semantic-tokens/core-tags/__snapshots__/core-tags.expected/index.html @@ -0,0 +1,10 @@ +
    +
    +
  • placeholder placeholder
  • +
    +
+ + placeholder +
+ none +
\ No newline at end of file diff --git a/packages/language-server/src/__tests__/fixtures/semantic-tokens/core-tags/__snapshots__/core-tags.expected/index.md b/packages/language-server/src/__tests__/fixtures/semantic-tokens/core-tags/__snapshots__/core-tags.expected/index.md new file mode 100644 index 00000000..e69de29b diff --git a/packages/language-server/src/__tests__/fixtures/semantic-tokens/core-tags/__snapshots__/core-tags.expected/index.tokens.md b/packages/language-server/src/__tests__/fixtures/semantic-tokens/core-tags/__snapshots__/core-tags.expected/index.tokens.md new file mode 100644 index 00000000..8f32d1b4 --- /dev/null +++ b/packages/language-server/src/__tests__/fixtures/semantic-tokens/core-tags/__snapshots__/core-tags.expected/index.tokens.md @@ -0,0 +1,16 @@ +Ln 1, Col 2 (len 3) macro `let` +Ln 1, Col 6 (len 5) variable [declaration local] `count` +Ln 3, Col 4 (len 3) keyword `for` +Ln 3, Col 8 (len 4) parameter [declaration] `item` +Ln 4, Col 11 (len 4) parameter `item` +Ln 4, Col 19 (len 5) variable [local] `count` +Ln 5, Col 5 (len 3) keyword `for` +Ln 7, Col 2 (len 2) keyword `if` +Ln 7, Col 5 (len 5) variable [local] `count` +Ln 8, Col 4 (len 5) macro `const` +Ln 8, Col 10 (len 7) variable [declaration readonly local] `doubled` +Ln 8, Col 20 (len 5) variable [local] `count` +Ln 9, Col 5 (len 7) variable [readonly local] `doubled` +Ln 10, Col 3 (len 2) keyword `if` +Ln 11, Col 2 (len 4) keyword `else` +Ln 13, Col 3 (len 4) keyword `else` diff --git a/packages/language-server/src/__tests__/fixtures/semantic-tokens/core-tags/__snapshots__/core-tags.expected/index.ts b/packages/language-server/src/__tests__/fixtures/semantic-tokens/core-tags/__snapshots__/core-tags.expected/index.ts new file mode 100644 index 00000000..3eb34131 --- /dev/null +++ b/packages/language-server/src/__tests__/fixtures/semantic-tokens/core-tags/__snapshots__/core-tags.expected/index.ts @@ -0,0 +1,109 @@ +export interface Input {} +(function (this: void) { + const input = Marko._.any as Input; + const $signal = Marko._.any as AbortSignal; + const $global = Marko._.getGlobal( + // @ts-expect-error We expect the compiler to error because we are checking if the MarkoRun.Context is defined. + (Marko._.error, Marko._.any as MarkoRun.Context), + ); + const count = Marko._.hoist(() => __marko_internal_hoist__count); + const doubled = Marko._.hoist(() => __marko_internal_hoist__doubled); + const __marko_internal_tag_1 = Marko._.resolveTemplate( + import("@marko/runtime-tags/tags/let.d.marko"), + ); + { + const count = Marko._.returned(() => __marko_internal_rendered_1); + const __marko_internal_rendered_1 = Marko._.renderTemplate( + __marko_internal_tag_1, + )()()({ + value: 1, + }); + Marko._.renderNativeTag("ul")()()({ + [Marko._.content]: (() => { + Marko._.forOfTag( + { + of: ["a", "b"], + }, + (item) => { + Marko._.renderNativeTag("li")()()({ + [Marko._.content]: (() => { + item; + count; + return () => { + return Marko._.voidReturn; + }; + })(), + }); + return Marko._.voidReturn; + }, + ); + return () => { + return Marko._.voidReturn; + }; + })(), + }); + const __marko_internal_rendered_2 = (() => { + if (count) { + const __marko_internal_tag_3 = Marko._.resolveTemplate( + import("@marko/runtime-tags/tags/const.d.marko"), + ); + { + const doubled = Marko._.returned(() => __marko_internal_rendered_3); + const __marko_internal_rendered_3 = Marko._.renderTemplate( + __marko_internal_tag_3, + )()()({ + value: count * 2, + }); + doubled; + return { + scope: { doubled }, + }; + } + } else { + } + })(); + var __marko_internal_hoist__count = count; + var { doubled: __marko_internal_hoist__doubled } = Marko._.readScope( + __marko_internal_rendered_2, + ); + } + Marko._.noop({ count, doubled, input, $global, $signal }); + return; +})(); +const __marko_internal_api = "tags"; +export { __marko_internal_api as "~api" }; +export default new (class Template extends Marko._.Template<{ + render( + input: Marko.TemplateInput, + stream?: { + write: (chunk: string) => void; + end: (chunk?: string) => void; + }, + ): Marko.Out; + + render( + input: Marko.TemplateInput, + cb?: (err: Error | null, result: Marko.RenderResult) => void, + ): Marko.Out; + + renderSync(input: Marko.TemplateInput): Marko.RenderResult; + + renderToString(input: Marko.TemplateInput): string; + + stream( + input: Marko.TemplateInput, + ): ReadableStream & NodeJS.ReadableStream; + + mount( + input: Marko.TemplateInput, + reference: Node, + position?: "afterbegin" | "afterend" | "beforebegin" | "beforeend", + ): Marko.MountedTemplate; + + api: typeof __marko_internal_api; + _(): () => <__marko_internal_input extends unknown>( + input: Marko.Directives & + Input & + Marko._.Relate<__marko_internal_input, Marko.Directives & Input>, + ) => Marko._.ReturnWithScope<__marko_internal_input, void>; +}> {})(); diff --git a/packages/language-server/src/__tests__/fixtures/semantic-tokens/core-tags/index.marko b/packages/language-server/src/__tests__/fixtures/semantic-tokens/core-tags/index.marko new file mode 100644 index 00000000..c99535a6 --- /dev/null +++ b/packages/language-server/src/__tests__/fixtures/semantic-tokens/core-tags/index.marko @@ -0,0 +1,13 @@ + +
    + +
  • ${item} ${count}
  • + +
+ + + ${doubled} + + + none + diff --git a/packages/language-server/src/__tests__/fixtures/semantic-tokens/custom-tag/__snapshots__/custom-tag.expected/components/my-button.html b/packages/language-server/src/__tests__/fixtures/semantic-tokens/custom-tag/__snapshots__/custom-tag.expected/components/my-button.html new file mode 100644 index 00000000..88afc79b --- /dev/null +++ b/packages/language-server/src/__tests__/fixtures/semantic-tokens/custom-tag/__snapshots__/custom-tag.expected/components/my-button.html @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/packages/language-server/src/__tests__/fixtures/semantic-tokens/custom-tag/__snapshots__/custom-tag.expected/components/my-button.md b/packages/language-server/src/__tests__/fixtures/semantic-tokens/custom-tag/__snapshots__/custom-tag.expected/components/my-button.md new file mode 100644 index 00000000..e69de29b diff --git a/packages/language-server/src/__tests__/fixtures/semantic-tokens/custom-tag/__snapshots__/custom-tag.expected/components/my-button.tokens.md b/packages/language-server/src/__tests__/fixtures/semantic-tokens/custom-tag/__snapshots__/custom-tag.expected/components/my-button.tokens.md new file mode 100644 index 00000000..eeddf87f --- /dev/null +++ b/packages/language-server/src/__tests__/fixtures/semantic-tokens/custom-tag/__snapshots__/custom-tag.expected/components/my-button.tokens.md @@ -0,0 +1,4 @@ +Ln 1, Col 18 (len 5) interface [declaration] `Input` +Ln 2, Col 3 (len 5) property [declaration] `label` +Ln 5, Col 11 (len 5) variable [readonly local] `input` +Ln 5, Col 17 (len 5) property `label` diff --git a/packages/language-server/src/__tests__/fixtures/semantic-tokens/custom-tag/__snapshots__/custom-tag.expected/components/my-button.ts b/packages/language-server/src/__tests__/fixtures/semantic-tokens/custom-tag/__snapshots__/custom-tag.expected/components/my-button.ts new file mode 100644 index 00000000..8e4a7ed1 --- /dev/null +++ b/packages/language-server/src/__tests__/fixtures/semantic-tokens/custom-tag/__snapshots__/custom-tag.expected/components/my-button.ts @@ -0,0 +1,64 @@ +export interface Input { + label: string; +} +abstract class Component extends Marko.Component {} +export { type Component }; +(function (this: void) { + const input = Marko._.any as Input; + const component = Marko._.any as Component; + const state = Marko._.state(component); + const out = Marko._.any as Marko.Out; + const $signal = Marko._.any as AbortSignal; + const $global = Marko._.getGlobal( + // @ts-expect-error We expect the compiler to error because we are checking if the MarkoRun.Context is defined. + (Marko._.error, Marko._.any as MarkoRun.Context), + ); + Marko._.renderNativeTag("button")()()({ + [Marko._.content]: (() => { + input.label; + return () => { + return Marko._.voidReturn; + }; + })(), + }); + Marko._.noop({ component, state, out, input, $global, $signal }); + return; +})(); +const __marko_internal_api = "class"; +export { __marko_internal_api as "~api" }; +const MyButton = new (class Template extends Marko._.Template<{ + render( + input: Marko.TemplateInput, + stream?: { + write: (chunk: string) => void; + end: (chunk?: string) => void; + }, + ): Marko.Out; + + render( + input: Marko.TemplateInput, + cb?: (err: Error | null, result: Marko.RenderResult) => void, + ): Marko.Out; + + renderSync(input: Marko.TemplateInput): Marko.RenderResult; + + renderToString(input: Marko.TemplateInput): string; + + stream( + input: Marko.TemplateInput, + ): ReadableStream & NodeJS.ReadableStream; + + mount( + input: Marko.TemplateInput, + reference: Node, + position?: "afterbegin" | "afterend" | "beforebegin" | "beforeend", + ): Marko.MountedTemplate; + + api: typeof __marko_internal_api; + _(): () => <__marko_internal_input extends unknown>( + input: Marko.Directives & + Input & + Marko._.Relate<__marko_internal_input, Marko.Directives & Input>, + ) => Marko._.ReturnWithScope<__marko_internal_input, void>; +}> {})(); +export default MyButton; diff --git a/packages/language-server/src/__tests__/fixtures/semantic-tokens/custom-tag/__snapshots__/custom-tag.expected/index.html b/packages/language-server/src/__tests__/fixtures/semantic-tokens/custom-tag/__snapshots__/custom-tag.expected/index.html new file mode 100644 index 00000000..9719322b --- /dev/null +++ b/packages/language-server/src/__tests__/fixtures/semantic-tokens/custom-tag/__snapshots__/custom-tag.expected/index.html @@ -0,0 +1,4 @@ +
+ + +
\ No newline at end of file diff --git a/packages/language-server/src/__tests__/fixtures/semantic-tokens/custom-tag/__snapshots__/custom-tag.expected/index.md b/packages/language-server/src/__tests__/fixtures/semantic-tokens/custom-tag/__snapshots__/custom-tag.expected/index.md new file mode 100644 index 00000000..e69de29b diff --git a/packages/language-server/src/__tests__/fixtures/semantic-tokens/custom-tag/__snapshots__/custom-tag.expected/index.tokens.md b/packages/language-server/src/__tests__/fixtures/semantic-tokens/custom-tag/__snapshots__/custom-tag.expected/index.tokens.md new file mode 100644 index 00000000..351bbd3d --- /dev/null +++ b/packages/language-server/src/__tests__/fixtures/semantic-tokens/custom-tag/__snapshots__/custom-tag.expected/index.tokens.md @@ -0,0 +1,2 @@ +Ln 4, Col 4 (len 9) class `my-button` +Ln 5, Col 4 (len 8) class `MyButton` diff --git a/packages/language-server/src/__tests__/fixtures/semantic-tokens/custom-tag/__snapshots__/custom-tag.expected/index.ts b/packages/language-server/src/__tests__/fixtures/semantic-tokens/custom-tag/__snapshots__/custom-tag.expected/index.ts new file mode 100644 index 00000000..425b8dc2 --- /dev/null +++ b/packages/language-server/src/__tests__/fixtures/semantic-tokens/custom-tag/__snapshots__/custom-tag.expected/index.ts @@ -0,0 +1,71 @@ +import MyButton from "./components/my-button.marko"; +export interface Input {} +abstract class Component extends Marko.Component {} +export { type Component }; +(function (this: void) { + const input = Marko._.any as Input; + const component = Marko._.any as Component; + const state = Marko._.state(component); + const out = Marko._.any as Marko.Out; + const $signal = Marko._.any as AbortSignal; + const $global = Marko._.getGlobal( + // @ts-expect-error We expect the compiler to error because we are checking if the MarkoRun.Context is defined. + (Marko._.error, Marko._.any as MarkoRun.Context), + ); + Marko._.renderNativeTag("div")()()({ + [Marko._.content]: (() => { + const __marko_internal_tag_1 = Marko._.resolveTemplate( + import("./components/my-button.marko"), + ); + Marko._.renderTemplate(__marko_internal_tag_1)()()({ + label: "first", + }); + const __marko_internal_tag_2 = MyButton; + Marko._.renderDynamicTag(__marko_internal_tag_2)()()({ + label: "second", + }); + return () => { + return Marko._.voidReturn; + }; + })(), + }); + Marko._.noop({ component, state, out, input, $global, $signal }); + return; +})(); +const __marko_internal_api = "class"; +export { __marko_internal_api as "~api" }; +export default new (class Template extends Marko._.Template<{ + render( + input: Marko.TemplateInput, + stream?: { + write: (chunk: string) => void; + end: (chunk?: string) => void; + }, + ): Marko.Out; + + render( + input: Marko.TemplateInput, + cb?: (err: Error | null, result: Marko.RenderResult) => void, + ): Marko.Out; + + renderSync(input: Marko.TemplateInput): Marko.RenderResult; + + renderToString(input: Marko.TemplateInput): string; + + stream( + input: Marko.TemplateInput, + ): ReadableStream & NodeJS.ReadableStream; + + mount( + input: Marko.TemplateInput, + reference: Node, + position?: "afterbegin" | "afterend" | "beforebegin" | "beforeend", + ): Marko.MountedTemplate; + + api: typeof __marko_internal_api; + _(): () => <__marko_internal_input extends unknown>( + input: Marko.Directives & + Input & + Marko._.Relate<__marko_internal_input, Marko.Directives & Input>, + ) => Marko._.ReturnWithScope<__marko_internal_input, void>; +}> {})(); diff --git a/packages/language-server/src/__tests__/fixtures/semantic-tokens/custom-tag/components/my-button.marko b/packages/language-server/src/__tests__/fixtures/semantic-tokens/custom-tag/components/my-button.marko new file mode 100644 index 00000000..8b946df7 --- /dev/null +++ b/packages/language-server/src/__tests__/fixtures/semantic-tokens/custom-tag/components/my-button.marko @@ -0,0 +1,5 @@ +export interface Input { + label: string; +} + + diff --git a/packages/language-server/src/__tests__/fixtures/semantic-tokens/custom-tag/index.marko b/packages/language-server/src/__tests__/fixtures/semantic-tokens/custom-tag/index.marko new file mode 100644 index 00000000..de5e5b9d --- /dev/null +++ b/packages/language-server/src/__tests__/fixtures/semantic-tokens/custom-tag/index.marko @@ -0,0 +1,6 @@ +import MyButton from ""; + +
+ + +
diff --git a/packages/language-server/src/__tests__/fixtures/semantic-tokens/edge-tags/__snapshots__/edge-tags.expected/index.html b/packages/language-server/src/__tests__/fixtures/semantic-tokens/edge-tags/__snapshots__/edge-tags.expected/index.html new file mode 100644 index 00000000..3888d694 --- /dev/null +++ b/packages/language-server/src/__tests__/fixtures/semantic-tokens/edge-tags/__snapshots__/edge-tags.expected/index.html @@ -0,0 +1,3 @@ +
+ placeholder +
\ No newline at end of file diff --git a/packages/language-server/src/__tests__/fixtures/semantic-tokens/edge-tags/__snapshots__/edge-tags.expected/index.md b/packages/language-server/src/__tests__/fixtures/semantic-tokens/edge-tags/__snapshots__/edge-tags.expected/index.md new file mode 100644 index 00000000..e69de29b diff --git a/packages/language-server/src/__tests__/fixtures/semantic-tokens/edge-tags/__snapshots__/edge-tags.expected/index.tokens.md b/packages/language-server/src/__tests__/fixtures/semantic-tokens/edge-tags/__snapshots__/edge-tags.expected/index.tokens.md new file mode 100644 index 00000000..aae3183b --- /dev/null +++ b/packages/language-server/src/__tests__/fixtures/semantic-tokens/edge-tags/__snapshots__/edge-tags.expected/index.tokens.md @@ -0,0 +1,4 @@ +Ln 1, Col 2 (len 3) macro `let` +Ln 1, Col 6 (len 3) variable [declaration local] `tag` +Ln 2, Col 4 (len 3) variable [local] `tag` +Ln 3, Col 5 (len 3) variable [local] `tag` diff --git a/packages/language-server/src/__tests__/fixtures/semantic-tokens/edge-tags/__snapshots__/edge-tags.expected/index.ts b/packages/language-server/src/__tests__/fixtures/semantic-tokens/edge-tags/__snapshots__/edge-tags.expected/index.ts new file mode 100644 index 00000000..aa1718f5 --- /dev/null +++ b/packages/language-server/src/__tests__/fixtures/semantic-tokens/edge-tags/__snapshots__/edge-tags.expected/index.ts @@ -0,0 +1,71 @@ +export interface Input {} +(function (this: void) { + const input = Marko._.any as Input; + const $signal = Marko._.any as AbortSignal; + const $global = Marko._.getGlobal( + // @ts-expect-error We expect the compiler to error because we are checking if the MarkoRun.Context is defined. + (Marko._.error, Marko._.any as MarkoRun.Context), + ); + const tag = Marko._.hoist(() => __marko_internal_hoist__tag); + const __marko_internal_tag_1 = Marko._.resolveTemplate( + import("@marko/runtime-tags/tags/let.d.marko"), + ); + { + const tag = Marko._.returned(() => __marko_internal_rendered_1); + const __marko_internal_rendered_1 = Marko._.renderTemplate( + __marko_internal_tag_1, + )()()({ + value: "section", + }); + const __marko_internal_tag_2 = tag; + Marko._.renderDynamicTag(__marko_internal_tag_2)()()({ + class: "box", + [Marko._.contentFor(__marko_internal_tag_2)]: (() => { + tag; + return () => { + return Marko._.voidReturn; + }; + })(), + }); + var __marko_internal_hoist__tag = tag; + } + Marko._.noop({ tag, input, $global, $signal }); + return; +})(); +const __marko_internal_api = "tags"; +export { __marko_internal_api as "~api" }; +export default new (class Template extends Marko._.Template<{ + render( + input: Marko.TemplateInput, + stream?: { + write: (chunk: string) => void; + end: (chunk?: string) => void; + }, + ): Marko.Out; + + render( + input: Marko.TemplateInput, + cb?: (err: Error | null, result: Marko.RenderResult) => void, + ): Marko.Out; + + renderSync(input: Marko.TemplateInput): Marko.RenderResult; + + renderToString(input: Marko.TemplateInput): string; + + stream( + input: Marko.TemplateInput, + ): ReadableStream & NodeJS.ReadableStream; + + mount( + input: Marko.TemplateInput, + reference: Node, + position?: "afterbegin" | "afterend" | "beforebegin" | "beforeend", + ): Marko.MountedTemplate; + + api: typeof __marko_internal_api; + _(): () => <__marko_internal_input extends unknown>( + input: Marko.Directives & + Input & + Marko._.Relate<__marko_internal_input, Marko.Directives & Input>, + ) => Marko._.ReturnWithScope<__marko_internal_input, void>; +}> {})(); diff --git a/packages/language-server/src/__tests__/fixtures/semantic-tokens/edge-tags/index.marko b/packages/language-server/src/__tests__/fixtures/semantic-tokens/edge-tags/index.marko new file mode 100644 index 00000000..58b10079 --- /dev/null +++ b/packages/language-server/src/__tests__/fixtures/semantic-tokens/edge-tags/index.marko @@ -0,0 +1,4 @@ + +<${tag} class="box"> + ${tag} + diff --git a/packages/language-server/src/__tests__/fixtures/semantic-tokens/expressions/__snapshots__/expressions.expected/index.html b/packages/language-server/src/__tests__/fixtures/semantic-tokens/expressions/__snapshots__/expressions.expected/index.html new file mode 100644 index 00000000..56eee24d --- /dev/null +++ b/packages/language-server/src/__tests__/fixtures/semantic-tokens/expressions/__snapshots__/expressions.expected/index.html @@ -0,0 +1,3 @@ + \ No newline at end of file diff --git a/packages/language-server/src/__tests__/fixtures/semantic-tokens/expressions/__snapshots__/expressions.expected/index.md b/packages/language-server/src/__tests__/fixtures/semantic-tokens/expressions/__snapshots__/expressions.expected/index.md new file mode 100644 index 00000000..e69de29b diff --git a/packages/language-server/src/__tests__/fixtures/semantic-tokens/expressions/__snapshots__/expressions.expected/index.tokens.md b/packages/language-server/src/__tests__/fixtures/semantic-tokens/expressions/__snapshots__/expressions.expected/index.tokens.md new file mode 100644 index 00000000..b362f47a --- /dev/null +++ b/packages/language-server/src/__tests__/fixtures/semantic-tokens/expressions/__snapshots__/expressions.expected/index.tokens.md @@ -0,0 +1,6 @@ +Ln 1, Col 14 (len 8) variable [declaration readonly] `GREETING` +Ln 3, Col 2 (len 3) macro `let` +Ln 3, Col 6 (len 5) variable [declaration local] `count` +Ln 4, Col 29 (len 5) variable [local] `count` +Ln 5, Col 5 (len 8) variable [readonly] `GREETING` +Ln 5, Col 25 (len 5) variable [local] `count` diff --git a/packages/language-server/src/__tests__/fixtures/semantic-tokens/expressions/__snapshots__/expressions.expected/index.ts b/packages/language-server/src/__tests__/fixtures/semantic-tokens/expressions/__snapshots__/expressions.expected/index.ts new file mode 100644 index 00000000..4808ecf1 --- /dev/null +++ b/packages/language-server/src/__tests__/fixtures/semantic-tokens/expressions/__snapshots__/expressions.expected/index.ts @@ -0,0 +1,79 @@ +const GREETING = "hello"; +export interface Input {} +(function (this: void) { + const input = Marko._.any as Input; + const $signal = Marko._.any as AbortSignal; + const $global = Marko._.getGlobal( + // @ts-expect-error We expect the compiler to error because we are checking if the MarkoRun.Context is defined. + (Marko._.error, Marko._.any as MarkoRun.Context), + ); + const count = Marko._.hoist(() => __marko_internal_hoist__count); + const __marko_internal_tag_1 = Marko._.resolveTemplate( + import("@marko/runtime-tags/tags/let.d.marko"), + ); + { + const count = Marko._.returned(() => __marko_internal_rendered_1); + const __marko_internal_rendered_1 = Marko._.renderTemplate( + __marko_internal_tag_1, + )()()({ + value: 0, + }); + const __marko_internal_change__count = Marko._.change( + "count", + "value", + __marko_internal_rendered_1.return, + ); + Marko._.renderNativeTag("button")()()({ + onClick() { + __marko_internal_change__count.count = count + 1; + }, + [Marko._.content]: (() => { + GREETING; + count; + return () => { + return Marko._.voidReturn; + }; + })(), + }); + var __marko_internal_hoist__count = count; + } + Marko._.noop({ count, input, $global, $signal }); + return; +})(); +const __marko_internal_api = "tags"; +export { __marko_internal_api as "~api" }; +export default new (class Template extends Marko._.Template<{ + render( + input: Marko.TemplateInput, + stream?: { + write: (chunk: string) => void; + end: (chunk?: string) => void; + }, + ): Marko.Out; + + render( + input: Marko.TemplateInput, + cb?: (err: Error | null, result: Marko.RenderResult) => void, + ): Marko.Out; + + renderSync(input: Marko.TemplateInput): Marko.RenderResult; + + renderToString(input: Marko.TemplateInput): string; + + stream( + input: Marko.TemplateInput, + ): ReadableStream & NodeJS.ReadableStream; + + mount( + input: Marko.TemplateInput, + reference: Node, + position?: "afterbegin" | "afterend" | "beforebegin" | "beforeend", + ): Marko.MountedTemplate; + + api: typeof __marko_internal_api; + _(): () => <__marko_internal_input extends unknown>( + input: Marko.Directives & + Input & + Marko._.Relate<__marko_internal_input, Marko.Directives & Input>, + ) => Marko._.ReturnWithScope<__marko_internal_input, void>; +}> {})(); diff --git a/packages/language-server/src/__tests__/fixtures/semantic-tokens/expressions/index.marko b/packages/language-server/src/__tests__/fixtures/semantic-tokens/expressions/index.marko new file mode 100644 index 00000000..66423790 --- /dev/null +++ b/packages/language-server/src/__tests__/fixtures/semantic-tokens/expressions/index.marko @@ -0,0 +1,6 @@ +static const GREETING = "hello"; + + + diff --git a/packages/language-server/src/__tests__/fixtures/semantic-tokens/script-blocks/__snapshots__/script-blocks.expected/index.html b/packages/language-server/src/__tests__/fixtures/semantic-tokens/script-blocks/__snapshots__/script-blocks.expected/index.html new file mode 100644 index 00000000..0ed0861f --- /dev/null +++ b/packages/language-server/src/__tests__/fixtures/semantic-tokens/script-blocks/__snapshots__/script-blocks.expected/index.html @@ -0,0 +1 @@ +

placeholder placeholder

\ No newline at end of file diff --git a/packages/language-server/src/__tests__/fixtures/semantic-tokens/script-blocks/__snapshots__/script-blocks.expected/index.md b/packages/language-server/src/__tests__/fixtures/semantic-tokens/script-blocks/__snapshots__/script-blocks.expected/index.md new file mode 100644 index 00000000..e69de29b diff --git a/packages/language-server/src/__tests__/fixtures/semantic-tokens/script-blocks/__snapshots__/script-blocks.expected/index.tokens.md b/packages/language-server/src/__tests__/fixtures/semantic-tokens/script-blocks/__snapshots__/script-blocks.expected/index.tokens.md new file mode 100644 index 00000000..0293175e --- /dev/null +++ b/packages/language-server/src/__tests__/fixtures/semantic-tokens/script-blocks/__snapshots__/script-blocks.expected/index.tokens.md @@ -0,0 +1,9 @@ +Ln 3, Col 14 (len 6) variable [declaration readonly] `banner` +Ln 3, Col 23 (len 5) function `greet` +Ln 5, Col 17 (len 5) function [declaration] `shout` +Ln 5, Col 23 (len 4) parameter [declaration] `name` +Ln 6, Col 10 (len 5) function `greet` +Ln 6, Col 16 (len 4) parameter `name` +Ln 6, Col 22 (len 11) method [defaultLibrary] `toUpperCase` +Ln 9, Col 6 (len 5) function `shout` +Ln 9, Col 24 (len 6) variable [readonly] `banner` diff --git a/packages/language-server/src/__tests__/fixtures/semantic-tokens/script-blocks/__snapshots__/script-blocks.expected/index.ts b/packages/language-server/src/__tests__/fixtures/semantic-tokens/script-blocks/__snapshots__/script-blocks.expected/index.ts new file mode 100644 index 00000000..85e02594 --- /dev/null +++ b/packages/language-server/src/__tests__/fixtures/semantic-tokens/script-blocks/__snapshots__/script-blocks.expected/index.ts @@ -0,0 +1,67 @@ +import { greet } from "./util"; +const banner = greet("server"); +function shout(name: string) { + return greet(name).toUpperCase(); +} +export interface Input {} +abstract class Component extends Marko.Component {} +export { type Component }; +(function (this: void) { + const input = Marko._.any as Input; + const component = Marko._.any as Component; + const state = Marko._.state(component); + const out = Marko._.any as Marko.Out; + const $signal = Marko._.any as AbortSignal; + const $global = Marko._.getGlobal( + // @ts-expect-error We expect the compiler to error because we are checking if the MarkoRun.Context is defined. + (Marko._.error, Marko._.any as MarkoRun.Context), + ); + Marko._.renderNativeTag("p")()()({ + [Marko._.content]: (() => { + shout("world"); + banner; + return () => { + return Marko._.voidReturn; + }; + })(), + }); + Marko._.noop({ component, state, out, input, $global, $signal }); + return; +})(); +const __marko_internal_api = "class"; +export { __marko_internal_api as "~api" }; +export default new (class Template extends Marko._.Template<{ + render( + input: Marko.TemplateInput, + stream?: { + write: (chunk: string) => void; + end: (chunk?: string) => void; + }, + ): Marko.Out; + + render( + input: Marko.TemplateInput, + cb?: (err: Error | null, result: Marko.RenderResult) => void, + ): Marko.Out; + + renderSync(input: Marko.TemplateInput): Marko.RenderResult; + + renderToString(input: Marko.TemplateInput): string; + + stream( + input: Marko.TemplateInput, + ): ReadableStream & NodeJS.ReadableStream; + + mount( + input: Marko.TemplateInput, + reference: Node, + position?: "afterbegin" | "afterend" | "beforebegin" | "beforeend", + ): Marko.MountedTemplate; + + api: typeof __marko_internal_api; + _(): () => <__marko_internal_input extends unknown>( + input: Marko.Directives & + Input & + Marko._.Relate<__marko_internal_input, Marko.Directives & Input>, + ) => Marko._.ReturnWithScope<__marko_internal_input, void>; +}> {})(); diff --git a/packages/language-server/src/__tests__/fixtures/semantic-tokens/script-blocks/index.marko b/packages/language-server/src/__tests__/fixtures/semantic-tokens/script-blocks/index.marko new file mode 100644 index 00000000..a76249d3 --- /dev/null +++ b/packages/language-server/src/__tests__/fixtures/semantic-tokens/script-blocks/index.marko @@ -0,0 +1,9 @@ +import { greet } from "./util"; + +server const banner = greet("server"); + +static function shout(name: string) { + return greet(name).toUpperCase(); +} + +

${shout("world")} ${banner}

diff --git a/packages/language-server/src/__tests__/fixtures/semantic-tokens/script-blocks/util.ts b/packages/language-server/src/__tests__/fixtures/semantic-tokens/script-blocks/util.ts new file mode 100644 index 00000000..f073512d --- /dev/null +++ b/packages/language-server/src/__tests__/fixtures/semantic-tokens/script-blocks/util.ts @@ -0,0 +1,3 @@ +export function greet(name: string) { + return `hello ${name}`; +} diff --git a/packages/language-server/src/__tests__/fixtures/semantic-tokens/style-block/__snapshots__/style-block.expected/index.html b/packages/language-server/src/__tests__/fixtures/semantic-tokens/style-block/__snapshots__/style-block.expected/index.html new file mode 100644 index 00000000..203a965f --- /dev/null +++ b/packages/language-server/src/__tests__/fixtures/semantic-tokens/style-block/__snapshots__/style-block.expected/index.html @@ -0,0 +1 @@ +
\ No newline at end of file diff --git a/packages/language-server/src/__tests__/fixtures/semantic-tokens/style-block/__snapshots__/style-block.expected/index.md b/packages/language-server/src/__tests__/fixtures/semantic-tokens/style-block/__snapshots__/style-block.expected/index.md new file mode 100644 index 00000000..e69de29b diff --git a/packages/language-server/src/__tests__/fixtures/semantic-tokens/style-block/__snapshots__/style-block.expected/index.tokens.md b/packages/language-server/src/__tests__/fixtures/semantic-tokens/style-block/__snapshots__/style-block.expected/index.tokens.md new file mode 100644 index 00000000..68e0c4e9 --- /dev/null +++ b/packages/language-server/src/__tests__/fixtures/semantic-tokens/style-block/__snapshots__/style-block.expected/index.tokens.md @@ -0,0 +1 @@ +(no semantic tokens) diff --git a/packages/language-server/src/__tests__/fixtures/semantic-tokens/style-block/__snapshots__/style-block.expected/index.ts b/packages/language-server/src/__tests__/fixtures/semantic-tokens/style-block/__snapshots__/style-block.expected/index.ts new file mode 100644 index 00000000..e4d5cd30 --- /dev/null +++ b/packages/language-server/src/__tests__/fixtures/semantic-tokens/style-block/__snapshots__/style-block.expected/index.ts @@ -0,0 +1,56 @@ +export interface Input {} +abstract class Component extends Marko.Component {} +export { type Component }; +(function (this: void) { + const input = Marko._.any as Input; + const component = Marko._.any as Component; + const state = Marko._.state(component); + const out = Marko._.any as Marko.Out; + const $signal = Marko._.any as AbortSignal; + const $global = Marko._.getGlobal( + // @ts-expect-error We expect the compiler to error because we are checking if the MarkoRun.Context is defined. + (Marko._.error, Marko._.any as MarkoRun.Context), + ); + Marko._.renderNativeTag("div")()()({ + class: "content", + }); + Marko._.noop({ component, state, out, input, $global, $signal }); + return; +})(); +const __marko_internal_api = "class"; +export { __marko_internal_api as "~api" }; +export default new (class Template extends Marko._.Template<{ + render( + input: Marko.TemplateInput, + stream?: { + write: (chunk: string) => void; + end: (chunk?: string) => void; + }, + ): Marko.Out; + + render( + input: Marko.TemplateInput, + cb?: (err: Error | null, result: Marko.RenderResult) => void, + ): Marko.Out; + + renderSync(input: Marko.TemplateInput): Marko.RenderResult; + + renderToString(input: Marko.TemplateInput): string; + + stream( + input: Marko.TemplateInput, + ): ReadableStream & NodeJS.ReadableStream; + + mount( + input: Marko.TemplateInput, + reference: Node, + position?: "afterbegin" | "afterend" | "beforebegin" | "beforeend", + ): Marko.MountedTemplate; + + api: typeof __marko_internal_api; + _(): () => <__marko_internal_input extends unknown>( + input: Marko.Directives & + Input & + Marko._.Relate<__marko_internal_input, Marko.Directives & Input>, + ) => Marko._.ReturnWithScope<__marko_internal_input, void>; +}> {})(); diff --git a/packages/language-server/src/__tests__/fixtures/semantic-tokens/style-block/index.marko b/packages/language-server/src/__tests__/fixtures/semantic-tokens/style-block/index.marko new file mode 100644 index 00000000..8d5df9ae --- /dev/null +++ b/packages/language-server/src/__tests__/fixtures/semantic-tokens/style-block/index.marko @@ -0,0 +1,7 @@ +
+ +style { + .content { + color: blue; + } +} diff --git a/packages/language-server/src/__tests__/fixtures/semantic-tokens/tag-params/__snapshots__/tag-params.expected/index.html b/packages/language-server/src/__tests__/fixtures/semantic-tokens/tag-params/__snapshots__/tag-params.expected/index.html new file mode 100644 index 00000000..2e00632a --- /dev/null +++ b/packages/language-server/src/__tests__/fixtures/semantic-tokens/tag-params/__snapshots__/tag-params.expected/index.html @@ -0,0 +1,5 @@ +
    +
    +
  • placeholder: placeholder
  • +
    +
\ No newline at end of file diff --git a/packages/language-server/src/__tests__/fixtures/semantic-tokens/tag-params/__snapshots__/tag-params.expected/index.md b/packages/language-server/src/__tests__/fixtures/semantic-tokens/tag-params/__snapshots__/tag-params.expected/index.md new file mode 100644 index 00000000..e69de29b diff --git a/packages/language-server/src/__tests__/fixtures/semantic-tokens/tag-params/__snapshots__/tag-params.expected/index.tokens.md b/packages/language-server/src/__tests__/fixtures/semantic-tokens/tag-params/__snapshots__/tag-params.expected/index.tokens.md new file mode 100644 index 00000000..2bee7b71 --- /dev/null +++ b/packages/language-server/src/__tests__/fixtures/semantic-tokens/tag-params/__snapshots__/tag-params.expected/index.tokens.md @@ -0,0 +1,7 @@ +Ln 2, Col 4 (len 3) keyword `for` +Ln 2, Col 8 (len 4) parameter [declaration] `item` +Ln 2, Col 14 (len 5) parameter [declaration] `index` +Ln 3, Col 11 (len 5) parameter `index` +Ln 3, Col 21 (len 4) parameter `item` +Ln 3, Col 26 (len 11) method [defaultLibrary] `toUpperCase` +Ln 4, Col 5 (len 3) keyword `for` diff --git a/packages/language-server/src/__tests__/fixtures/semantic-tokens/tag-params/__snapshots__/tag-params.expected/index.ts b/packages/language-server/src/__tests__/fixtures/semantic-tokens/tag-params/__snapshots__/tag-params.expected/index.ts new file mode 100644 index 00000000..99ab17bc --- /dev/null +++ b/packages/language-server/src/__tests__/fixtures/semantic-tokens/tag-params/__snapshots__/tag-params.expected/index.ts @@ -0,0 +1,77 @@ +export interface Input {} +abstract class Component extends Marko.Component {} +export { type Component }; +(function (this: void) { + const input = Marko._.any as Input; + const component = Marko._.any as Component; + const state = Marko._.state(component); + const out = Marko._.any as Marko.Out; + const $signal = Marko._.any as AbortSignal; + const $global = Marko._.getGlobal( + // @ts-expect-error We expect the compiler to error because we are checking if the MarkoRun.Context is defined. + (Marko._.error, Marko._.any as MarkoRun.Context), + ); + Marko._.renderNativeTag("ul")()()({ + [Marko._.content]: (() => { + Marko._.forOfTag( + { + of: ["a", "b", "c"], + }, + (item, index) => { + Marko._.renderNativeTag("li")()()({ + [Marko._.content]: (() => { + index; + item.toUpperCase(); + return () => { + return Marko._.voidReturn; + }; + })(), + }); + return Marko._.voidReturn; + }, + ); + return () => { + return Marko._.voidReturn; + }; + })(), + }); + Marko._.noop({ component, state, out, input, $global, $signal }); + return; +})(); +const __marko_internal_api = "class"; +export { __marko_internal_api as "~api" }; +export default new (class Template extends Marko._.Template<{ + render( + input: Marko.TemplateInput, + stream?: { + write: (chunk: string) => void; + end: (chunk?: string) => void; + }, + ): Marko.Out; + + render( + input: Marko.TemplateInput, + cb?: (err: Error | null, result: Marko.RenderResult) => void, + ): Marko.Out; + + renderSync(input: Marko.TemplateInput): Marko.RenderResult; + + renderToString(input: Marko.TemplateInput): string; + + stream( + input: Marko.TemplateInput, + ): ReadableStream & NodeJS.ReadableStream; + + mount( + input: Marko.TemplateInput, + reference: Node, + position?: "afterbegin" | "afterend" | "beforebegin" | "beforeend", + ): Marko.MountedTemplate; + + api: typeof __marko_internal_api; + _(): () => <__marko_internal_input extends unknown>( + input: Marko.Directives & + Input & + Marko._.Relate<__marko_internal_input, Marko.Directives & Input>, + ) => Marko._.ReturnWithScope<__marko_internal_input, void>; +}> {})(); diff --git a/packages/language-server/src/__tests__/fixtures/semantic-tokens/tag-params/index.marko b/packages/language-server/src/__tests__/fixtures/semantic-tokens/tag-params/index.marko new file mode 100644 index 00000000..a6a652e3 --- /dev/null +++ b/packages/language-server/src/__tests__/fixtures/semantic-tokens/tag-params/index.marko @@ -0,0 +1,5 @@ +
    + +
  • ${index}: ${item.toUpperCase()}
  • + +
diff --git a/packages/language-server/src/__tests__/index.test.ts b/packages/language-server/src/__tests__/index.test.ts index 006436d4..de106268 100644 --- a/packages/language-server/src/__tests__/index.test.ts +++ b/packages/language-server/src/__tests__/index.test.ts @@ -22,6 +22,7 @@ import { getFixCandidates, } from "../service/marko/code-actions"; import { codeFrame } from "./util/code-frame"; +import { formatSemanticTokens } from "./util/semantic-tokens"; Project.setDefaultTypePaths({ internalTypesFile: @@ -187,6 +188,29 @@ for (const subdir of fs.readdirSync(FIXTURE_DIR)) { results += `## Code Actions\n${codeActionResults}`; } + // Token snapshots are opt-in by fixture group to avoid churning every + // existing fixture; put new semantic-token cases under this subdir. + if (subdir === "semantic-tokens") { + const tokens = await MarkoLanguageService.getSemanticTokens( + doc, + { textDocument: { uri: doc.uri } }, + CancellationToken.None, + ); + await snapshot( + formatSemanticTokens( + code, + tokens && !Array.isArray(tokens) ? tokens : undefined, + ), + { + file: path.relative( + fixtureDir, + filename.replace(/\.marko$/, ".tokens.md"), + ), + dir: fixtureDir, + }, + ); + } + documents.doClose(params); await snapshot(results, { diff --git a/packages/language-server/src/__tests__/semantic-tokens.test.ts b/packages/language-server/src/__tests__/semantic-tokens.test.ts new file mode 100644 index 00000000..8569a8b3 --- /dev/null +++ b/packages/language-server/src/__tests__/semantic-tokens.test.ts @@ -0,0 +1,235 @@ +import assert from "node:assert/strict"; + +import { CancellationToken, type SemanticTokens } from "vscode-languageserver"; +import { TextDocument } from "vscode-languageserver-textdocument"; + +import { createService } from "../service/create-service"; +import { + decodeTsClassification, + TokenModifier, + TokenType, +} from "../service/semantic-tokens"; +import type { SemanticToken } from "../service/types"; +import { formatSemanticTokens } from "./util/semantic-tokens"; + +const doc = TextDocument.create("file:///test.marko", "marko", 0, ""); +const fullParams = { textDocument: { uri: doc.uri } }; + +function token( + line: number, + start: number, + end: number, + type = TokenType.variable, + modifiers = 0, +): SemanticToken { + return { + range: { start: { line, character: start }, end: { line, character: end } }, + type, + modifiers, + }; +} + +function getTokens( + pluginTokens: (SemanticToken[] | undefined)[], + params: typeof fullParams & { range?: SemanticToken["range"] } = fullParams, + forDoc = doc, +) { + const service = createService( + pluginTokens.map((tokens) => ({ getSemanticTokens: () => tokens })), + ); + return service.getSemanticTokens( + forDoc, + params, + CancellationToken.None, + ) as Promise; +} + +describe("semantic tokens facade", () => { + it("merges, sorts, and delta-encodes tokens across plugins", async () => { + const result = await getTokens([ + [token(2, 4, 6, TokenType.class)], + [ + token(0, 5, 8, TokenType.function, TokenModifier.async), + token(0, 0, 3, TokenType.keyword), + ], + ]); + + assert.deepEqual(result?.data, [ + ...[0, 0, 3, TokenType.keyword, 0], + ...[0, 5, 3, TokenType.function, TokenModifier.async], + ...[2, 4, 2, TokenType.class, 0], + ]); + assert.equal(result?.resultId, undefined); + }); + + it("drops a token overlapping one from an earlier plugin", async () => { + const result = await getTokens([ + [token(0, 2, 6, TokenType.class)], + [token(0, 4, 8, TokenType.variable), token(0, 6, 9, TokenType.property)], + ]); + + assert.deepEqual(result?.data, [ + ...[0, 2, 4, TokenType.class, 0], + ...[0, 4, 3, TokenType.property, 0], + ]); + }); + + it("drops a token duplicating an earlier plugin's start", async () => { + const result = await getTokens([ + [token(1, 1, 4, TokenType.class)], + [token(1, 1, 4, TokenType.variable)], + ]); + + assert.deepEqual(result?.data, [...[1, 1, 3, TokenType.class, 0]]); + }); + + it("splits multiline tokens per line and drops empty ones", async () => { + const multilineDoc = TextDocument.create( + "file:///multiline.marko", + "marko", + 0, + "hello\nworld\nok\n", + ); + const result = await getTokens( + [ + [ + { + range: { + start: { line: 0, character: 2 }, + end: { line: 2, character: 1 }, + }, + type: TokenType.string, + modifiers: 0, + }, + token(3, 3, 3), + token(4, 1, 2), + ], + ], + fullParams, + multilineDoc, + ); + + assert.deepEqual(result?.data, [ + ...[0, 2, 3, TokenType.string, 0], + ...[1, 0, 5, TokenType.string, 0], + ...[1, 0, 1, TokenType.string, 0], + ...[2, 1, 1, TokenType.variable, 0], + ]); + }); + + it("resolves overlaps identically for full and range requests", async () => { + const pluginTokens = [ + [token(0, 2, 6, TokenType.class)], + [token(0, 4, 8, TokenType.property)], + ]; + + const full = await getTokens(pluginTokens); + assert.deepEqual(full?.data, [...[0, 2, 4, TokenType.class, 0]]); + + const ranged = await getTokens(pluginTokens, { + ...fullParams, + range: { + start: { line: 0, character: 7 }, + end: { line: 0, character: 20 }, + }, + }); + assert.deepEqual(ranged?.data, []); + }); + + it("filters to the requested range", async () => { + const result = await getTokens( + [[token(0, 0, 2), token(2, 4, 8), token(5, 0, 3)]], + { + ...fullParams, + range: { + start: { line: 1, character: 0 }, + end: { line: 3, character: 0 }, + }, + }, + ); + + assert.deepEqual(result?.data, [...[2, 4, 4, TokenType.variable, 0]]); + }); + + it("ignores a throwing plugin and returns nothing when no plugin responds", async () => { + const service = createService([ + { + getSemanticTokens() { + throw new Error("boom"); + }, + }, + { getSemanticTokens: () => [token(4, 0, 2)] }, + ]); + const result = (await service.getSemanticTokens( + doc, + fullParams, + CancellationToken.None, + )) as SemanticTokens | undefined; + assert.deepEqual(result?.data, [...[4, 0, 2, TokenType.variable, 0]]); + + assert.equal(await getTokens([undefined, []]), undefined); + }); +}); + +describe("decodeTsClassification", () => { + it("splits the 2020 encoding into an aligned type index and modifier set", () => { + assert.deepEqual(decodeTsClassification(((7 + 1) << 8) | 0b100101), { + type: TokenType.variable, + modifiers: + TokenModifier.declaration | TokenModifier.async | TokenModifier.local, + }); + assert.deepEqual(decodeTsClassification((11 + 1) << 8), { + type: TokenType.method, + modifiers: 0, + }); + }); + + it("rejects unclassified and out-of-range values", () => { + assert.equal(decodeTsClassification(0), undefined); + assert.equal(decodeTsClassification(255), undefined); + assert.equal(decodeTsClassification((12 + 1) << 8), undefined); + }); +}); + +describe("formatSemanticTokens", () => { + it("renders one readable line per token", () => { + const code = "let a = 1;\nconst b = a;\n"; + assert.equal( + formatSemanticTokens(code, { + data: [ + ...[0, 4, 1, TokenType.variable, TokenModifier.declaration], + ...[ + 1, + 6, + 1, + TokenType.variable, + TokenModifier.declaration | TokenModifier.readonly, + ], + ...[0, 4, 1, TokenType.variable, 0], + ], + }), + "Ln 1, Col 5 (len 1) variable [declaration] `a`\n" + + "Ln 2, Col 7 (len 1) variable [declaration readonly] `b`\n" + + "Ln 2, Col 11 (len 1) variable `a`\n", + ); + }); + + it("throws on out-of-order or malformed data", () => { + assert.throws(() => + formatSemanticTokens("a b", { data: [0, -2, 1, TokenType.variable, 0] }), + ); + assert.throws(() => + formatSemanticTokens("a b", { data: [0, 0, 0, TokenType.variable, 0] }), + ); + assert.throws(() => + formatSemanticTokens("a b", { data: [0, 0, 1, 99, 0] }), + ); + assert.throws(() => + formatSemanticTokens("a b", { data: [0, 0, 3, TokenType.variable, 0] }), + ); + assert.equal( + formatSemanticTokens("", { data: [] }), + "(no semantic tokens)\n", + ); + }); +}); diff --git a/packages/language-server/src/__tests__/util/semantic-tokens.ts b/packages/language-server/src/__tests__/util/semantic-tokens.ts new file mode 100644 index 00000000..26a2b271 --- /dev/null +++ b/packages/language-server/src/__tests__/util/semantic-tokens.ts @@ -0,0 +1,68 @@ +import { tokenModifiers, tokenTypes } from "../../service/semantic-tokens"; + +/** + * Decode the LSP delta-encoded token data into one line per token, with the + * source text sliced at each token so snapshots read as plain english. Throws + * on malformed data (negative deltas, unknown legend indices) so an encoding + * bug fails the test instead of producing a plausible snapshot. + */ +export function formatSemanticTokens( + code: string, + tokens: { data: number[] } | undefined, +): string { + if (!tokens?.data.length) return "(no semantic tokens)\n"; + + const lineStarts = [0]; + for (let i = 0; i < code.length; i++) { + if (code.charCodeAt(i) === 10) lineStarts.push(i + 1); + } + + const { data } = tokens; + let result = ""; + let line = 0; + let char = 0; + + for (let i = 0; i < data.length; i += 5) { + const deltaLine = data[i]; + const deltaChar = data[i + 1]; + const length = data[i + 2]; + const type = tokenTypes[data[i + 3]]; + + if (deltaLine < 0 || (deltaLine === 0 && deltaChar < 0) || length <= 0) { + throw new Error( + `malformed semantic token data at index ${i}: [${deltaLine}, ${deltaChar}, ${length}]`, + ); + } + if (!type) { + throw new Error(`unknown token type index ${data[i + 3]} at index ${i}`); + } + + line += deltaLine; + char = deltaLine ? deltaChar : char + deltaChar; + + let modifiers = ""; + for (let bit = 0; 1 << bit <= data[i + 4]; bit++) { + if (data[i + 4] & (1 << bit)) { + const modifier = tokenModifiers[bit]; + if (!modifier) { + throw new Error(`unknown token modifier bit ${bit} at index ${i}`); + } + modifiers += modifiers ? ` ${modifier}` : modifier; + } + } + + const start = lineStarts[line] + char; + const text = code.slice(start, start + length); + if (text.length !== length || /\s/.test(text)) { + throw new Error( + `token at index ${i} does not cover a token-like slice: \`${text}\``, + ); + } + + result += `Ln ${line + 1}, Col ${char + 1} (len ${length}) ${type}${ + modifiers ? ` [${modifiers}]` : "" + } \`${text}\`\n`; + } + + return result; +} diff --git a/packages/language-server/src/browser.ts b/packages/language-server/src/browser.ts index e29d1864..25494bcd 100644 --- a/packages/language-server/src/browser.ts +++ b/packages/language-server/src/browser.ts @@ -9,8 +9,9 @@ export { createService } from "./service/create-service"; export { default as MarkoPlugin } from "./service/marko"; export { default as ScriptPlugin } from "./service/script"; +export { semanticTokensLegend } from "./service/semantic-tokens"; export { default as StylePlugin } from "./service/style"; -export type { Plugin } from "./service/types"; +export type { Plugin, SemanticToken } from "./service/types"; export { clearMarkoCacheForFile } from "./utils/file"; export { default as setupMessages } from "./utils/messages"; export * as documents from "./utils/text-documents"; diff --git a/packages/language-server/src/index.ts b/packages/language-server/src/index.ts index 5a1b898d..7d7d8e5a 100644 --- a/packages/language-server/src/index.ts +++ b/packages/language-server/src/index.ts @@ -3,17 +3,23 @@ import "./utils/project-defaults"; import { Project } from "@marko/language-tools"; import { inspect, isDeepStrictEqual } from "util"; import { + type CancellationToken, createConnection, type DefinitionLink, Diagnostic, DidChangeWatchedFilesNotification, + LSPErrorCodes, ProposedFeatures, + ResponseError, + type SemanticTokensParams, + type SemanticTokensRangeParams, TextDocumentSyncKind, } from "vscode-languageserver/node"; import type { TextDocument } from "vscode-languageserver-textdocument"; import service from "./service"; import { markoCodeActionKinds } from "./service/marko/code-actions"; +import { semanticTokensLegend } from "./service/semantic-tokens"; import { clearMarkoCacheForFile } from "./utils/file"; import setupMessages from "./utils/messages"; import * as documents from "./utils/text-documents"; @@ -30,7 +36,9 @@ if ( const connection = createConnection(ProposedFeatures.all); const prevDiags = new WeakMap(); let diagnosticTimeout: ReturnType | undefined; +let semanticTokensRefreshTimeout: ReturnType | undefined; let canRegisterFileWatchers = false; +let canRefreshSemanticTokens = false; // On-disk changes to these files affect Marko intellisense (imported // components, TS modules, styles, tag definitions, tsconfig, and dependency @@ -53,6 +61,9 @@ connection.onInitialize(async (params) => { canRegisterFileWatchers = Boolean( params.capabilities.workspace?.didChangeWatchedFiles?.dynamicRegistration, ); + canRefreshSemanticTokens = Boolean( + params.capabilities.workspace?.semanticTokens?.refreshSupport, + ); setupMessages(connection); await service.initialize(params); @@ -72,6 +83,11 @@ connection.onInitialize(async (params) => { colorProvider: true, documentHighlightProvider: true, documentSymbolProvider: true, + semanticTokensProvider: { + legend: semanticTokensLegend, + full: true, + range: true, + }, completionProvider: { resolveProvider: true, triggerCharacters: [ @@ -112,7 +128,10 @@ connection.onInitialized(() => { }); workspace.setup(connection); -workspace.onConfigChange(validateDocs); +workspace.onConfigChange(() => { + validateDocs(); + queueSemanticTokensRefresh(); +}); connection.onDidOpenTextDocument(async (params) => { documents.doOpen(params); @@ -136,7 +155,10 @@ documents.onFileChange((changeDoc) => { queueDiagnostic(); clearMarkoCacheForFile(changeDoc); } else { + // On-disk/config changes can alter tag and type classifications in every + // open document, which clients only re-request when told to. validateDocs(); + queueSemanticTokensRefresh(); } }); @@ -204,6 +226,38 @@ connection.onDocumentHighlight(async (params, cancel) => { ); }); +connection.languages.semanticTokens.on((params, cancel) => + getSemanticTokens({ textDocument: params.textDocument }, cancel), +); +connection.languages.semanticTokens.onRange(getSemanticTokens); + +async function getSemanticTokens( + params: SemanticTokensParams | SemanticTokensRangeParams, + cancel: CancellationToken, +) { + const doc = documents.get(params.textDocument.uri); + if (!doc) return null; + + const { version } = doc; + const result = await service.getSemanticTokens(doc, params, cancel); + + if (cancel.isCancellationRequested) { + throw new ResponseError(LSPErrorCodes.RequestCancelled, "request canceled"); + } + + // Returning an empty result would clear the editor's existing tokens, so a + // request that raced an edit (or a close/reopen) must fail with + // ContentModified instead -- the client keeps the old tokens and retries. + if ( + documents.get(params.textDocument.uri) !== doc || + doc.version !== version + ) { + throw new ResponseError(LSPErrorCodes.ContentModified, "content modified"); + } + + return result && !Array.isArray(result) ? result : { data: [] }; +} + connection.onDocumentColor(async (params, cancel) => { return ( (await service.findDocumentColors( @@ -287,6 +341,14 @@ function validateDocs() { Project.clearCaches(); } +function queueSemanticTokensRefresh() { + if (!canRefreshSemanticTokens) return; + clearTimeout(semanticTokensRefreshTimeout); + semanticTokensRefreshTimeout = setTimeout(() => { + void connection.languages.semanticTokens.refresh(); + }, 1000); +} + function queueDiagnostic() { clearTimeout(diagnosticTimeout); const id = (diagnosticTimeout = setTimeout(async () => { diff --git a/packages/language-server/src/service/create-service.ts b/packages/language-server/src/service/create-service.ts index d9491bb1..2c74f4c9 100644 --- a/packages/language-server/src/service/create-service.ts +++ b/packages/language-server/src/service/create-service.ts @@ -15,9 +15,14 @@ import type { SymbolInformation, WorkspaceEdit, } from "vscode-languageserver"; -import { MarkupContent, MarkupKind } from "vscode-languageserver"; +import { + MarkupContent, + MarkupKind, + SemanticTokensBuilder, +} from "vscode-languageserver"; +import type { TextDocument } from "vscode-languageserver-textdocument"; -import type { Plugin } from "./types"; +import type { Plugin, SemanticToken } from "./types"; const REG_MARKDOWN_CHARS = /[\\`*_{}[\]<>()#+.!|-]/g; @@ -170,6 +175,84 @@ export function createService(plugins: Partial[]): Plugin { return highlights; }, + async getSemanticTokens(doc, params, cancel) { + const results = await Promise.allSettled( + plugins.map(async (plugin) => + plugin.getSemanticTokens?.(doc, params, cancel), + ), + ); + + if (cancel.isCancellationRequested) return; + + let tokens: SemanticToken[] | undefined; + for (const result of results) { + if (result.status === "rejected") { + console.error(result.reason); + continue; + } + if (!Array.isArray(result.value)) continue; + + for (const token of result.value) { + const { start, end } = token.range; + if (start.line === end.line) { + if (start.character < end.character) { + (tokens ||= []).push(token); + } + continue; + } + + // No client this server has seen advertises multilineTokenSupport, + // so split multiline tokens into one per line rather than emit them. + for (let line = start.line; line <= end.line; line++) { + const startCharacter = line === start.line ? start.character : 0; + const endCharacter = + line === end.line ? end.character : lineLength(doc, line); + if (startCharacter < endCharacter) { + (tokens ||= []).push({ + range: { + start: { line, character: startCharacter }, + end: { line, character: endCharacter }, + }, + type: token.type, + modifiers: token.modifiers, + }); + } + } + } + } + + if (!tokens) return; + + // A stable sort keeps plugin order for equal starts, and the `prevEnd` + // watermark then drops any token overlapping one already kept -- so on + // conflicting ranges the earliest plugin wins, mirroring the overlap + // rules in findReferences/doRename. The watermark advances before the + // range filter so full and range requests agree on which overlapping + // token survives. + tokens.sort( + (a, b) => + a.range.start.line - b.range.start.line || + a.range.start.character - b.range.start.character, + ); + + const filterRange = "range" in params ? params.range : undefined; + const builder = new SemanticTokensBuilder(); + let prevEnd: Range["end"] | undefined; + for (const { range, type, modifiers } of tokens) { + if (prevEnd && positionBefore(range.start, prevEnd)) continue; + prevEnd = range.end; + if (filterRange && !rangesOverlap(range, filterRange)) continue; + builder.push( + range.start.line, + range.start.character, + range.end.character - range.start.character, + type, + modifiers, + ); + } + + return { data: builder.build().data }; + }, async findDocumentColors(doc, params, cancel) { const results = await Promise.allSettled( plugins.map((plugin) => @@ -343,6 +426,15 @@ function positionBefore(a: Range["start"], b: Range["start"]) { return a.line < b.line || (a.line === b.line && a.character < b.character); } +function lineLength(doc: TextDocument, line: number) { + return doc + .getText({ + start: { line, character: 0 }, + end: { line: line + 1, character: 0 }, + }) + .replace(/\r?\n$/, "").length; +} + /** Whether two ranges overlap (ie two plugins reported the same token). */ function rangesOverlap(a: Range, b: Range) { return positionBefore(a.start, b.end) && positionBefore(b.start, a.end); diff --git a/packages/language-server/src/service/marko/index.ts b/packages/language-server/src/service/marko/index.ts index 38a62216..208a285e 100644 --- a/packages/language-server/src/service/marko/index.ts +++ b/packages/language-server/src/service/marko/index.ts @@ -8,6 +8,7 @@ import { findDocumentLinks } from "./document-links"; import { findDocumentSymbols } from "./document-symbols"; import { format, formatDocument, type FormatOptions } from "./format"; import { doHover } from "./hover"; +import { getSemanticTokens } from "./semantic-tokens"; import { doValidate } from "./validate"; export default { @@ -20,6 +21,7 @@ export default { findDefinition, findDocumentLinks, findDocumentSymbols, + getSemanticTokens, format, commands: { "$/formatWithMode": async ({ diff --git a/packages/language-server/src/service/marko/semantic-tokens.ts b/packages/language-server/src/service/marko/semantic-tokens.ts new file mode 100644 index 00000000..a4563aca --- /dev/null +++ b/packages/language-server/src/service/marko/semantic-tokens.ts @@ -0,0 +1,118 @@ +import { type Node, NodeType, UNFINISHED } from "@marko/language-tools"; + +import { type MarkoFile, processDoc } from "../../utils/file"; +import { TokenModifier, TokenType } from "../semantic-tokens"; +import type { Plugin, SemanticToken } from "../types"; +import { isCoreTag } from "./util/is-core-tag"; + +export const getSemanticTokens: Plugin["getSemanticTokens"] = (doc) => + processDoc(doc, extractSemanticTokens); + +const localTagNameReg = /^[A-Z][a-zA-Z0-9_$]+$/; +const nodeModulesReg = /[\\/]node_modules[\\/]/; +// Mirrors the `#tag-name` rule in packages/vscode/syntaxes/marko.tmLanguage.json +// (`keyword.control.flow.marko`); keep the two lists in sync. +const controlFlowTagNames = new Set([ + "if", + "else", + "else-if", + "for", + "while", + "try", + "await", + "return", +]); +// The grammar scopes `html-comment` as `support.type.builtin.marko` alongside +// script/style (which the taglib flags `html`); leave its color to the grammar. +const grammarBuiltinTagNames = new Set(["html-comment"]); + +type TagClassification = Pick | undefined; + +function extractSemanticTokens({ + parsed, + lookup, + code, +}: MarkoFile): SemanticToken[] { + const tokens: SemanticToken[] = []; + const classifications = new Map(); + + const classify = (nameText: string): TagClassification => { + if (classifications.has(nameText)) return classifications.get(nameText); + + let classification: TagClassification; + const def = lookup.getTag(nameText); + if (def) { + if (!def.html && !grammarBuiltinTagNames.has(nameText)) { + const modifiers = def.deprecated ? TokenModifier.deprecated : 0; + if (isCoreTag(def)) { + classification = { + type: controlFlowTagNames.has(nameText) + ? TokenType.keyword + : TokenType.macro, + modifiers, + }; + } else { + const file = def.template || def.renderer || def.types; + if (file) { + classification = { + type: TokenType.class, + modifiers: nodeModulesReg.test(file) + ? modifiers | TokenModifier.defaultLibrary + : modifiers, + }; + } + } + } + } else if (localTagNameReg.test(nameText)) { + classification = { type: TokenType.class, modifiers: 0 }; + } + + classifications.set(nameText, classification); + return classification; + }; + + const stack: Node.ChildNode[] = [...parsed.program.body]; + while (stack.length) { + const node = stack.pop()!; + if (node.type !== NodeType.Tag && node.type !== NodeType.AttrTag) continue; + + const { nameText } = node; + if ( + node.type === NodeType.Tag && + nameText && + node.name.end !== UNFINISHED + ) { + const classification = classify(nameText); + if (classification) { + tokens.push({ + range: parsed.locationAt(node.name), + ...classification, + }); + + if (node.close && node.close.end !== UNFINISHED) { + const nameStart = code + .slice(node.close.start, node.close.end) + .indexOf(nameText); + if (nameStart !== -1) { + const start = node.close.start + nameStart; + tokens.push({ + range: parsed.locationAt({ + start, + end: start + nameText.length, + }), + ...classification, + }); + } + } + } + } + + if (node.body) { + for (const child of node.body) { + stack.push(child); + } + } + } + + return tokens; +} diff --git a/packages/language-server/src/service/marko/util/get-tag-name-completion.ts b/packages/language-server/src/service/marko/util/get-tag-name-completion.ts index 7ce2c469..10538cbc 100644 --- a/packages/language-server/src/service/marko/util/get-tag-name-completion.ts +++ b/packages/language-server/src/service/marko/util/get-tag-name-completion.ts @@ -11,6 +11,7 @@ import { } from "vscode-languageserver"; import { URI } from "vscode-uri"; +import { isCoreTag } from "./is-core-tag"; import { isHTML } from "./is-html"; const deprecated = [CompletionItemTag.Deprecated] as CompletionItemTag[]; @@ -34,15 +35,13 @@ export default function getTagNameCompletion({ const nodeModuleName = nodeModuleMatch && nodeModuleMatch[1].replace(/\\/g, "/"); - const isCoreTag = - /^@?marko[/-]/.test(tag.taglibId || tag.filePath) || - nodeModuleName === "marko"; + const coreTag = isCoreTag(tag); const html = isHTML(tag); const documentation = { kind: MarkupKind.Markdown, value: html ? `Built in [<${tag.name}>](https://developer.mozilla.org/en-US/docs/Web/HTML/Element/${tag.name}) HTML tag.` - : isCoreTag + : coreTag ? `Core Marko <${tag.name}> tag.` : nodeModuleName ? `Custom Marko tag discovered from the ["${nodeModuleName}"](${fileURIForTag}) npm package.` diff --git a/packages/language-server/src/service/marko/util/is-core-tag.ts b/packages/language-server/src/service/marko/util/is-core-tag.ts new file mode 100644 index 00000000..c9fb13bf --- /dev/null +++ b/packages/language-server/src/service/marko/util/is-core-tag.ts @@ -0,0 +1,18 @@ +import type { TagDefinition } from "@marko/compiler/babel-utils"; + +const coreTagReg = /^@?marko[/-]/; +const markoNodeModulesReg = /[\\/]node_modules[\\/]marko[\\/]/; + +/** + * Whether a tag ships with Marko itself. Checked against the taglib id, the + * runtime types module, and the resolved file path: which one identifies a + * core tag varies by how the taglib was discovered (and local `components/` + * dirs can shadow the taglib id while the types still point at the runtime). + */ +export function isCoreTag(tag: TagDefinition) { + return ( + coreTagReg.test(tag.taglibId || tag.filePath) || + (typeof tag.types === "string" && coreTagReg.test(tag.types)) || + markoNodeModulesReg.test(tag.template || tag.renderer || tag.filePath) + ); +} diff --git a/packages/language-server/src/service/script/index.ts b/packages/language-server/src/service/script/index.ts index a2eba4a9..f696cb43 100644 --- a/packages/language-server/src/service/script/index.ts +++ b/packages/language-server/src/service/script/index.ts @@ -43,7 +43,12 @@ import { import * as documents from "../../utils/text-documents"; import { system } from "../../utils/ts-system"; import * as workspace from "../../utils/workspace"; -import type { Plugin } from "../types"; +import { + decodeTsClassification, + TokenModifier, + TokenType, +} from "../semantic-tokens"; +import type { Plugin, SemanticToken } from "../types"; import printJSDocTag from "./util/print-jsdoc-tag"; // Filter out some syntax errors from the TS compiler which will be surfaced from the marko compiler. @@ -60,6 +65,11 @@ export interface TSProject { const extractCache = new Map(); const snapshotCache = new Map(); const insertModuleStatementLocCache = new WeakMap(); +const semanticTokensCache = new WeakMap< + Extracted, + { projectVersion: number; tokens: SemanticToken[] } +>(); +const maxSemanticTokensGeneratedLength = 256 * 1024; const markoFileReg = /\.marko$/; // Plain (non-Marko) script files: `.ts`/`.tsx`/`.js`/`.jsx` and their // `.mjs`/`.cjs`/`.mts`/`.cts` variants. These are already part of the @@ -483,6 +493,121 @@ const ScriptService: Partial = { return result.length ? result : undefined; }, + getSemanticTokens(doc, _params, cancel) { + const fileName = getFSPath(doc); + if (!fileName) return; + + const project = getTSProject(fileName); + const result: SemanticToken[] = []; + + if (plainScriptReg.test(fileName)) { + const { spans } = project.service.getEncodedSemanticClassifications( + fileName, + { start: 0, length: doc.getText().length }, + ts.SemanticClassificationFormat.TwentyTwenty, + ); + + for (let i = 0; i < spans.length; i += 3) { + const decoded = decodeTsClassification(spans[i + 2]); + if (!decoded) continue; + result.push({ + range: { + start: doc.positionAt(spans[i]), + end: doc.positionAt(spans[i] + spans[i + 1]), + }, + ...decoded, + }); + } + + return result; + } + + const extracted = processScript(doc, project); + const generated = extracted.toString(); + + // Classifications come from the whole TS program, so cache per extraction + // and re-classify only when any document changes. Without this the range + // and full requests a client sends per version would each re-classify. + const cached = semanticTokensCache.get(extracted); + if (cached && cached.projectVersion === documents.projectVersion) { + return cached.tokens; + } + + // At the measured ~380ns per generated char, classification passes 100ms + // of uninterruptible work around this size; beyond it, leave highlighting + // to the grammar. + if (generated.length > maxSemanticTokensGeneratedLength) return; + + const source = extracted.parsed.code; + const letTagVarNames = getLetTagVarNames(getMarkoFile(doc).parsed); + const { spans } = project.service.getEncodedSemanticClassifications( + fileName, + { start: 0, length: generated.length }, + ts.SemanticClassificationFormat.TwentyTwenty, + ); + + const resultByRange = new Map(); + for (let i = 0; i < spans.length; i += 3) { + if (cancel.isCancellationRequested) return; + + const genStart = spans[i]; + const genLength = spans[i + 1]; + const decoded = decodeTsClassification(spans[i + 2]); + if (!decoded) continue; + + // The extractor copies the same source range into several generated + // contexts and interleaves synthesized glue; only a classification that + // sits inside a single mapping, spans it exactly, and matches the source + // text verbatim is a real token of the .marko file. + const ranges = extracted.sourceRangesAt(genStart, genStart + genLength); + if (ranges.length !== 1) continue; + + const [sourceRange] = ranges; + if ( + sourceRange.end - sourceRange.start !== genLength || + source.slice(sourceRange.start, sourceRange.end) !== + generated.slice(genStart, genStart + genLength) + ) { + continue; + } + + const { type } = decoded; + let { modifiers } = decoded; + if ( + letTagVarNames && + type === TokenType.variable && + modifiers & TokenModifier.readonly && + letTagVarNames.has(source.slice(sourceRange.start, sourceRange.end)) + ) { + // A `` variable compiles to a generated `const`, so TypeScript + // reports mutable state as readonly; don't let it render like one. + modifiers &= ~TokenModifier.readonly; + } + + const range = extracted.parsed.locationAt(sourceRange); + const key = rangeKey(range); + const existing = resultByRange.get(key); + if (!existing) { + resultByRange.set(key, { range, type, modifiers }); + } else if ( + existing.type === TokenType.property && + type !== TokenType.property + ) { + // A mutated tag variable is also rewritten to a generated property + // access; the non-property classification is the one that reflects + // the source. + existing.type = type; + existing.modifiers = modifiers; + } + } + + result.push(...resultByRange.values()); + semanticTokensCache.set(extracted, { + projectVersion: documents.projectVersion, + tokens: result, + }); + return result; + }, doHover(doc, params) { const fileName = getFSPath(doc); if (!fileName) return; @@ -830,6 +955,32 @@ function rangeKey({ start, end }: Range) { return `${start.line}:${start.character}:${end.line}:${end.character}`; } +const identifierReg = /^[A-Za-z_$][A-Za-z0-9_$]*$/; + +function getLetTagVarNames(parsed: Parsed) { + let names: Set | undefined; + const stack: Node.ChildNode[] = [...parsed.program.body]; + while (stack.length) { + const node = stack.pop()!; + if (node.type !== NodeType.Tag && node.type !== NodeType.AttrTag) continue; + + if (node.type === NodeType.Tag && node.nameText === "let" && node.var) { + const name = parsed.read(node.var.value); + if (identifierReg.test(name)) { + (names ||= new Set()).add(name); + } + } + + if (node.body) { + for (const child of node.body) { + stack.push(child); + } + } + } + + return names; +} + function forEachSourceLocation( project: TSProject, { fileName, textSpan }: ts.DocumentSpan, diff --git a/packages/language-server/src/service/semantic-tokens.ts b/packages/language-server/src/service/semantic-tokens.ts new file mode 100644 index 00000000..80b03e93 --- /dev/null +++ b/packages/language-server/src/service/semantic-tokens.ts @@ -0,0 +1,57 @@ +import type { SemanticTokensLegend } from "vscode-languageserver"; + +// Type indices 0-11 and modifier bits 0-5 mirror TypeScript's classifier2020 +// encoding (with `member` renamed to LSP's `method`), so decoded script +// classifications pass through unchanged. Append new entries; never reorder. +export const tokenTypes = [ + "class", + "enum", + "interface", + "namespace", + "typeParameter", + "type", + "parameter", + "variable", + "enumMember", + "property", + "function", + "method", + "keyword", + "macro", +] as const; + +export const tokenModifiers = [ + "declaration", + "static", + "async", + "readonly", + "defaultLibrary", + "local", + "deprecated", +] as const; + +export const semanticTokensLegend: SemanticTokensLegend = { + tokenTypes: [...tokenTypes], + tokenModifiers: [...tokenModifiers], +}; + +export const TokenType = Object.fromEntries( + tokenTypes.map((name, index) => [name, index]), +) as { [Name in (typeof tokenTypes)[number]]: number }; + +export const TokenModifier = Object.fromEntries( + tokenModifiers.map((name, index) => [name, 1 << index]), +) as { [Name in (typeof tokenModifiers)[number]]: number }; + +const tsTokenTypeCount = 12; +const tsTokenModifierCount = 6; +const tsTokenModifierMask = (1 << tsTokenModifierCount) - 1; + +// TypeScript "2020" format: classification = ((type + 1) << 8) + modifierSet. +export function decodeTsClassification( + classification: number, +): { type: number; modifiers: number } | undefined { + const type = (classification >> 8) - 1; + if (type < 0 || type >= tsTokenTypeCount) return; + return { type, modifiers: classification & tsTokenModifierMask }; +} diff --git a/packages/language-server/src/service/types.ts b/packages/language-server/src/service/types.ts index 3b49adfc..6700e6e1 100644 --- a/packages/language-server/src/service/types.ts +++ b/packages/language-server/src/service/types.ts @@ -28,12 +28,21 @@ import type { Range, ReferenceParams, RenameParams, + SemanticTokens, + SemanticTokensParams, + SemanticTokensRangeParams, SymbolInformation, TextEdit, WorkspaceEdit, } from "vscode-languageserver"; import type { TextDocument } from "vscode-languageserver-textdocument"; +export interface SemanticToken { + range: Range; + type: number; + modifiers: number; +} + export type Result = Promise | V | void; type Handler = ( doc: TextDocument, @@ -69,6 +78,10 @@ export type Plugin = { findDocumentSymbols: Handler; findDocumentLinks: Handler; findDocumentHighlights: Handler; + getSemanticTokens: Handler< + SemanticTokensParams | SemanticTokensRangeParams, + SemanticToken[] | SemanticTokens + >; findDocumentColors: Handler; getColorPresentations: Handler; format: Handler; From 8719beef80fadbcf7465197da894c39de8d01cc1 Mon Sep 17 00:00:00 2001 From: Brian Carbone Date: Tue, 28 Jul 2026 09:32:58 -0400 Subject: [PATCH 03/11] chore: record review findings in agent-feedback --- agent-feedback/bugs.md | 48 +++++++++++++++++++++++++++++++++++++++ agent-feedback/cleanup.md | 24 ++++++++++++++++++++ agent-feedback/perf.md | 12 ++++++++++ agent-feedback/unclear.md | 12 ++++++++++ 4 files changed, 96 insertions(+) diff --git a/agent-feedback/bugs.md b/agent-feedback/bugs.md index 412b8d00..290b86eb 100644 --- a/agent-feedback/bugs.md +++ b/agent-feedback/bugs.md @@ -24,8 +24,56 @@ The runtime accepts a string OR number loop key — `assertValidLoopKey` throws A custom tag that fails to resolve (a component referenced by kebab tag name with no `import`, or a typo'd tag name) is a hard build error but produces ZERO diagnostics under `mtc` — the type-check tool agents are told to run. Reproduced in a real @marko/run scaffold: Marko 6 auto-discovers only `tags/` dirs (`runtime-tags` sets `tagDiscoveryDirs = ["tags"]` at `packages/runtime-tags/src/translator/index.ts:40`), so `src/components/*.marko` is NOT auto-registered — `` (no import) and the typo `` both pass `mtc` with exit 0, while `npm run compile -o html` / `marko-run build` throw `Unable to find entry point for custom tag ` (`resolveTagImport` at `tags.js:353`; `tagNotFoundError` at `packages/runtime-tags/src/translator/visitors/tag/custom-tag.ts:411`). The identical wrong attr on a RESOLVED tag correctly errors TS2322 (verified against ``), proving tag resolution — not the attribute — is the gate: `#writeTag` lowers an unresolvable tag to `renderDynamicTag(...)` whose input is `Record`, so every attribute and callback param goes unchecked. This is the worst shape for an agent whose deterministic verify loop is `mtc`: it creates a component, references it by tag (natural for anyone used to auto-registering `components/` dirs), sees a clean type-check, and ships wrong props or a misspelled tag; the build then fails with an "entry point" error that looks unrelated to the type loop. Direction: have @marko/language-tools emit a distinct "cannot resolve tag ``" diagnostic mirroring the compiler instead of degrading to an untyped dynamic tag. The compiler-side authority is marko-js/marko's `packages/compiler/src/babel-utils/tags.js` › `resolveTagImport` and `custom-tag.ts` › `tagNotFoundError`. Distinct from the run dx.md route-types entries (missing `Run` global / stale `routes.d.ts`), which concern generated route types, not custom-tag resolution. +## Guard `Extracted.sourceRangeAt` consumers against anchor-token expansion and cross-token spans + +`packages/language-tools/src/util/extractor.ts` › `Extracted` | 2026-07-27 | impact:med | effort:med + +Two hazards exist for a consumer that maps a generated range back to source and assumes the length is preserved. A zero-width `Mapping.anchor` token (`generatedLength === 0`, `sourceLength > 0`) matches a query at exactly its `generatedStart`, and `rangeAt`/`sourceRangesAt` then contribute the anchor's entire unrelated source range. Separately, `rangeAt` pairs the first overlapping start token with the last end token, spanning any unmapped glue between two different tokens, so the returned source range can be longer than and textually different from the queried generated span; `sourceLocationAtTextSpan` in `packages/language-server/src/service/script/index.ts` relies on `sourceLocationAt` and can therefore report oversized or misplaced ranges for TS results whose spans straddle mappings. Consumers that need precision should use `sourceRangesAt` and require a single result of exact width with byte-identical text, or `rangeAt` could learn to reject cross-token spans. Re-verify: query `sourceRangeAt` over a generated span that begins at an `anchor` offset (the anchors emitted in `extractors/script/index.ts` around tag names) and compare the returned width to the queried width. + +## Add `macro` and `effect` to the TextMate grammar's core-tag keyword lists + +`packages/vscode/syntaxes/marko.tmLanguage.json` › `#tag-name` | 2026-07-27 | impact:low | effort:low + +The `#tag-name` rule hardcodes two alternations for Marko core tags (`for|if|while|else-if|else|try|await|return` scoped `keyword.control.flow.marko` and `const|context|debug|define|id|let|log|lifecycle` scoped `support.function.marko`), and `macro` and `effect` appear in neither, so both color as ordinary user-defined tags (`entity.name.tag.marko`). `grep -n "macro" packages/vscode/syntaxes/marko.tmLanguage.json` returns no hits today, and both are real core tags: `macro` is recognized in `packages/language-tools/src/extractors/script/util/get-runtime-api.ts`. The fix is adding the two names to the alternations. Re-verify: open a `.marko` file containing `` and check the scope with VS Code's "Developer: Inspect Editor Tokens and Scopes". + ## Keep the `"use strict"` prologue when injecting the `import.meta.url` banner `packages/ts-plugin/build.mts` › `build` | 2026-07-24 | impact:low | effort:low The `banner.js` that defines `_importMetaUrl` is emitted above esbuild's own `"use strict"`, so that string is no longer a directive prologue and the bundle runs in sloppy mode: `head -2 packages/ts-plugin/dist/index.js` shows the `const` on line 1 and `"use strict";` on line 2. The same applies to all three `packages/vscode/dist/*.js` bundles, which share the pattern via `packages/vscode/build.mts`. Bundled TS rarely depends on strict semantics, so nothing is known to break today, but silently dropping strict mode is not what either build intends. Fix by prefixing the banner with `"use strict";\n` (as `packages/language-tools/build.mts` does); re-verify with `head -2` on each bundle. + +## Make `clearMarkoCacheForFile` clear the same cache `getMarkoFile` populates for non-file documents + +`packages/language-server/src/utils/file.ts` › `clearMarkoCacheForFile` | 2026-07-28 | impact:med | effort:low + +`getMarkoFile` caches a non-`file:` (e.g. `untitled:`) document under `Project.getCache(process.cwd())`, but `clearMarkoCacheForFile` deletes from `Project.getCache(getFSDir(doc))`, which is `getCache(undefined)` for those documents. When the cwd resolves a local marko install those are different maps, the delete is a no-op, and edits to untitled documents keep serving the stale parse; reproduced live, where document symbols requested after an edit that inserted a new tag still returned only the pre-edit symbols. Fix by extracting one shared helper that computes the cache directory identically for both call sites. Re-verify: open an untitled marko doc, request symbols, apply an edit adding a tag, and request symbols again; the new tag is currently missing. + +## Invalidate the compiler-diagnostics cache on watched-file changes, not just document versions + +`packages/language-server/src/service/marko/validate.ts` › `getMarkoDiagnostics` | 2026-07-28 | impact:med | effort:low + +The compiler diagnostics cache is keyed per `(doc, doc.version)`, but a watched-file event (e.g. creating the component a template references) triggers revalidation without changing the document's version, so stale cached diagnostics such as an unresolved-tag error the on-disk change just fixed are re-sent until the user edits the document itself. Key the cache additionally on a cache generation such as `documents.projectVersion`, or clear it from the watched-files path. Re-verify: open a doc referencing a missing component, create the component file on disk, fire the watched-files event, and re-validate; the error should clear without editing the doc. + +## Recover from a whitespace-containing close tag instead of swallowing the rest of the file + +`packages/language-tools/src/parser.ts` › `Builder` | 2026-07-28 | impact:med | effort:med + +A close tag with trailing whitespace before `>` (e.g. ``) leaves the close range `UNFINISHED` and the parser absorbs the entire remainder of the file into it, so every feature that walks the CST (symbols, links, semantic tokens) loses everything below that point; mid-keystroke this visibly strips highlighting for the rest of the file. The runtime compiler accepts the same input, so the gap is in parser recovery rather than in the language definition. Re-verify: parse `a\n${x}` and observe that the `` tag is missing from `program.body` because it lives inside the `if` close range. + +## Bound recursion (or go iterative) in the script extractor and remaining CST walkers + +`packages/language-tools/src/extractors/script/index.ts` › `ScriptExtractor` | 2026-07-28 | impact:low | effort:med + +Roughly 5,000 levels of nested unclosed tags overflow the call stack in the recursive script extractor, as well as in the remaining recursive walkers in `packages/language-server/src/service/marko/document-symbols.ts` and `document-links.ts`, and the facade's `Promise.allSettled` swallows the `RangeError`, so features return empty results for the document with no error surfaced. The semantic-token walkers were converted to explicit stacks during review; the extractor and the two remaining walkers still recurse. Re-verify: use `"".repeat(5000)` as a document body and request document symbols, which currently returns empty with a swallowed RangeError. + +## Wrap plugin calls in `createService` merge arms so synchronous throws are isolated + +`packages/language-server/src/service/create-service.ts` › `createService` | 2026-07-28 | impact:low | effort:low + +Every merge arm except `getSemanticTokens` calls `plugin.method?.(...)` directly inside the `plugins.map(...)` passed to `Promise.allSettled`, so a plugin method that throws synchronously (several script-plugin handlers are synchronous) throws out of the `.map` callback before `allSettled` can isolate it, rejecting the entire feature instead of that one plugin's contribution. `getSemanticTokens` wraps the call in an `async` closure to isolate such throws; the same one-word change applies to the other arms. Re-verify: a test plugin whose `findDocumentHighlights` throws synchronously currently rejects the whole facade call rather than being skipped. + +## Classify mutation targets of tag variables in the extracted script + +`packages/language-tools/marko.internal.d.ts` › `change` | 2026-07-28 | impact:low | effort:med + +The assignment target of a mutated `` variable is rewritten to a generated property access (`__marko_internal_change__x.x = ...`), and TypeScript emits no semantic classification for that property because the `Marko._.change(...)` type does not resolve it to a named property symbol, even though plain TS classifies write targets (`o.p = 2` classifies `p` as property). In editors with semantic highlighting, the one place a reactive variable is written is therefore the one occurrence left uncolored. A candidate fix is making the change-handler type in the internal types carry a real named property. Re-verify: request semantic tokens for `
- 82 | -``` - diff --git a/packages/language-server/src/__tests__/fixtures/script/class-api-basic/__snapshots__/class-api-basic.expected/index.ts b/packages/language-server/src/__tests__/fixtures/script/class-api-basic/__snapshots__/class-api-basic.expected/index.ts index 655e1de7..ffc34b33 100644 --- a/packages/language-server/src/__tests__/fixtures/script/class-api-basic/__snapshots__/class-api-basic.expected/index.ts +++ b/packages/language-server/src/__tests__/fixtures/script/class-api-basic/__snapshots__/class-api-basic.expected/index.ts @@ -151,7 +151,7 @@ export { type Component }; }; })(), }); - const __marko_internal_tag_3 = Missing; + const __marko_internal_tag_3 = Marko._.interpolated`Missing`; Marko._.renderDynamicTag(__marko_internal_tag_3)()()({}); const __marko_internal_tag_4 = Marko._.interpolated`complex-missing`; Marko._.renderDynamicTag(__marko_internal_tag_4)()()({}); diff --git a/packages/language-server/src/__tests__/fixtures/script/prefer-local-identifier-tag-name/__snapshots__/prefer-local-identifier-tag-name.expected/index.ts b/packages/language-server/src/__tests__/fixtures/script/prefer-local-identifier-tag-name/__snapshots__/prefer-local-identifier-tag-name.expected/index.ts index 91a30ce0..20a25948 100644 --- a/packages/language-server/src/__tests__/fixtures/script/prefer-local-identifier-tag-name/__snapshots__/prefer-local-identifier-tag-name.expected/index.ts +++ b/packages/language-server/src/__tests__/fixtures/script/prefer-local-identifier-tag-name/__snapshots__/prefer-local-identifier-tag-name.expected/index.ts @@ -26,11 +26,10 @@ export interface Input {} Marko._.renderDynamicTag(__marko_internal_tag_2)()()({ a: "hello", }); - const __marko_internal_tag_3 = Marko._.fallbackTemplate( - TestTagB, + const __marko_internal_tag_3 = Marko._.resolveTemplate( import("./components/TestTagB.marko"), ); - Marko._.renderDynamicTag(__marko_internal_tag_3)()()({ + Marko._.renderTemplate(__marko_internal_tag_3)()()({ b: "hello", }); Marko._.renderNativeTag("div")()()({ diff --git a/packages/language-server/src/__tests__/fixtures/semantic-tokens/single-letter/__snapshots__/single-letter.expected/components/A.html b/packages/language-server/src/__tests__/fixtures/semantic-tokens/single-letter/__snapshots__/single-letter.expected/components/A.html new file mode 100644 index 00000000..2b0fba8c --- /dev/null +++ b/packages/language-server/src/__tests__/fixtures/semantic-tokens/single-letter/__snapshots__/single-letter.expected/components/A.html @@ -0,0 +1 @@ +
A
\ No newline at end of file diff --git a/packages/language-server/src/__tests__/fixtures/semantic-tokens/single-letter/__snapshots__/single-letter.expected/components/A.md b/packages/language-server/src/__tests__/fixtures/semantic-tokens/single-letter/__snapshots__/single-letter.expected/components/A.md new file mode 100644 index 00000000..e69de29b diff --git a/packages/language-server/src/__tests__/fixtures/semantic-tokens/single-letter/__snapshots__/single-letter.expected/components/A.tokens.md b/packages/language-server/src/__tests__/fixtures/semantic-tokens/single-letter/__snapshots__/single-letter.expected/components/A.tokens.md new file mode 100644 index 00000000..68e0c4e9 --- /dev/null +++ b/packages/language-server/src/__tests__/fixtures/semantic-tokens/single-letter/__snapshots__/single-letter.expected/components/A.tokens.md @@ -0,0 +1 @@ +(no semantic tokens) diff --git a/packages/language-server/src/__tests__/fixtures/semantic-tokens/single-letter/__snapshots__/single-letter.expected/components/A.ts b/packages/language-server/src/__tests__/fixtures/semantic-tokens/single-letter/__snapshots__/single-letter.expected/components/A.ts new file mode 100644 index 00000000..c77c7440 --- /dev/null +++ b/packages/language-server/src/__tests__/fixtures/semantic-tokens/single-letter/__snapshots__/single-letter.expected/components/A.ts @@ -0,0 +1,60 @@ +export interface Input {} +abstract class Component extends Marko.Component {} +export { type Component }; +(function (this: void) { + const input = Marko._.any as Input; + const component = Marko._.any as Component; + const state = Marko._.state(component); + const out = Marko._.any as Marko.Out; + const $signal = Marko._.any as AbortSignal; + const $global = Marko._.getGlobal( + // @ts-expect-error We expect the compiler to error because we are checking if the MarkoRun.Context is defined. + (Marko._.error, Marko._.any as MarkoRun.Context), + ); + Marko._.renderNativeTag("div")()()({ + [Marko._.content]: (() => { + return () => { + return Marko._.voidReturn; + }; + })(), + }); + Marko._.noop({ component, state, out, input, $global, $signal }); + return; +})(); +const __marko_internal_api = "class"; +export { __marko_internal_api as "~api" }; +export default new (class Template extends Marko._.Template<{ + render( + input: Marko.TemplateInput, + stream?: { + write: (chunk: string) => void; + end: (chunk?: string) => void; + }, + ): Marko.Out; + + render( + input: Marko.TemplateInput, + cb?: (err: Error | null, result: Marko.RenderResult) => void, + ): Marko.Out; + + renderSync(input: Marko.TemplateInput): Marko.RenderResult; + + renderToString(input: Marko.TemplateInput): string; + + stream( + input: Marko.TemplateInput, + ): ReadableStream & NodeJS.ReadableStream; + + mount( + input: Marko.TemplateInput, + reference: Node, + position?: "afterbegin" | "afterend" | "beforebegin" | "beforeend", + ): Marko.MountedTemplate; + + api: typeof __marko_internal_api; + _(): () => <__marko_internal_input extends unknown>( + input: Marko.Directives & + Input & + Marko._.Relate<__marko_internal_input, Marko.Directives & Input>, + ) => Marko._.ReturnWithScope<__marko_internal_input, void>; +}> {})(); diff --git a/packages/language-server/src/__tests__/fixtures/semantic-tokens/single-letter/__snapshots__/single-letter.expected/components/Ab.html b/packages/language-server/src/__tests__/fixtures/semantic-tokens/single-letter/__snapshots__/single-letter.expected/components/Ab.html new file mode 100644 index 00000000..0ef105d1 --- /dev/null +++ b/packages/language-server/src/__tests__/fixtures/semantic-tokens/single-letter/__snapshots__/single-letter.expected/components/Ab.html @@ -0,0 +1 @@ +
Ab
\ No newline at end of file diff --git a/packages/language-server/src/__tests__/fixtures/semantic-tokens/single-letter/__snapshots__/single-letter.expected/components/Ab.md b/packages/language-server/src/__tests__/fixtures/semantic-tokens/single-letter/__snapshots__/single-letter.expected/components/Ab.md new file mode 100644 index 00000000..e69de29b diff --git a/packages/language-server/src/__tests__/fixtures/semantic-tokens/single-letter/__snapshots__/single-letter.expected/components/Ab.tokens.md b/packages/language-server/src/__tests__/fixtures/semantic-tokens/single-letter/__snapshots__/single-letter.expected/components/Ab.tokens.md new file mode 100644 index 00000000..68e0c4e9 --- /dev/null +++ b/packages/language-server/src/__tests__/fixtures/semantic-tokens/single-letter/__snapshots__/single-letter.expected/components/Ab.tokens.md @@ -0,0 +1 @@ +(no semantic tokens) diff --git a/packages/language-server/src/__tests__/fixtures/semantic-tokens/single-letter/__snapshots__/single-letter.expected/components/Ab.ts b/packages/language-server/src/__tests__/fixtures/semantic-tokens/single-letter/__snapshots__/single-letter.expected/components/Ab.ts new file mode 100644 index 00000000..c77c7440 --- /dev/null +++ b/packages/language-server/src/__tests__/fixtures/semantic-tokens/single-letter/__snapshots__/single-letter.expected/components/Ab.ts @@ -0,0 +1,60 @@ +export interface Input {} +abstract class Component extends Marko.Component {} +export { type Component }; +(function (this: void) { + const input = Marko._.any as Input; + const component = Marko._.any as Component; + const state = Marko._.state(component); + const out = Marko._.any as Marko.Out; + const $signal = Marko._.any as AbortSignal; + const $global = Marko._.getGlobal( + // @ts-expect-error We expect the compiler to error because we are checking if the MarkoRun.Context is defined. + (Marko._.error, Marko._.any as MarkoRun.Context), + ); + Marko._.renderNativeTag("div")()()({ + [Marko._.content]: (() => { + return () => { + return Marko._.voidReturn; + }; + })(), + }); + Marko._.noop({ component, state, out, input, $global, $signal }); + return; +})(); +const __marko_internal_api = "class"; +export { __marko_internal_api as "~api" }; +export default new (class Template extends Marko._.Template<{ + render( + input: Marko.TemplateInput, + stream?: { + write: (chunk: string) => void; + end: (chunk?: string) => void; + }, + ): Marko.Out; + + render( + input: Marko.TemplateInput, + cb?: (err: Error | null, result: Marko.RenderResult) => void, + ): Marko.Out; + + renderSync(input: Marko.TemplateInput): Marko.RenderResult; + + renderToString(input: Marko.TemplateInput): string; + + stream( + input: Marko.TemplateInput, + ): ReadableStream & NodeJS.ReadableStream; + + mount( + input: Marko.TemplateInput, + reference: Node, + position?: "afterbegin" | "afterend" | "beforebegin" | "beforeend", + ): Marko.MountedTemplate; + + api: typeof __marko_internal_api; + _(): () => <__marko_internal_input extends unknown>( + input: Marko.Directives & + Input & + Marko._.Relate<__marko_internal_input, Marko.Directives & Input>, + ) => Marko._.ReturnWithScope<__marko_internal_input, void>; +}> {})(); diff --git a/packages/language-server/src/__tests__/fixtures/semantic-tokens/single-letter/__snapshots__/single-letter.expected/index.html b/packages/language-server/src/__tests__/fixtures/semantic-tokens/single-letter/__snapshots__/single-letter.expected/index.html new file mode 100644 index 00000000..74aa865b --- /dev/null +++ b/packages/language-server/src/__tests__/fixtures/semantic-tokens/single-letter/__snapshots__/single-letter.expected/index.html @@ -0,0 +1 @@ +
bound to the import, so an identifier tag
not bound, so an uppercase native tag
\ No newline at end of file diff --git a/packages/language-server/src/__tests__/fixtures/semantic-tokens/single-letter/__snapshots__/single-letter.expected/index.md b/packages/language-server/src/__tests__/fixtures/semantic-tokens/single-letter/__snapshots__/single-letter.expected/index.md new file mode 100644 index 00000000..e69de29b diff --git a/packages/language-server/src/__tests__/fixtures/semantic-tokens/single-letter/__snapshots__/single-letter.expected/index.tokens.md b/packages/language-server/src/__tests__/fixtures/semantic-tokens/single-letter/__snapshots__/single-letter.expected/index.tokens.md new file mode 100644 index 00000000..83a75227 --- /dev/null +++ b/packages/language-server/src/__tests__/fixtures/semantic-tokens/single-letter/__snapshots__/single-letter.expected/index.tokens.md @@ -0,0 +1,8 @@ +Ln 4, Col 2 (len 1) class `A` +Ln 5, Col 2 (len 2) class `Ab` +Ln 6, Col 2 (len 1) class `B` +Ln 7, Col 2 (len 2) class `Bc` +Ln 9, Col 2 (len 1) class `B` +Ln 9, Col 47 (len 1) class `B` +Ln 10, Col 2 (len 1) class `I` +Ln 10, Col 43 (len 1) class `I` diff --git a/packages/language-server/src/__tests__/fixtures/semantic-tokens/single-letter/__snapshots__/single-letter.expected/index.ts b/packages/language-server/src/__tests__/fixtures/semantic-tokens/single-letter/__snapshots__/single-letter.expected/index.ts new file mode 100644 index 00000000..438f5aa6 --- /dev/null +++ b/packages/language-server/src/__tests__/fixtures/semantic-tokens/single-letter/__snapshots__/single-letter.expected/index.ts @@ -0,0 +1,83 @@ +import B from "./components/A.marko"; +import Bc from "./components/Ab.marko"; +export interface Input {} +abstract class Component extends Marko.Component {} +export { type Component }; +(function (this: void) { + const input = Marko._.any as Input; + const component = Marko._.any as Component; + const state = Marko._.state(component); + const out = Marko._.any as Marko.Out; + const $signal = Marko._.any as AbortSignal; + const $global = Marko._.getGlobal( + // @ts-expect-error We expect the compiler to error because we are checking if the MarkoRun.Context is defined. + (Marko._.error, Marko._.any as MarkoRun.Context), + ); + const __marko_internal_tag_1 = Marko._.resolveTemplate( + import("./components/A.marko"), + ); + Marko._.renderTemplate(__marko_internal_tag_1)()()({}); + const __marko_internal_tag_2 = Marko._.resolveTemplate( + import("./components/Ab.marko"), + ); + Marko._.renderTemplate(__marko_internal_tag_2)()()({}); + const __marko_internal_tag_3 = B; + Marko._.renderDynamicTag(__marko_internal_tag_3)()()({}); + const __marko_internal_tag_4 = Bc; + Marko._.renderDynamicTag(__marko_internal_tag_4)()()({}); + const __marko_internal_tag_5 = B; + Marko._.renderDynamicTag(__marko_internal_tag_5)()()({ + [Marko._.contentFor(__marko_internal_tag_5)]: (() => { + return () => { + return Marko._.voidReturn; + }; + })(), + }); + const __marko_internal_tag_6 = Marko._.interpolated`I`; + Marko._.renderDynamicTag(__marko_internal_tag_6)()()({ + [Marko._.contentFor(__marko_internal_tag_6)]: (() => { + return () => { + return Marko._.voidReturn; + }; + })(), + }); + Marko._.noop({ component, state, out, input, $global, $signal }); + return; +})(); +const __marko_internal_api = "class"; +export { __marko_internal_api as "~api" }; +export default new (class Template extends Marko._.Template<{ + render( + input: Marko.TemplateInput, + stream?: { + write: (chunk: string) => void; + end: (chunk?: string) => void; + }, + ): Marko.Out; + + render( + input: Marko.TemplateInput, + cb?: (err: Error | null, result: Marko.RenderResult) => void, + ): Marko.Out; + + renderSync(input: Marko.TemplateInput): Marko.RenderResult; + + renderToString(input: Marko.TemplateInput): string; + + stream( + input: Marko.TemplateInput, + ): ReadableStream & NodeJS.ReadableStream; + + mount( + input: Marko.TemplateInput, + reference: Node, + position?: "afterbegin" | "afterend" | "beforebegin" | "beforeend", + ): Marko.MountedTemplate; + + api: typeof __marko_internal_api; + _(): () => <__marko_internal_input extends unknown>( + input: Marko.Directives & + Input & + Marko._.Relate<__marko_internal_input, Marko.Directives & Input>, + ) => Marko._.ReturnWithScope<__marko_internal_input, void>; +}> {})(); diff --git a/packages/language-server/src/__tests__/fixtures/semantic-tokens/single-letter/components/A.marko b/packages/language-server/src/__tests__/fixtures/semantic-tokens/single-letter/components/A.marko new file mode 100644 index 00000000..ef04d626 --- /dev/null +++ b/packages/language-server/src/__tests__/fixtures/semantic-tokens/single-letter/components/A.marko @@ -0,0 +1 @@ +
A
diff --git a/packages/language-server/src/__tests__/fixtures/semantic-tokens/single-letter/components/Ab.marko b/packages/language-server/src/__tests__/fixtures/semantic-tokens/single-letter/components/Ab.marko new file mode 100644 index 00000000..db2ab880 --- /dev/null +++ b/packages/language-server/src/__tests__/fixtures/semantic-tokens/single-letter/components/Ab.marko @@ -0,0 +1 @@ +
Ab
diff --git a/packages/language-server/src/__tests__/fixtures/semantic-tokens/single-letter/index.marko b/packages/language-server/src/__tests__/fixtures/semantic-tokens/single-letter/index.marko new file mode 100644 index 00000000..a97e63ca --- /dev/null +++ b/packages/language-server/src/__tests__/fixtures/semantic-tokens/single-letter/index.marko @@ -0,0 +1,10 @@ +import B from ""; +import Bc from ""; + + + + + + +bound to the import, so an identifier tag +not bound, so an uppercase native tag diff --git a/packages/language-server/src/service/marko/semantic-tokens.ts b/packages/language-server/src/service/marko/semantic-tokens.ts index a4563aca..6326c879 100644 --- a/packages/language-server/src/service/marko/semantic-tokens.ts +++ b/packages/language-server/src/service/marko/semantic-tokens.ts @@ -8,7 +8,7 @@ import { isCoreTag } from "./util/is-core-tag"; export const getSemanticTokens: Plugin["getSemanticTokens"] = (doc) => processDoc(doc, extractSemanticTokens); -const localTagNameReg = /^[A-Z][a-zA-Z0-9_$]+$/; +const localTagNameReg = /^[A-Z][a-zA-Z0-9_$]*$/; const nodeModulesReg = /[\\/]node_modules[\\/]/; // Mirrors the `#tag-name` rule in packages/vscode/syntaxes/marko.tmLanguage.json // (`keyword.control.flow.marko`); keep the two lists in sync. diff --git a/packages/language-tools/src/extractors/script/index.ts b/packages/language-tools/src/extractors/script/index.ts index 128ef8d8..16ee91dc 100644 --- a/packages/language-tools/src/extractors/script/index.ts +++ b/packages/language-tools/src/extractors/script/index.ts @@ -1,4 +1,4 @@ -import type { types as t } from "@marko/compiler"; +import { types as t } from "@marko/compiler"; import type { TagDefinition, TaglibLookup } from "@marko/compiler/babel-utils"; import path from "path"; import { relativeImportPath } from "relative-import-path"; @@ -52,7 +52,7 @@ const REG_INPUT_TYPE = /\s*(interface|type)\s+Input\b/y; const REG_OBJECT_PROPERTY = /^[_$a-z][_$a-z0-9]*$/i; // Match https://www.typescriptlang.org/docs/handbook/triple-slash-directives.html#-reference-path- and https://www.typescriptlang.org/docs/handbook/intro-to-js-ts.html#ts-check const REG_COMMENT_PRAGMA = /\/\/(?:\s*@ts-|\/\s*<)/y; -const REG_TAG_NAME_IDENTIFIER = /^[A-Z][a-zA-Z0-9_$]+$/; +const REG_TAG_NAME_IDENTIFIER = /^[A-Z][a-zA-Z0-9_$]*$/; const REG_NODE_MODULES = /[\\/]node_modules[\\/]/; const IF_TAG_ALTERNATES = new WeakMap(); const TAG_ID = new WeakMap(); @@ -128,6 +128,7 @@ class ScriptExtractor { #ts: ExtractScriptOptions["ts"]; #runtimeTypes: ExtractScriptOptions["runtimeTypesCode"]; #mutations: Repeatable; + #bindingNames: Set | undefined; #tagId = 1; #closeBrackets: number[] = [0]; constructor(opts: ExtractScriptOptions) { @@ -790,6 +791,71 @@ constructor(_) {} return returnTag !== undefined; } + /** + * Every name a tag name could resolve to in the generated module: the + * declarations of the module level statements plus every Marko tag variable + * and tag parameter. + * + * Deliberately a superset of any single tag's lexical scope. Over-reporting + * leaves a tag on the identifier path it has always taken, while + * under-reporting would silently retype a tag that does resolve, so the + * cheaper error is to include too much. Built on first use, since a template + * with no capitalized tag names never needs it. + */ + #getBindingNames() { + if (this.#bindingNames) return this.#bindingNames; + + const names = (this.#bindingNames = new Set()); + const add = (node: t.Node | null | undefined) => { + if (!node) return; + + let found = false; + for (const name in t.getBindingIdentifiers(node)) { + names.add(name); + found = true; + } + + if (!found) { + // A TypeScript type only declaration (`type`/`interface`/`enum`/ + // `namespace`) has no binding identifier but still shadows a tag name. + const { id } = node as { id?: t.Node }; + if (id && t.isIdentifier(id)) names.add(id.name); + } + }; + + for (const node of this.#parsed.program.static) { + switch (node.type) { + case NodeType.Import: + add(this.#ast.import(node)); + break; + case NodeType.Export: + add(this.#ast.export(node)); + break; + case NodeType.Static: + for (const statement of this.#ast.static(node) || []) add(statement); + break; + } + } + + const stack: Node.ChildNode[] = [...this.#parsed.program.body]; + while (stack.length) { + const node = stack.pop()!; + if (node.type !== NodeType.Tag && node.type !== NodeType.AttrTag) + continue; + + if (node.var) add(this.#ast.tagVar(node.var)); + if (node.params) { + for (const param of this.#ast.tagParams(node.params) || []) add(param); + } + + if (node.body) { + for (const child of node.body) stack.push(child); + } + } + + return names; + } + #writeTag(tag: Node.Tag) { const tagName = tag.nameText; const def = tagName ? this.#lookup.getTag(tagName) : undefined; @@ -802,7 +868,16 @@ constructor(_) {} let templateVar: undefined | string; if (!def || importPath) { - const isIdentifier = tagName && REG_TAG_NAME_IDENTIFIER.test(tagName); + // Both translators only treat a tag name as an identifier when it also + // resolves to a binding (`TAG_NAME_IDENTIFIER_REG.test(name) && + // tag.scope.getBinding(name)`); an unbound name stays a native/dynamic + // tag. Matching that here keeps markup the compiler accepts -- eg the + // uppercase HTML tags ``, ``, `` -- from being emitted as a bare + // reference that TypeScript reports `Cannot find name` for. + const isIdentifier = + tagName && + REG_TAG_NAME_IDENTIFIER.test(tagName) && + this.#getBindingNames().has(tagName); const isMarkoFile = importPath?.endsWith(".marko"); if (isIdentifier || isMarkoFile || !importPath) { From d350f69ee5e073f9cf8b62fca96e27acdfd3b5bc Mon Sep 17 00:00:00 2001 From: Brian Carbone Date: Tue, 28 Jul 2026 15:27:28 -0400 Subject: [PATCH 07/11] fix: derive tag variable writability from the tag's return type --- .../components/my-input.html | 1 + .../components/my-input.md | 0 .../components/my-input.tokens.md | 17 ++ .../components/my-input.ts | 82 +++++++ .../components/my-ro.html | 1 + .../components/my-ro.md | 0 .../components/my-ro.tokens.md | 10 + .../components/my-ro.ts | 80 +++++++ .../writable-vars.expected/index.html | 6 + .../writable-vars.expected/index.md | 0 .../writable-vars.expected/index.tokens.md | 35 +++ .../writable-vars.expected/index.ts | 193 +++++++++++++++++ .../writable-vars/components/my-input.marko | 7 + .../writable-vars/components/my-ro.marko | 6 + .../semantic-tokens/writable-vars/index.marko | 21 ++ .../src/service/script/index.ts | 203 +++++++++++++++--- 16 files changed, 631 insertions(+), 31 deletions(-) create mode 100644 packages/language-server/src/__tests__/fixtures/semantic-tokens/writable-vars/__snapshots__/writable-vars.expected/components/my-input.html create mode 100644 packages/language-server/src/__tests__/fixtures/semantic-tokens/writable-vars/__snapshots__/writable-vars.expected/components/my-input.md create mode 100644 packages/language-server/src/__tests__/fixtures/semantic-tokens/writable-vars/__snapshots__/writable-vars.expected/components/my-input.tokens.md create mode 100644 packages/language-server/src/__tests__/fixtures/semantic-tokens/writable-vars/__snapshots__/writable-vars.expected/components/my-input.ts create mode 100644 packages/language-server/src/__tests__/fixtures/semantic-tokens/writable-vars/__snapshots__/writable-vars.expected/components/my-ro.html create mode 100644 packages/language-server/src/__tests__/fixtures/semantic-tokens/writable-vars/__snapshots__/writable-vars.expected/components/my-ro.md create mode 100644 packages/language-server/src/__tests__/fixtures/semantic-tokens/writable-vars/__snapshots__/writable-vars.expected/components/my-ro.tokens.md create mode 100644 packages/language-server/src/__tests__/fixtures/semantic-tokens/writable-vars/__snapshots__/writable-vars.expected/components/my-ro.ts create mode 100644 packages/language-server/src/__tests__/fixtures/semantic-tokens/writable-vars/__snapshots__/writable-vars.expected/index.html create mode 100644 packages/language-server/src/__tests__/fixtures/semantic-tokens/writable-vars/__snapshots__/writable-vars.expected/index.md create mode 100644 packages/language-server/src/__tests__/fixtures/semantic-tokens/writable-vars/__snapshots__/writable-vars.expected/index.tokens.md create mode 100644 packages/language-server/src/__tests__/fixtures/semantic-tokens/writable-vars/__snapshots__/writable-vars.expected/index.ts create mode 100644 packages/language-server/src/__tests__/fixtures/semantic-tokens/writable-vars/components/my-input.marko create mode 100644 packages/language-server/src/__tests__/fixtures/semantic-tokens/writable-vars/components/my-ro.marko create mode 100644 packages/language-server/src/__tests__/fixtures/semantic-tokens/writable-vars/index.marko diff --git a/packages/language-server/src/__tests__/fixtures/semantic-tokens/writable-vars/__snapshots__/writable-vars.expected/components/my-input.html b/packages/language-server/src/__tests__/fixtures/semantic-tokens/writable-vars/__snapshots__/writable-vars.expected/components/my-input.html new file mode 100644 index 00000000..7693b577 --- /dev/null +++ b/packages/language-server/src/__tests__/fixtures/semantic-tokens/writable-vars/__snapshots__/writable-vars.expected/components/my-input.html @@ -0,0 +1 @@ +
placeholder
\ No newline at end of file diff --git a/packages/language-server/src/__tests__/fixtures/semantic-tokens/writable-vars/__snapshots__/writable-vars.expected/components/my-input.md b/packages/language-server/src/__tests__/fixtures/semantic-tokens/writable-vars/__snapshots__/writable-vars.expected/components/my-input.md new file mode 100644 index 00000000..e69de29b diff --git a/packages/language-server/src/__tests__/fixtures/semantic-tokens/writable-vars/__snapshots__/writable-vars.expected/components/my-input.tokens.md b/packages/language-server/src/__tests__/fixtures/semantic-tokens/writable-vars/__snapshots__/writable-vars.expected/components/my-input.tokens.md new file mode 100644 index 00000000..67ddcc8a --- /dev/null +++ b/packages/language-server/src/__tests__/fixtures/semantic-tokens/writable-vars/__snapshots__/writable-vars.expected/components/my-input.tokens.md @@ -0,0 +1,17 @@ +Ln 1, Col 18 (len 5) interface [declaration] `Input` +Ln 1, Col 24 (len 1) typeParameter [declaration] `T` +Ln 2, Col 3 (len 5) property [declaration] `value` +Ln 2, Col 10 (len 1) typeParameter `T` +Ln 3, Col 3 (len 11) method [declaration] `valueChange` +Ln 3, Col 18 (len 5) parameter [declaration] `value` +Ln 3, Col 25 (len 1) typeParameter `T` +Ln 6, Col 8 (len 6) class [defaultLibrary] `String` +Ln 6, Col 15 (len 5) variable [readonly local] `input` +Ln 6, Col 21 (len 5) property `value` +Ln 7, Col 2 (len 6) keyword `return` +Ln 7, Col 9 (len 5) variable [readonly local] `input` +Ln 7, Col 15 (len 5) property `value` +Ln 7, Col 34 (len 5) variable [readonly local] `input` +Ln 7, Col 40 (len 11) method `valueChange` +Ln 7, Col 56 (len 5) parameter [declaration] `value` +Ln 7, Col 63 (len 1) typeParameter `T` diff --git a/packages/language-server/src/__tests__/fixtures/semantic-tokens/writable-vars/__snapshots__/writable-vars.expected/components/my-input.ts b/packages/language-server/src/__tests__/fixtures/semantic-tokens/writable-vars/__snapshots__/writable-vars.expected/components/my-input.ts new file mode 100644 index 00000000..4118ac90 --- /dev/null +++ b/packages/language-server/src/__tests__/fixtures/semantic-tokens/writable-vars/__snapshots__/writable-vars.expected/components/my-input.ts @@ -0,0 +1,82 @@ +export interface Input { + value: T; + valueChange?: (value: T) => void; +} +function __marko_internal_template(this: void) { + const input = Marko._.any as Input; + const $signal = Marko._.any as AbortSignal; + const $global = Marko._.getGlobal( + // @ts-expect-error We expect the compiler to error because we are checking if the MarkoRun.Context is defined. + (Marko._.error, Marko._.any as MarkoRun.Context), + ); + Marko._.renderNativeTag("div")()()({ + [Marko._.content]: (() => { + String(input.value); + return () => { + return Marko._.voidReturn; + }; + })(), + }); + var __marko_internal_return = Marko._.returnTag({ + value: input.value, + valueChange: input.valueChange as (value: T) => void, + }); + Marko._.noop({ input, $global, $signal }); + return __marko_internal_return; +} +const __marko_internal_api = "tags"; +export { __marko_internal_api as "~api" }; +const MyInput = new (class Template extends Marko._.Template<{ + render( + input: Marko.TemplateInput>, + stream?: { + write: (chunk: string) => void; + end: (chunk?: string) => void; + }, + ): Marko.Out; + + render( + input: Marko.TemplateInput>, + cb?: (err: Error | null, result: Marko.RenderResult) => void, + ): Marko.Out; + + renderSync( + input: Marko.TemplateInput>, + ): Marko.RenderResult; + + renderToString(input: Marko.TemplateInput>): string; + + stream( + input: Marko.TemplateInput>, + ): ReadableStream & NodeJS.ReadableStream; + + mount( + input: Marko.TemplateInput>, + reference: Node, + position?: "afterbegin" | "afterend" | "beforebegin" | "beforeend", + ): Marko.MountedTemplate; + + api: typeof __marko_internal_api; + _<__marko_internal_apply = 1>(): __marko_internal_apply extends 0 + ? () => <__marko_internal_input extends unknown>( + input: Marko.Directives & + Input & + Marko._.Relate<__marko_internal_input, Marko.Directives & Input>, + ) => Marko._.ReturnWithScope< + __marko_internal_input, + typeof __marko_internal_template extends () => infer Return + ? Return + : never + > + : () => <__marko_internal_input extends unknown, T>( + input: Marko.Directives & + Input & + Marko._.Relate<__marko_internal_input, Marko.Directives & Input>, + ) => Marko._.ReturnWithScope< + __marko_internal_input, + typeof __marko_internal_template extends () => infer Return + ? Return + : never + >; +}> {})(); +export default MyInput; diff --git a/packages/language-server/src/__tests__/fixtures/semantic-tokens/writable-vars/__snapshots__/writable-vars.expected/components/my-ro.html b/packages/language-server/src/__tests__/fixtures/semantic-tokens/writable-vars/__snapshots__/writable-vars.expected/components/my-ro.html new file mode 100644 index 00000000..7693b577 --- /dev/null +++ b/packages/language-server/src/__tests__/fixtures/semantic-tokens/writable-vars/__snapshots__/writable-vars.expected/components/my-ro.html @@ -0,0 +1 @@ +
placeholder
\ No newline at end of file diff --git a/packages/language-server/src/__tests__/fixtures/semantic-tokens/writable-vars/__snapshots__/writable-vars.expected/components/my-ro.md b/packages/language-server/src/__tests__/fixtures/semantic-tokens/writable-vars/__snapshots__/writable-vars.expected/components/my-ro.md new file mode 100644 index 00000000..e69de29b diff --git a/packages/language-server/src/__tests__/fixtures/semantic-tokens/writable-vars/__snapshots__/writable-vars.expected/components/my-ro.tokens.md b/packages/language-server/src/__tests__/fixtures/semantic-tokens/writable-vars/__snapshots__/writable-vars.expected/components/my-ro.tokens.md new file mode 100644 index 00000000..f5026ee0 --- /dev/null +++ b/packages/language-server/src/__tests__/fixtures/semantic-tokens/writable-vars/__snapshots__/writable-vars.expected/components/my-ro.tokens.md @@ -0,0 +1,10 @@ +Ln 1, Col 18 (len 5) interface [declaration] `Input` +Ln 1, Col 24 (len 1) typeParameter [declaration] `T` +Ln 2, Col 3 (len 5) property [declaration] `value` +Ln 2, Col 10 (len 1) typeParameter `T` +Ln 5, Col 8 (len 6) class [defaultLibrary] `String` +Ln 5, Col 15 (len 5) variable [readonly local] `input` +Ln 5, Col 21 (len 5) property `value` +Ln 6, Col 2 (len 6) keyword `return` +Ln 6, Col 9 (len 5) variable [readonly local] `input` +Ln 6, Col 15 (len 5) property `value` diff --git a/packages/language-server/src/__tests__/fixtures/semantic-tokens/writable-vars/__snapshots__/writable-vars.expected/components/my-ro.ts b/packages/language-server/src/__tests__/fixtures/semantic-tokens/writable-vars/__snapshots__/writable-vars.expected/components/my-ro.ts new file mode 100644 index 00000000..6df1148f --- /dev/null +++ b/packages/language-server/src/__tests__/fixtures/semantic-tokens/writable-vars/__snapshots__/writable-vars.expected/components/my-ro.ts @@ -0,0 +1,80 @@ +export interface Input { + value: T; +} +function __marko_internal_template(this: void) { + const input = Marko._.any as Input; + const $signal = Marko._.any as AbortSignal; + const $global = Marko._.getGlobal( + // @ts-expect-error We expect the compiler to error because we are checking if the MarkoRun.Context is defined. + (Marko._.error, Marko._.any as MarkoRun.Context), + ); + Marko._.renderNativeTag("div")()()({ + [Marko._.content]: (() => { + String(input.value); + return () => { + return Marko._.voidReturn; + }; + })(), + }); + var __marko_internal_return = Marko._.returnTag({ + value: input.value, + }); + Marko._.noop({ input, $global, $signal }); + return __marko_internal_return; +} +const __marko_internal_api = "tags"; +export { __marko_internal_api as "~api" }; +const MyRo = new (class Template extends Marko._.Template<{ + render( + input: Marko.TemplateInput>, + stream?: { + write: (chunk: string) => void; + end: (chunk?: string) => void; + }, + ): Marko.Out; + + render( + input: Marko.TemplateInput>, + cb?: (err: Error | null, result: Marko.RenderResult) => void, + ): Marko.Out; + + renderSync( + input: Marko.TemplateInput>, + ): Marko.RenderResult; + + renderToString(input: Marko.TemplateInput>): string; + + stream( + input: Marko.TemplateInput>, + ): ReadableStream & NodeJS.ReadableStream; + + mount( + input: Marko.TemplateInput>, + reference: Node, + position?: "afterbegin" | "afterend" | "beforebegin" | "beforeend", + ): Marko.MountedTemplate; + + api: typeof __marko_internal_api; + _<__marko_internal_apply = 1>(): __marko_internal_apply extends 0 + ? () => <__marko_internal_input extends unknown>( + input: Marko.Directives & + Input & + Marko._.Relate<__marko_internal_input, Marko.Directives & Input>, + ) => Marko._.ReturnWithScope< + __marko_internal_input, + typeof __marko_internal_template extends () => infer Return + ? Return + : never + > + : () => <__marko_internal_input extends unknown, T>( + input: Marko.Directives & + Input & + Marko._.Relate<__marko_internal_input, Marko.Directives & Input>, + ) => Marko._.ReturnWithScope< + __marko_internal_input, + typeof __marko_internal_template extends () => infer Return + ? Return + : never + >; +}> {})(); +export default MyRo; diff --git a/packages/language-server/src/__tests__/fixtures/semantic-tokens/writable-vars/__snapshots__/writable-vars.expected/index.html b/packages/language-server/src/__tests__/fixtures/semantic-tokens/writable-vars/__snapshots__/writable-vars.expected/index.html new file mode 100644 index 00000000..841fb6d9 --- /dev/null +++ b/packages/language-server/src/__tests__/fixtures/semantic-tokens/writable-vars/__snapshots__/writable-vars.expected/index.html @@ -0,0 +1,6 @@ +
+ + placeholder +
\ No newline at end of file diff --git a/packages/language-server/src/__tests__/fixtures/semantic-tokens/writable-vars/__snapshots__/writable-vars.expected/index.md b/packages/language-server/src/__tests__/fixtures/semantic-tokens/writable-vars/__snapshots__/writable-vars.expected/index.md new file mode 100644 index 00000000..e69de29b diff --git a/packages/language-server/src/__tests__/fixtures/semantic-tokens/writable-vars/__snapshots__/writable-vars.expected/index.tokens.md b/packages/language-server/src/__tests__/fixtures/semantic-tokens/writable-vars/__snapshots__/writable-vars.expected/index.tokens.md new file mode 100644 index 00000000..4141e3ab --- /dev/null +++ b/packages/language-server/src/__tests__/fixtures/semantic-tokens/writable-vars/__snapshots__/writable-vars.expected/index.tokens.md @@ -0,0 +1,35 @@ +Ln 1, Col 18 (len 5) interface [declaration] `Input` +Ln 2, Col 3 (len 4) property [declaration] `pair` +Ln 2, Col 11 (len 1) property [declaration] `a` +Ln 2, Col 22 (len 1) property [declaration] `b` +Ln 5, Col 14 (len 5) variable [declaration readonly] `value` +Ln 6, Col 14 (len 7) variable [declaration readonly] `doubled` +Ln 6, Col 24 (len 5) variable [readonly] `value` +Ln 8, Col 2 (len 3) macro `let` +Ln 8, Col 6 (len 7) variable [declaration local] `mutable` +Ln 9, Col 2 (len 3) macro `let` +Ln 9, Col 8 (len 1) variable [declaration local] `a` +Ln 9, Col 11 (len 1) variable [declaration local] `b` +Ln 9, Col 17 (len 5) variable [readonly local] `input` +Ln 9, Col 23 (len 4) property `pair` +Ln 10, Col 2 (len 5) macro `const` +Ln 10, Col 8 (len 6) variable [declaration readonly local] `frozen` +Ln 11, Col 2 (len 8) class `my-input` +Ln 11, Col 11 (len 5) variable [declaration local] `typed` +Ln 12, Col 2 (len 5) class `my-ro` +Ln 12, Col 8 (len 5) variable [declaration readonly local] `plain` +Ln 14, Col 31 (len 7) variable [local] `mutable` +Ln 15, Col 5 (len 7) variable [readonly] `doubled` +Ln 15, Col 16 (len 7) variable [local] `mutable` +Ln 15, Col 27 (len 1) variable [local] `a` +Ln 15, Col 32 (len 1) variable [local] `b` +Ln 15, Col 37 (len 6) variable [readonly local] `frozen` +Ln 15, Col 47 (len 5) variable [local] `typed` +Ln 15, Col 56 (len 5) variable [readonly local] `plain` +Ln 18, Col 2 (len 3) keyword `for` +Ln 18, Col 6 (len 1) parameter [declaration] `i` +Ln 19, Col 4 (len 3) macro `let` +Ln 19, Col 8 (len 5) variable [declaration local] `value` +Ln 19, Col 16 (len 1) parameter `i` +Ln 20, Col 5 (len 5) variable [local] `value` +Ln 21, Col 3 (len 3) keyword `for` diff --git a/packages/language-server/src/__tests__/fixtures/semantic-tokens/writable-vars/__snapshots__/writable-vars.expected/index.ts b/packages/language-server/src/__tests__/fixtures/semantic-tokens/writable-vars/__snapshots__/writable-vars.expected/index.ts new file mode 100644 index 00000000..ce296b1c --- /dev/null +++ b/packages/language-server/src/__tests__/fixtures/semantic-tokens/writable-vars/__snapshots__/writable-vars.expected/index.ts @@ -0,0 +1,193 @@ +export interface Input { + pair: { a: string; b: number }; +} +const value = 1; +const doubled = value * 2; +(function (this: void) { + const input = Marko._.any as Input; + const $signal = Marko._.any as AbortSignal; + const $global = Marko._.getGlobal( + // @ts-expect-error We expect the compiler to error because we are checking if the MarkoRun.Context is defined. + (Marko._.error, Marko._.any as MarkoRun.Context), + ); + const mutable = Marko._.hoist(() => __marko_internal_hoist__mutable); + const a = Marko._.hoist(() => __marko_internal_hoist__a); + const b = Marko._.hoist(() => __marko_internal_hoist__b); + const frozen = Marko._.hoist(() => __marko_internal_hoist__frozen); + const typed = Marko._.hoist(() => __marko_internal_hoist__typed); + const plain = Marko._.hoist(() => __marko_internal_hoist__plain); + const value = Marko._.hoist(() => __marko_internal_hoist__value); + const __marko_internal_tag_1 = Marko._.resolveTemplate( + import("@marko/runtime-tags/tags/let.d.marko"), + ); + { + const mutable = Marko._.returned(() => __marko_internal_rendered_1); + const __marko_internal_rendered_1 = Marko._.renderTemplate( + __marko_internal_tag_1, + )()()({ + value: 2, + }); + const __marko_internal_change__mutable = Marko._.change( + "mutable", + "value", + __marko_internal_rendered_1.return, + ); + const __marko_internal_tag_2 = Marko._.resolveTemplate( + import("@marko/runtime-tags/tags/let.d.marko"), + ); + { + const { a, b } = Marko._.returned(() => __marko_internal_rendered_2); + const __marko_internal_rendered_2 = Marko._.renderTemplate( + __marko_internal_tag_2, + )()()({ + value: input.pair, + }); + const __marko_internal_change__a = Marko._.change( + "a", + __marko_internal_rendered_2.return.a, + ); + const __marko_internal_tag_3 = Marko._.resolveTemplate( + import("@marko/runtime-tags/tags/const.d.marko"), + ); + { + const frozen = Marko._.returned(() => __marko_internal_rendered_3); + const __marko_internal_rendered_3 = Marko._.renderTemplate( + __marko_internal_tag_3, + )()()({ + value: 3, + }); + const __marko_internal_tag_4 = Marko._.resolveTemplate( + import("./components/my-input.marko"), + ); + { + const typed = Marko._.returned(() => __marko_internal_rendered_4); + const __marko_internal_rendered_4 = Marko._.renderTemplate( + __marko_internal_tag_4, + )()()({ + value: "x", + }); + const __marko_internal_change__typed = Marko._.change( + "typed", + "value", + __marko_internal_rendered_4.return, + ); + const __marko_internal_tag_5 = Marko._.resolveTemplate( + import("./components/my-ro.marko"), + ); + { + const plain = Marko._.returned(() => __marko_internal_rendered_5); + const __marko_internal_rendered_5 = Marko._.renderTemplate( + __marko_internal_tag_5, + )()()({ + value: "y", + }); + Marko._.renderNativeTag("button")()()({ + onClick() { + __marko_internal_change__mutable.mutable = mutable + 1; + __marko_internal_change__typed.typed = "z"; + __marko_internal_change__a.a = "b"; + }, + [Marko._.content]: (() => { + doubled; + mutable; + a; + b; + frozen; + typed; + plain; + return () => { + return Marko._.voidReturn; + }; + })(), + }); + const __marko_internal_rendered_6 = Marko._.forOfTag( + { + of: [1, 2], + }, + (i) => { + const __marko_internal_tag_7 = Marko._.resolveTemplate( + import("@marko/runtime-tags/tags/let.d.marko"), + ); + { + const value = Marko._.returned( + () => __marko_internal_rendered_7, + ); + const __marko_internal_rendered_7 = Marko._.renderTemplate( + __marko_internal_tag_7, + )()()({ + value: i, + }); + value; + return new (class MarkoReturn { + readonly [Marko._.scope] = { value }; + declare return: Return; + constructor(_?: Return) {} + })(); + } + }, + ); + var __marko_internal_hoist__mutable = mutable; + var __marko_internal_hoist__a = a; + var __marko_internal_hoist__b = b; + var __marko_internal_hoist__frozen = frozen; + var __marko_internal_hoist__typed = typed; + var __marko_internal_hoist__plain = plain; + var { value: __marko_internal_hoist__value } = Marko._.readScope( + __marko_internal_rendered_6, + ); + } + } + } + } + } + Marko._.noop({ + mutable, + a, + b, + frozen, + typed, + plain, + value, + input, + $global, + $signal, + }); + return; +})(); +const __marko_internal_api = "tags"; +export { __marko_internal_api as "~api" }; +export default new (class Template extends Marko._.Template<{ + render( + input: Marko.TemplateInput, + stream?: { + write: (chunk: string) => void; + end: (chunk?: string) => void; + }, + ): Marko.Out; + + render( + input: Marko.TemplateInput, + cb?: (err: Error | null, result: Marko.RenderResult) => void, + ): Marko.Out; + + renderSync(input: Marko.TemplateInput): Marko.RenderResult; + + renderToString(input: Marko.TemplateInput): string; + + stream( + input: Marko.TemplateInput, + ): ReadableStream & NodeJS.ReadableStream; + + mount( + input: Marko.TemplateInput, + reference: Node, + position?: "afterbegin" | "afterend" | "beforebegin" | "beforeend", + ): Marko.MountedTemplate; + + api: typeof __marko_internal_api; + _(): () => <__marko_internal_input extends unknown>( + input: Marko.Directives & + Input & + Marko._.Relate<__marko_internal_input, Marko.Directives & Input>, + ) => Marko._.ReturnWithScope<__marko_internal_input, void>; +}> {})(); diff --git a/packages/language-server/src/__tests__/fixtures/semantic-tokens/writable-vars/components/my-input.marko b/packages/language-server/src/__tests__/fixtures/semantic-tokens/writable-vars/components/my-input.marko new file mode 100644 index 00000000..3e4ab88c --- /dev/null +++ b/packages/language-server/src/__tests__/fixtures/semantic-tokens/writable-vars/components/my-input.marko @@ -0,0 +1,7 @@ +export interface Input { + value: T; + valueChange?: (value: T) => void; +} + +
${String(input.value)}
+ void)/> diff --git a/packages/language-server/src/__tests__/fixtures/semantic-tokens/writable-vars/components/my-ro.marko b/packages/language-server/src/__tests__/fixtures/semantic-tokens/writable-vars/components/my-ro.marko new file mode 100644 index 00000000..418d13a9 --- /dev/null +++ b/packages/language-server/src/__tests__/fixtures/semantic-tokens/writable-vars/components/my-ro.marko @@ -0,0 +1,6 @@ +export interface Input { + value: T; +} + +
${String(input.value)}
+ diff --git a/packages/language-server/src/__tests__/fixtures/semantic-tokens/writable-vars/index.marko b/packages/language-server/src/__tests__/fixtures/semantic-tokens/writable-vars/index.marko new file mode 100644 index 00000000..e8207606 --- /dev/null +++ b/packages/language-server/src/__tests__/fixtures/semantic-tokens/writable-vars/index.marko @@ -0,0 +1,21 @@ +export interface Input { + pair: { a: string; b: number }; +} + +static const value = 1; +static const doubled = value * 2; + + + + + + + + + + + + ${value} + diff --git a/packages/language-server/src/service/script/index.ts b/packages/language-server/src/service/script/index.ts index f696cb43..d696b053 100644 --- a/packages/language-server/src/service/script/index.ts +++ b/packages/language-server/src/service/script/index.ts @@ -539,12 +539,12 @@ const ScriptService: Partial = { if (generated.length > maxSemanticTokensGeneratedLength) return; const source = extracted.parsed.code; - const letTagVarNames = getLetTagVarNames(getMarkoFile(doc).parsed); const { spans } = project.service.getEncodedSemanticClassifications( fileName, { start: 0, length: generated.length }, ts.SemanticClassificationFormat.TwentyTwenty, ); + const writability = getTagVarWritability(project, fileName, extracted); const resultByRange = new Map(); for (let i = 0; i < spans.length; i += 3) { @@ -574,30 +574,19 @@ const ScriptService: Partial = { const { type } = decoded; let { modifiers } = decoded; if ( - letTagVarNames && type === TokenType.variable && modifiers & TokenModifier.readonly && - letTagVarNames.has(source.slice(sourceRange.start, sourceRange.end)) + writability?.isWritable(genStart) ) { - // A `` variable compiles to a generated `const`, so TypeScript - // reports mutable state as readonly; don't let it render like one. modifiers &= ~TokenModifier.readonly; } const range = extracted.parsed.locationAt(sourceRange); const key = rangeKey(range); - const existing = resultByRange.get(key); - if (!existing) { + // Duplicate ranges with differing classifications are routine (the same + // source is copied into several generated contexts); the first span wins. + if (!resultByRange.has(key)) { resultByRange.set(key, { range, type, modifiers }); - } else if ( - existing.type === TokenType.property && - type !== TokenType.property - ) { - // A mutated tag variable is also rewritten to a generated property - // access; the non-property classification is the one that reflects - // the source. - existing.type = type; - existing.modifiers = modifiers; } } @@ -955,30 +944,182 @@ function rangeKey({ start, end }: Range) { return `${start.line}:${start.character}:${end.line}:${end.character}`; } -const identifierReg = /^[A-Za-z_$][A-Za-z0-9_$]*$/; +const anyOrUnknownFlags = ts.TypeFlags.Any | ts.TypeFlags.Unknown; -function getLetTagVarNames(parsed: Parsed) { - let names: Set | undefined; - const stack: Node.ChildNode[] = [...parsed.program.body]; - while (stack.length) { - const node = stack.pop()!; - if (node.type !== NodeType.Tag && node.type !== NodeType.AttrTag) continue; +/** + * Every tag variable is emitted as a generated `const`, so TypeScript reports + * all of them as readonly. Whether one can actually be assigned depends on the + * tag: a mutation compiles into a write through the object `Marko._.change` + * builds, and that object's property is mutable exactly when the tag returns a + * `valueChange` callback. Ask the tag's return type that same question, + * resolved from the token's own symbol so the answer follows scope instead of + * the variable's name. + */ +class TagVarWritability { + #checker: ts.TypeChecker; + #sourceFile: ts.SourceFile; + #extracted: Extracted; + #byBinding = new Map(); + + constructor( + checker: ts.TypeChecker, + sourceFile: ts.SourceFile, + extracted: Extracted, + ) { + this.#checker = checker; + this.#sourceFile = sourceFile; + this.#extracted = extracted; + } - if (node.type === NodeType.Tag && node.nameText === "let" && node.var) { - const name = parsed.read(node.var.value); - if (identifierReg.test(name)) { - (names ||= new Set()).add(name); + isWritable(generatedOffset: number) { + try { + const node = generatedNodeAt(this.#sourceFile, generatedOffset); + if (!ts.isIdentifier(node)) return false; + const symbol = this.#checker.getSymbolAtLocation(node); + if (!symbol) return false; + + let result = this.#byBinding.get(symbol); + if (result === undefined) { + result = this.#resolve(symbol, true); + this.#byBinding.set(symbol, result); } + return result; + } catch { + // A checker failure must not cost the file its tokens; keep readonly. + return false; + } + } + + #resolve(symbol: ts.Symbol, followHoist: boolean): boolean { + const decl = enclosingVariableDeclaration(symbol.valueDeclaration); + const init = decl?.initializer; + if ( + !decl || + !init || + !ts.isCallExpression(init) || + !ts.isPropertyAccessExpression(init.expression) || + !isMarkoInternalsAccess(init.expression.expression) + ) { + return false; } - if (node.body) { - for (const child of node.body) { - stack.push(child); + // Read outside its own tag body, a tag variable resolves to a program + // scope alias. The alias declaration maps back to the same source + // identifier as the real binding, so re-resolve through the identifier's + // other generated copies to reach it. + if (followHoist && init.expression.name.text === "hoist") { + if (!ts.isIdentifier(decl.name)) return false; + const aliasOffset = decl.name.getStart(this.#sourceFile); + const sourceRanges = this.#extracted.sourceRangesAt( + aliasOffset, + aliasOffset + decl.name.text.length, + ); + if (sourceRanges.length !== 1) return false; + + for (const offset of this.#extracted.generatedOffsetsAt( + sourceRanges[0].start, + )) { + if (offset === aliasOffset) continue; + const node = generatedNodeAt(this.#sourceFile, offset); + if (!ts.isIdentifier(node)) continue; + const bound = this.#checker.getSymbolAtLocation(node); + if (bound && bound !== symbol && this.#resolve(bound, false)) { + return true; + } } + return false; + } + + // Anything that is not `const = Marko._.returned(() => )` + // (or a destructuring of one) is an ordinary binding and keeps its + // readonly. + if (init.expression.name.text !== "returned") return false; + const thunk = init.arguments[0]; + if ( + !thunk || + !ts.isArrowFunction(thunk) || + ts.isBlock(thunk.body) || + !ts.isIdentifier(thunk.body) + ) { + return false; } + + const rendered = thunk.body; + const checker = this.#checker; + const renderedType = checker.getTypeAtLocation(rendered); + if (renderedType.flags & anyOrUnknownFlags) return true; + const returnSymbol = renderedType.getProperty("return"); + if (!returnSymbol) return false; + const returnType = checker.getTypeOfSymbolAtLocation( + returnSymbol, + rendered, + ); + if (returnType.flags & anyOrUnknownFlags) return true; + const changeSymbol = returnType.getProperty("valueChange"); + if (!changeSymbol) return false; + return isCallable( + checker.getTypeOfSymbolAtLocation(changeSymbol, rendered), + ); + } +} + +function getTagVarWritability( + project: TSProject, + fileName: string, + extracted: Extracted, +) { + try { + const program = project.service.getProgram(); + const sourceFile = program?.getSourceFile(fileName); + if (!program || !sourceFile) return; + return new TagVarWritability( + program.getTypeChecker(), + sourceFile, + extracted, + ); + } catch { + // Without a checker every tag variable keeps its readonly; better than + // failing the request and losing the file's script tokens entirely. } +} + +/** + * Whether an expression is the extractor's internals namespace access + * (`Marko._`), so a user's own `.returned(...)`/`.hoist(...)` methods cannot + * spoof a tag variable binding. + */ +function isMarkoInternalsAccess(expression: ts.Expression) { + return ( + ts.isPropertyAccessExpression(expression) && + expression.name.text === "_" && + ts.isIdentifier(expression.expression) && + expression.expression.text === "Marko" + ); +} + +function isCallable(type: ts.Type): boolean { + if (type.flags & anyOrUnknownFlags) return true; + if (type.isUnion()) return type.types.some(isCallable); + return type.getCallSignatures().length > 0; +} - return names; +/** The deepest node containing a generated offset. */ +function generatedNodeAt(node: ts.Node, pos: number): ts.Node { + let found: ts.Node | undefined; + ts.forEachChild(node, (child) => { + if (child.pos <= pos && pos < child.end) { + found = child; + return true; + } + }); + return found ? generatedNodeAt(found, pos) : node; +} + +function enclosingVariableDeclaration(node: ts.Node | undefined) { + for (let cur = node; cur; cur = cur.parent) { + if (ts.isVariableDeclaration(cur)) return cur; + if (ts.isStatement(cur) || ts.isSourceFile(cur)) return undefined; + } } function forEachSourceLocation( From 00f6af51eacd15b938233b49f807ae0d4f54a826 Mon Sep 17 00:00:00 2001 From: Brian Carbone Date: Tue, 28 Jul 2026 15:27:49 -0400 Subject: [PATCH 08/11] fix: cache, bound, and cancel plain-script semantic tokens --- .../src/__tests__/plain-script.test.ts | 113 ++++++++++++++++++ .../src/service/script/index.ts | 24 +++- 2 files changed, 136 insertions(+), 1 deletion(-) diff --git a/packages/language-server/src/__tests__/plain-script.test.ts b/packages/language-server/src/__tests__/plain-script.test.ts index ddf5ba18..37850ee1 100644 --- a/packages/language-server/src/__tests__/plain-script.test.ts +++ b/packages/language-server/src/__tests__/plain-script.test.ts @@ -7,6 +7,7 @@ import { URI } from "vscode-uri"; import { documents } from "../service"; import ScriptPlugin from "../service/script"; +import { tokenModifiers, tokenTypes } from "../service/semantic-tokens"; Project.setDefaultTypePaths({ internalTypesFile: @@ -100,4 +101,116 @@ describe("plain script files", () => { documents.doClose({ textDocument: { uri } }); } }); + + it("classifies semantic tokens straight from the TypeScript classifier", async () => { + // The plain branch skips Marko extraction entirely, so the classifier's + // offsets are the document's own -- a token must land on the identifier it + // describes with TypeScript's classification passed through unchanged. + const text = [ + "const answer = 42;", + "let mutable = answer;", + "function twice(n: number) {", + " return n * 2;", + "}", + "class Thing {}", + "export { mutable, twice, Thing };", + "", + ].join("\n"); + const { uri, doc } = openScript(text); + try { + const tokens = await ScriptPlugin.getSemanticTokens!( + doc, + { textDocument: { uri } } as never, + CancellationToken.None, + ); + assert.ok(tokens?.length, "expected semantic tokens"); + + // Index by source offset so an assertion names a position in `text` + // rather than an ordinal in the (delta-encoded upstream) token list. + const byOffset = new Map(); + for (const token of tokens!) { + const start = doc.offsetAt(token.range.start); + const end = doc.offsetAt(token.range.end); + assert.equal( + text.slice(start, end), + text.slice(start, end).trim(), + `token at ${start} covers whitespace`, + ); + const modifiers = tokenModifiers.filter( + (_, bit) => token.modifiers & (1 << bit), + ); + byOffset.set( + start, + `${tokenTypes[token.type]}${ + modifiers.length ? ` [${modifiers.join(" ")}]` : "" + } \`${text.slice(start, end)}\``, + ); + } + + const at = (needle: string, from = 0) => + byOffset.get(text.indexOf(needle, from)); + + assert.equal(at("answer"), "variable [declaration readonly] `answer`"); + assert.equal(at("mutable"), "variable [declaration] `mutable`"); + // The read of the `const` keeps `readonly`; the `let` read has no + // modifiers at all. + assert.equal(at("answer", 20), "variable [readonly] `answer`"); + assert.equal(at("twice"), "function [declaration] `twice`"); + assert.equal(at("n: number"), "parameter [declaration] `n`"); + assert.equal(at("Thing"), "class [declaration] `Thing`"); + assert.equal(at("mutable, twice"), "variable `mutable`"); + } finally { + documents.doClose({ textDocument: { uri } }); + } + }); + + it("caches semantic tokens per version and honors cancellation", async () => { + const { uri, doc } = openScript(`const first = 1;\n`); + try { + const params = { textDocument: { uri } } as never; + const initial = await ScriptPlugin.getSemanticTokens!( + doc, + params, + CancellationToken.None, + ); + // A repeat request at the same version serves the cached array itself. + assert.equal( + await ScriptPlugin.getSemanticTokens!( + doc, + params, + CancellationToken.None, + ), + initial, + ); + + documents.doChange({ + textDocument: { uri, version: 2 }, + contentChanges: [{ text: `const second = 2;\n` }], + }); + const changed = await ScriptPlugin.getSemanticTokens!( + doc, + params, + CancellationToken.None, + ); + assert.notEqual(changed, initial); + assert.ok(changed?.length, "expected tokens after the edit"); + + // A request cancelled before the classification loop finishes must not + // produce (or cache) a result. + documents.doChange({ + textDocument: { uri, version: 3 }, + contentChanges: [{ text: `const third = 3;\n` }], + }); + assert.equal( + await ScriptPlugin.getSemanticTokens!( + doc, + params, + CancellationToken.Cancelled, + ), + undefined, + ); + } finally { + documents.doClose({ textDocument: { uri } }); + } + }); }); diff --git a/packages/language-server/src/service/script/index.ts b/packages/language-server/src/service/script/index.ts index d696b053..2db7a8f6 100644 --- a/packages/language-server/src/service/script/index.ts +++ b/packages/language-server/src/service/script/index.ts @@ -69,6 +69,10 @@ const semanticTokensCache = new WeakMap< Extracted, { projectVersion: number; tokens: SemanticToken[] } >(); +const plainSemanticTokensCache = new WeakMap< + TextDocument, + { version: number; projectVersion: number; tokens: SemanticToken[] } +>(); const maxSemanticTokensGeneratedLength = 256 * 1024; const markoFileReg = /\.marko$/; // Plain (non-Marko) script files: `.ts`/`.tsx`/`.js`/`.jsx` and their @@ -501,13 +505,26 @@ const ScriptService: Partial = { const result: SemanticToken[] = []; if (plainScriptReg.test(fileName)) { + const cached = plainSemanticTokensCache.get(doc); + if ( + cached && + cached.version === doc.version && + cached.projectVersion === documents.projectVersion + ) { + return cached.tokens; + } + + const { length } = doc.getText(); + if (length > maxSemanticTokensGeneratedLength) return; + const { spans } = project.service.getEncodedSemanticClassifications( fileName, - { start: 0, length: doc.getText().length }, + { start: 0, length }, ts.SemanticClassificationFormat.TwentyTwenty, ); for (let i = 0; i < spans.length; i += 3) { + if (cancel.isCancellationRequested) return; const decoded = decodeTsClassification(spans[i + 2]); if (!decoded) continue; result.push({ @@ -519,6 +536,11 @@ const ScriptService: Partial = { }); } + plainSemanticTokensCache.set(doc, { + version: doc.version, + projectVersion: documents.projectVersion, + tokens: result, + }); return result; } From 6b9d846ac632bdef4987a6902fd7c7de641bf66c Mon Sep 17 00:00:00 2001 From: Brian Carbone Date: Tue, 28 Jul 2026 15:27:51 -0400 Subject: [PATCH 09/11] test: use a real legend type in the multiline split test --- .../language-server/src/__tests__/semantic-tokens.test.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/language-server/src/__tests__/semantic-tokens.test.ts b/packages/language-server/src/__tests__/semantic-tokens.test.ts index 8569a8b3..1e7153dd 100644 --- a/packages/language-server/src/__tests__/semantic-tokens.test.ts +++ b/packages/language-server/src/__tests__/semantic-tokens.test.ts @@ -98,7 +98,7 @@ describe("semantic tokens facade", () => { start: { line: 0, character: 2 }, end: { line: 2, character: 1 }, }, - type: TokenType.string, + type: TokenType.type, modifiers: 0, }, token(3, 3, 3), @@ -110,9 +110,9 @@ describe("semantic tokens facade", () => { ); assert.deepEqual(result?.data, [ - ...[0, 2, 3, TokenType.string, 0], - ...[1, 0, 5, TokenType.string, 0], - ...[1, 0, 1, TokenType.string, 0], + ...[0, 2, 3, TokenType.type, 0], + ...[1, 0, 5, TokenType.type, 0], + ...[1, 0, 1, TokenType.type, 0], ...[2, 1, 1, TokenType.variable, 0], ]); }); From c5937cb5f571df8c58e7e2b969dded5d5d3e4bf7 Mon Sep 17 00:00:00 2001 From: Brian Carbone Date: Tue, 28 Jul 2026 15:27:52 -0400 Subject: [PATCH 10/11] chore: document mask intent and record review findings --- .changeset/semantic-tokens.md | 3 ++- agent-feedback/bugs.md | 14 +++++++++++++- cspell.json | 3 ++- .../language-server/src/service/semantic-tokens.ts | 3 +++ 4 files changed, 20 insertions(+), 3 deletions(-) diff --git a/.changeset/semantic-tokens.md b/.changeset/semantic-tokens.md index ba839fa8..26979415 100644 --- a/.changeset/semantic-tokens.md +++ b/.changeset/semantic-tokens.md @@ -1,6 +1,7 @@ --- "@marko/language-server": minor +"@marko/language-tools": patch "marko-vscode": minor --- -Add semantic token support (`textDocument/semanticTokens` full and range). Custom component tags stop looking like native HTML — `` and `` both render in the component/type color, matching how `.tsx` files color ``, while `
` keeps its tag color. Core tags the grammar doesn't recognize (``, ``, ``) pick up core-tag coloring, and identifiers in placeholders, attribute values, tag params/vars, and `static`/`server`/`class` blocks are colored from TypeScript's view of the code. +Add semantic token support (`textDocument/semanticTokens` full and range). Custom component tags stop looking like native HTML — `` and `` both render in the component/type color, matching how `.tsx` files color ``, while `
` keeps its tag color. Core tags the grammar doesn't recognize (``, ``, ``) pick up core-tag coloring, and identifiers in placeholders, attribute values, tag params/vars, and `static`/`server`/`class` blocks are colored from TypeScript's view of the code. Core-tag detection is now shared with tag-name completions, which as a side effect improves completion documentation for core tags resolved through nested `node_modules`. diff --git a/agent-feedback/bugs.md b/agent-feedback/bugs.md index 9e06e067..b35dfd76 100644 --- a/agent-feedback/bugs.md +++ b/agent-feedback/bugs.md @@ -76,4 +76,16 @@ Every merge arm except `getSemanticTokens` calls `plugin.method?.(...)` directly `packages/language-tools/marko.internal.d.ts` › `change` | 2026-07-28 | impact:low | effort:med -The assignment target of a mutated `` variable is rewritten to a generated property access (`__marko_internal_change__x.x = ...`), and TypeScript emits no semantic classification for that property because the `Marko._.change(...)` type does not resolve it to a named property symbol, even though plain TS classifies write targets (`o.p = 2` classifies `p` as property). In editors with semantic highlighting, the one place a reactive variable is written is therefore the one occurrence left uncolored. A candidate fix is making the change-handler type in the internal types carry a real named property. Re-verify: request semantic tokens for `
+ 82 | +``` + diff --git a/packages/language-server/src/__tests__/fixtures/script/class-api-basic/__snapshots__/class-api-basic.expected/index.ts b/packages/language-server/src/__tests__/fixtures/script/class-api-basic/__snapshots__/class-api-basic.expected/index.ts index ffc34b33..655e1de7 100644 --- a/packages/language-server/src/__tests__/fixtures/script/class-api-basic/__snapshots__/class-api-basic.expected/index.ts +++ b/packages/language-server/src/__tests__/fixtures/script/class-api-basic/__snapshots__/class-api-basic.expected/index.ts @@ -151,7 +151,7 @@ export { type Component }; }; })(), }); - const __marko_internal_tag_3 = Marko._.interpolated`Missing`; + const __marko_internal_tag_3 = Missing; Marko._.renderDynamicTag(__marko_internal_tag_3)()()({}); const __marko_internal_tag_4 = Marko._.interpolated`complex-missing`; Marko._.renderDynamicTag(__marko_internal_tag_4)()()({}); diff --git a/packages/language-server/src/__tests__/fixtures/semantic-tokens/single-letter/__snapshots__/single-letter.expected/index.html b/packages/language-server/src/__tests__/fixtures/semantic-tokens/single-letter/__snapshots__/single-letter.expected/index.html index 74aa865b..286bf4e2 100644 --- a/packages/language-server/src/__tests__/fixtures/semantic-tokens/single-letter/__snapshots__/single-letter.expected/index.html +++ b/packages/language-server/src/__tests__/fixtures/semantic-tokens/single-letter/__snapshots__/single-letter.expected/index.html @@ -1 +1 @@ -
bound to the import, so an identifier tag
not bound, so an uppercase native tag
\ No newline at end of file +
bound to the import, so an identifier tag
not bound, so an uppercase native tag
not bound and no tag file, so a missing component error
\ No newline at end of file diff --git a/packages/language-server/src/__tests__/fixtures/semantic-tokens/single-letter/__snapshots__/single-letter.expected/index.md b/packages/language-server/src/__tests__/fixtures/semantic-tokens/single-letter/__snapshots__/single-letter.expected/index.md index e69de29b..52e5d1dd 100644 --- a/packages/language-server/src/__tests__/fixtures/semantic-tokens/single-letter/__snapshots__/single-letter.expected/index.md +++ b/packages/language-server/src/__tests__/fixtures/semantic-tokens/single-letter/__snapshots__/single-letter.expected/index.md @@ -0,0 +1,10 @@ +## Diagnostics +### Ln 11, Col 2 +```marko + 9 | bound to the import, so an identifier tag + 10 | not bound, so an uppercase native tag +> 11 | not bound and no tag file, so a missing component error + | ^^^^^^^ Cannot find name 'Missing'. + 12 | +``` + diff --git a/packages/language-server/src/__tests__/fixtures/semantic-tokens/single-letter/__snapshots__/single-letter.expected/index.tokens.md b/packages/language-server/src/__tests__/fixtures/semantic-tokens/single-letter/__snapshots__/single-letter.expected/index.tokens.md index 83a75227..d686ab68 100644 --- a/packages/language-server/src/__tests__/fixtures/semantic-tokens/single-letter/__snapshots__/single-letter.expected/index.tokens.md +++ b/packages/language-server/src/__tests__/fixtures/semantic-tokens/single-letter/__snapshots__/single-letter.expected/index.tokens.md @@ -6,3 +6,5 @@ Ln 9, Col 2 (len 1) class `B` Ln 9, Col 47 (len 1) class `B` Ln 10, Col 2 (len 1) class `I` Ln 10, Col 43 (len 1) class `I` +Ln 11, Col 2 (len 7) class `Missing` +Ln 11, Col 67 (len 7) class `Missing` diff --git a/packages/language-server/src/__tests__/fixtures/semantic-tokens/single-letter/__snapshots__/single-letter.expected/index.ts b/packages/language-server/src/__tests__/fixtures/semantic-tokens/single-letter/__snapshots__/single-letter.expected/index.ts index 438f5aa6..8a967888 100644 --- a/packages/language-server/src/__tests__/fixtures/semantic-tokens/single-letter/__snapshots__/single-letter.expected/index.ts +++ b/packages/language-server/src/__tests__/fixtures/semantic-tokens/single-letter/__snapshots__/single-letter.expected/index.ts @@ -41,6 +41,14 @@ export { type Component }; }; })(), }); + const __marko_internal_tag_7 = Missing; + Marko._.renderDynamicTag(__marko_internal_tag_7)()()({ + [Marko._.contentFor(__marko_internal_tag_7)]: (() => { + return () => { + return Marko._.voidReturn; + }; + })(), + }); Marko._.noop({ component, state, out, input, $global, $signal }); return; })(); diff --git a/packages/language-server/src/__tests__/fixtures/semantic-tokens/single-letter/index.marko b/packages/language-server/src/__tests__/fixtures/semantic-tokens/single-letter/index.marko index a97e63ca..440f0d7a 100644 --- a/packages/language-server/src/__tests__/fixtures/semantic-tokens/single-letter/index.marko +++ b/packages/language-server/src/__tests__/fixtures/semantic-tokens/single-letter/index.marko @@ -8,3 +8,4 @@ import Bc from ""; bound to the import, so an identifier tag not bound, so an uppercase native tag +not bound and no tag file, so a missing component error diff --git a/packages/language-tools/src/extractors/script/index.ts b/packages/language-tools/src/extractors/script/index.ts index 16ee91dc..ff3f8105 100644 --- a/packages/language-tools/src/extractors/script/index.ts +++ b/packages/language-tools/src/extractors/script/index.ts @@ -871,13 +871,16 @@ constructor(_) {} // Both translators only treat a tag name as an identifier when it also // resolves to a binding (`TAG_NAME_IDENTIFIER_REG.test(name) && // tag.scope.getBinding(name)`); an unbound name stays a native/dynamic - // tag. Matching that here keeps markup the compiler accepts -- eg the - // uppercase HTML tags `
`, ``, `` -- from being emitted as a bare - // reference that TypeScript reports `Cannot find name` for. + // tag. Applying that gate everywhere would also silence TypeScript's + // `Cannot find name` on a misspelled component (``), so it is + // limited to the unbound names the compiler gives a meaning of their + // own: a single-letter name is uppercase HTML (``, ``, ``) and + // a name that resolves to a tag file renders that file. const isIdentifier = tagName && REG_TAG_NAME_IDENTIFIER.test(tagName) && - this.#getBindingNames().has(tagName); + (this.#getBindingNames().has(tagName) || + (!importPath && tagName.length > 1)); const isMarkoFile = importPath?.endsWith(".marko"); if (isIdentifier || isMarkoFile || !importPath) {