From ed169bd5f865e4964ac6f7f149b2a85d28d40b8a Mon Sep 17 00:00:00 2001 From: "haozhe.yang" Date: Tue, 21 Jul 2026 13:48:59 +0800 Subject: [PATCH 1/7] feat(tree-sitter-bash): scaffold pure-TypeScript bash parser package Add packages/tree-sitter-bash with the SyntaxNode tree model (UTF-16 code-unit offsets, tree-sitter-bash named-node type names), a parse budget (deadline + node cap) with an aborted ParseResult variant, and a placeholder parse() to be replaced by the real lexer/parser. --- flake.nix | 2 + packages/tree-sitter-bash/package.json | 47 +++++++++ packages/tree-sitter-bash/src/budget.ts | 56 ++++++++++ packages/tree-sitter-bash/src/grammar.ts | 30 ++++++ packages/tree-sitter-bash/src/index.ts | 4 + packages/tree-sitter-bash/src/node.ts | 105 +++++++++++++++++++ packages/tree-sitter-bash/src/parse.ts | 34 ++++++ packages/tree-sitter-bash/test/node.test.ts | 98 +++++++++++++++++ packages/tree-sitter-bash/test/parse.test.ts | 68 ++++++++++++ packages/tree-sitter-bash/tsconfig.json | 5 + packages/tree-sitter-bash/tsdown.config.ts | 13 +++ packages/tree-sitter-bash/vitest.config.ts | 8 ++ pnpm-lock.yaml | 44 ++++++++ 13 files changed, 514 insertions(+) create mode 100644 packages/tree-sitter-bash/package.json create mode 100644 packages/tree-sitter-bash/src/budget.ts create mode 100644 packages/tree-sitter-bash/src/grammar.ts create mode 100644 packages/tree-sitter-bash/src/index.ts create mode 100644 packages/tree-sitter-bash/src/node.ts create mode 100644 packages/tree-sitter-bash/src/parse.ts create mode 100644 packages/tree-sitter-bash/test/node.test.ts create mode 100644 packages/tree-sitter-bash/test/parse.test.ts create mode 100644 packages/tree-sitter-bash/tsconfig.json create mode 100644 packages/tree-sitter-bash/tsdown.config.ts create mode 100644 packages/tree-sitter-bash/vitest.config.ts diff --git a/flake.nix b/flake.nix index 7f56b35994..15c9107f7a 100644 --- a/flake.nix +++ b/flake.nix @@ -77,6 +77,7 @@ ./packages/protocol ./packages/telemetry ./packages/transcript + ./packages/tree-sitter-bash ./apps/kimi-code ./apps/vscode ./apps/kimi-inspect @@ -103,6 +104,7 @@ "@moonshot-ai/protocol" "@moonshot-ai/kimi-telemetry" "@moonshot-ai/transcript" + "@moonshot-ai/tree-sitter-bash" "@moonshot-ai/kimi-code" "kimi-code" "@moonshot-ai/kimi-inspect" diff --git a/packages/tree-sitter-bash/package.json b/packages/tree-sitter-bash/package.json new file mode 100644 index 0000000000..dab858aa31 --- /dev/null +++ b/packages/tree-sitter-bash/package.json @@ -0,0 +1,47 @@ +{ + "name": "@moonshot-ai/tree-sitter-bash", + "version": "0.1.0", + "private": true, + "description": "A pure-TypeScript bash parser producing a syntax tree whose named node types match tree-sitter-bash one-to-one.", + "license": "MIT", + "author": "Moonshot AI", + "homepage": "https://github.com/MoonshotAI/kimi-code/tree/main/packages/tree-sitter-bash#readme", + "repository": { + "type": "git", + "url": "git+https://github.com/MoonshotAI/kimi-code.git", + "directory": "packages/tree-sitter-bash" + }, + "bugs": { + "url": "https://github.com/MoonshotAI/kimi-code/issues" + }, + "keywords": [ + "kimi", + "bash", + "shell", + "parser", + "tree-sitter" + ], + "files": [ + "dist" + ], + "type": "module", + "imports": { + "#/*": "./src/*.ts" + }, + "exports": { + ".": { + "types": "./src/index.ts", + "default": "./src/index.ts" + } + }, + "scripts": { + "build": "tsdown", + "typecheck": "tsc -p tsconfig.json --noEmit", + "test": "vitest run", + "clean": "rm -rf dist" + }, + "devDependencies": { + "tree-sitter-bash": "0.25.0", + "web-tree-sitter": "0.25.10" + } +} diff --git a/packages/tree-sitter-bash/src/budget.ts b/packages/tree-sitter-bash/src/budget.ts new file mode 100644 index 0000000000..ed0cf9b734 --- /dev/null +++ b/packages/tree-sitter-bash/src/budget.ts @@ -0,0 +1,56 @@ +// src/budget.ts +// +// Parse budget: a hard cap on wall-clock time and on the number of syntax +// nodes a single parse may create. The parser calls `budget.tick()` every time +// it creates a node; when either limit is exceeded `tick` throws `Aborted`, +// which the `parse` entry point catches and reports as +// `{ ok: false, reason: 'aborted' }`. + +export const DEFAULT_TIMEOUT_MS = 50; +export const DEFAULT_MAX_NODES = 50_000; + +/** Internal control-flow error. Never escapes the `parse` entry point. */ +export class Aborted extends Error { + constructor(message: string) { + super(message); + this.name = 'Aborted'; + } +} + +export interface BudgetOptions { + /** Wall-clock limit in milliseconds. `Infinity` disables the time check. */ + timeoutMs?: number; + /** Maximum number of nodes the parse may create. */ + maxNodes?: number; +} + +export class ParseBudget { + private readonly deadline: number; + private readonly maxNodes: number; + private nodeCount = 0; + + constructor(options: BudgetOptions = {}) { + const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS; + this.deadline = Date.now() + timeoutMs; + this.maxNodes = options.maxNodes ?? DEFAULT_MAX_NODES; + } + + /** Number of nodes created so far. */ + get nodesUsed(): number { + return this.nodeCount; + } + + /** + * Account for one created node and re-check the deadline. Throws `Aborted` + * when the node cap is exceeded or the deadline has been reached. + */ + tick(): void { + this.nodeCount++; + if (this.nodeCount > this.maxNodes) { + throw new Aborted(`parse aborted: node budget exceeded (${this.nodeCount} > ${this.maxNodes})`); + } + if (Date.now() >= this.deadline) { + throw new Aborted(`parse aborted: timeout`); + } + } +} diff --git a/packages/tree-sitter-bash/src/grammar.ts b/packages/tree-sitter-bash/src/grammar.ts new file mode 100644 index 0000000000..0103f33387 --- /dev/null +++ b/packages/tree-sitter-bash/src/grammar.ts @@ -0,0 +1,30 @@ +// src/grammar.ts +// +// Static grammar tables for bash. M0 only carries the two tables the lexer +// will need first; later milestones extend this file (operator tables, node +// type registry, etc.) as the real parser lands. + +/** Reserved words of POSIX bash (recognition is context-dependent: a word is + * only a keyword where a command name is expected). */ +export const SHELL_KEYWORDS = [ + 'if', + 'then', + 'else', + 'elif', + 'fi', + 'for', + 'while', + 'until', + 'do', + 'done', + 'case', + 'esac', + 'in', + 'function', + 'select', + 'time', + 'coproc', +] as const; + +/** Single-character special parameters: $@ $* $# $? $- $$ $! $0 and $_. */ +export const SPECIAL_VARIABLES = ['@', '*', '#', '?', '-', '$', '!', '0', '_'] as const; diff --git a/packages/tree-sitter-bash/src/index.ts b/packages/tree-sitter-bash/src/index.ts new file mode 100644 index 0000000000..2737d6529e --- /dev/null +++ b/packages/tree-sitter-bash/src/index.ts @@ -0,0 +1,4 @@ +export * from './node'; +export * from './budget'; +export * from './parse'; +export * from './grammar'; diff --git a/packages/tree-sitter-bash/src/node.ts b/packages/tree-sitter-bash/src/node.ts new file mode 100644 index 0000000000..fdebc359fd --- /dev/null +++ b/packages/tree-sitter-bash/src/node.ts @@ -0,0 +1,105 @@ +// src/node.ts +// +// Syntax tree node model. Named node types correspond one-to-one with +// tree-sitter-bash's named node types, so consumers can write tree queries +// against either implementation. Two deliberate deviations from tree-sitter: +// +// - startIndex / endIndex are UTF-16 code unit offsets (native JS string +// indexing), not tree-sitter's UTF-8 byte offsets. `node.text` therefore +// always equals `source.slice(node.startIndex, node.endIndex)`. +// - `text` is pre-computed when the node is created instead of being +// re-sliced on every access. +// +// Child semantics: `children` contains every child in source order, including +// anonymous (punctuation / keyword token) nodes; `namedChildren` contains only +// the named ones, in the same relative order. Anonymous nodes never appear in +// `namedChildren`. `descendantsOfType` walks named descendants only, matching +// how tree-sitter queries see the tree. + +export interface SyntaxNode { + /** Node type, e.g. 'program', 'command', 'word'. Matches tree-sitter-bash. */ + readonly type: string; + /** Source text covered by this node (UTF-16 slice of the original source). */ + readonly text: string; + /** Start offset in UTF-16 code units, inclusive. */ + readonly startIndex: number; + /** End offset in UTF-16 code units, exclusive. */ + readonly endIndex: number; + /** Whether this is a named node (false for punctuation/keyword tokens). */ + readonly isNamed: boolean; + readonly parent: SyntaxNode | null; + /** All children in source order, named and anonymous. */ + readonly children: readonly SyntaxNode[]; + /** Named children only, in source order. */ + readonly namedChildren: readonly SyntaxNode[]; +} + +export interface NodeInit { + type: string; + source: string; + startIndex: number; + endIndex: number; + isNamed?: boolean; +} + +/** + * Mutable node under construction. The parser builds trees with this class and + * exposes them through the readonly `SyntaxNode` interface; once a node is + * handed out it must be treated as immutable. + */ +export class SyntaxNodeBuilder { + readonly type: string; + readonly text: string; + readonly startIndex: number; + readonly endIndex: number; + readonly isNamed: boolean; + parent: SyntaxNodeBuilder | null = null; + readonly children: SyntaxNodeBuilder[] = []; + readonly namedChildren: SyntaxNodeBuilder[] = []; + + constructor(init: NodeInit) { + if (init.startIndex < 0 || init.endIndex < init.startIndex || init.endIndex > init.source.length) { + throw new RangeError( + `invalid node range [${init.startIndex}, ${init.endIndex}) for source of length ${init.source.length}`, + ); + } + this.type = init.type; + this.startIndex = init.startIndex; + this.endIndex = init.endIndex; + this.isNamed = init.isNamed ?? true; + this.text = init.source.slice(init.startIndex, init.endIndex); + } + + /** Attach a child, wiring its parent pointer. Named children are also added + * to `namedChildren`. Returns the child for chaining. */ + addChild(child: T): T { + if (child.parent !== null) throw new Error(`node '${child.type}' already has a parent`); + child.parent = this; + this.children.push(child); + if (child.isNamed) this.namedChildren.push(child); + return child; + } +} + +/** Convenience factory for a detached node. */ +export function createNode(init: NodeInit): SyntaxNodeBuilder { + return new SyntaxNodeBuilder(init); +} + +/** + * Pre-order traversal of the named descendants of `root` (not including + * `root` itself), filtered to the given types. With no types, returns every + * named descendant in pre-order. + */ +export function descendantsOfType(root: SyntaxNode, ...types: string[]): SyntaxNode[] { + const wanted = types.length > 0 ? new Set(types) : null; + const out: SyntaxNode[] = []; + const walk = (node: SyntaxNode): void => { + for (const child of node.namedChildren) { + if (wanted === null || wanted.has(child.type)) out.push(child); + walk(child); + } + }; + walk(root); + return out; +} diff --git a/packages/tree-sitter-bash/src/parse.ts b/packages/tree-sitter-bash/src/parse.ts new file mode 100644 index 0000000000..3248323f07 --- /dev/null +++ b/packages/tree-sitter-bash/src/parse.ts @@ -0,0 +1,34 @@ +// src/parse.ts +// +// Public parse entry point. M0 ships a placeholder parser: the whole source is +// wrapped in a `program` root with a single `word` child covering the entire +// text. The budget mechanism is real, and later milestones keep this exact +// entry/exit contract while replacing the placeholder with the actual +// lexer + parser. + +import { Aborted, ParseBudget } from '#/budget'; +import type { BudgetOptions } from '#/budget'; +import { SyntaxNodeBuilder } from '#/node'; +import type { SyntaxNode } from '#/node'; + +export type ParseResult = + | { ok: true; rootNode: SyntaxNode; hasError: boolean } + | { ok: false; reason: 'aborted' }; + +export type ParseOptions = BudgetOptions; + +export function parse(source: string, options: ParseOptions = {}): ParseResult { + const budget = new ParseBudget(options); + try { + const root = new SyntaxNodeBuilder({ type: 'program', source, startIndex: 0, endIndex: source.length }); + budget.tick(); + if (source.length > 0) { + root.addChild(new SyntaxNodeBuilder({ type: 'word', source, startIndex: 0, endIndex: source.length })); + budget.tick(); + } + return { ok: true, rootNode: root, hasError: false }; + } catch (error) { + if (error instanceof Aborted) return { ok: false, reason: 'aborted' }; + throw error; + } +} diff --git a/packages/tree-sitter-bash/test/node.test.ts b/packages/tree-sitter-bash/test/node.test.ts new file mode 100644 index 0000000000..4c6736a59a --- /dev/null +++ b/packages/tree-sitter-bash/test/node.test.ts @@ -0,0 +1,98 @@ +import { describe, expect, it } from 'vitest'; + +import { createNode, descendantsOfType } from '#/node'; + +const SOURCE = 'echo foo; echo bar'; + +/** program + * ├─ command "echo foo" + * │ ├─ command_name "echo" + * │ │ └─ word "echo" + * │ └─ word "foo" + * ├─ ";" (anonymous) + * └─ command "echo bar" + * └─ word "bar" + */ +function buildTree() { + const program = createNode({ type: 'program', source: SOURCE, startIndex: 0, endIndex: SOURCE.length }); + const cmd1 = program.addChild(createNode({ type: 'command', source: SOURCE, startIndex: 0, endIndex: 8 })); + const name1 = cmd1.addChild(createNode({ type: 'command_name', source: SOURCE, startIndex: 0, endIndex: 4 })); + name1.addChild(createNode({ type: 'word', source: SOURCE, startIndex: 0, endIndex: 4 })); + cmd1.addChild(createNode({ type: 'word', source: SOURCE, startIndex: 5, endIndex: 8 })); + program.addChild(createNode({ type: ';', source: SOURCE, startIndex: 8, endIndex: 9, isNamed: false })); + const cmd2 = program.addChild(createNode({ type: 'command', source: SOURCE, startIndex: 10, endIndex: 18 })); + cmd2.addChild(createNode({ type: 'word', source: SOURCE, startIndex: 15, endIndex: 18 })); + return program; +} + +describe('createNode', () => { + it('pre-stores text as the UTF-16 slice of the source', () => { + const node = createNode({ type: 'word', source: SOURCE, startIndex: 5, endIndex: 8 }); + expect(node.text).toBe('foo'); + expect(node.startIndex).toBe(5); + expect(node.endIndex).toBe(8); + expect(node.isNamed).toBe(true); + expect(node.parent).toBeNull(); + }); + + it('rejects out-of-range offsets', () => { + expect(() => createNode({ type: 'word', source: SOURCE, startIndex: -1, endIndex: 2 })).toThrow(RangeError); + expect(() => createNode({ type: 'word', source: SOURCE, startIndex: 3, endIndex: 2 })).toThrow(RangeError); + expect(() => createNode({ type: 'word', source: SOURCE, startIndex: 0, endIndex: 100 })).toThrow(RangeError); + }); + + it('rejects attaching a child that already has a parent', () => { + const a = createNode({ type: 'program', source: SOURCE, startIndex: 0, endIndex: SOURCE.length }); + const b = createNode({ type: 'program', source: SOURCE, startIndex: 0, endIndex: SOURCE.length }); + const child = createNode({ type: 'word', source: SOURCE, startIndex: 0, endIndex: 4 }); + a.addChild(child); + expect(() => b.addChild(child)).toThrow(/already has a parent/); + }); +}); + +describe('children vs namedChildren', () => { + it('keeps anonymous nodes out of namedChildren but in children', () => { + const program = buildTree(); + expect(program.children.map((c) => c.type)).toEqual(['command', ';', 'command']); + expect(program.namedChildren.map((c) => c.type)).toEqual(['command', 'command']); + expect(program.children[1]?.isNamed).toBe(false); + }); + + it('wires parent pointers', () => { + const program = buildTree(); + const [cmd1] = program.namedChildren; + expect(cmd1?.parent).toBe(program); + expect(cmd1?.namedChildren[0]?.parent).toBe(cmd1); + }); +}); + +describe('descendantsOfType', () => { + it('returns matching named descendants in pre-order', () => { + const program = buildTree(); + const words = descendantsOfType(program, 'word'); + expect(words.map((w) => w.text)).toEqual(['echo', 'foo', 'bar']); + }); + + it('matches multiple types and skips anonymous nodes', () => { + const program = buildTree(); + const nodes = descendantsOfType(program, 'command', ';'); + expect(nodes.map((n) => n.type)).toEqual(['command', 'command']); + }); + + it('returns every named descendant when no type is given', () => { + const program = buildTree(); + expect(descendantsOfType(program).map((n) => n.type)).toEqual([ + 'command', + 'command_name', + 'word', + 'word', + 'command', + 'word', + ]); + }); + + it('does not include the root itself', () => { + const leaf = createNode({ type: 'word', source: SOURCE, startIndex: 0, endIndex: 4 }); + expect(descendantsOfType(leaf, 'word')).toEqual([]); + }); +}); diff --git a/packages/tree-sitter-bash/test/parse.test.ts b/packages/tree-sitter-bash/test/parse.test.ts new file mode 100644 index 0000000000..ab1399bf7f --- /dev/null +++ b/packages/tree-sitter-bash/test/parse.test.ts @@ -0,0 +1,68 @@ +import { describe, expect, it } from 'vitest'; + +import { Aborted, ParseBudget } from '#/budget'; +import { parse } from '#/parse'; + +describe('ParseBudget', () => { + it('counts nodes and stays under the cap', () => { + const budget = new ParseBudget({ timeoutMs: 60_000, maxNodes: 3 }); + budget.tick(); + budget.tick(); + expect(budget.nodesUsed).toBe(2); + }); + + it('throws Aborted when the node cap is exceeded', () => { + const budget = new ParseBudget({ timeoutMs: 60_000, maxNodes: 1 }); + budget.tick(); + expect(() => budget.tick()).toThrow(Aborted); + }); + + it('throws Aborted once the deadline is reached', () => { + const budget = new ParseBudget({ timeoutMs: 0, maxNodes: 1_000 }); + expect(() => budget.tick()).toThrow(Aborted); + }); + + it('applies documented defaults', () => { + const budget = new ParseBudget(); + // Default node cap is 50_000: the 50_000th node is still allowed. + for (let i = 0; i < 50_000; i++) budget.tick(); + expect(() => budget.tick()).toThrow(Aborted); + }); +}); + +describe('parse (M0 placeholder)', () => { + it('wraps the whole source in a program root with a single word child', () => { + const source = 'echo hello'; + const result = parse(source); + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.hasError).toBe(false); + expect(result.rootNode.type).toBe('program'); + expect(result.rootNode.text).toBe(source); + expect(result.rootNode.startIndex).toBe(0); + expect(result.rootNode.endIndex).toBe(source.length); + expect(result.rootNode.namedChildren).toHaveLength(1); + const word = result.rootNode.namedChildren[0]!; + expect(word.type).toBe('word'); + expect(word.text).toBe(source); + expect(word.parent).toBe(result.rootNode); + }); + + it('parses an empty source into an empty program', () => { + const result = parse(''); + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.rootNode.type).toBe('program'); + expect(result.rootNode.text).toBe(''); + expect(result.rootNode.children).toHaveLength(0); + }); + + it('returns { ok: false, reason: "aborted" } when the node budget is exceeded', () => { + // The placeholder parse creates 2 nodes (program + word). + expect(parse('echo hello', { timeoutMs: 60_000, maxNodes: 1 })).toEqual({ ok: false, reason: 'aborted' }); + }); + + it('returns { ok: false, reason: "aborted" } when the deadline has passed', () => { + expect(parse('echo hello', { timeoutMs: 0 })).toEqual({ ok: false, reason: 'aborted' }); + }); +}); diff --git a/packages/tree-sitter-bash/tsconfig.json b/packages/tree-sitter-bash/tsconfig.json new file mode 100644 index 0000000000..46f71c10fe --- /dev/null +++ b/packages/tree-sitter-bash/tsconfig.json @@ -0,0 +1,5 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": {}, + "include": ["src", "test"] +} diff --git a/packages/tree-sitter-bash/tsdown.config.ts b/packages/tree-sitter-bash/tsdown.config.ts new file mode 100644 index 0000000000..b27e00afd7 --- /dev/null +++ b/packages/tree-sitter-bash/tsdown.config.ts @@ -0,0 +1,13 @@ +import { defineConfig } from 'tsdown'; + +export default defineConfig({ + entry: ['./src/index.ts'], + format: ['esm'], + dts: false, + outDir: 'dist', + clean: true, + deps: { + alwaysBundle: [/^@moonshot-ai\//], + neverBundle: [], + }, +}); diff --git a/packages/tree-sitter-bash/vitest.config.ts b/packages/tree-sitter-bash/vitest.config.ts new file mode 100644 index 0000000000..85c8b889e8 --- /dev/null +++ b/packages/tree-sitter-bash/vitest.config.ts @@ -0,0 +1,8 @@ +import { defineConfig } from 'vitest/config'; + +export default defineConfig({ + test: { + name: 'tree-sitter-bash', + include: ['test/**/*.test.ts'], + }, +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 57ea9f82e7..65a2e23fa5 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1018,6 +1018,15 @@ importers: specifier: 'catalog:' version: 4.3.6 + packages/tree-sitter-bash: + devDependencies: + tree-sitter-bash: + specifier: 0.25.0 + version: 0.25.0 + web-tree-sitter: + specifier: 0.25.10 + version: 0.25.10 + packages: '@agentclientprotocol/sdk@0.23.0': @@ -7666,6 +7675,10 @@ packages: node-addon-api@7.1.1: resolution: {integrity: sha512-5m3bsyrjFWE1xf7nz7YXdN4udnVtXK6/Yfgn5qnahL6bCkf2yKt4k3nuTKAtT4r3IG8JNR2ncsIMdZuAzJjHQQ==} + node-addon-api@8.9.0: + resolution: {integrity: sha512-ekZMeaaIzSQTSpr7X2X3iJM7lTzgnx8ahAG9pJfT/7+14mlEM8ZYQ9cgCDvSSRbReFK0oHli3WrZdCiRsgAT9Q==} + engines: {node: ^18 || ^20 || >= 21} + node-domexception@1.0.0: resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==} engines: {node: '>=10.5.0'} @@ -7688,6 +7701,10 @@ packages: resolution: {integrity: sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + node-gyp-build@4.8.4: + resolution: {integrity: sha512-LA4ZjwlnUblHVgq0oBF3Jl/6h/Nvs5fzBLwdEF4nuxnFdsfajde4WfxtJr3CaiH+F6ewcIB/q4jQ4UzPyid+CQ==} + hasBin: true + node-pty@1.1.0: resolution: {integrity: sha512-20JqtutY6JPXTUnL0ij1uad7Qe1baT46lyolh2sSENDd4sTzKZ4nmAFkeAARDKwmlLjPx6XKRlwRUxwjOy+lUg==} @@ -9155,6 +9172,14 @@ packages: resolution: {integrity: sha512-L0Orpi8qGpRG//Nd+H90vFB+3iHnue1zSSGmNOOCh1GLJ7rUKVwV2HvijphGQS2UmhUZewS9VgvxYIdgr+fG1A==} hasBin: true + tree-sitter-bash@0.25.0: + resolution: {integrity: sha512-gZtlj9+qFS81qKxpLfD6H0UssQ3QBc/F0nKkPsiFDyfQF2YBqYvglFJUzchrPpVhZe9kLZTrJ9n2J6lmka69Vg==} + peerDependencies: + tree-sitter: ^0.25.0 + peerDependenciesMeta: + tree-sitter: + optional: true + trim-lines@3.0.1: resolution: {integrity: sha512-kRj8B+YHZCc9kQYdWfJB2/oUl9rA99qbowYYBtr4ui4mZyAQ2JpvVBd/6U2YloATfqBhBTSMhTpgBHtU0Mf3Rg==} @@ -9714,6 +9739,14 @@ packages: resolution: {integrity: sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==} engines: {node: '>= 8'} + web-tree-sitter@0.25.10: + resolution: {integrity: sha512-Y09sF44/13XvgVKgO2cNDw5rGk6s26MgoZPXLESvMXeefBf7i6/73eFurre0IsTW6E14Y0ArIzhUMmjoc7xyzA==} + peerDependencies: + '@types/emscripten': ^1.40.0 + peerDependenciesMeta: + '@types/emscripten': + optional: true + webidl-conversions@3.0.1: resolution: {integrity: sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==} @@ -17187,6 +17220,8 @@ snapshots: node-addon-api@7.1.1: {} + node-addon-api@8.9.0: {} + node-domexception@1.0.0: {} node-emoji@2.2.0: @@ -17208,6 +17243,8 @@ snapshots: fetch-blob: 3.2.0 formdata-polyfill: 4.0.10 + node-gyp-build@4.8.4: {} + node-pty@1.1.0: dependencies: node-addon-api: 7.1.1 @@ -19004,6 +19041,11 @@ snapshots: tree-kill@1.2.2: {} + tree-sitter-bash@0.25.0: + dependencies: + node-addon-api: 8.9.0 + node-gyp-build: 4.8.4 + trim-lines@3.0.1: {} trough@2.2.0: {} @@ -19547,6 +19589,8 @@ snapshots: web-streams-polyfill@3.3.3: {} + web-tree-sitter@0.25.10: {} + webidl-conversions@3.0.1: {} webidl-conversions@7.0.0: From d306c0c038ad890933a21cafab86d3bc09c82c7b Mon Sep 17 00:00:00 2001 From: "haozhe.yang" Date: Tue, 21 Jul 2026 15:29:48 +0800 Subject: [PATCH 2/7] feat(tree-sitter-bash): add lexer and core recursive-descent parser Cover the permission-analysis grammar subset: lists, pipelines, commands, words, quotes, expansions, command/process substitution, subshells, the full redirect operator set, heredocs and comments, with tree-sitter-bash named-node type names. Long scan loops check the parse deadline via budget.progress() so large literals cannot exhaust the node cap; parse depth is bounded on all recursion chains. --- packages/tree-sitter-bash/README.md | 75 ++ packages/tree-sitter-bash/src/budget.ts | 21 +- packages/tree-sitter-bash/src/grammar.ts | 64 +- packages/tree-sitter-bash/src/lexer.ts | 437 +++++++ packages/tree-sitter-bash/src/node.ts | 34 +- packages/tree-sitter-bash/src/parse.ts | 38 +- packages/tree-sitter-bash/src/parser.ts | 1167 ++++++++++++++++++ packages/tree-sitter-bash/test/node.test.ts | 34 + packages/tree-sitter-bash/test/parse.test.ts | 503 +++++++- 9 files changed, 2296 insertions(+), 77 deletions(-) create mode 100644 packages/tree-sitter-bash/README.md create mode 100644 packages/tree-sitter-bash/src/lexer.ts create mode 100644 packages/tree-sitter-bash/src/parser.ts diff --git a/packages/tree-sitter-bash/README.md b/packages/tree-sitter-bash/README.md new file mode 100644 index 0000000000..e4ea71b590 --- /dev/null +++ b/packages/tree-sitter-bash/README.md @@ -0,0 +1,75 @@ +# @moonshot-ai/tree-sitter-bash + +A pure-TypeScript bash parser that produces a syntax tree whose named node +types match [tree-sitter-bash](https://github.com/tree-sitter/tree-sitter-bash) +one-to-one, built for agent-side command permission analysis. + +- No native addons; offsets are UTF-16 code units (`node.text` is always a + direct `source.slice(startIndex, endIndex)`). +- Parsing runs under a hard budget (default 50 ms / 50 000 nodes); exceeding + it returns `{ ok: false, reason: 'aborted' }` instead of throwing. +- Malformed input never throws either. Unterminated constructs (quotes, + expansions, substitutions, heredocs) are kept as partial nodes and flagged + with `hasError: true`; tokens that cannot start or continue a statement + (stray `)`, a leading `&&`, …) are wrapped in `ERROR` nodes and parsing + continues. + +```ts +import { parse } from '@moonshot-ai/tree-sitter-bash'; + +const result = parse('git status && rm -rf /'); +if (result.ok) { + // result.rootNode: program → list → command … +} +``` + +## Known differences from tree-sitter-bash + +Named node types always come from tree-sitter-bash's `node-types.json`, but +for the following constructs the tree shape deliberately deviates from what +tree-sitter-bash 0.25.0 produces (verified against the real parser): + +- `<>` (read-write redirect) is parsed as a normal `file_redirect` + operator; tree-sitter-bash 0.25.0 fails to parse it. +- Several heredocs on one line (`cat < a cmd x` makes + `cmd` the `command_name`), while redirects after the command name consume + every following word (`cmd > out arg` puts both words in the redirect) — + this matches tree-sitter-bash's actual disambiguation. diff --git a/packages/tree-sitter-bash/src/budget.ts b/packages/tree-sitter-bash/src/budget.ts index ed0cf9b734..35ffe9db76 100644 --- a/packages/tree-sitter-bash/src/budget.ts +++ b/packages/tree-sitter-bash/src/budget.ts @@ -1,10 +1,12 @@ // src/budget.ts // // Parse budget: a hard cap on wall-clock time and on the number of syntax -// nodes a single parse may create. The parser calls `budget.tick()` every time -// it creates a node; when either limit is exceeded `tick` throws `Aborted`, -// which the `parse` entry point catches and reports as -// `{ ok: false, reason: 'aborted' }`. +// nodes a single parse may create. The parser calls `budget.tick()` every +// time it creates a node; long scan loops (word runs, strings, heredoc +// bodies) call `budget.progress()` periodically so a pathological single +// token still hits the deadline without inflating the node count. When +// either limit is exceeded the methods throw `Aborted`, which the `parse` +// entry point catches and reports as `{ ok: false, reason: 'aborted' }`. export const DEFAULT_TIMEOUT_MS = 50; export const DEFAULT_MAX_NODES = 50_000; @@ -53,4 +55,15 @@ export class ParseBudget { throw new Aborted(`parse aborted: timeout`); } } + + /** + * Re-check the deadline WITHOUT counting a node. For long scan loops that + * run many iterations per produced node (character-level scanning), called + * at intervals so the deadline is still enforced promptly. + */ + progress(): void { + if (Date.now() >= this.deadline) { + throw new Aborted(`parse aborted: timeout`); + } + } } diff --git a/packages/tree-sitter-bash/src/grammar.ts b/packages/tree-sitter-bash/src/grammar.ts index 0103f33387..a53b900caa 100644 --- a/packages/tree-sitter-bash/src/grammar.ts +++ b/packages/tree-sitter-bash/src/grammar.ts @@ -1,30 +1,44 @@ // src/grammar.ts // -// Static grammar tables for bash. M0 only carries the two tables the lexer -// will need first; later milestones extend this file (operator tables, node -// type registry, etc.) as the real parser lands. - -/** Reserved words of POSIX bash (recognition is context-dependent: a word is - * only a keyword where a command name is expected). */ -export const SHELL_KEYWORDS = [ - 'if', - 'then', - 'else', - 'elif', - 'fi', - 'for', - 'while', - 'until', - 'do', - 'done', - 'case', - 'esac', - 'in', - 'function', - 'select', - 'time', - 'coproc', -] as const; +// Static grammar tables for bash: operator tables and variable-name sets +// shared by the lexer and the parser. /** Single-character special parameters: $@ $* $# $? $- $$ $! $0 and $_. */ export const SPECIAL_VARIABLES = ['@', '*', '#', '?', '-', '$', '!', '0', '_'] as const; + +/** Special parameter characters that may follow `$` directly, as a string + * for `includes` checks. Derived from SPECIAL_VARIABLES by dropping `0` + * and `_`, which are word characters matched by the `\w+` rule instead. */ +export const SPECIAL_VARIABLE_CHARS = SPECIAL_VARIABLES.filter((ch) => !/\w/.test(ch)).join(''); + +/** Operators that open a `file_redirect` (heredoc and herestring operators + * are handled separately). `<>` is included even though tree-sitter-bash + * 0.25.0 fails to parse it — it is a real bash operator (`exec 3<>file`). */ +export const FILE_REDIRECT_OPERATORS = ['<', '>', '>>', '>&', '<&', '&>', '&>>', '>|', '<>', '>&-', '<&-'] as const; + +/** Infix operators inside `${...}` expansions, longest first so the parser + * can match them greedily (`##` before `#`, `:-` before `-`). */ +export const EXPANSION_OPERATORS = [ + ':-', + ':=', + ':?', + ':+', + '##', + '%%', + '/#', + '/%', + '//', + '^^', + ',,', + '-', + '+', + '?', + '=', + '#', + '%', + '/', + '^', + ',', + '@', + ':', +] as const; diff --git a/packages/tree-sitter-bash/src/lexer.ts b/packages/tree-sitter-bash/src/lexer.ts new file mode 100644 index 0000000000..16f991194e --- /dev/null +++ b/packages/tree-sitter-bash/src/lexer.ts @@ -0,0 +1,437 @@ +// src/lexer.ts +// +// Hand-written bash tokenizer. Produces a flat token stream of: +// +// word a maximal run of adjacent word material: bare characters, +// '...' / "..." quotes, $var / ${...} / $(...) / `...` expansions, +// <(...) / >(...) process substitutions. Quotes and substitutions +// are skipped over with nesting awareness; the parser re-scans +// the token range to build the actual sub-tree. The single +// characters { } [ ] are emitted as their own word tokens (they +// are not word characters in bash), matching tree-sitter-bash's +// _special_character behaviour. +// op statement operators (&& || | |& ; ;; & ( )) and redirect +// operators (< > >> >& <& &> &>> >| <> >&- <&- <<< << <<-). +// io_number a run of digits immediately followed by < or > — the file +// descriptor prefix of a redirect (2>/dev/null). +// newline a statement-terminating \n. Carries any heredoc bodies that +// were queued on the lexer and are scanned right after the line. +// comment # ... to end of line (only at token start; # inside a word is +// a plain word character). +// eof end of the lexer's range. Also carries pending heredoc bodies. +// +// The lexer is range-bounded: sub-parsers lex $(...) / `...` bodies with +// their own Lexer over the same source but a narrower [start, end) window. +// +// Heredoc queue: the parser registers a HeredocSpec for every << / <<- it +// accepts. When the lexer produces the next newline (or eof) token it scans +// one body per queued spec, in registration order, and attaches them to the +// token; the parser completes the matching heredoc_redirect nodes when it +// consumes that token. +// +// Budget: the lexer never creates nodes, so it only checks the deadline — +// once per produced token and periodically (every SCAN_TICK_INTERVAL +// characters) inside every long scan loop (word runs, quote/paren skipping, +// comments, blanks, heredoc bodies) — via budget.progress(), so a +// pathological single token cannot starve the deadline check. Node counting +// (budget.tick()) is the parser's job. + +import type { ParseBudget } from '#/budget'; +import { SPECIAL_VARIABLE_CHARS } from '#/grammar'; + +export type TokenType = 'word' | 'op' | 'io_number' | 'newline' | 'comment' | 'eof'; + +export interface Token { + readonly type: TokenType; + readonly start: number; + readonly end: number; + /** Heredoc bodies scanned when this newline/eof token was produced. */ + readonly heredocBodies: HeredocBody[]; +} + +export interface HeredocSpec { + /** Delimiter text with any quoting removed — what must appear at line start. */ + readonly delimiter: string; + /** `<<-`: leading tabs of the first body line are stripped. */ + readonly stripTabs: boolean; + /** The delimiter was quoted ('EOF', "EOF", \EOF): the body is not expanded. */ + readonly quoted: boolean; +} + +export interface HeredocBody { + /** Start of the heredoc_body node (after first-line tab stripping). */ + readonly bodyStart: number; + /** End of the heredoc_body node: right before the end marker, so the final + * newline and any tabs preceding the marker stay inside the body. */ + readonly bodyEnd: number; + /** Start of the heredoc_end marker (the delimiter word itself). */ + readonly endStart: number; + /** End of the heredoc_end marker. */ + readonly endEnd: number; + /** False when the delimiter line never appeared: body runs to the end of + * the lexer's range and there is no heredoc_end. */ + readonly found: boolean; +} + +/** How often long scan loops tick the budget, in scanned characters. */ +const SCAN_TICK_INTERVAL = 2048; + +const CONTROL_OPERATORS = ['&>>', '&>', '&&', '&', '|&', '||', '|', ';;', ';', '(', ')'] as const; +const REDIRECT_OPERATORS = ['<<-', '<<<', '<<', '<&-', '<&', '<>', '<', '>&-', '>&', '>>', '>|', '>'] as const; + +function isWordChar(ch: string | undefined): boolean { + return ch !== undefined && /[\w]/.test(ch); +} + +function isBlank(ch: string | undefined): boolean { + return ch === ' ' || ch === '\t' || ch === '\r'; +} + +function isDigitAt(source: string, i: number): boolean { + const ch = source[i]!; + return ch >= '0' && ch <= '9'; +} + +/** Skip a "..." quoted region starting at `i` (which points at the opening + * quote). Returns the index just past the closing quote, or `end` when the + * string is unterminated. Substitution-aware: $(...), ${...} and `...` + * inside the string may themselves contain quotes. */ +export function skipDoubleQuoted(source: string, budget: ParseBudget, i: number, end: number): number { + let j = i + 1; + let sinceTick = 0; + while (j < end) { + if (++sinceTick >= SCAN_TICK_INTERVAL) { + budget.progress(); + sinceTick = 0; + } + const ch = source[j]!; + if (ch === '\\') { + j += 2; + continue; + } + if (ch === '"') return j + 1; + if (ch === '`') { + j = skipBacktick(source, budget, j, end); + continue; + } + if (ch === '$') { + const next = source[j + 1]; + if (next === '(') { + j = scanBalanced(source, budget, j + 1, end, '(', ')').end; + continue; + } + if (next === '{') { + j = scanBalanced(source, budget, j + 1, end, '{', '}').end; + continue; + } + } + j++; + } + return end; +} + +/** Skip a '...' region starting at `i`. No escapes exist in raw strings. */ +export function skipSingleQuoted(source: string, _budget: ParseBudget, i: number, end: number): number { + const close = source.indexOf("'", i + 1); + if (close === -1 || close >= end) return end; + return close + 1; +} + +/** Skip a `...` region starting at `i`; \` is an escaped backtick. */ +export function skipBacktick(source: string, budget: ParseBudget, i: number, end: number): number { + let j = i + 1; + let sinceTick = 0; + while (j < end) { + if (++sinceTick >= SCAN_TICK_INTERVAL) { + budget.progress(); + sinceTick = 0; + } + const ch = source[j]!; + if (ch === '\\') { + j += 2; + continue; + } + if (ch === '`') return j + 1; + j++; + } + return end; +} + +/** Result of scanning a balanced region. */ +export interface BalancedScan { + /** Index just past the matching close, or `end` when unbalanced. */ + end: number; + /** Whether the matching close was actually found. */ + balanced: boolean; +} + +/** Scan a balanced open/close region starting at `i` (which points at the + * opening character). Quote- and escape-aware. */ +export function scanBalanced( + source: string, + budget: ParseBudget, + i: number, + end: number, + open: string, + close: string, +): BalancedScan { + let depth = 0; + let j = i; + let sinceTick = 0; + while (j < end) { + if (++sinceTick >= SCAN_TICK_INTERVAL) { + budget.progress(); + sinceTick = 0; + } + const ch = source[j]!; + if (ch === '\\') { + j += 2; + continue; + } + if (ch === open) { + depth++; + } else if (ch === close) { + depth--; + if (depth === 0) return { end: j + 1, balanced: true }; + } else if (ch === '"') { + j = skipDoubleQuoted(source, budget, j, end); + continue; + } else if (ch === "'") { + j = skipSingleQuoted(source, budget, j, end); + continue; + } else if (ch === '`') { + j = skipBacktick(source, budget, j, end); + continue; + } + j++; + } + return { end, balanced: false }; +} + +/** Skip a $-construct starting at `i` (which points at the `$`). Handles + * $(...), $((...)), ${...}, $name and the single-character specials. A `$` + * followed by anything else (including a quote) consumes just the `$`. */ +export function skipDollar(source: string, budget: ParseBudget, i: number, end: number): number { + const next = source[i + 1]; + if (next === '(') return scanBalanced(source, budget, i + 1, end, '(', ')').end; + if (next === '{') return scanBalanced(source, budget, i + 1, end, '{', '}').end; + if (isWordChar(next)) { + let j = i + 1; + while (j < end && isWordChar(source[j])) j++; + return j; + } + if (next !== undefined && SPECIAL_VARIABLE_CHARS.includes(next)) return i + 2; + return i + 1; +} + +export class Lexer { + /** Current scan position; exposed for the parser's heredoc bookkeeping. */ + pos: number; + private lookahead: Token | null = null; + private readonly pendingHeredocs: HeredocSpec[] = []; + + constructor( + private readonly source: string, + private readonly budget: ParseBudget, + start = 0, + private readonly end = source.length, + ) { + this.pos = start; + } + + /** Queue a heredoc body to be scanned after the current line. */ + queueHeredoc(spec: HeredocSpec): void { + this.pendingHeredocs.push(spec); + } + + peek(): Token { + this.lookahead ??= this.scanToken(); + return this.lookahead; + } + + next(): Token { + const token = this.peek(); + this.lookahead = null; + return token; + } + + text(token: Token): string { + return this.source.slice(token.start, token.end); + } + + private scanToken(): Token { + this.budget.progress(); + this.skipBlanks(); + const start = this.pos; + if (this.pos >= this.end) return this.scanBoundary('eof', start, start); + const ch = this.source[this.pos]!; + if (ch === '\n') { + this.pos++; + return this.scanBoundary('newline', start, this.pos); + } + if (ch === '#') { + let sinceTick = 0; + while (this.pos < this.end && this.source[this.pos] !== '\n') { + if (++sinceTick >= SCAN_TICK_INTERVAL) { + this.budget.progress(); + sinceTick = 0; + } + this.pos++; + } + return { type: 'comment', start, end: this.pos, heredocBodies: [] }; + } + if (ch === '<' || ch === '>') { + // <( / >( start a process substitution, which is word material. The + // heredoc operators were already excluded: <<( is << + ( … + if (this.source[this.pos + 1] === '(') return this.scanWord(); + return this.scanOp(REDIRECT_OPERATORS); + } + if (ch === '&' || ch === '|' || ch === ';' || ch === '(' || ch === ')') { + return this.scanOp(CONTROL_OPERATORS); + } + if (ch === '{' || ch === '}' || ch === '[' || ch === ']') { + this.pos++; + return { type: 'word', start, end: this.pos, heredocBodies: [] }; + } + if (ch >= '0' && ch <= '9') { + let i = this.pos; + while (i < this.end && isDigitAt(this.source, i)) i++; + const next = this.source[i]; + if (next === '<' || next === '>') { + this.pos = i; + return { type: 'io_number', start, end: i, heredocBodies: [] }; + } + return this.scanWord(); + } + return this.scanWord(); + } + + /** Produce a newline/eof token, scanning any queued heredoc bodies that + * start right after it. */ + private scanBoundary(type: 'newline' | 'eof', start: number, end: number): Token { + const bodies: HeredocBody[] = []; + while (this.pendingHeredocs.length > 0) { + bodies.push(this.readHeredocBody(this.pendingHeredocs.shift()!)); + } + return { type, start, end, heredocBodies: bodies }; + } + + private skipBlanks(): void { + let sinceTick = 0; + for (;;) { + if (++sinceTick >= SCAN_TICK_INTERVAL) { + this.budget.progress(); + sinceTick = 0; + } + const ch = this.source[this.pos]; + if (isBlank(ch)) { + this.pos++; + continue; + } + // Line continuation is whitespace, not word material. + if (ch === '\\' && this.source[this.pos + 1] === '\n') { + this.pos += 2; + continue; + } + return; + } + } + + private scanOp(table: readonly string[]): Token { + const start = this.pos; + for (const op of table) { + if (this.source.startsWith(op, this.pos) && this.pos + op.length <= this.end) { + this.pos += op.length; + return { type: 'op', start, end: this.pos, heredocBodies: [] }; + } + } + // Unreachable for the callers above, but never loop forever. + this.pos++; + return { type: 'op', start, end: this.pos, heredocBodies: [] }; + } + + private scanWord(): Token { + const start = this.pos; + let i = this.pos; + let sinceTick = 0; + while (i < this.end) { + if (++sinceTick >= SCAN_TICK_INTERVAL) { + this.budget.progress(); + sinceTick = 0; + } + const ch = this.source[i]!; + if (isBlank(ch) || ch === '\n') break; + if (ch === '&' || ch === '|' || ch === ';' || ch === '(' || ch === ')') break; + if (ch === '{' || ch === '}' || ch === '[' || ch === ']') break; + if (ch === '<' || ch === '>') { + if (this.source[i + 1] === '(') { + i = scanBalanced(this.source, this.budget, i + 1, this.end, '(', ')').end; + continue; + } + break; + } + if (ch === '\\') { + // A line continuation ends the run (it acts as whitespace); a lone + // trailing backslash at end of range is consumed as word text. + if (this.source[i + 1] === '\n') break; + i += 2; + continue; + } + if (ch === '"') { + i = skipDoubleQuoted(this.source, this.budget, i, this.end); + continue; + } + if (ch === "'") { + i = skipSingleQuoted(this.source, this.budget, i, this.end); + continue; + } + if (ch === '`') { + i = skipBacktick(this.source, this.budget, i, this.end); + continue; + } + if (ch === '$') { + i = skipDollar(this.source, this.budget, i, this.end); + continue; + } + i++; + } + if (i === start) i++; // defensive: never emit a zero-width word token + this.pos = i; + return { type: 'word', start, end: i, heredocBodies: [] }; + } + + /** + * Scan one heredoc body, starting at the current position (right after the + * newline that ended the command line). Matches tree-sitter-bash's layout: + * for `<<-` only the first body line's leading tabs are stripped; the end + * marker is the bare delimiter word and any tabs before it belong to the + * body. + */ + private readHeredocBody(spec: HeredocSpec): HeredocBody { + let bodyStart = this.pos; + if (spec.stripTabs) { + while (bodyStart < this.end && this.source[bodyStart] === '\t') bodyStart++; + } + let lineStart = this.pos; + while (lineStart < this.end) { + this.budget.progress(); + let marker = lineStart; + if (spec.stripTabs) { + while (marker < this.end && this.source[marker] === '\t') marker++; + } + if (spec.delimiter.length > 0 && this.source.startsWith(spec.delimiter, marker)) { + const after = marker + spec.delimiter.length; + if (after >= this.end || this.source[after] === '\n') { + this.pos = after; + return { bodyStart, bodyEnd: marker, endStart: marker, endEnd: after, found: true }; + } + } + const newline = this.source.indexOf('\n', lineStart); + if (newline === -1) break; + lineStart = newline + 1; + } + // Unterminated: the body runs to the end of this lexer's range. + this.pos = this.end; + return { bodyStart, bodyEnd: this.end, endStart: this.end, endEnd: this.end, found: false }; + } +} diff --git a/packages/tree-sitter-bash/src/node.ts b/packages/tree-sitter-bash/src/node.ts index fdebc359fd..09e0001892 100644 --- a/packages/tree-sitter-bash/src/node.ts +++ b/packages/tree-sitter-bash/src/node.ts @@ -71,9 +71,24 @@ export class SyntaxNodeBuilder { } /** Attach a child, wiring its parent pointer. Named children are also added - * to `namedChildren`. Returns the child for chaining. */ + * to `namedChildren`. Returns the child for chaining. + * + * Children must lie inside the parent's range and be appended in source + * order without overlapping the previous sibling (a zero-width child may + * start exactly where the previous sibling ends). */ addChild(child: T): T { if (child.parent !== null) throw new Error(`node '${child.type}' already has a parent`); + if (child.startIndex < this.startIndex || child.endIndex > this.endIndex) { + throw new RangeError( + `child '${child.type}' [${child.startIndex}, ${child.endIndex}) escapes parent '${this.type}' [${this.startIndex}, ${this.endIndex})`, + ); + } + const last = this.children.at(-1); + if (last !== undefined && child.startIndex < last.endIndex) { + throw new RangeError( + `child '${child.type}' [${child.startIndex}, ${child.endIndex}) overlaps sibling '${last.type}' [${last.startIndex}, ${last.endIndex})`, + ); + } child.parent = this; this.children.push(child); if (child.isNamed) this.namedChildren.push(child); @@ -89,17 +104,18 @@ export function createNode(init: NodeInit): SyntaxNodeBuilder { /** * Pre-order traversal of the named descendants of `root` (not including * `root` itself), filtered to the given types. With no types, returns every - * named descendant in pre-order. + * named descendant in pre-order. Iterative with an explicit stack so that + * pathologically deep trees cannot overflow the call stack. */ export function descendantsOfType(root: SyntaxNode, ...types: string[]): SyntaxNode[] { const wanted = types.length > 0 ? new Set(types) : null; const out: SyntaxNode[] = []; - const walk = (node: SyntaxNode): void => { - for (const child of node.namedChildren) { - if (wanted === null || wanted.has(child.type)) out.push(child); - walk(child); - } - }; - walk(root); + const stack: SyntaxNode[] = []; + for (let i = root.namedChildren.length - 1; i >= 0; i--) stack.push(root.namedChildren[i]!); + while (stack.length > 0) { + const node = stack.pop()!; + if (wanted === null || wanted.has(node.type)) out.push(node); + for (let i = node.namedChildren.length - 1; i >= 0; i--) stack.push(node.namedChildren[i]!); + } return out; } diff --git a/packages/tree-sitter-bash/src/parse.ts b/packages/tree-sitter-bash/src/parse.ts index 3248323f07..1e28a83c9d 100644 --- a/packages/tree-sitter-bash/src/parse.ts +++ b/packages/tree-sitter-bash/src/parse.ts @@ -1,15 +1,26 @@ // src/parse.ts // -// Public parse entry point. M0 ships a placeholder parser: the whole source is -// wrapped in a `program` root with a single `word` child covering the entire -// text. The budget mechanism is real, and later milestones keep this exact -// entry/exit contract while replacing the placeholder with the actual -// lexer + parser. +// Public parse entry point. Runs the hand-written lexer + recursive-descent +// parser under a ParseBudget and returns the materialized syntax tree. +// +// Exit contract: +// - Budget exhaustion (time or node cap) → { ok: false, reason: 'aborted' }. +// `Aborted` never escapes. +// - Malformed input → { ok: true, hasError: true } with ERROR nodes and/or +// partial nodes where the parser recovered. +// - Any other parser-internal exception (a bug, not user input) is caught +// here as a last resort: the caller still gets a usable tree — a program +// root with a single ERROR child spanning the whole source and +// hasError: true — instead of an exception. Chosen over reporting +// 'aborted' because the parse did not hit its budget; downstream +// consumers keep working on a degraded tree and hasError signals the +// failure. (See the catch block below.) import { Aborted, ParseBudget } from '#/budget'; import type { BudgetOptions } from '#/budget'; import { SyntaxNodeBuilder } from '#/node'; import type { SyntaxNode } from '#/node'; +import { Parser, materialize } from '#/parser'; export type ParseResult = | { ok: true; rootNode: SyntaxNode; hasError: boolean } @@ -20,15 +31,16 @@ export type ParseOptions = BudgetOptions; export function parse(source: string, options: ParseOptions = {}): ParseResult { const budget = new ParseBudget(options); try { - const root = new SyntaxNodeBuilder({ type: 'program', source, startIndex: 0, endIndex: source.length }); - budget.tick(); - if (source.length > 0) { - root.addChild(new SyntaxNodeBuilder({ type: 'word', source, startIndex: 0, endIndex: source.length })); - budget.tick(); - } - return { ok: true, rootNode: root, hasError: false }; + const parser = new Parser(source, budget); + const root = parser.parseProgram(); + const rootNode = materialize(root, source); + return { ok: true, rootNode, hasError: parser.hasError }; } catch (error) { if (error instanceof Aborted) return { ok: false, reason: 'aborted' }; - throw error; + // Last-resort guard for parser bugs: degrade to an ERROR root instead of + // throwing into the caller (see the file header for why). + const root = new SyntaxNodeBuilder({ type: 'program', source, startIndex: 0, endIndex: source.length }); + root.addChild(new SyntaxNodeBuilder({ type: 'ERROR', source, startIndex: 0, endIndex: source.length })); + return { ok: true, rootNode: root, hasError: true }; } } diff --git a/packages/tree-sitter-bash/src/parser.ts b/packages/tree-sitter-bash/src/parser.ts new file mode 100644 index 0000000000..deb0061c91 --- /dev/null +++ b/packages/tree-sitter-bash/src/parser.ts @@ -0,0 +1,1167 @@ +// src/parser.ts +// +// Recursive-descent bash parser. One method per tree-sitter-bash grammar +// rule (parseList ↔ list, parsePipeline ↔ pipeline, parseCommand ↔ command, +// …), producing node ranges and child layouts that match the real +// tree-sitter-bash 0.25.0 tree for the M1 syntax subset. +// +// Construction strategy: the parser builds lightweight `Frame`s, not +// SyntaxNodeBuilders, because heredoc bodies are scanned only when the line +// ends — a heredoc_redirect node (and every ancestor) gets its final range +// after the fact. Frames are mutable and carry parent pointers so the +// heredoc drain can extend the ancestor chain. `materialize` converts the +// finished frame tree into SyntaxNodeBuilders iteratively (explicit stack, +// safe for pathologically deep trees) once parsing is done. +// +// Error recovery: unterminated constructs (quote, expansion, substitution, +// heredoc) keep their partial node and set hasError; tokens that cannot +// start or continue a statement are wrapped in an ERROR node and parsing +// continues. No parser-internal exception is expected to escape; parse() +// still guards against it. +// +// Depth guard: word-level substitutions ($( … ) / `…` / <( … )) recurse via +// fresh sub-Parser instances bounded to their source range (tracked by +// `depth`), subshells recurse within one parser (`scopeDepth`), and the +// parseLiteral ↔ parseString / parseExpansion chain behind ${ … } nesting +// has its own counter (`literalDepth`). All three are capped at +// MAX_PARSE_DEPTH; beyond it the construct is skipped and reported with an +// ERROR node instead of risking a stack overflow. + +import type { ParseBudget } from '#/budget'; +import { EXPANSION_OPERATORS, FILE_REDIRECT_OPERATORS, SPECIAL_VARIABLE_CHARS } from '#/grammar'; +import { Lexer, scanBalanced, skipBacktick, skipSingleQuoted } from '#/lexer'; +import type { BalancedScan, HeredocBody, HeredocSpec, Token } from '#/lexer'; +import { SyntaxNodeBuilder } from '#/node'; + +/** Maximum nesting of scopes (subshells, command substitutions, …). */ +export const MAX_PARSE_DEPTH = 500; + +const FILE_REDIRECT_OP_SET: ReadonlySet = new Set(FILE_REDIRECT_OPERATORS); +const NUMBER_RE = /^-?(0x)?[0-9]+(#[0-9A-Za-z@_]+)?$/; +const ASSIGNMENT_RE = /^[A-Za-z_][A-Za-z0-9_]*\+?=/; +const ASSIGNMENT_SPLIT_RE = /^([A-Za-z_][A-Za-z0-9_]*)(\+?=)/; + +/** Mutable node under construction; see the file header. */ +export interface Frame { + type: string; + start: number; + end: number; + isNamed: boolean; + parent: Frame | null; + children: Frame[]; +} + +interface PendingHeredoc { + frame: Frame; + spec: HeredocSpec; +} + +function isFileRedirectOp(text: string): boolean { + return FILE_REDIRECT_OP_SET.has(text); +} + +/** Strip quoting from a heredoc delimiter word; any quote or backslash in + * the raw text marks the body as quoted (no expansions). */ +function extractHeredocSpec(raw: string, stripTabs: boolean): HeredocSpec { + let delimiter = ''; + let quoted = false; + for (let i = 0; i < raw.length; i++) { + const ch = raw[i]!; + if (ch === '\\' && i + 1 < raw.length) { + delimiter += raw[i + 1]; + quoted = true; + i++; + } else if (ch === '"' || ch === "'") { + quoted = true; + } else { + delimiter += ch; + } + } + return { delimiter, stripTabs, quoted }; +} + +export class Parser { + /** Set when any recovery path was taken. */ + hasError = false; + private lexer!: Lexer; + private readonly heredocQueue: PendingHeredoc[] = []; + private scopeDepth = 0; + /** Recursion depth of parseLiteral ↔ parseString / parseExpansion (the + * ${} nesting chain that stays inside this Parser instance). */ + private literalDepth = 0; + /** True while parsing inside a heredoc's line tail: a second heredoc + * cannot be represented and degrades to ERROR (see parseHeredocRedirect). */ + private noHeredoc = false; + + constructor( + private readonly source: string, + private readonly budget: ParseBudget, + private readonly depth = 0, + ) {} + + // ---------------------------------------------------------------- helpers + + private frame(type: string, start: number, end: number, children: Frame[] = [], isNamed = true): Frame { + this.budget.tick(); + const frame: Frame = { type, start, end, isNamed, parent: null, children }; + for (const child of children) child.parent = frame; + return frame; + } + + private anon(type: string, start: number, end: number): Frame { + return this.frame(type, start, end, [], false); + } + + private addKid(parent: Frame, child: Frame): void { + child.parent = parent; + parent.children.push(child); + } + + private text(start: number, end: number): string { + return this.source.slice(start, end); + } + + private tokenText(token: Token): string { + return this.source.slice(token.start, token.end); + } + + private isStatementStart(token: Token): boolean { + if (token.type === 'word' || token.type === 'io_number') return true; + if (token.type !== 'op') return false; + const text = this.tokenText(token); + return text === '(' || text === '<<<' || text === '<<' || text === '<<-' || isFileRedirectOp(text); + } + + // ------------------------------------------------------------ entry point + + /** program: the whole source as one statement list. */ + parseProgram(): Frame { + this.lexer = new Lexer(this.source, this.budget, 0, this.source.length); + const children = this.parseStatementList(false); + return this.frame('program', 0, this.source.length, children); + } + + /** Parse a sub-range as a statement list (body of $( … ), ` … `, <( … )). */ + private parseScopedStatements(start: number, end: number): Frame[] { + if (this.depth + 1 >= MAX_PARSE_DEPTH) { + this.hasError = true; + return [this.frame('ERROR', start, end)]; + } + const sub = new Parser(this.source, this.budget, this.depth + 1); + sub.lexer = new Lexer(this.source, this.budget, start, end); + const children = sub.parseStatementList(false); + if (sub.hasError) this.hasError = true; + return children; + } + + // -------------------------------------------------------- statement lists + + /** _statements: statements separated/terminated by ; & and newlines. */ + private parseStatementList(stopAtParen: boolean): Frame[] { + const children: Frame[] = []; + let needTerminator = false; + for (;;) { + const token = this.lexer.peek(); + if (token.type === 'eof') { + this.completeHeredocs(token.heredocBodies); + this.failOpenHeredocs(); + break; + } + if (token.type === 'newline') { + this.lexer.next(); + this.completeHeredocs(token.heredocBodies); + // A newline that scanned heredoc bodies belongs to the heredoc + // redirects (it sits inside their range); it is not a program-level + // terminator node. + if (token.heredocBodies.length === 0) { + children.push(this.anon('\n', token.start, token.end)); + } + needTerminator = false; + continue; + } + if (token.type === 'comment') { + this.lexer.next(); + children.push(this.frame('comment', token.start, token.end)); + continue; + } + const op = token.type === 'op' ? this.tokenText(token) : ''; + if (token.type === 'op' && op === ')' && stopAtParen) { + this.failOpenHeredocs(); + break; + } + if (token.type === 'op' && (op === ';' || op === '&' || op === ';;')) { + this.lexer.next(); + children.push(this.anon(op, token.start, token.end)); + needTerminator = false; + continue; + } + if (token.type === 'op' && (op === ')' || op === '&&' || op === '||' || op === '|' || op === '|&')) { + // A binary operator or closer with no left-hand side: recover. + this.hasError = true; + this.lexer.next(); + children.push(this.frame('ERROR', token.start, token.end, [this.anon(op, token.start, token.end)])); + needTerminator = false; + continue; + } + if (needTerminator) this.hasError = true; // missing separator, e.g. `cmd (sub)` + children.push(this.parseList()); + needTerminator = true; + } + return children; + } + + /** Consume newlines and comments in operator-continuation position + * (`a &&\n b`). Returns the comment nodes; newlines are dropped. */ + private skipContinuation(): Frame[] { + const comments: Frame[] = []; + for (;;) { + const token = this.lexer.peek(); + if (token.type === 'newline') { + this.lexer.next(); + this.completeHeredocs(token.heredocBodies); + continue; + } + if (token.type === 'comment') { + this.lexer.next(); + comments.push(this.frame('comment', token.start, token.end)); + continue; + } + return comments; + } + } + + // ------------------------------------------------------------- statements + + /** list: statement ((&& | ||) statement)* — left associative. */ + private parseList(): Frame { + let left = this.parsePipeline(); + for (;;) { + const token = this.lexer.peek(); + if (token.type !== 'op') break; + const op = this.tokenText(token); + if (op !== '&&' && op !== '||') break; + this.lexer.next(); + const extras = this.skipContinuation(); + if (this.isStatementStart(this.lexer.peek())) { + const right = this.parsePipeline(); + left = this.frame('list', left.start, right.end, [left, this.anon(op, token.start, token.end), ...extras, right]); + } else { + // Trailing connector (`ls &&`): keep the partial list, flag it. + this.hasError = true; + left = this.frame('list', left.start, token.end, [left, this.anon(op, token.start, token.end)]); + break; + } + } + return left; + } + + /** pipeline: statement ((| | |&) statement)* */ + private parsePipeline(): Frame { + return this.parsePipelineTail(this.parseStatementNotPipeline()); + } + + private parsePipelineTail(first: Frame): Frame { + const kids: Frame[] = [first]; + let end = first.end; + let pipes = 0; + for (;;) { + const token = this.lexer.peek(); + if (token.type !== 'op') break; + const op = this.tokenText(token); + if (op !== '|' && op !== '|&') break; + this.lexer.next(); + pipes++; + kids.push(this.anon(op, token.start, token.end)); + end = token.end; + kids.push(...this.skipContinuation()); + if (this.isStatementStart(this.lexer.peek())) { + const next = this.parseStatementNotPipeline(); + kids.push(next); + end = next.end; + } else { + this.hasError = true; // dangling pipe (`ls |`) + break; + } + } + if (pipes === 0) return first; + return this.frame('pipeline', first.start, end, kids); + } + + /** _statement_not_pipeline for the M1 subset: command, subshell, + * negated_command, variable_assignment(s), redirected_statement. */ + private parseStatementNotPipeline(): Frame { + const token = this.lexer.peek(); + let inner: Frame | null = null; + if (token.type === 'op' && this.tokenText(token) === '(') { + inner = this.parseSubshell(); + } else if (token.type === 'word' && this.tokenText(token) === '!') { + inner = this.parseNegatedCommand(); + } else { + inner = this.parseCommand(); + } + const trailing: Frame[] = []; + for (;;) { + const next = this.lexer.peek(); + if (next.type === 'io_number') { + trailing.push(this.parseRedirect()); + continue; + } + if (next.type === 'op') { + const op = this.tokenText(next); + if (isFileRedirectOp(op) || op === '<<<' || op === '<<' || op === '<<-') { + trailing.push(this.parseRedirect()); + continue; + } + } + break; + } + if (trailing.length > 0) { + const kids = inner === null ? trailing : [inner, ...trailing]; + inner = this.frame('redirected_statement', kids[0]!.start, kids.at(-1)!.end, kids); + } + if (inner === null) { + // Unreachable from well-formed dispatch; recover without looping. + this.hasError = true; + const skipped = this.lexer.next(); + inner = this.frame('ERROR', skipped.start, skipped.end); + } + return inner; + } + + /** negated_command: `!` followed by a command or subshell. */ + private parseNegatedCommand(): Frame { + const bang = this.lexer.next(); + const kids: Frame[] = [this.anon('!', bang.start, bang.end)]; + let end = bang.end; + const token = this.lexer.peek(); + if (token.type === 'op' && this.tokenText(token) === '(') { + const subshell = this.parseSubshell(); + kids.push(subshell); + end = subshell.end; + } else { + const command = this.parseCommand(); + if (command === null) { + this.hasError = true; + } else { + kids.push(command); + end = command.end; + } + } + return this.frame('negated_command', bang.start, end, kids); + } + + /** subshell: `(` _statements `)` */ + private parseSubshell(): Frame { + const open = this.lexer.next(); + if (this.scopeDepth >= MAX_PARSE_DEPTH) { + // Absurd nesting: skip token-wise to the matching close paren. Word + // tokens hide their internal parens, so token-level counting is exact. + this.hasError = true; + let depth = 1; + let end = open.end; + for (;;) { + const token = this.lexer.next(); + end = token.end; + if (token.type === 'eof') break; + this.completeHeredocs(token.heredocBodies); + if (token.type === 'op') { + const op = this.tokenText(token); + if (op === '(') depth++; + else if (op === ')') { + depth--; + if (depth === 0) break; + } + } + } + return this.frame('ERROR', open.start, end, [this.anon('(', open.start, open.end)]); + } + const kids: Frame[] = [this.anon('(', open.start, open.end)]; + this.scopeDepth++; + const inner = this.parseStatementList(true); + this.scopeDepth--; + kids.push(...inner); + let end = inner.length > 0 ? inner.at(-1)!.end : open.end; + const token = this.lexer.peek(); + if (token.type === 'op' && this.tokenText(token) === ')') { + this.lexer.next(); + kids.push(this.anon(')', token.start, token.end)); + end = token.end; + } else { + this.hasError = true; // unterminated subshell + } + return this.frame('subshell', open.start, end, kids); + } + + // --------------------------------------------------------------- commands + + /** + * command: leading variable_assignment / redirect prefix, command_name, + * then arguments and inline herestring redirects. Returns null when no + * command name is present and nothing was consumed; a nameless prefix is + * assembled into variable_assignment(s) / redirected_statement here. + */ + private parseCommand(): Frame | null { + const prefix: Frame[] = []; + for (;;) { + const token = this.lexer.peek(); + if (token.type === 'word' && ASSIGNMENT_RE.test(this.tokenText(token))) { + this.lexer.next(); + prefix.push(this.parseVariableAssignment(token)); + continue; + } + // Prefix redirects take exactly one destination so a following word + // still becomes the command_name (`> a cmd x`). Heredoc operators are + // left for the trailing-redirect path. + if (token.type === 'io_number') { + prefix.push(this.parseRedirect(1)); + continue; + } + if (token.type === 'op') { + const op = this.tokenText(token); + if (isFileRedirectOp(op) || op === '<<<') { + prefix.push(this.parseRedirect(1)); + continue; + } + } + break; + } + if (this.lexer.peek().type !== 'word') { + if (prefix.length === 0) return null; + return this.assembleNamelessPrefix(prefix); + } + const start = prefix.length > 0 ? prefix[0]!.start : this.lexer.peek().start; + const nameToken = this.lexer.next(); + const name = this.frame('command_name', nameToken.start, nameToken.end, [ + this.parseLiteral(nameToken.start, nameToken.end), + ]); + const command = this.frame('command', start, nameToken.end, [...prefix, name]); + for (;;) { + const token = this.lexer.peek(); + if (token.type === 'word') { + const argument = this.parseWordArgument(); + this.addKid(command, argument); + command.end = argument.end; + continue; + } + if (token.type === 'op' && this.tokenText(token) === '<<<') { + const herestring = this.parseHerestringRedirect(null); + this.addKid(command, herestring); + command.end = herestring.end; + continue; + } + break; + } + return command; + } + + /** A prefix of assignments/redirects with no command name: + * `FOO=bar`, `A=1 B=2`, `> out`, `> out FOO=bar`. */ + private assembleNamelessPrefix(prefix: Frame[]): Frame { + const assignments = prefix.filter((f) => f.type === 'variable_assignment'); + if (assignments.length === prefix.length) { + if (prefix.length === 1) return prefix[0]!; + return this.frame('variable_assignments', prefix[0]!.start, prefix.at(-1)!.end, prefix); + } + return this.frame('redirected_statement', prefix[0]!.start, prefix.at(-1)!.end, prefix); + } + + /** variable_assignment: NAME ( = | += ) value — the word token is split + * into variable_name, the operator, and a value parsed from the rest of + * the token (which may hold quotes/expansions). */ + private parseVariableAssignment(token: Token): Frame { + const text = this.tokenText(token); + const match = ASSIGNMENT_SPLIT_RE.exec(text)!; + const nameEnd = token.start + match[1]!.length; + const opEnd = nameEnd + match[2]!.length; + const kids: Frame[] = [ + this.frame('variable_name', token.start, nameEnd), + this.anon(match[2]!, nameEnd, opEnd), + ]; + if (opEnd < token.end) { + kids.push(this.parseLiteral(opEnd, token.end)); + } + return this.frame('variable_assignment', token.start, token.end, kids); + } + + // --------------------------------------------------------------- redirect + + /** Parse one redirect with an optional io_number descriptor prefix already + * peeked. Dispatches to file_redirect / herestring / heredoc handling. + * `maxDestinations` limits greedy destination consumption (command-prefix + * redirects take exactly one). */ + private parseRedirect(maxDestinations = Number.POSITIVE_INFINITY): Frame { + let descriptor: Frame | null = null; + let token = this.lexer.peek(); + if (token.type === 'io_number') { + this.lexer.next(); + descriptor = this.frame('file_descriptor', token.start, token.end); + token = this.lexer.peek(); + } + if (token.type !== 'op') { + // io_number not followed by an operator — recover. + this.hasError = true; + const stray = descriptor ?? this.frame('ERROR', token.start, token.end); + return this.frame('ERROR', stray.start, stray.end, descriptor === null ? [] : [stray]); + } + const op = this.tokenText(token); + if (op === '<<' || op === '<<-') { + return this.noHeredoc ? this.parseBrokenHeredoc(descriptor) : this.parseHeredocRedirect(descriptor); + } + if (op === '<<<') return this.parseHerestringRedirect(descriptor); + if (isFileRedirectOp(op)) return this.parseFileRedirect(descriptor, maxDestinations); + this.hasError = true; + const stray = descriptor ?? this.anon(op, token.start, token.end); + return this.frame('ERROR', stray.start, stray.end, [stray]); + } + + /** file_redirect: [fd] op destination* — `>&-`/`<&-` take no destination. + * In command-prefix position only one destination is taken (`> a cmd x` + * makes `cmd` the command_name); after the command name the redirect is + * greedy, matching tree-sitter-bash (`cmd > out arg` puts both words in + * the redirect). */ + private parseFileRedirect(descriptor: Frame | null, maxDestinations = Number.POSITIVE_INFINITY): Frame { + const opToken = this.lexer.next(); + const op = this.tokenText(opToken); + const kids: Frame[] = []; + if (descriptor !== null) kids.push(descriptor); + kids.push(this.anon(op, opToken.start, opToken.end)); + let end = opToken.end; + if (op !== '>&-' && op !== '<&-') { + let destinations = 0; + while (destinations < maxDestinations && this.lexer.peek().type === 'word') { + const destination = this.parseWordArgument(); + kids.push(destination); + end = destination.end; + destinations++; + } + if (destinations === 0) this.hasError = true; // missing destination + } + return this.frame('file_redirect', descriptor?.start ?? opToken.start, end, kids); + } + + /** herestring_redirect: [fd] `<<<` literal */ + private parseHerestringRedirect(descriptor: Frame | null): Frame { + const opToken = this.lexer.next(); + const kids: Frame[] = []; + if (descriptor !== null) kids.push(descriptor); + kids.push(this.anon('<<<', opToken.start, opToken.end)); + let end = opToken.end; + if (this.lexer.peek().type === 'word') { + const value = this.parseWordArgument(); + kids.push(value); + end = value.end; + } else { + this.hasError = true; + } + return this.frame('herestring_redirect', descriptor?.start ?? opToken.start, end, kids); + } + + /** + * heredoc_redirect: [fd] (`<<` | `<<-`) heredoc_start, then the rest of + * the line (arguments, redirects, a pipeline or &&/|| tail, and `;`/`&` + * separated follow-up statements) absorbed into the redirect — matching + * tree-sitter-bash's grammar, which swallows all of these into + * heredoc_redirect. + * + * Only ONE heredoc may be registered per line region: a second `<<` + * (or a `<<` inside a swallowed follow-up statement) cannot be + * represented as a tree — its body would have to sit inside a node whose + * range interleaves with the first heredoc's siblings. tree-sitter-bash + * 0.25.0 errors on `cat <= MAX_PARSE_DEPTH) { + this.hasError = true; + return this.frame('ERROR', start, end); + } + this.literalDepth++; + try { + return this.parseLiteralPieces(start, end); + } finally { + this.literalDepth--; + } + } + + private parseLiteralPieces(start: number, end: number): Frame { + const pieces: Frame[] = []; + let i = start; + while (i < end) { + this.budget.progress(); + const ch = this.source[i]!; + if (ch === '"') { + const [piece, next] = this.parseString(i, end); + pieces.push(piece); + i = next; + continue; + } + if (ch === "'") { + const close = skipSingleQuoted(this.source, this.budget, i, end); + if (close >= end && this.source[close - 1] !== "'") this.hasError = true; // unterminated raw string + pieces.push(this.frame('raw_string', i, close)); + i = close; + continue; + } + if (ch === '`') { + const [piece, next] = this.parseBacktickSubstitution(i, end); + pieces.push(piece); + i = next; + continue; + } + if (ch === '$') { + const dollar = this.parseDollar(i, end); + if (dollar !== null) { + pieces.push(dollar[0]); + i = dollar[1]; + } else { + // Bare `$` (not followed by anything expandable). + pieces.push(this.anon('$', i, i + 1)); + i++; + } + continue; + } + if ((ch === '<' || ch === '>') && this.source[i + 1] === '(' && i + 1 < end) { + const [piece, next] = this.parseProcessSubstitution(i, end); + pieces.push(piece); + i = next; + continue; + } + if (ch === '{' || ch === '}' || ch === '[' || ch === ']') { + pieces.push(this.frame('word', i, i + 1)); + i++; + continue; + } + // Bare run of ordinary word characters (escapes included as-is). + let j = i; + let sinceTick = 0; + while (j < end) { + if (++sinceTick >= 2048) { + this.budget.progress(); + sinceTick = 0; + } + const next = this.source[j]!; + if (next === '"' || next === "'" || next === '`' || next === '$') break; + if (next === '{' || next === '}' || next === '[' || next === ']') break; + if ((next === '<' || next === '>') && this.source[j + 1] === '(' && j + 1 < end) break; + if (next === '\\' && j + 1 < end) { + j += 2; + continue; + } + j++; + } + if (j === i) j++; // defensive: never stall + pieces.push(this.frame('word', i, j)); + i = j; + } + if (pieces.length === 1) { + const only = pieces[0]!; + if (only.type === 'word' && NUMBER_RE.test(this.text(only.start, only.end))) { + return this.frame('number', only.start, only.end); + } + return only; + } + return this.frame('concatenation', start, end, pieces); + } + + /** string: " … " with string_content chunks and expansions inside. */ + private parseString(start: number, rangeEnd: number): [Frame, number] { + const string = this.frame('string', start, start + 1, [this.anon('"', start, start + 1)]); + let chunkStart = start + 1; + let i = start + 1; + const flushChunk = (upto: number): void => { + if (upto > chunkStart) { + this.addKid(string, this.frame('string_content', chunkStart, upto)); + } + }; + let sinceTick = 0; + while (i < rangeEnd) { + if (++sinceTick >= 2048) { + this.budget.progress(); + sinceTick = 0; + } + const ch = this.source[i]!; + if (ch === '"') { + flushChunk(i); + this.addKid(string, this.anon('"', i, i + 1)); + string.end = i + 1; + return [string, i + 1]; + } + if (ch === '\\') { + i += 2; // escaped characters stay inside the current content chunk + continue; + } + if (ch === '$') { + const dollar = this.parseDollar(i, rangeEnd); + if (dollar !== null) { + flushChunk(i); + this.addKid(string, dollar[0]); + i = dollar[1]; + chunkStart = i; + continue; + } + i++; + continue; + } + if (ch === '`') { + flushChunk(i); + const [piece, next] = this.parseBacktickSubstitution(i, rangeEnd); + this.addKid(string, piece); + i = next; + chunkStart = i; + continue; + } + i++; + } + // Unterminated string: keep the partial node, flag the error. + this.hasError = true; + flushChunk(rangeEnd); + string.end = rangeEnd; + return [string, rangeEnd]; + } + + /** Dispatch a $-construct at `i`: simple_expansion, expansion, + * command_substitution, or the arithmetic placeholder. Returns null for a + * bare `$` (including the M2 forms $"…" and $'…'). */ + private parseDollar(i: number, rangeEnd: number): [Frame, number] | null { + const next = this.source[i + 1]; + if (next === '(' && i + 1 < rangeEnd) { + if (this.source[i + 2] === '(') return this.parseArithmeticExpansion(i, rangeEnd); + return this.parseCommandSubstitution(i, rangeEnd); + } + if (next === '{' && i + 1 < rangeEnd) return this.parseExpansion(i, rangeEnd); + if (next !== undefined && /[\w]/.test(next)) { + let j = i + 1; + while (j < rangeEnd && /[\w]/.test(this.source[j]!)) j++; + const expansion = this.frame('simple_expansion', i, j, [ + this.anon('$', i, i + 1), + this.frame('variable_name', i + 1, j), + ]); + return [expansion, j]; + } + if (next !== undefined && i + 1 < rangeEnd && SPECIAL_VARIABLE_CHARS.includes(next)) { + const expansion = this.frame('simple_expansion', i, i + 2, [ + this.anon('$', i, i + 1), + this.frame('special_variable_name', i + 1, i + 2), + ]); + return [expansion, i + 2]; + } + return null; + } + + /** expansion: ${ … } with optional prefix operators (! #), a variable_name + * / special_variable_name / subscript, an optional infix operator + * (:- ## % / …), and a value parsed as a literal. */ + private parseExpansion(i: number, rangeEnd: number): [Frame, number] { + const scan = this.scanBalanced(i + 1, rangeEnd, '{', '}'); + const close = scan.end; + const terminated = scan.balanced; + if (!terminated) this.hasError = true; + const innerEnd = terminated ? close - 1 : close; + const expansion = this.frame('expansion', i, close, [this.anon('${', i, i + 2)]); + let j = i + 2; + // Prefix operators: ${#v} (length), ${!v} (indirect). + while (j < innerEnd && (this.source[j] === '#' || this.source[j] === '!')) { + const after = this.source[j + 1]; + if (after === undefined || j + 1 >= innerEnd || !/[\w@*#?$!-]/.test(after)) break; + this.addKid(expansion, this.anon(this.source[j]!, j, j + 1)); + j++; + } + // The variable itself. + let hasName = false; + if (j < innerEnd && /[\w]/.test(this.source[j]!)) { + let nameEnd = j; + while (nameEnd < innerEnd && /[\w]/.test(this.source[nameEnd]!)) nameEnd++; + this.addKid(expansion, this.frame('variable_name', j, nameEnd)); + j = nameEnd; + hasName = true; + } else if (j < innerEnd && SPECIAL_VARIABLE_CHARS.includes(this.source[j]!)) { + this.addKid(expansion, this.frame('special_variable_name', j, j + 1)); + j++; + hasName = true; + } + // Subscript: ${a[0]} — replaces the bare variable_name child. + if (j < innerEnd && this.source[j] === '[' && hasName) { + const sub = this.scanBalanced(j, rangeEnd, '[', ']'); + const subEnd = sub.end; + if (!sub.balanced) this.hasError = true; + const indexEnd = sub.balanced ? subEnd - 1 : subEnd; + const variable = expansion.children.pop()!; + const subscript = this.frame('subscript', variable.start, subEnd, [ + variable, + this.anon('[', j, j + 1), + ]); + if (indexEnd > j + 1) { + this.addKid(subscript, this.parseLiteral(j + 1, indexEnd)); + } else { + this.hasError = true; + } + if (sub.balanced) this.addKid(subscript, this.anon(']', subEnd - 1, subEnd)); + this.addKid(expansion, subscript); + j = subEnd; + } + // Infix operator plus an optional value. + if (j < innerEnd) { + for (const operator of EXPANSION_OPERATORS) { + if (j + operator.length <= innerEnd && this.source.startsWith(operator, j)) { + this.addKid(expansion, this.anon(operator, j, j + operator.length)); + j += operator.length; + break; + } + } + if (j < innerEnd) { + this.addKid(expansion, this.parseLiteral(j, innerEnd)); + } + } + if (terminated) this.addKid(expansion, this.anon('}', close - 1, close)); + return [expansion, close]; + } + + /** command_substitution: $( _statements ) */ + private parseCommandSubstitution(i: number, rangeEnd: number): [Frame, number] { + const scan = this.scanBalanced(i + 1, rangeEnd, '(', ')'); + const close = scan.end; + if (!scan.balanced) this.hasError = true; + const innerEnd = scan.balanced ? close - 1 : close; + const substitution = this.frame('command_substitution', i, close, [this.anon('$(', i, i + 2)]); + for (const child of this.parseScopedStatements(i + 2, innerEnd)) { + this.addKid(substitution, child); + } + if (scan.balanced) this.addKid(substitution, this.anon(')', close - 1, close)); + return [substitution, close]; + } + + /** command_substitution: ` _statements ` (legacy backtick form). */ + private parseBacktickSubstitution(i: number, rangeEnd: number): [Frame, number] { + const close = skipBacktick(this.source, this.budget, i, rangeEnd); + const terminated = close > i + 1 && this.source[close - 1] === '`'; + if (!terminated) this.hasError = true; + const innerEnd = terminated ? close - 1 : close; + const substitution = this.frame('command_substitution', i, close, [this.anon('`', i, i + 1)]); + if (innerEnd > i + 1) { + for (const child of this.parseScopedStatements(i + 1, innerEnd)) { + this.addKid(substitution, child); + } + } + if (terminated) this.addKid(substitution, this.anon('`', close - 1, close)); + return [substitution, close]; + } + + /** process_substitution: ( <( | >( ) _statements ) */ + private parseProcessSubstitution(i: number, rangeEnd: number): [Frame, number] { + const scan = this.scanBalanced(i + 1, rangeEnd, '(', ')'); + const close = scan.end; + if (!scan.balanced) this.hasError = true; + const innerEnd = scan.balanced ? close - 1 : close; + const opener = this.source[i]!; + const substitution = this.frame('process_substitution', i, close, [this.anon(`${opener}(`, i, i + 2)]); + for (const child of this.parseScopedStatements(i + 2, innerEnd)) { + this.addKid(substitution, child); + } + if (scan.balanced) this.addKid(substitution, this.anon(')', close - 1, close)); + return [substitution, close]; + } + + /** + * TODO(M2): real arithmetic expansion parsing ($(( … )) with + * binary_expression / number / … children). For M1 the whole construct is + * surfaced as an arithmetic_expansion whose raw inner text sits in a + * single word child, so the node type and range are already correct and + * downstream consumers can rely on both. + */ + private parseArithmeticExpansion(i: number, rangeEnd: number): [Frame, number] { + const scan = this.scanBalanced(i + 1, rangeEnd, '(', ')'); + const close = scan.end; + if (!scan.balanced) this.hasError = true; + // The content sits between `$((` and the final `))`. + const closed = scan.balanced && close - 2 >= i + 3 && this.source[close - 2] === ')'; + const innerStart = Math.min(i + 3, close); + const innerEnd = closed ? close - 2 : close; + const expansion = this.frame('arithmetic_expansion', i, close, [this.anon('$((', i, innerStart)]); + if (innerEnd > innerStart) { + this.addKid(expansion, this.frame('word', innerStart, innerEnd)); + } + if (closed) { + this.addKid(expansion, this.anon('))', innerEnd, close)); + } + return [expansion, close]; + } + + /** heredoc_body content: heredoc_content chunks interleaved with + * expansions and command substitutions (unquoted delimiters only). + * Unlike tree-sitter-bash's scanner, every plain chunk — including the + * leading one — becomes a heredoc_content node. */ + private parseHeredocContent(start: number, end: number): Frame[] { + const pieces: Frame[] = []; + let chunkStart = start; + let i = start; + const flush = (upto: number): void => { + if (upto > chunkStart) { + pieces.push(this.frame('heredoc_content', chunkStart, upto)); + } + }; + let sinceTick = 0; + while (i < end) { + if (++sinceTick >= 2048) { + this.budget.progress(); + sinceTick = 0; + } + const ch = this.source[i]!; + if (ch === '\\') { + i += 2; // escaped characters stay literal (and suppress expansion) + continue; + } + if (ch === '$') { + const dollar = this.parseDollar(i, end); + if (dollar !== null) { + flush(i); + pieces.push(dollar[0]); + i = dollar[1]; + chunkStart = i; + continue; + } + i++; + continue; + } + if (ch === '`') { + flush(i); + const [piece, next] = this.parseBacktickSubstitution(i, end); + pieces.push(piece); + i = next; + chunkStart = i; + continue; + } + i++; + } + flush(end); + return pieces; + } + + /** Balanced-scan wrapper returning whether the close was actually found. */ + private scanBalanced(i: number, end: number, open: string, close: string): BalancedScan { + return scanBalanced(this.source, this.budget, i, end, open, close); + } +} + +/** + * Convert a finished frame tree into SyntaxNodeBuilders. Iterative + * (explicit stack) so deep trees cannot overflow the call stack; node + * creation here is not budget-ticked because every frame already ticked the + * budget when it was created (one frame ↔ one node). + */ +export function materialize(root: Frame, source: string): SyntaxNodeBuilder { + const nodes = new Map(); + const order: Frame[] = []; + const stack: Frame[] = [root]; + while (stack.length > 0) { + const frame = stack.pop()!; + order.push(frame); + for (const child of frame.children) stack.push(child); + } + for (const frame of order) { + nodes.set( + frame, + new SyntaxNodeBuilder({ + type: frame.type, + source, + startIndex: frame.start, + endIndex: frame.end, + isNamed: frame.isNamed, + }), + ); + } + for (const frame of order) { + const node = nodes.get(frame)!; + for (const child of frame.children) { + node.addChild(nodes.get(child)!); + } + } + return nodes.get(root)!; +} diff --git a/packages/tree-sitter-bash/test/node.test.ts b/packages/tree-sitter-bash/test/node.test.ts index 4c6736a59a..16d5219808 100644 --- a/packages/tree-sitter-bash/test/node.test.ts +++ b/packages/tree-sitter-bash/test/node.test.ts @@ -48,6 +48,27 @@ describe('createNode', () => { a.addChild(child); expect(() => b.addChild(child)).toThrow(/already has a parent/); }); + + it('rejects children outside the parent range', () => { + const parent = createNode({ type: 'command', source: SOURCE, startIndex: 0, endIndex: 8 }); + expect(() => + parent.addChild(createNode({ type: 'word', source: SOURCE, startIndex: 0, endIndex: 9 })), + ).toThrow(RangeError); + }); + + it('rejects overlapping or out-of-order siblings', () => { + const parent = createNode({ type: 'command', source: SOURCE, startIndex: 0, endIndex: 8 }); + parent.addChild(createNode({ type: 'word', source: SOURCE, startIndex: 2, endIndex: 5 })); + expect(() => parent.addChild(createNode({ type: 'word', source: SOURCE, startIndex: 4, endIndex: 7 }))).toThrow( + RangeError, + ); + expect(() => parent.addChild(createNode({ type: 'word', source: SOURCE, startIndex: 0, endIndex: 2 }))).toThrow( + RangeError, + ); + // Adjacent (start == previous end) is fine. + parent.addChild(createNode({ type: 'word', source: SOURCE, startIndex: 5, endIndex: 8 })); + expect(parent.children).toHaveLength(2); + }); }); describe('children vs namedChildren', () => { @@ -95,4 +116,17 @@ describe('descendantsOfType', () => { const leaf = createNode({ type: 'word', source: SOURCE, startIndex: 0, endIndex: 4 }); expect(descendantsOfType(leaf, 'word')).toEqual([]); }); + + it('handles pathologically deep trees without overflowing the stack', () => { + // 200k-deep chain; a recursive walk would overflow the call stack. + const depth = 200_000; + const nodes = []; + for (let i = 0; i < depth; i++) { + nodes.push(createNode({ type: 'word', source: SOURCE, startIndex: 0, endIndex: 1 })); + } + for (let i = depth - 2; i >= 0; i--) { + nodes[i]!.addChild(nodes[i + 1]!); + } + expect(descendantsOfType(nodes[0]!, 'word')).toHaveLength(depth - 1); + }); }); diff --git a/packages/tree-sitter-bash/test/parse.test.ts b/packages/tree-sitter-bash/test/parse.test.ts index ab1399bf7f..9a8b25f8a8 100644 --- a/packages/tree-sitter-bash/test/parse.test.ts +++ b/packages/tree-sitter-bash/test/parse.test.ts @@ -1,8 +1,35 @@ import { describe, expect, it } from 'vitest'; import { Aborted, ParseBudget } from '#/budget'; +import type { SyntaxNode } from '#/node'; +import { descendantsOfType } from '#/node'; import { parse } from '#/parse'; +/** Compact S-expression of the tree: named leaves are (type "text"), + * anonymous leaves are just their quoted type, inner nodes nest. */ +function sexp(node: SyntaxNode): string { + if (node.children.length === 0) { + return node.isNamed ? `(${node.type} ${JSON.stringify(node.text)})` : JSON.stringify(node.type); + } + const label = node.isNamed ? node.type : JSON.stringify(node.type); + return `(${label} ${node.children.map(sexp).join(' ')})`; +} + +function parseOk(source: string): { rootNode: SyntaxNode; hasError: boolean } { + const result = parse(source); + expect(result.ok).toBe(true); + if (!result.ok) throw new Error('unreachable'); + return result; +} + +/** Parse and assert the exact tree shape plus the hasError flag. */ +function expectTree(source: string, expected: string, hasError = false): SyntaxNode { + const { rootNode, hasError: actual } = parseOk(source); + expect(actual).toBe(hasError); + expect(sexp(rootNode)).toBe(expected); + return rootNode; +} + describe('ParseBudget', () => { it('counts nodes and stays under the cap', () => { const budget = new ParseBudget({ timeoutMs: 60_000, maxNodes: 3 }); @@ -14,55 +41,479 @@ describe('ParseBudget', () => { it('throws Aborted when the node cap is exceeded', () => { const budget = new ParseBudget({ timeoutMs: 60_000, maxNodes: 1 }); budget.tick(); - expect(() => budget.tick()).toThrow(Aborted); + expect(() => { + budget.tick(); + }).toThrow(Aborted); }); it('throws Aborted once the deadline is reached', () => { const budget = new ParseBudget({ timeoutMs: 0, maxNodes: 1_000 }); - expect(() => budget.tick()).toThrow(Aborted); + expect(() => { + budget.tick(); + }).toThrow(Aborted); }); it('applies documented defaults', () => { const budget = new ParseBudget(); // Default node cap is 50_000: the 50_000th node is still allowed. for (let i = 0; i < 50_000; i++) budget.tick(); - expect(() => budget.tick()).toThrow(Aborted); + expect(() => { + budget.tick(); + }).toThrow(Aborted); }); }); -describe('parse (M0 placeholder)', () => { - it('wraps the whole source in a program root with a single word child', () => { - const source = 'echo hello'; - const result = parse(source); +describe('parse entry contract', () => { + it('parses an empty source into an empty program', () => { + const { rootNode, hasError } = parseOk(''); + expect(rootNode.type).toBe('program'); + expect(rootNode.text).toBe(''); + expect(rootNode.children).toHaveLength(0); + expect(hasError).toBe(false); + }); + + it('returns { ok: false, reason: "aborted" } when the node budget is exceeded', () => { + expect(parse('echo hello', { timeoutMs: 60_000, maxNodes: 1 })).toEqual({ ok: false, reason: 'aborted' }); + }); + + it('returns { ok: false, reason: "aborted" } when the deadline has passed', () => { + expect(parse('echo hello', { timeoutMs: 0 })).toEqual({ ok: false, reason: 'aborted' }); + }); + + it('aborts instead of hanging on tens of thousands of statements', () => { + expect(parse('ls; '.repeat(20_000))).toEqual({ ok: false, reason: 'aborted' }); + }); +}); + +describe('simple commands', () => { + it('parses a command with arguments', () => { + expectTree('ls -la', `(program (command (command_name (word "ls")) (word "-la")))`); + }); + + it('parses double-quoted strings with expansions and raw strings', () => { + expectTree( + `echo "hello $NAME" 'raw'`, + `(program (command (command_name (word "echo")) (string "\\"" (string_content "hello ") (simple_expansion "$" (variable_name "NAME")) "\\"") (raw_string "'raw'")))`, + ); + }); + + it('parses a pipeline inside && / || lists, left associative', () => { + expectTree( + 'FOO=bar env | grep FOO && echo ok || echo fail', + `(program (list (list (pipeline (command (variable_assignment (variable_name "FOO") "=" (word "bar")) (command_name (word "env"))) "|" (command (command_name (word "grep")) (word "FOO"))) "&&" (command (command_name (word "echo")) (word "ok"))) "||" (command (command_name (word "echo")) (word "fail"))))`, + ); + }); + + it('parses ! as negated_command', () => { + expectTree( + '! grep -q x file', + `(program (negated_command "!" (command (command_name (word "grep")) (word "-q") (word "x") (word "file"))))`, + ); + }); + + it('attaches comments as named children', () => { + expectTree( + '# a comment\necho hi # trailing', + `(program (comment "# a comment") "\\n" (command (command_name (word "echo")) (word "hi")) (comment "# trailing"))`, + ); + }); + + it('distinguishes number from word', () => { + expectTree( + 'echo 123 45.6', + `(program (command (command_name (word "echo")) (number "123") (word "45.6")))`, + ); + }); + + it('parses concatenations of adjacent pieces', () => { + expectTree( + `echo a"b"c'd'$e\${f}g`, + `(program (command (command_name (word "echo")) (concatenation (word "a") (string "\\"" (string_content "b") "\\"") (word "c") (raw_string "'d'") (simple_expansion "$" (variable_name "e")) (expansion "\${" (variable_name "f") "}") (word "g"))))`, + ); + }); + + it('splits { } into single-character word pieces', () => { + expectTree( + 'echo {a,b}', + `(program (command (command_name (word "echo")) (concatenation (word "{") (word "a,b") (word "}"))))`, + ); + }); + + it('treats a lone $ as an anonymous argument', () => { + expectTree('echo $', `(program (command (command_name (word "echo")) "$"))`); + }); +}); + +describe('expansions and substitutions', () => { + it('parses backtick and $() command substitutions', () => { + expectTree( + 'echo `hostname` $(date)', + `(program (command (command_name (word "echo")) (command_substitution "\`" (command (command_name (word "hostname"))) "\`") (command_substitution "$(" (command (command_name (word "date"))) ")")))`, + ); + }); + + it('parses nested command substitutions recursively', () => { + expectTree( + 'echo $(a $(b c))', + `(program (command (command_name (word "echo")) (command_substitution "$(" (command (command_name (word "a")) (command_substitution "$(" (command (command_name (word "b")) (word "c")) ")")) ")")))`, + ); + }); + + it('parses ${...} expansions with operators and defaults', () => { + expectTree( + 'echo ${v:-d} ${x} $1 $@', + `(program (command (command_name (word "echo")) (expansion "\${" (variable_name "v") ":-" (word "d") "}") (expansion "\${" (variable_name "x") "}") (simple_expansion "$" (variable_name "1")) (simple_expansion "$" (special_variable_name "@"))))`, + ); + }); + + it('parses ${#v}, ${v:=word} and ${a[0]}', () => { + expectTree('echo ${#v}', `(program (command (command_name (word "echo")) (expansion "\${" "#" (variable_name "v") "}")))`); + expectTree( + 'echo ${v:=word}', + `(program (command (command_name (word "echo")) (expansion "\${" (variable_name "v") ":=" (word "word") "}")))`, + ); + expectTree( + 'echo ${a[0]}', + `(program (command (command_name (word "echo")) (expansion "\${" (subscript (variable_name "a") "[" (number "0") "]") "}")))`, + ); + }); + + it('parses expansions nested inside expansions', () => { + expectTree( + 'echo ${v:-$(cmd)}', + `(program (command (command_name (word "echo")) (expansion "\${" (variable_name "v") ":-" (command_substitution "$(" (command (command_name (word "cmd"))) ")") "}")))`, + ); + }); + + it('parses process substitutions', () => { + expectTree( + 'diff <(cmd) >(other)', + `(program (command (command_name (word "diff")) (process_substitution "<(" (command (command_name (word "cmd"))) ")") (process_substitution ">(" (command (command_name (word "other"))) ")")))`, + ); + }); + + it('parses $((...)) as an arithmetic_expansion placeholder (TODO(M2))', () => { + expectTree( + 'echo $((1<<2))', + `(program (command (command_name (word "echo")) (arithmetic_expansion "$((" (word "1<<2") "))")))`, + ); + }); +}); + +describe('redirects', () => { + it('parses output redirect and fd duplication', () => { + expectTree( + 'cmd > out 2>&1', + `(program (redirected_statement (command (command_name (word "cmd"))) (file_redirect ">" (word "out")) (file_redirect (file_descriptor "2") ">&" (number "1"))))`, + ); + }); + + it('parses &>> appending both streams', () => { + expectTree( + 'cmd &>> log', + `(program (redirected_statement (command (command_name (word "cmd"))) (file_redirect "&>>" (word "log"))))`, + ); + }); + + it('parses <> read-write redirect with a file descriptor', () => { + expectTree( + 'exec 3<>file', + `(program (redirected_statement (command (command_name (word "exec"))) (file_redirect (file_descriptor "3") "<>" (word "file"))))`, + ); + }); + + it('parses >|, <&-, < and fd redirects', () => { + expectTree( + 'cmd >| file', + `(program (redirected_statement (command (command_name (word "cmd"))) (file_redirect ">|" (word "file"))))`, + ); + expectTree('cmd >&-', `(program (redirected_statement (command (command_name (word "cmd"))) (file_redirect ">&-")))`); + expectTree( + 'cmd < in', + `(program (redirected_statement (command (command_name (word "cmd"))) (file_redirect "<" (word "in"))))`, + ); + expectTree( + 'cmd 2>/dev/null', + `(program (redirected_statement (command (command_name (word "cmd"))) (file_redirect (file_descriptor "2") ">" (word "/dev/null"))))`, + ); + }); + + it('keeps a herestring inside the command node', () => { + expectTree( + 'cmd <<< "$str"', + `(program (command (command_name (word "cmd")) (herestring_redirect "<<<" (string "\\"" (simple_expansion "$" (variable_name "str")) "\\""))))`, + ); + }); + + it('wraps a command carrying both a herestring and a file redirect', () => { + expectTree( + 'cmd <<< x > out', + `(program (redirected_statement (command (command_name (word "cmd")) (herestring_redirect "<<<" (word "x"))) (file_redirect ">" (word "out"))))`, + ); + }); + + it('parses a redirect-only statement', () => { + expectTree('> out', `(program (redirected_statement (file_redirect ">" (word "out"))))`); + }); + + it('keeps a prefix redirect inside the command', () => { + expectTree( + '> a cmd x', + `(program (command (file_redirect ">" (word "a")) (command_name (word "cmd")) (word "x")))`, + ); + }); + + it('wraps subshells and negated commands with their redirects', () => { + expectTree( + '(ls) > out', + `(program (redirected_statement (subshell "(" (command (command_name (word "ls"))) ")") (file_redirect ">" (word "out"))))`, + ); + expectTree( + '! ls > out', + `(program (redirected_statement (negated_command "!" (command (command_name (word "ls")))) (file_redirect ">" (word "out"))))`, + ); + }); +}); + +describe('heredocs', () => { + it('parses a heredoc with expansions in the body', () => { + expectTree( + 'cat < { + expectTree( + 'cat <<-EOF\n\tindented\n\tEOF', + `(program (redirected_statement (command (command_name (word "cat"))) (heredoc_redirect "<<-" (heredoc_start "EOF") (heredoc_body (heredoc_content "indented\\n\\t")) (heredoc_end "EOF"))))`, + ); + }); + + it('does not expand the body of a quoted delimiter', () => { + expectTree( + `cat <<'EOF'\nraw $notexpanded\nEOF`, + `(program (redirected_statement (command (command_name (word "cat"))) (heredoc_redirect "<<" (heredoc_start "'EOF'") (heredoc_body "raw $notexpanded\\n") (heredoc_end "EOF"))))`, + ); + }); + + it('absorbs trailing words and redirects into the heredoc_redirect', () => { + expectTree( + 'cat < out\nbody\nEOF', + `(program (redirected_statement (command (command_name (word "cat"))) (heredoc_redirect "<<" (heredoc_start "EOF") (word "arg") (file_redirect ">" (word "out")) (heredoc_body (heredoc_content "body\\n")) (heredoc_end "EOF"))))`, + ); + }); + + it('absorbs a pipeline tail into the heredoc_redirect', () => { + expectTree( + 'cat < { + expectTree( + 'cat < { + expectTree( + 'cat < { + expectTree( + 'cat < { + it('mixes semicolons and newlines as terminators', () => { + expectTree( + 'ls; echo a\necho b', + `(program (command (command_name (word "ls"))) ";" (command (command_name (word "echo")) (word "a")) "\\n" (command (command_name (word "echo")) (word "b")))`, + ); + }); + + it('parses & background terminators', () => { + expectTree('a & b &', `(program (command (command_name (word "a"))) "&" (command (command_name (word "b"))) "&")`); + }); + + it('parses |& pipelines', () => { + expectTree( + 'cmd1 |& cmd2', + `(program (pipeline (command (command_name (word "cmd1"))) "|&" (command (command_name (word "cmd2")))))`, + ); + }); + + it('parses a subshell', () => { + expectTree('(ls)', `(program (subshell "(" (command (command_name (word "ls"))) ")"))`); + }); + + it('parses standalone assignments', () => { + expectTree('FOO=bar ls', `(program (command (variable_assignment (variable_name "FOO") "=" (word "bar")) (command_name (word "ls"))))`); + expectTree('VAR=value', `(program (variable_assignment (variable_name "VAR") "=" (word "value")))`); + expectTree( + 'A=1 B=2', + `(program (variable_assignments (variable_assignment (variable_name "A") "=" (number "1")) (variable_assignment (variable_name "B") "=" (number "2"))))`, + ); + }); + + it('recovers a trailing && with a partial list and hasError', () => { + expectTree('ls &&', `(program (list (command (command_name (word "ls"))) "&&"))`, true); + }); + + it('recovers a trailing | with a partial pipeline and hasError', () => { + expectTree('ls |', `(program (pipeline (command (command_name (word "ls"))) "|"))`, true); + }); + + it('continues a list across a newline after &&', () => { + expectTree( + 'a &&\nb', + `(program (list (command (command_name (word "a"))) "&&" (command (command_name (word "b")))))`, + ); + }); + + it('splits compound commands into separate command nodes', () => { + const root = expectTree( + 'git status && rm -rf /', + `(program (list (command (command_name (word "git")) (word "status")) "&&" (command (command_name (word "rm")) (word "-rf") (word "/"))))`, + ); + const commands = descendantsOfType(root, 'command'); + expect(commands).toHaveLength(2); + const argv = commands.map((cmd) => + cmd.namedChildren.map((part) => (part.type === 'command_name' ? part.namedChildren[0]!.text : part.text)), + ); + expect(argv).toEqual([ + ['git', 'status'], + ['rm', '-rf', '/'], + ]); + }); +}); + +describe('offsets', () => { + it('uses UTF-16 code unit offsets', () => { + // 🎉 is two UTF-16 code units. + const { rootNode } = parseOk('x🎉y; ls'); + expect(rootNode.endIndex).toBe(8); + const commands = descendantsOfType(rootNode, 'command'); + expect(commands).toHaveLength(2); + expect(commands[0]!.startIndex).toBe(0); + expect(commands[0]!.endIndex).toBe(4); + expect(commands[1]!.startIndex).toBe(6); + expect(commands[1]!.endIndex).toBe(8); + expect(commands[1]!.text).toBe('ls'); + }); + + it('node.text always equals the source slice', () => { + const source = 'FOO=bar env | grep FOO && echo "ok $X"'; + const { rootNode } = parseOk(source); + const walk = (node: SyntaxNode): void => { + expect(node.text).toBe(source.slice(node.startIndex, node.endIndex)); + node.children.forEach(walk); + }; + walk(rootNode); + }); +}); + +describe('adversarial input', () => { + it('recovers unterminated quotes, substitutions and expansions', () => { + for (const source of ['echo "abc', "echo 'abc", 'echo $(foo', 'echo ${foo', 'echo `foo']) { + const result = parse(source); + expect(result.ok).toBe(true); + if (result.ok) expect(result.hasError).toBe(true); + } + }); + + it('recovers stray operators without throwing', () => { + for (const source of ['echo )', '&& echo', '| echo', ')']) { + const result = parse(source); + expect(result.ok).toBe(true); + if (result.ok) expect(result.hasError).toBe(true); + } + }); + + it('parses a 500KB double-quoted string under the default budget', () => { + // Regression: per-character budget.tick() used to burn the 50k node cap + // on long strings even though only a handful of nodes are created. + const body = 'a'.repeat(500 * 1024); + const result = parse(`echo "${body}"`); expect(result.ok).toBe(true); if (!result.ok) return; expect(result.hasError).toBe(false); - expect(result.rootNode.type).toBe('program'); - expect(result.rootNode.text).toBe(source); - expect(result.rootNode.startIndex).toBe(0); - expect(result.rootNode.endIndex).toBe(source.length); - expect(result.rootNode.namedChildren).toHaveLength(1); - const word = result.rootNode.namedChildren[0]!; - expect(word.type).toBe('word'); - expect(word.text).toBe(source); - expect(word.parent).toBe(result.rootNode); + const content = descendantsOfType(result.rootNode, 'string_content'); + expect(content).toHaveLength(1); + expect(content[0]!.text).toBe(body); }); - it('parses an empty source into an empty program', () => { - const result = parse(''); + it('parses a 500KB heredoc body under the default budget', () => { + const body = 'b'.repeat(500 * 1024); + const result = parse(`cat < { - // The placeholder parse creates 2 nodes (program + word). - expect(parse('echo hello', { timeoutMs: 60_000, maxNodes: 1 })).toEqual({ ok: false, reason: 'aborted' }); + it('handles 1000-level nested ${} expansions within the depth cap', () => { + const source = 'echo ' + '${a:-'.repeat(1000) + 'x' + '}'.repeat(1000); + const result = parse(source, { timeoutMs: 10_000 }); + expect(result.ok).toBe(true); + if (result.ok) expect(result.hasError).toBe(true); }); - it('returns { ok: false, reason: "aborted" } when the deadline has passed', () => { - expect(parse('echo hello', { timeoutMs: 0 })).toEqual({ ok: false, reason: 'aborted' }); + it('handles 1000-level nested command substitutions within the depth cap', () => { + const source = '$('.repeat(1000) + 'x' + ')'.repeat(1000); + const result = parse(source, { timeoutMs: 10_000 }); + expect(result.ok).toBe(true); + if (result.ok) expect(result.hasError).toBe(true); + }); + + it('handles 1000-level nested subshells within the depth cap', () => { + const source = '('.repeat(1000) + 'x' + ')'.repeat(1000); + const result = parse(source, { timeoutMs: 10_000 }); + expect(result.ok).toBe(true); + if (result.ok) expect(result.hasError).toBe(true); + }); + + it('parses a multi-megabyte single word without throwing', () => { + const source = 'a'.repeat(2 * 1024 * 1024); + const result = parse(source, { timeoutMs: 10_000 }); + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.hasError).toBe(false); + const word = result.rootNode.namedChildren[0]!.namedChildren[0]!.namedChildren[0]!; + expect(word.type).toBe('word'); + expect(word.text).toBe(source); + }); + + it('does not throw on random binary garbage', () => { + let source = ''; + for (let i = 0; i < 20_000; i++) { + source += String.fromCodePoint(Math.floor(Math.random() * 0xd800)); + } + // The call itself must never throw; either it parses (possibly with + // errors) or the budget cuts it off. + const result = parse(source); + if (result.ok) { + expect(result.rootNode.type).toBe('program'); + } else { + expect(result.reason).toBe('aborted'); + } + }); + + it('does not throw on NUL bytes', () => { + const result = parse('echo \0\0 test \0'); + expect(result.ok).toBe(true); }); }); From 9a9f9cafb1904de7fd3e7d6fcdf0d0403cb023df Mon Sep 17 00:00:00 2001 From: "haozhe.yang" Date: Tue, 21 Jul 2026 18:56:22 +0800 Subject: [PATCH 3/7] feat(tree-sitter-bash): support the full bash grammar Add compound commands (if/while/until/for/c-style-for/select/case/ function/compound/do_group), test commands with reference-exact extglob/regex right-hand-side rules, a Pratt expression engine for arithmetic expansion shared across arith/c-for/test modes, arrays and subscripts, declaration/unset commands, ansi-c/translated strings and brace expressions. Reserved words are recognized only in statement position. Case-aware balanced scanning keeps command substitutions intact around case items, expression leftovers are kept as ERROR nodes instead of dropped, and recursion depth is bounded per chain (substitution 150, parse 500, lexer scan 1024). --- packages/tree-sitter-bash/README.md | 123 +- packages/tree-sitter-bash/src/grammar.ts | 136 +- packages/tree-sitter-bash/src/lexer.ts | 207 +- packages/tree-sitter-bash/src/parser.ts | 2454 +++++++++++++++-- packages/tree-sitter-bash/test/parse.test.ts | 10 +- .../test/parser-compound.test.ts | 1232 +++++++++ 6 files changed, 3865 insertions(+), 297 deletions(-) create mode 100644 packages/tree-sitter-bash/test/parser-compound.test.ts diff --git a/packages/tree-sitter-bash/README.md b/packages/tree-sitter-bash/README.md index e4ea71b590..c677c35ee5 100644 --- a/packages/tree-sitter-bash/README.md +++ b/packages/tree-sitter-bash/README.md @@ -7,12 +7,29 @@ one-to-one, built for agent-side command permission analysis. - No native addons; offsets are UTF-16 code units (`node.text` is always a direct `source.slice(startIndex, endIndex)`). - Parsing runs under a hard budget (default 50 ms / 50 000 nodes); exceeding - it returns `{ ok: false, reason: 'aborted' }` instead of throwing. + it returns `{ ok: false, reason: 'aborted' }` instead of throwing. The + budget caps total work, not input size: a multi-hundred-KB string or + heredoc body parses fine (it produces only a handful of nodes), but a + source whose tree would exceed 50 000 nodes — a megabyte-long `case` + statement, tens of thousands of arithmetic expressions — is aborted + under the defaults. Pass `timeoutMs` / `maxNodes` to raise the limits. - Malformed input never throws either. Unterminated constructs (quotes, - expansions, substitutions, heredocs) are kept as partial nodes and flagged - with `hasError: true`; tokens that cannot start or continue a statement - (stray `)`, a leading `&&`, …) are wrapped in `ERROR` nodes and parsing - continues. + expansions, substitutions, heredocs, compound commands) are kept as + partial nodes and flagged with `hasError: true`; tokens that cannot start + or continue a statement (stray `)`, a leading `&&`, …) are wrapped in + `ERROR` nodes and parsing continues. +- Newlines never appear in the tree: statement-terminating `\n`s produce no + nodes at all (matching tree-sitter-bash, whose scanner only emits `\n` + tokens in heredoc position), while `;` / `&` terminators are anonymous + children. +- Recursion is depth-capped so pathological nesting degrades locally + (ERROR nodes, `hasError: true`) instead of overflowing the stack: + `MAX_SUBSTITUTION_DEPTH = 150` for the $( … ) / `…` / <( … ) sub-parser + chain (the deepest per level, ~13 frames — measured stack overflow at + ~380–500 levels on a default Node stack, so the cap keeps a ≥2.5× + margin), `MAX_PARSE_DEPTH = 500` for subshell/compound/`${…}`/expression + nesting (verified safe at those caps), and `MAX_SCAN_DEPTH = 1024` for + the lexer's own scan recursion. ```ts import { parse } from '@moonshot-ai/tree-sitter-bash'; @@ -48,27 +65,89 @@ tree-sitter-bash 0.25.0 produces (verified against the real parser): into the content. - A heredoc redirect at statement start (`< a cmd x` makes `cmd` the `command_name`), while redirects after the command name consume every following word (`cmd > out arg` puts both words in the redirect) — diff --git a/packages/tree-sitter-bash/src/grammar.ts b/packages/tree-sitter-bash/src/grammar.ts index a53b900caa..8a16268490 100644 --- a/packages/tree-sitter-bash/src/grammar.ts +++ b/packages/tree-sitter-bash/src/grammar.ts @@ -1,7 +1,7 @@ // src/grammar.ts // -// Static grammar tables for bash: operator tables and variable-name sets -// shared by the lexer and the parser. +// Static grammar tables for bash: operator tables, keyword sets and +// variable-name sets shared by the lexer and the parser. /** Single-character special parameters: $@ $* $# $? $- $$ $! $0 and $_. */ export const SPECIAL_VARIABLES = ['@', '*', '#', '?', '-', '$', '!', '0', '_'] as const; @@ -16,29 +16,119 @@ export const SPECIAL_VARIABLE_CHARS = SPECIAL_VARIABLES.filter((ch) => !/\w/.tes * 0.25.0 fails to parse it — it is a real bash operator (`exec 3<>file`). */ export const FILE_REDIRECT_OPERATORS = ['<', '>', '>>', '>&', '<&', '&>', '&>>', '>|', '<>', '>&-', '<&-'] as const; -/** Infix operators inside `${...}` expansions, longest first so the parser - * can match them greedily (`##` before `#`, `:-` before `-`). */ -export const EXPANSION_OPERATORS = [ - ':-', - ':=', - ':?', - ':+', - '##', - '%%', - '/#', - '/%', - '//', - '^^', - ',,', - '-', - '+', - '?', +/** Keywords that open a declaration_command at statement position. */ +export const DECLARATION_COMMAND_KEYWORDS = ['declare', 'typeset', 'export', 'readonly', 'local'] as const; + +/** Keywords that open an unset_command at statement position. */ +export const UNSET_COMMAND_KEYWORDS = ['unset', 'unsetenv'] as const; + +/** Reserved words that may not be used as a function name in the + * `name() { ...; }` form (they are keywords in command position). */ +export const RESERVED_WORDS = [ + 'if', + 'then', + 'elif', + 'else', + 'fi', + 'while', + 'until', + 'do', + 'done', + 'for', + 'select', + 'in', + 'case', + 'esac', + 'function', + '{', + '}', + '!', + ...DECLARATION_COMMAND_KEYWORDS, + ...UNSET_COMMAND_KEYWORDS, +] as const; + +/** Infix operators inside arithmetic / test expressions with their + * tree-sitter-bash precedence level (grammar.js PREC table; higher binds + * tighter). `test_operator` binaries sit at level 10 (TEST) and are handled + * separately. Ternary `? :` is level 2, postfix `++`/`--` level 18, prefix + * `++`/`--` level 17, prefix `!`/`~`/`+`/`-` level 11. */ +export const EXPRESSION_PRECEDENCE: Readonly> = { + '+=': 0, + '-=': 0, + '*=': 0, + '/=': 0, + '%=': 0, + '**=': 0, + '<<=': 0, + '>>=': 0, + '&=': 0, + '^=': 0, + '|=': 0, + '=': 1, + '=~': 1, + '||': 3, + '&&': 4, + '|': 5, + '^': 6, + '&': 7, + '==': 8, + '!=': 8, + '<': 9, + '<=': 9, + '>': 9, + '>=': 9, + '<<': 12, + '>>': 12, + '+': 13, + '-': 13, + '*': 14, + '/': 14, + '%': 14, + '**': 15, +}; + +/** All operator texts the expression lexer recognizes, longest first so + * matching is greedy (`**=` before `**` before `*`). `;` and `,` only act + * as separators (c-style for headers and comma lists), not as operators. */ +export const EXPRESSION_OPERATORS = [ + '**=', + '<<=', + '>>=', + '++', + '--', + '**', + '<<', + '>>', + '<=', + '>=', + '==', + '!=', + '&&', + '||', + '+=', + '-=', + '*=', + '/=', + '%=', + '&=', + '^=', + '|=', + '=~', '=', - '#', - '%', + '+', + '-', + '*', '/', + '%', '^', - ',', - '@', + '&', + '|', + '<', + '>', + '!', + '~', + '?', ':', + ',', + ';', ] as const; diff --git a/packages/tree-sitter-bash/src/lexer.ts b/packages/tree-sitter-bash/src/lexer.ts index 16f991194e..200db43ab7 100644 --- a/packages/tree-sitter-bash/src/lexer.ts +++ b/packages/tree-sitter-bash/src/lexer.ts @@ -20,6 +20,18 @@ // a plain word character). // eof end of the lexer's range. Also carries pending heredoc bodies. // +// Additionally: +// - `;&` and `;;&` are single op tokens (case_item fallthrough +// terminators). +// - `((...))` at token start is scanned as one word token (arithmetic +// command / c-style for header), so the parser can re-scan the range +// with its expression parser; the word ends right after the balanced +// close and does not merge with following word characters. +// - peekAt(n) looks arbitrarily far ahead (function_definition +// detection); reposition(pos) rewinds the stream (test commands parse +// their [[ ... ]] range character-wise and then resume tokenizing past +// the closer). +// // The lexer is range-bounded: sub-parsers lex $(...) / `...` bodies with // their own Lexer over the same source but a narrower [start, end) window. // @@ -76,7 +88,17 @@ export interface HeredocBody { /** How often long scan loops tick the budget, in scanned characters. */ const SCAN_TICK_INTERVAL = 2048; -const CONTROL_OPERATORS = ['&>>', '&>', '&&', '&', '|&', '||', '|', ';;', ';', '(', ')'] as const; +/** + * Depth cap for the scanBalanced ↔ skipDoubleQuoted mutual recursion + * (`${a#"${a#"…` nests two scan frames per level). Beyond the cap the scan + * reports "unbalanced", letting the parser degrade the construct with + * hasError instead of overflowing the call stack. Sized above the parser's + * own nesting guards (which trigger first for parser-driven recursion); + * this cap only protects lexer-driven scans of pathological input. + */ +const MAX_SCAN_DEPTH = 1024; + +const CONTROL_OPERATORS = ['&>>', '&>', '&&', '&', '|&', '||', '|', ';;&', ';;', ';&', ';', '(', ')'] as const; const REDIRECT_OPERATORS = ['<<-', '<<<', '<<', '<&-', '<&', '<>', '<', '>&-', '>&', '>>', '>|', '>'] as const; function isWordChar(ch: string | undefined): boolean { @@ -95,8 +117,10 @@ function isDigitAt(source: string, i: number): boolean { /** Skip a "..." quoted region starting at `i` (which points at the opening * quote). Returns the index just past the closing quote, or `end` when the * string is unterminated. Substitution-aware: $(...), ${...} and `...` - * inside the string may themselves contain quotes. */ -export function skipDoubleQuoted(source: string, budget: ParseBudget, i: number, end: number): number { + * inside the string may themselves contain quotes. `depth` tracks the + * scanBalanced ↔ skipDoubleQuoted recursion (see MAX_SCAN_DEPTH). */ +export function skipDoubleQuoted(source: string, budget: ParseBudget, i: number, end: number, depth = 0): number { + if (depth >= MAX_SCAN_DEPTH) return end; let j = i + 1; let sinceTick = 0; while (j < end) { @@ -117,11 +141,11 @@ export function skipDoubleQuoted(source: string, budget: ParseBudget, i: number, if (ch === '$') { const next = source[j + 1]; if (next === '(') { - j = scanBalanced(source, budget, j + 1, end, '(', ')').end; + j = scanBalancedStatements(source, budget, j + 1, end, depth + 1).end; continue; } if (next === '{') { - j = scanBalanced(source, budget, j + 1, end, '{', '}').end; + j = scanBalanced(source, budget, j + 1, end, '{', '}', depth + 1).end; continue; } } @@ -166,7 +190,8 @@ export interface BalancedScan { } /** Scan a balanced open/close region starting at `i` (which points at the - * opening character). Quote- and escape-aware. */ + * opening character). Quote- and escape-aware. `depth` tracks the + * scanBalanced ↔ skipDoubleQuoted recursion (see MAX_SCAN_DEPTH). */ export function scanBalanced( source: string, budget: ParseBudget, @@ -174,8 +199,10 @@ export function scanBalanced( end: number, open: string, close: string, + depth = 0, ): BalancedScan { - let depth = 0; + if (depth >= MAX_SCAN_DEPTH) return { end, balanced: false }; + let nesting = 0; let j = i; let sinceTick = 0; while (j < end) { @@ -189,12 +216,12 @@ export function scanBalanced( continue; } if (ch === open) { - depth++; + nesting++; } else if (ch === close) { - depth--; - if (depth === 0) return { end: j + 1, balanced: true }; + nesting--; + if (nesting === 0) return { end: j + 1, balanced: true }; } else if (ch === '"') { - j = skipDoubleQuoted(source, budget, j, end); + j = skipDoubleQuoted(source, budget, j, end, depth + 1); continue; } else if (ch === "'") { j = skipSingleQuoted(source, budget, j, end); @@ -208,12 +235,128 @@ export function scanBalanced( return { end, balanced: false }; } +/** Words after which a `case` word opens a case_statement (statement + * position) rather than being an ordinary argument. */ +const CASE_ENABLING_WORDS: ReadonlySet = new Set(['if', 'then', 'elif', 'else', 'while', 'until', 'do']); + +/** + * Case-aware variant of scanBalanced for `(` … `)` regions that may hold + * full statements (command/process substitution bodies). A naive paren + * count closes the region early on a case_item pattern (`a) 1;; esac`): + * every item's `)` would decrement the depth. While a `case` is open (seen + * in statement position, not yet closed by `esac`), a `)` at the paren + * depth where the `case` started is a pattern close and does NOT count; + * parens from other constructs inside the case (subshells, $(…), the + * optional `(` of an item) balance themselves normally. + * + * Statement position is approximated lexically: `case` counts as a keyword + * after `(`, `;`, `&`, `|`, a newline, `{`, or one of the compound + * keywords in CASE_ENABLING_WORDS — so `echo case` does not confuse the + * scan. `esac` pops the innermost open case regardless of position (the + * reference scanner emits the esac token even in argument position). + */ +export function scanBalancedStatements( + source: string, + budget: ParseBudget, + i: number, + end: number, + depth = 0, +): BalancedScan { + if (depth >= MAX_SCAN_DEPTH) return { end, balanced: false }; + let nesting = 0; + /** Paren depths at which each open case_statement started. */ + const caseDepths: number[] = []; + let j = i; + /** What preceded the current position: 'start' | 'sep' | 'keyword' | 'word'. */ + let previous: 'start' | 'sep' | 'keyword' | 'word' = 'start'; + let sinceTick = 0; + while (j < end) { + if (++sinceTick >= SCAN_TICK_INTERVAL) { + budget.progress(); + sinceTick = 0; + } + const ch = source[j]!; + if (ch === '\\') { + j += 2; + continue; + } + if (ch === '"') { + j = skipDoubleQuoted(source, budget, j, end, depth + 1); + previous = 'word'; + continue; + } + if (ch === "'") { + j = skipSingleQuoted(source, budget, j, end); + previous = 'word'; + continue; + } + if (ch === '`') { + j = skipBacktick(source, budget, j, end); + previous = 'word'; + continue; + } + if (ch === ' ' || ch === '\t' || ch === '\r') { + j++; + continue; + } + if (ch === '\n' || ch === ';' || ch === '&' || ch === '|') { + previous = 'sep'; + j++; + continue; + } + if (ch === '{') { + previous = 'sep'; + j++; + continue; + } + if (ch === '(') { + nesting++; + previous = 'sep'; + j++; + continue; + } + if (ch === ')') { + if (caseDepths.length > 0 && caseDepths.at(-1) === nesting) { + // A case_item pattern close: not a paren. + previous = 'sep'; + j++; + continue; + } + nesting--; + if (nesting === 0) return { end: j + 1, balanced: true }; + previous = 'word'; + j++; + continue; + } + // A word run: check for the case/esac keywords. + let k = j; + while (k < end && isWordChar(source[k])) k++; + if (k > j) { + const word = source.slice(j, k); + if (word === 'esac' && caseDepths.length > 0) { + caseDepths.pop(); + previous = 'sep'; + } else if (word === 'case' && previous !== 'word') { + caseDepths.push(nesting); + previous = 'word'; + } else { + previous = CASE_ENABLING_WORDS.has(word) ? 'keyword' : 'word'; + } + j = k; + continue; + } + previous = 'word'; + j++; + } + return { end, balanced: false }; +} + /** Skip a $-construct starting at `i` (which points at the `$`). Handles * $(...), $((...)), ${...}, $name and the single-character specials. A `$` * followed by anything else (including a quote) consumes just the `$`. */ export function skipDollar(source: string, budget: ParseBudget, i: number, end: number): number { const next = source[i + 1]; - if (next === '(') return scanBalanced(source, budget, i + 1, end, '(', ')').end; + if (next === '(') return scanBalancedStatements(source, budget, i + 1, end).end; if (next === '{') return scanBalanced(source, budget, i + 1, end, '{', '}').end; if (isWordChar(next)) { let j = i + 1; @@ -227,7 +370,7 @@ export function skipDollar(source: string, budget: ParseBudget, i: number, end: export class Lexer { /** Current scan position; exposed for the parser's heredoc bookkeeping. */ pos: number; - private lookahead: Token | null = null; + private lookahead: Token[] = []; private readonly pendingHeredocs: HeredocSpec[] = []; constructor( @@ -239,20 +382,37 @@ export class Lexer { this.pos = start; } + /** End of the lexer's range (exclusive). */ + get rangeEnd(): number { + return this.end; + } + /** Queue a heredoc body to be scanned after the current line. */ queueHeredoc(spec: HeredocSpec): void { this.pendingHeredocs.push(spec); } peek(): Token { - this.lookahead ??= this.scanToken(); - return this.lookahead; + return this.peekAt(0); + } + + /** Look `n` tokens ahead (0 = the next token). Tokens are scanned + * on demand and buffered, so lookahead never re-scans. */ + peekAt(n: number): Token { + while (this.lookahead.length <= n) this.lookahead.push(this.scanToken()); + return this.lookahead[n]!; } next(): Token { - const token = this.peek(); - this.lookahead = null; - return token; + if (this.lookahead.length > 0) return this.lookahead.shift()!; + return this.scanToken(); + } + + /** Rewind the stream to `pos`, discarding buffered lookahead. Used when + * the parser consumed a region character-wise (test commands). */ + reposition(pos: number): void { + this.pos = pos; + this.lookahead = []; } text(token: Token): string { @@ -287,6 +447,9 @@ export class Lexer { return this.scanOp(REDIRECT_OPERATORS); } if (ch === '&' || ch === '|' || ch === ';' || ch === '(' || ch === ')') { + // `((` at token start opens arithmetic (an arithmetic command or a + // c-style for header), which is word material, not two parens. + if (ch === '(' && this.source[this.pos + 1] === '(') return this.scanWord(); return this.scanOp(CONTROL_OPERATORS); } if (ch === '{' || ch === '}' || ch === '[' || ch === ']') { @@ -352,6 +515,12 @@ export class Lexer { private scanWord(): Token { const start = this.pos; + // `((...))` at token start: one word token ending right after the + // balanced close (the parser re-scans the range as arithmetic). + if (this.source[start] === '(' && this.source[start + 1] === '(') { + this.pos = scanBalanced(this.source, this.budget, start, this.end, '(', ')').end; + return { type: 'word', start, end: this.pos, heredocBodies: [] }; + } let i = this.pos; let sinceTick = 0; while (i < this.end) { @@ -365,7 +534,7 @@ export class Lexer { if (ch === '{' || ch === '}' || ch === '[' || ch === ']') break; if (ch === '<' || ch === '>') { if (this.source[i + 1] === '(') { - i = scanBalanced(this.source, this.budget, i + 1, this.end, '(', ')').end; + i = scanBalancedStatements(this.source, this.budget, i + 1, this.end).end; continue; } break; diff --git a/packages/tree-sitter-bash/src/parser.ts b/packages/tree-sitter-bash/src/parser.ts index deb0061c91..83f5917984 100644 --- a/packages/tree-sitter-bash/src/parser.ts +++ b/packages/tree-sitter-bash/src/parser.ts @@ -3,7 +3,7 @@ // Recursive-descent bash parser. One method per tree-sitter-bash grammar // rule (parseList ↔ list, parsePipeline ↔ pipeline, parseCommand ↔ command, // …), producing node ranges and child layouts that match the real -// tree-sitter-bash 0.25.0 tree for the M1 syntax subset. +// tree-sitter-bash 0.25.0 tree. // // Construction strategy: the parser builds lightweight `Frame`s, not // SyntaxNodeBuilders, because heredoc bodies are scanned only when the line @@ -14,32 +14,103 @@ // safe for pathologically deep trees) once parsing is done. // // Error recovery: unterminated constructs (quote, expansion, substitution, -// heredoc) keep their partial node and set hasError; tokens that cannot -// start or continue a statement are wrapped in an ERROR node and parsing -// continues. No parser-internal exception is expected to escape; parse() -// still guards against it. +// heredoc, compound commands) keep their partial node and set hasError; +// tokens that cannot start or continue a statement are wrapped in an ERROR +// node and parsing continues. No parser-internal exception is expected to +// escape; parse() still guards against it. // -// Depth guard: word-level substitutions ($( … ) / `…` / <( … )) recurse via -// fresh sub-Parser instances bounded to their source range (tracked by -// `depth`), subshells recurse within one parser (`scopeDepth`), and the -// parseLiteral ↔ parseString / parseExpansion chain behind ${ … } nesting -// has its own counter (`literalDepth`). All three are capped at -// MAX_PARSE_DEPTH; beyond it the construct is skipped and reported with an -// ERROR node instead of risking a stack overflow. +// Depth guards, all capped so pathological nesting degrades locally +// (ERROR nodes, hasError) instead of overflowing the call stack: +// - word-level substitutions ($( … ) / `…` / <( … )) recurse via fresh +// sub-Parser instances bounded to their source range — the deepest +// chain per level (~13 frames), capped by MAX_SUBSTITUTION_DEPTH; +// - subshells and compound commands (if/while/for/case/{…}/function +// bodies) recurse within one parser (`scopeDepth`); +// - the parseLiteral ↔ parseString ↔ parseExpansion chain behind ${ … } +// nesting — including the pattern → string → $ cycle — has its own +// counter (`literalDepth`, incremented by both parseLiteral and +// parseExpansion); +// - parenthesized_expression nesting inside arithmetic/test expressions +// uses `exprDepth`. +// The last three are capped by MAX_PARSE_DEPTH; beyond the caps the +// construct degrades (compound keywords fall back to plain commands, +// expressions and substitutions to ERROR nodes). The lexer's own scan +// recursion (scanBalanced ↔ skipDoubleQuoted) is cheaper per level and +// capped separately (MAX_SCAN_DEPTH in lexer.ts). +// +// Expression engine: arithmetic expansions ($((…)), ((…)), $[…]), c-style +// for headers and test commands ([[ … ]] / [ … ]) share a small Pratt +// parser (parseExpression over an ExprState) driven by the precedence table +// in grammar.ts, mirroring grammar.js's PREC levels and the tree shapes they +// produce (e.g. prefix -/+/~/! grab a full tighter-precedence expression: +// $((-x + ~y)) is unary(-, binary(x, +, unary(~, y))) in the reference). import type { ParseBudget } from '#/budget'; -import { EXPANSION_OPERATORS, FILE_REDIRECT_OPERATORS, SPECIAL_VARIABLE_CHARS } from '#/grammar'; -import { Lexer, scanBalanced, skipBacktick, skipSingleQuoted } from '#/lexer'; +import { + DECLARATION_COMMAND_KEYWORDS, + EXPRESSION_OPERATORS, + EXPRESSION_PRECEDENCE, + FILE_REDIRECT_OPERATORS, + RESERVED_WORDS, + SPECIAL_VARIABLE_CHARS, + UNSET_COMMAND_KEYWORDS, +} from '#/grammar'; +import { Lexer, scanBalanced, scanBalancedStatements, skipBacktick, skipDoubleQuoted, skipSingleQuoted } from '#/lexer'; import type { BalancedScan, HeredocBody, HeredocSpec, Token } from '#/lexer'; import { SyntaxNodeBuilder } from '#/node'; -/** Maximum nesting of scopes (subshells, command substitutions, …). */ +/** Maximum nesting of scopes (subshells, compound commands, expressions, + * ${…} chains). These chains cost only a handful of stack frames per + * level and their guards skip iteratively, so 500 is comfortably safe — + * verified: inputs thousands of levels deep degrade locally instead of + * overflowing. */ export const MAX_PARSE_DEPTH = 500; +/** + * Maximum nesting of the $( … ) / `…` / <( … ) sub-parser chain (each + * level spawns a fresh Parser and costs ~13 stack frames through + * parseCommandSubstitution → parseScopedStatements → parseStatementList → + * parseCommand → parseLiteral → parseDollar). Measured overflow on a + * default Node stack: ~380–500 levels depending on environment, BEFORE the + * old 500 cap could fire. 150 keeps a ≥2.5× margin; beyond it the + * substitution degrades to a local ERROR node (hasError) instead of + * taking the whole tree down via a RangeError. + */ +export const MAX_SUBSTITUTION_DEPTH = 150; + const FILE_REDIRECT_OP_SET: ReadonlySet = new Set(FILE_REDIRECT_OPERATORS); +const DECLARATION_COMMAND_SET: ReadonlySet = new Set(DECLARATION_COMMAND_KEYWORDS); +const UNSET_COMMAND_SET: ReadonlySet = new Set(UNSET_COMMAND_KEYWORDS); +const RESERVED_WORD_SET: ReadonlySet = new Set(RESERVED_WORDS); const NUMBER_RE = /^-?(0x)?[0-9]+(#[0-9A-Za-z@_]+)?$/; const ASSIGNMENT_RE = /^[A-Za-z_][A-Za-z0-9_]*\+?=/; const ASSIGNMENT_SPLIT_RE = /^([A-Za-z_][A-Za-z0-9_]*)(\+?=)/; +const SUBSCRIPT_ASSIGNMENT_RE = /^(\w+)\[([^\]\n]*)\](\+?=)/; +const IDENTIFIER_RE = /^[A-Za-z_]\w*$/; +const BRACE_EXPRESSION_RE = /^\{(\d+)\.\.(\d+)\}/; +/** Statement-position `((…))` that tree-sitter-bash parses as a + * test_command: inner text starting with an identifier-ish token + * (letters/digits/`_`/`-`) ending in `++`/`--`. */ +const PAREN_TEST_RE = /^\(\(\s*[A-Za-z_][\w-]*(\+\+|--)/; +const STOP_THEN: ReadonlySet = new Set(['then']); +const STOP_DO: ReadonlySet = new Set(['do']); +const STOP_DONE: ReadonlySet = new Set(['done']); +const STOP_IF_BODY: ReadonlySet = new Set(['elif', 'else', 'fi']); +const STOP_FI: ReadonlySet = new Set(['fi']); +const STOP_CLOSE_BRACE: ReadonlySet = new Set(['}']); +const STOP_ESAC: ReadonlySet = new Set(['esac']); +const CASE_TERMINATION_OPS: ReadonlySet = new Set([';;', ';&', ';;&']); + +/** How often long scan loops tick the budget, in scanned characters. */ +const SCAN_TICK_INTERVAL = 2048; + +// Precedence levels from grammar.js's PREC table that are not in +// EXPRESSION_PRECEDENCE. +const PREC_TERNARY = 2; +const PREC_TEST = 10; +const PREC_UNARY = 11; +const PREC_PREFIX = 17; +const PREC_POSTFIX = 18; /** Mutable node under construction; see the file header. */ export interface Frame { @@ -56,6 +127,59 @@ interface PendingHeredoc { spec: HeredocSpec; } +interface StatementListOptions { + readonly stopAtParen?: boolean; + readonly stopWords?: ReadonlySet; + readonly stopOps?: ReadonlySet; + /** True when the grammar requires a terminator before the stop keyword + * (if/while/do bodies, `{ … }`): stopping with a pending statement and + * no separator flags an error (`if a; then b fi` is an error in the + * reference too). Case items and subshells do not require one. */ + readonly terminatorRequired?: boolean; +} + +/** Where the expression engine runs: arithmetic expansions, c-style for + * headers, or test commands. */ +type ExprMode = 'arith' | 'c' | 'test'; + +type ExprTokenKind = + | 'number' + | 'ident' + | 'word' + | 'subst' + | 'string' + | 'testop' + | 'op' + | 'lparen' + | 'rparen' + | 'unknown' + | 'end'; + +interface ExprToken { + readonly kind: ExprTokenKind; + readonly start: number; + readonly end: number; + readonly text: string; + /** Pre-built node for subst/string/subscript tokens. */ + readonly frame?: Frame; +} + +interface ExprState { + pos: number; + readonly end: number; + readonly mode: ExprMode; + lookahead: ExprToken | null; + /** Nesting of parenthesized_expression inside test commands (the + * reference emits word, not extglob_pattern, for a non-glob ==/!= + * right-hand side inside parentheses). */ + parenDepth: number; + /** Test mode only: true after a complete operand, where `=`/`!`-family + * characters are comparison operators even when attached to the next + * word (`a ==b` is `a == b`); false at operand position, where they are + * word characters (`=b`, `!x`, `a!=b` are words in the reference). */ + expectOperator: boolean; +} + function isFileRedirectOp(text: string): boolean { return FILE_REDIRECT_OP_SET.has(text); } @@ -89,9 +213,18 @@ export class Parser { /** Recursion depth of parseLiteral ↔ parseString / parseExpansion (the * ${} nesting chain that stays inside this Parser instance). */ private literalDepth = 0; + /** Nesting of parenthesized_expression inside the expression parser. */ + private exprDepth = 0; /** True while parsing inside a heredoc's line tail: a second heredoc * cannot be represented and degrades to ERROR (see parseHeredocRedirect). */ private noHeredoc = false; + /** Nesting of case_item statement lists (see parseCaseItemStatements). */ + private caseItemDepth = 0; + /** End of the most recently consumed `;`/`&`/`;;`/newline terminator in + * any statement list. case_item uses it to extend its range over a + * trailing terminator before `esac`, matching the reference (whose + * last_case_item includes the final _terminator in its range). */ + private lastTerminatorEnd = 0; constructor( private readonly source: string, @@ -125,6 +258,10 @@ export class Parser { return this.source.slice(token.start, token.end); } + private endOf(kids: Frame[], fallback: number): number { + return kids.length > 0 ? kids.at(-1)!.end : fallback; + } + private isStatementStart(token: Token): boolean { if (token.type === 'word' || token.type === 'io_number') return true; if (token.type !== 'op') return false; @@ -132,32 +269,56 @@ export class Parser { return text === '(' || text === '<<<' || text === '<<' || text === '<<-' || isFileRedirectOp(text); } + /** Peeked word token whose text equals `keyword`. */ + private peekKeyword(keyword: string): boolean { + const token = this.lexer.peek(); + return token.type === 'word' && this.tokenText(token) === keyword; + } + + /** Consume a keyword as an anonymous child; false (no consume) when the + * next token is not that keyword. */ + private consumeKeyword(kids: Frame[], keyword: string): boolean { + if (!this.peekKeyword(keyword)) return false; + const token = this.lexer.next(); + kids.push(this.anon(keyword, token.start, token.end)); + return true; + } + // ------------------------------------------------------------ entry point /** program: the whole source as one statement list. */ parseProgram(): Frame { this.lexer = new Lexer(this.source, this.budget, 0, this.source.length); - const children = this.parseStatementList(false); + const children = this.parseStatementList(); return this.frame('program', 0, this.source.length, children); } - /** Parse a sub-range as a statement list (body of $( … ), ` … `, <( … )). */ + /** Parse a sub-range as a statement list (body of $( … ), ` … `, <( … )). + * The sub-parser chain is the deepest per-level recursion in the + * parser, so it has its own, tighter cap — see MAX_SUBSTITUTION_DEPTH. */ private parseScopedStatements(start: number, end: number): Frame[] { - if (this.depth + 1 >= MAX_PARSE_DEPTH) { + if (this.depth + 1 >= MAX_SUBSTITUTION_DEPTH) { this.hasError = true; return [this.frame('ERROR', start, end)]; } const sub = new Parser(this.source, this.budget, this.depth + 1); sub.lexer = new Lexer(this.source, this.budget, start, end); - const children = sub.parseStatementList(false); + const children = sub.parseStatementList(); if (sub.hasError) this.hasError = true; return children; } // -------------------------------------------------------- statement lists - /** _statements: statements separated/terminated by ; & and newlines. */ - private parseStatementList(stopAtParen: boolean): Frame[] { + /** + * _statements: statements separated/terminated by ; & and newlines. + * `;`/`&`/`;;` terminators become anonymous children; newlines and + * comments do not (comments stay named children) — tree-sitter-bash never + * emits `\n` terminator nodes either. Parsing stops (without consuming) + * at a word in stopWords (then/do/done/fi/esac/…), an op in stopOps + * (case_item terminators), or a `)` when stopAtParen is set. + */ + private parseStatementList(options: StatementListOptions = {}): Frame[] { const children: Frame[] = []; let needTerminator = false; for (;;) { @@ -170,12 +331,7 @@ export class Parser { if (token.type === 'newline') { this.lexer.next(); this.completeHeredocs(token.heredocBodies); - // A newline that scanned heredoc bodies belongs to the heredoc - // redirects (it sits inside their range); it is not a program-level - // terminator node. - if (token.heredocBodies.length === 0) { - children.push(this.anon('\n', token.start, token.end)); - } + this.lastTerminatorEnd = token.end; needTerminator = false; continue; } @@ -184,18 +340,27 @@ export class Parser { children.push(this.frame('comment', token.start, token.end)); continue; } + if (token.type === 'word' && options.stopWords?.has(this.tokenText(token)) === true) { + if (needTerminator && options.terminatorRequired === true) this.hasError = true; // missing separator + break; + } const op = token.type === 'op' ? this.tokenText(token) : ''; - if (token.type === 'op' && op === ')' && stopAtParen) { + if (token.type === 'op' && options.stopOps?.has(op) === true) break; + if (token.type === 'op' && op === ')' && options.stopAtParen === true) { this.failOpenHeredocs(); break; } if (token.type === 'op' && (op === ';' || op === '&' || op === ';;')) { this.lexer.next(); children.push(this.anon(op, token.start, token.end)); + this.lastTerminatorEnd = token.end; needTerminator = false; continue; } - if (token.type === 'op' && (op === ')' || op === '&&' || op === '||' || op === '|' || op === '|&')) { + if ( + token.type === 'op' && + (op === ')' || op === '&&' || op === '||' || op === '|' || op === '|&' || op === ';&' || op === ';;&') + ) { // A binary operator or closer with no left-hand side: recover. this.hasError = true; this.lexer.next(); @@ -287,18 +452,9 @@ export class Parser { return this.frame('pipeline', first.start, end, kids); } - /** _statement_not_pipeline for the M1 subset: command, subshell, - * negated_command, variable_assignment(s), redirected_statement. */ + /** _statement_not_pipeline: any statement form plus trailing redirects. */ private parseStatementNotPipeline(): Frame { - const token = this.lexer.peek(); - let inner: Frame | null = null; - if (token.type === 'op' && this.tokenText(token) === '(') { - inner = this.parseSubshell(); - } else if (token.type === 'word' && this.tokenText(token) === '!') { - inner = this.parseNegatedCommand(); - } else { - inner = this.parseCommand(); - } + let inner: Frame | null = this.parseStatementCore(); const trailing: Frame[] = []; for (;;) { const next = this.lexer.peek(); @@ -328,7 +484,79 @@ export class Parser { return inner; } - /** negated_command: `!` followed by a command or subshell. */ + /** + * Dispatch one statement (no pipeline, no trailing redirects) on its + * leading token: subshell, negation, compound commands, test commands, + * declaration/unset commands, function definitions, or a plain command. + * Reserved words only count at statement position — `echo if` keeps `if` + * as a word, and a prefix assignment (`x=1 if …`) disables the keyword + * reading just like in the reference. + */ + private parseStatementCore(): Frame | null { + const token = this.lexer.peek(); + if (token.type === 'op' && this.tokenText(token) === '(') { + return this.parseSubshell(); + } + if (token.type === 'word') { + const text = this.tokenText(token); + if (text === '!') return this.parseNegatedCommand(); + if (text === '{') { + // `{1..3}` at statement position is a brace_expression word. + if (!BRACE_EXPRESSION_RE.test(this.source.slice(token.start))) { + return this.parseCompoundGuarded(() => this.parseCompoundStatement()); + } + } else if (text === 'if') { + return this.parseCompoundGuarded(() => this.parseIfStatement()); + } else if (text === 'while' || text === 'until') { + return this.parseCompoundGuarded(() => this.parseWhileStatement()); + } else if (text === 'for' || text === 'select') { + return this.parseCompoundGuarded(() => this.parseForStatement()); + } else if (text === 'case') { + return this.parseCompoundGuarded(() => this.parseCaseStatement()); + } else if (text === 'function') { + return this.parseCompoundGuarded(() => this.parseFunctionDefinition()); + } else if (text === '[') { + return this.parseTestCommand(); + } else if (text.startsWith('((') && PAREN_TEST_RE.test(text)) { + // `((word++…))` / `((word--…))` at statement position is a + // test_command in the reference (its statement-position arithmetic + // grammar does not accept a leading postfix operand), while other + // `((…))` forms are arithmetic_expansion command names. + return this.parseParenTestCommand(); + } else if (DECLARATION_COMMAND_SET.has(text)) { + return this.parseDeclarationCommand(); + } else if (UNSET_COMMAND_SET.has(text)) { + return this.parseUnsetCommand(); + } else if (this.isFunctionDefinitionAhead()) { + return this.parseCompoundGuarded(() => this.parseFunctionDefinition()); + } + } + return this.parseCommand(); + } + + /** + * Run a compound-statement parser under the scope-depth guard: beyond + * MAX_PARSE_DEPTH the keyword is parsed as a plain command word instead, + * which bounds the recursion chain (if → statements → if → …). + */ + private parseCompoundGuarded(parse: () => Frame): Frame { + if (this.scopeDepth >= MAX_PARSE_DEPTH) { + this.hasError = true; + const fallback = this.parseCommand(); + if (fallback !== null) return fallback; + const token = this.lexer.next(); + return this.frame('ERROR', token.start, token.end); + } + this.scopeDepth++; + try { + return parse(); + } finally { + this.scopeDepth--; + } + } + + /** negated_command: `!` followed by a command, test command, assignment + * or subshell (the grammar's full choice). */ private parseNegatedCommand(): Frame { const bang = this.lexer.next(); const kids: Frame[] = [this.anon('!', bang.start, bang.end)]; @@ -338,6 +566,14 @@ export class Parser { const subshell = this.parseSubshell(); kids.push(subshell); end = subshell.end; + } else if (token.type === 'word' && this.tokenText(token) === '[') { + const test = this.parseTestCommand(); + kids.push(test); + end = test.end; + } else if (token.type === 'word' && this.tokenText(token).startsWith('((') && PAREN_TEST_RE.test(this.tokenText(token))) { + const test = this.parseParenTestCommand(); + kids.push(test); + end = test.end; } else { const command = this.parseCommand(); if (command === null) { @@ -377,10 +613,11 @@ export class Parser { } const kids: Frame[] = [this.anon('(', open.start, open.end)]; this.scopeDepth++; - const inner = this.parseStatementList(true); + const inner = this.parseStatementList({ stopAtParen: true }); this.scopeDepth--; + if (inner.length === 0) this.hasError = true; // empty subshell (grammar requires statements) kids.push(...inner); - let end = inner.length > 0 ? inner.at(-1)!.end : open.end; + let end = this.endOf(inner, open.end); const token = this.lexer.peek(); if (token.type === 'op' && this.tokenText(token) === ')') { this.lexer.next(); @@ -392,176 +629,846 @@ export class Parser { return this.frame('subshell', open.start, end, kids); } - // --------------------------------------------------------------- commands + // ------------------------------------------------------ compound commands - /** - * command: leading variable_assignment / redirect prefix, command_name, - * then arguments and inline herestring redirects. Returns null when no - * command name is present and nothing was consumed; a nameless prefix is - * assembled into variable_assignment(s) / redirected_statement here. - */ - private parseCommand(): Frame | null { - const prefix: Frame[] = []; + /** compound_statement: `{` _terminated_statement `}` */ + private parseCompoundStatement(): Frame { + const open = this.lexer.next(); + const kids: Frame[] = [ + this.anon('{', open.start, open.end), + ...this.parseStatementList({ stopWords: STOP_CLOSE_BRACE, terminatorRequired: true }), + ]; + let end = this.endOf(kids, open.end); + const token = this.lexer.peek(); + if (token.type === 'word' && this.tokenText(token) === '}') { + this.lexer.next(); + kids.push(this.anon('}', token.start, token.end)); + end = token.end; + } else { + this.hasError = true; // unterminated { … + } + return this.frame('compound_statement', open.start, end, kids); + } + + /** if_statement: `if` condition `then` body elif* else? `fi` */ + private parseIfStatement(): Frame { + const ifToken = this.lexer.next(); + const kids: Frame[] = [ + this.anon('if', ifToken.start, ifToken.end), + ...this.parseStatementList({ stopWords: STOP_THEN, terminatorRequired: true }), + ]; + if (!this.consumeKeyword(kids, 'then')) { + this.hasError = true; // missing then + return this.frame('if_statement', ifToken.start, this.endOf(kids, ifToken.end), kids); + } + kids.push(...this.parseStatementList({ stopWords: STOP_IF_BODY, terminatorRequired: true })); + while (this.peekKeyword('elif')) { + kids.push(this.parseElifClause()); + } + if (this.peekKeyword('else')) { + kids.push(this.parseElseClause()); + } + let end = this.endOf(kids, ifToken.end); + if (this.consumeKeyword(kids, 'fi')) { + end = kids.at(-1)!.end; + } else { + this.hasError = true; // unterminated if + } + return this.frame('if_statement', ifToken.start, end, kids); + } + + /** elif_clause: `elif` condition `then` body */ + private parseElifClause(): Frame { + const elifToken = this.lexer.next(); + const kids: Frame[] = [ + this.anon('elif', elifToken.start, elifToken.end), + ...this.parseStatementList({ stopWords: STOP_THEN, terminatorRequired: true }), + ]; + if (!this.consumeKeyword(kids, 'then')) { + this.hasError = true; + return this.frame('elif_clause', elifToken.start, this.endOf(kids, elifToken.end), kids); + } + kids.push(...this.parseStatementList({ stopWords: STOP_IF_BODY, terminatorRequired: true })); + // Like case_item, an elif/else clause extends over its trailing + // terminator (usually a newline) in the reference. + const end = Math.max(this.endOf(kids, elifToken.end), this.lastTerminatorEnd); + return this.frame('elif_clause', elifToken.start, end, kids); + } + + /** else_clause: `else` body */ + private parseElseClause(): Frame { + const elseToken = this.lexer.next(); + const kids: Frame[] = [ + this.anon('else', elseToken.start, elseToken.end), + ...this.parseStatementList({ stopWords: STOP_FI, terminatorRequired: true }), + ]; + const end = Math.max(this.endOf(kids, elseToken.end), this.lastTerminatorEnd); + return this.frame('else_clause', elseToken.start, end, kids); + } + + /** while_statement: (`while` | `until`) condition do_group */ + private parseWhileStatement(): Frame { + const kwToken = this.lexer.next(); + const keyword = this.tokenText(kwToken); + const kids: Frame[] = [ + this.anon(keyword, kwToken.start, kwToken.end), + ...this.parseStatementList({ stopWords: STOP_DO, terminatorRequired: true }), + ]; + if (this.peekKeyword('do')) { + kids.push(this.parseDoGroup()); + } else { + this.hasError = true; // missing do + } + return this.frame('while_statement', kwToken.start, this.endOf(kids, kwToken.end), kids); + } + + /** do_group: `do` body `done` */ + private parseDoGroup(): Frame { + const doToken = this.lexer.next(); + const kids: Frame[] = [ + this.anon('do', doToken.start, doToken.end), + ...this.parseStatementList({ stopWords: STOP_DONE, terminatorRequired: true }), + ]; + let end = this.endOf(kids, doToken.end); + if (this.consumeKeyword(kids, 'done')) { + end = kids.at(-1)!.end; + } else { + this.hasError = true; // unterminated do + } + return this.frame('do_group', doToken.start, end, kids); + } + + /** Consume the terminator between a for-header and its body: `;`/`&` + * become anonymous children, newlines are dropped, comments are kept. */ + private consumeForTerminator(kids: Frame[]): void { for (;;) { const token = this.lexer.peek(); - if (token.type === 'word' && ASSIGNMENT_RE.test(this.tokenText(token))) { + if (token.type === 'newline') { this.lexer.next(); - prefix.push(this.parseVariableAssignment(token)); + this.completeHeredocs(token.heredocBodies); continue; } - // Prefix redirects take exactly one destination so a following word - // still becomes the command_name (`> a cmd x`). Heredoc operators are - // left for the trailing-redirect path. - if (token.type === 'io_number') { - prefix.push(this.parseRedirect(1)); + if (token.type === 'comment') { + this.lexer.next(); + kids.push(this.frame('comment', token.start, token.end)); continue; } if (token.type === 'op') { const op = this.tokenText(token); - if (isFileRedirectOp(op) || op === '<<<') { - prefix.push(this.parseRedirect(1)); - continue; + if (op === ';' || op === '&') { + this.lexer.next(); + kids.push(this.anon(op, token.start, token.end)); } } break; } - if (this.lexer.peek().type !== 'word') { - if (prefix.length === 0) return null; - return this.assembleNamelessPrefix(prefix); + } + + /** for_statement: (`for` | `select`) variable (`in` values)? terminator + * do_group — or, for `for ((…))`, a c_style_for_statement. */ + private parseForStatement(): Frame { + const kwToken = this.lexer.next(); + const keyword = this.tokenText(kwToken); + const kids: Frame[] = [this.anon(keyword, kwToken.start, kwToken.end)]; + const next = this.lexer.peek(); + if (keyword === 'for' && next.type === 'word' && this.tokenText(next).startsWith('((')) { + return this.parseCStyleForStatement(kwToken, kids); } - const start = prefix.length > 0 ? prefix[0]!.start : this.lexer.peek().start; - const nameToken = this.lexer.next(); - const name = this.frame('command_name', nameToken.start, nameToken.end, [ - this.parseLiteral(nameToken.start, nameToken.end), - ]); - const command = this.frame('command', start, nameToken.end, [...prefix, name]); - for (;;) { - const token = this.lexer.peek(); - if (token.type === 'word') { - const argument = this.parseWordArgument(); - this.addKid(command, argument); - command.end = argument.end; - continue; + const varToken = this.lexer.peek(); + if (varToken.type === 'word' && /^\w+$/.test(this.tokenText(varToken))) { + this.lexer.next(); + kids.push(this.frame('variable_name', varToken.start, varToken.end)); + } else { + this.hasError = true; // missing loop variable + } + if (this.peekKeyword('in')) { + const inToken = this.lexer.next(); + const values: Frame[] = []; + while (this.lexer.peek().type === 'word') { + values.push(this.parseWordArgument()); } - if (token.type === 'op' && this.tokenText(token) === '<<<') { - const herestring = this.parseHerestringRedirect(null); - this.addKid(command, herestring); - command.end = herestring.end; - continue; + if (values.length === 0) { + // `for x in; do` — the reference wraps the bare `in` in ERROR. + this.hasError = true; + kids.push(this.frame('ERROR', inToken.start, inToken.end, [this.anon('in', inToken.start, inToken.end)])); + } else { + kids.push(this.anon('in', inToken.start, inToken.end)); + kids.push(...values); } - break; - } - return command; - } - - /** A prefix of assignments/redirects with no command name: - * `FOO=bar`, `A=1 B=2`, `> out`, `> out FOO=bar`. */ - private assembleNamelessPrefix(prefix: Frame[]): Frame { - const assignments = prefix.filter((f) => f.type === 'variable_assignment'); - if (assignments.length === prefix.length) { - if (prefix.length === 1) return prefix[0]!; - return this.frame('variable_assignments', prefix[0]!.start, prefix.at(-1)!.end, prefix); } - return this.frame('redirected_statement', prefix[0]!.start, prefix.at(-1)!.end, prefix); - } - - /** variable_assignment: NAME ( = | += ) value — the word token is split - * into variable_name, the operator, and a value parsed from the rest of - * the token (which may hold quotes/expansions). */ - private parseVariableAssignment(token: Token): Frame { - const text = this.tokenText(token); - const match = ASSIGNMENT_SPLIT_RE.exec(text)!; - const nameEnd = token.start + match[1]!.length; - const opEnd = nameEnd + match[2]!.length; - const kids: Frame[] = [ - this.frame('variable_name', token.start, nameEnd), - this.anon(match[2]!, nameEnd, opEnd), - ]; - if (opEnd < token.end) { - kids.push(this.parseLiteral(opEnd, token.end)); + this.consumeForTerminator(kids); + if (this.peekKeyword('do')) { + kids.push(this.parseDoGroup()); + } else { + this.hasError = true; // missing do } - return this.frame('variable_assignment', token.start, token.end, kids); + return this.frame('for_statement', kwToken.start, this.endOf(kids, kwToken.end), kids); } - // --------------------------------------------------------------- redirect - - /** Parse one redirect with an optional io_number descriptor prefix already - * peeked. Dispatches to file_redirect / herestring / heredoc handling. - * `maxDestinations` limits greedy destination consumption (command-prefix - * redirects take exactly one). */ - private parseRedirect(maxDestinations = Number.POSITIVE_INFINITY): Frame { - let descriptor: Frame | null = null; - let token = this.lexer.peek(); - if (token.type === 'io_number') { - this.lexer.next(); - descriptor = this.frame('file_descriptor', token.start, token.end); - token = this.lexer.peek(); - } - if (token.type !== 'op') { - // io_number not followed by an operator — recover. - this.hasError = true; - const stray = descriptor ?? this.frame('ERROR', token.start, token.end); - return this.frame('ERROR', stray.start, stray.end, descriptor === null ? [] : [stray]); + /** c_style_for_statement: `for` `((` init `;` cond `;` update `))` + * terminator (do_group | compound_statement). The whole ((…)) header + * arrived as one word token (see the lexer). */ + private parseCStyleForStatement(kwToken: Token, kids: Frame[]): Frame { + const header = this.lexer.next(); + const closed = + header.end - header.start >= 4 && this.source[header.end - 2] === ')' && this.source[header.end - 1] === ')'; + if (!closed) this.hasError = true; + kids.push(this.anon('((', header.start, header.start + 2)); + const innerEnd = closed ? header.end - 2 : header.end; + kids.push(...this.parseCForBody(header.start + 2, innerEnd)); + if (closed) { + kids.push(this.anon('))', header.end - 2, header.end)); } - const op = this.tokenText(token); - if (op === '<<' || op === '<<-') { - return this.noHeredoc ? this.parseBrokenHeredoc(descriptor) : this.parseHeredocRedirect(descriptor); + this.consumeForTerminator(kids); + if (this.peekKeyword('do')) { + kids.push(this.parseDoGroup()); + } else if (this.peekKeyword('{')) { + kids.push(this.parseCompoundStatement()); + } else { + this.hasError = true; // missing body } - if (op === '<<<') return this.parseHerestringRedirect(descriptor); - if (isFileRedirectOp(op)) return this.parseFileRedirect(descriptor, maxDestinations); - this.hasError = true; - const stray = descriptor ?? this.anon(op, token.start, token.end); - return this.frame('ERROR', stray.start, stray.end, [stray]); + return this.frame('c_style_for_statement', kwToken.start, this.endOf(kids, kwToken.end), kids); } - /** file_redirect: [fd] op destination* — `>&-`/`<&-` take no destination. - * In command-prefix position only one destination is taken (`> a cmd x` - * makes `cmd` the command_name); after the command name the redirect is - * greedy, matching tree-sitter-bash (`cmd > out arg` puts both words in - * the redirect). */ - private parseFileRedirect(descriptor: Frame | null, maxDestinations = Number.POSITIVE_INFINITY): Frame { - const opToken = this.lexer.next(); - const op = this.tokenText(opToken); + /** The init/condition/update parts of a c-style for header: three + * comma-separated c-expression lists divided by `;`. */ + private parseCForBody(start: number, end: number): Frame[] { + const st = this.newExprState(start, end, 'c'); const kids: Frame[] = []; - if (descriptor !== null) kids.push(descriptor); - kids.push(this.anon(op, opToken.start, opToken.end)); - let end = opToken.end; - if (op !== '>&-' && op !== '<&-') { - let destinations = 0; - while (destinations < maxDestinations && this.lexer.peek().type === 'word') { - const destination = this.parseWordArgument(); - kids.push(destination); - end = destination.end; - destinations++; + for (let part = 0; part < 3; part++) { + for (;;) { + const expression = this.parseExpression(st, 0); + if (expression !== null) kids.push(expression); + const token = this.exprPeek(st); + if (token.kind === 'op' && token.text === ',') { + this.exprNext(st); + kids.push(this.anon(',', token.start, token.end)); + continue; + } + break; + } + if (part < 2) { + const token = this.exprPeek(st); + if (token.kind === 'op' && token.text === ';') { + this.exprNext(st); + kids.push(this.anon(';', token.start, token.end)); + } else { + this.hasError = true; // missing ; in for-header + break; + } } - if (destinations === 0) this.hasError = true; // missing destination } - return this.frame('file_redirect', descriptor?.start ?? opToken.start, end, kids); + const leftover = this.exprLeftover(st); + if (leftover !== null) kids.push(leftover); + return kids; } - /** herestring_redirect: [fd] `<<<` literal */ - private parseHerestringRedirect(descriptor: Frame | null): Frame { - const opToken = this.lexer.next(); - const kids: Frame[] = []; - if (descriptor !== null) kids.push(descriptor); - kids.push(this.anon('<<<', opToken.start, opToken.end)); - let end = opToken.end; + /** case_statement: `case` value `in` case_item* `esac` */ + private parseCaseStatement(): Frame { + const caseToken = this.lexer.next(); + const kids: Frame[] = [this.anon('case', caseToken.start, caseToken.end)]; if (this.lexer.peek().type === 'word') { - const value = this.parseWordArgument(); - kids.push(value); - end = value.end; + kids.push(this.parseWordArgument()); } else { + this.hasError = true; // missing value + } + this.skipCaseTerminators(kids); + if (!this.consumeKeyword(kids, 'in')) { this.hasError = true; + return this.frame('case_statement', caseToken.start, this.endOf(kids, caseToken.end), kids); } - return this.frame('herestring_redirect', descriptor?.start ?? opToken.start, end, kids); + this.skipCaseTerminators(kids); + for (;;) { + // Between items: newlines are dropped, comments kept as children. + for (;;) { + const token = this.lexer.peek(); + if (token.type === 'newline') { + this.lexer.next(); + this.completeHeredocs(token.heredocBodies); + continue; + } + if (token.type === 'comment') { + this.lexer.next(); + kids.push(this.frame('comment', token.start, token.end)); + continue; + } + break; + } + const token = this.lexer.peek(); + if (token.type === 'eof') { + this.hasError = true; // missing esac + break; + } + if (token.type === 'word' && this.tokenText(token) === 'esac') break; + const before = this.lexer.pos; + kids.push(this.parseCaseItem()); + if (this.lexer.pos === before) { + // Defensive: a case_item that consumed nothing would loop forever. + this.hasError = true; + this.lexer.next(); + } + if (this.peekKeyword('esac')) { + // A fallthrough terminator on the LAST item is a grammar error in + // the reference (last_case_item only allows `;;`). + const last = kids.at(-1)!.children.at(-1); + if (last !== undefined && !last.isNamed && (last.type === ';&' || last.type === ';;&')) { + this.hasError = true; + } + } + } + let end = this.endOf(kids, caseToken.end); + if (this.consumeKeyword(kids, 'esac')) { + end = kids.at(-1)!.end; + } + return this.frame('case_statement', caseToken.start, end, kids); } - /** - * heredoc_redirect: [fd] (`<<` | `<<-`) heredoc_start, then the rest of - * the line (arguments, redirects, a pipeline or &&/|| tail, and `;`/`&` - * separated follow-up statements) absorbed into the redirect — matching - * tree-sitter-bash's grammar, which swallows all of these into - * heredoc_redirect. + /** Terminators around `in` / between case items: newlines dropped, `;`/`&` + * kept as anonymous children, comments kept. */ + private skipCaseTerminators(kids: Frame[]): void { + for (;;) { + const token = this.lexer.peek(); + if (token.type === 'newline') { + this.lexer.next(); + this.completeHeredocs(token.heredocBodies); + continue; + } + if (token.type === 'comment') { + this.lexer.next(); + kids.push(this.frame('comment', token.start, token.end)); + continue; + } + if (token.type === 'op') { + const op = this.tokenText(token); + if (op === ';' || op === '&') { + this.lexer.next(); + kids.push(this.anon(op, token.start, token.end)); + continue; + } + } + break; + } + } + + /** + * case_item: `(`? pattern (`|` pattern)* `)` statements? terminator? + * Patterns are literals, except bare words that contain a glob character + * (* ? [) or that are followed by `|` — those are extglob_pattern nodes, + * matching the reference scanner. + */ + private parseCaseItem(): Frame { + const kids: Frame[] = []; + let token = this.lexer.peek(); + if (token.type === 'op' && this.tokenText(token) === '(') { + this.lexer.next(); + kids.push(this.anon('(', token.start, token.end)); + } + // Patterns. + for (;;) { + token = this.lexer.peek(); + if (token.type !== 'word') { + this.hasError = true; // missing pattern + break; + } + const [start, end] = this.consumeWordRun(); + const raw = this.text(start, end); + const bare = !/["'$`\\]/.test(raw); + const followedByAlternate = this.lexer.peek().type === 'op' && this.tokenText(this.lexer.peek()) === '|'; + if (bare && (/[*?[\]]/.test(raw) || followedByAlternate)) { + kids.push(this.frame('extglob_pattern', start, end)); + } else { + kids.push(this.parseLiteral(start, end)); + } + token = this.lexer.peek(); + if (token.type === 'op' && this.tokenText(token) === '|') { + this.lexer.next(); + kids.push(this.anon('|', token.start, token.end)); + continue; + } + break; + } + token = this.lexer.peek(); + if (token.type === 'op' && this.tokenText(token) === ')') { + this.lexer.next(); + kids.push(this.anon(')', token.start, token.end)); + } else { + this.hasError = true; // missing ) + } + kids.push(...this.parseCaseItemStatements()); + token = this.lexer.peek(); + if (token.type === 'op') { + const op = this.tokenText(token); + if (op === ';;' || op === ';&' || op === ';;&') { + this.lexer.next(); + kids.push(this.anon(op, token.start, token.end)); + } + } + const start = kids.length > 0 ? kids[0]!.start : token.start; + // An item closed by `esac` (no `;;`) extends over its trailing + // terminator (usually a newline) in the reference. + const end = Math.max(this.endOf(kids, token.start), this.lastTerminatorEnd); + return this.frame('case_item', start, end, kids); + } + + /** case_item statements with the esac-argument guard active (see + * caseItemDepth). */ + private parseCaseItemStatements(): Frame[] { + this.caseItemDepth++; + try { + return this.parseStatementList({ stopWords: STOP_ESAC, stopOps: CASE_TERMINATION_OPS }); + } finally { + this.caseItemDepth--; + } + } + + /** function_definition: (`function`)? name (`()`)? body redirect? + * The body is a compound_statement, subshell, test_command or + * if_statement (the grammar's full choice). */ + private parseFunctionDefinition(): Frame { + const kids: Frame[] = []; + const first = this.lexer.next(); + if (this.tokenText(first) === 'function') { + kids.push(this.anon('function', first.start, first.end)); + const nameToken = this.lexer.peek(); + if (nameToken.type === 'word') { + this.lexer.next(); + kids.push(this.frame('word', nameToken.start, nameToken.end)); + } else { + this.hasError = true; // missing function name + } + this.consumeParenPair(kids); + } else { + kids.push(this.frame('word', first.start, first.end)); + this.consumeParenPair(kids); + } + // Blank lines/comments may sit between the header and the body. + for (;;) { + const token = this.lexer.peek(); + if (token.type === 'newline') { + this.lexer.next(); + this.completeHeredocs(token.heredocBodies); + continue; + } + if (token.type === 'comment') { + this.lexer.next(); + kids.push(this.frame('comment', token.start, token.end)); + continue; + } + break; + } + const token = this.lexer.peek(); + if (token.type === 'word' && this.tokenText(token) === '{') { + kids.push(this.parseCompoundStatement()); + } else if (token.type === 'op' && this.tokenText(token) === '(') { + kids.push(this.parseSubshell()); + } else if (token.type === 'word' && this.tokenText(token) === '[') { + kids.push(this.parseTestCommand()); + } else if (token.type === 'word' && this.tokenText(token) === 'if') { + kids.push(this.parseIfStatement()); + } else { + this.hasError = true; // missing function body + } + // Optional single redirect (file_redirect or herestring). + const redirect = this.lexer.peek(); + if (redirect.type === 'io_number') { + kids.push(this.parseRedirect()); + } else if (redirect.type === 'op') { + const op = this.tokenText(redirect); + if (isFileRedirectOp(op) || op === '<<<') { + kids.push(this.parseRedirect()); + } + } + return this.frame('function_definition', kids[0]!.start, this.endOf(kids, first.end), kids); + } + + /** Consume a `(` `)` pair (blanks allowed between) as anonymous children + * when present. */ + private consumeParenPair(kids: Frame[]): void { + const open = this.lexer.peek(); + if (open.type !== 'op' || this.tokenText(open) !== '(') return; + const close = this.lexer.peekAt(1); + if (close.type !== 'op' || this.tokenText(close) !== ')') return; + this.lexer.next(); + this.lexer.next(); + kids.push(this.anon('(', open.start, open.end)); + kids.push(this.anon(')', close.start, close.end)); + } + + /** Lookahead for the `name() body` function form: a non-reserved word + * followed by `(` `)` and a valid body start (`{`, `(`, `[`, `if`), + * possibly across newlines. Without a valid body start the tokens belong + * to a plain command (`foo () ls` is command + ERROR in the reference). */ + private isFunctionDefinitionAhead(): boolean { + const name = this.lexer.peek(); + const text = this.tokenText(name); + if (!IDENTIFIER_RE.test(text) || RESERVED_WORD_SET.has(text)) return false; + const open = this.lexer.peekAt(1); + if (open.type !== 'op' || this.tokenText(open) !== '(') return false; + const close = this.lexer.peekAt(2); + if (close.type !== 'op' || this.tokenText(close) !== ')') return false; + for (let n = 3; ; n++) { + const token = this.lexer.peekAt(n); + if (token.type === 'newline' || token.type === 'comment') continue; + if (token.type === 'op' && this.tokenText(token) === '(') return true; + if (token.type === 'word') { + const word = this.tokenText(token); + return word === '{' || word === '[' || word === 'if'; + } + return false; + } + } + + /** declaration_command: (`declare` | `typeset` | `export` | `readonly` | + * `local`) (variable_assignment | literal | variable_name)* */ + private parseDeclarationCommand(): Frame { + const kwToken = this.lexer.next(); + const kids: Frame[] = [this.anon(this.tokenText(kwToken), kwToken.start, kwToken.end)]; + for (;;) { + const token = this.lexer.peek(); + if (token.type !== 'word') break; + const text = this.tokenText(token); + if (ASSIGNMENT_RE.test(text)) { + this.lexer.next(); + kids.push(this.parseVariableAssignment(token)); + } else if (this.isSubscriptAssignmentAhead(token)) { + kids.push(this.parseSubscriptAssignment()); + } else if (IDENTIFIER_RE.test(text)) { + this.lexer.next(); + kids.push(this.frame('variable_name', token.start, token.end)); + } else { + kids.push(this.parseWordArgument()); + } + } + return this.frame('declaration_command', kwToken.start, this.endOf(kids, kwToken.end), kids); + } + + /** unset_command: (`unset` | `unsetenv`) (literal | variable_name)* */ + private parseUnsetCommand(): Frame { + const kwToken = this.lexer.next(); + const kids: Frame[] = [this.anon(this.tokenText(kwToken), kwToken.start, kwToken.end)]; + for (;;) { + const token = this.lexer.peek(); + if (token.type !== 'word') break; + const text = this.tokenText(token); + if (IDENTIFIER_RE.test(text)) { + this.lexer.next(); + kids.push(this.frame('variable_name', token.start, token.end)); + } else { + kids.push(this.parseWordArgument()); + } + } + return this.frame('unset_command', kwToken.start, this.endOf(kids, kwToken.end), kids); + } + + // --------------------------------------------------------------- commands + + /** + * command: leading variable_assignment / redirect prefix, command_name, + * then arguments, inline herestring redirects and a trailing subshell + * (`foo (ls)` keeps the subshell inside the command, as the reference + * does). Returns null when no command name is present and nothing was + * consumed; a nameless prefix is assembled into variable_assignment(s) / + * redirected_statement here. + */ + private parseCommand(): Frame | null { + const prefix: Frame[] = []; + for (;;) { + const token = this.lexer.peek(); + if (token.type === 'word' && ASSIGNMENT_RE.test(this.tokenText(token))) { + this.lexer.next(); + prefix.push(this.parseVariableAssignment(token)); + continue; + } + if (token.type === 'word' && this.isSubscriptAssignmentAhead(token)) { + prefix.push(this.parseSubscriptAssignment()); + continue; + } + // Prefix redirects take exactly one destination so a following word + // still becomes the command_name (`> a cmd x`). Heredoc operators are + // left for the trailing-redirect path. + if (token.type === 'io_number') { + prefix.push(this.parseRedirect(1)); + continue; + } + if (token.type === 'op') { + const op = this.tokenText(token); + if (isFileRedirectOp(op) || op === '<<<') { + prefix.push(this.parseRedirect(1)); + continue; + } + } + break; + } + if (this.lexer.peek().type !== 'word') { + if (prefix.length === 0) return null; + return this.assembleNamelessPrefix(prefix); + } + const start = prefix.length > 0 ? prefix[0]!.start : this.lexer.peek().start; + // The name merges adjacent word tokens (the lexer splits { } [ ] out as + // single-character tokens): `{1..3}` or `cmd{x}` is one name. + const [nameStart, nameEnd] = this.consumeWordRun(); + const name = this.frame('command_name', nameStart, nameEnd, [this.parseLiteral(nameStart, nameEnd)]); + const command = this.frame('command', start, nameEnd, [...prefix, name]); + for (;;) { + const token = this.lexer.peek(); + if (token.type === 'word') { + // Inside a case_item, `esac` closes the item rather than becoming + // an argument (see caseItemDepth). + if (this.caseItemDepth > 0 && this.tokenText(token) === 'esac') break; + for (const argument of this.parseCommandArgument()) { + this.addKid(command, argument); + command.end = argument.end; + } + continue; + } + if (token.type === 'op' && this.tokenText(token) === '<<<') { + const herestring = this.parseHerestringRedirect(null); + this.addKid(command, herestring); + command.end = herestring.end; + continue; + } + if (token.type === 'op' && this.tokenText(token) === '(') { + const subshell = this.parseSubshell(); + this.addKid(command, subshell); + command.end = subshell.end; + continue; + } + break; + } + return command; + } + + /** Consume a run of adjacent word tokens; returns [start, end). */ + private consumeWordRun(): [number, number] { + const first = this.lexer.next(); + let end = first.end; + for (;;) { + const token = this.lexer.peek(); + if (token.type !== 'word' || token.start !== end) break; + this.lexer.next(); + end = token.end; + } + return [first.start, end]; + } + + /** One command argument: normally one literal, but `$"…"` (translated + * string) is TWO arguments — an anonymous `$` plus the string — because + * a bare dollar cannot start a concatenation in the reference grammar + * (tree-sitter-bash 0.25.0 never produces translated_string). */ + private parseCommandArgument(): Frame[] { + const [start, end] = this.consumeWordRun(); + if (this.source[start] === '$' && this.source[start + 1] === '"' && start + 1 < end) { + return [this.anon('$', start, start + 1), this.parseLiteral(start + 1, end)]; + } + return [this.parseLiteral(start, end)]; + } + + /** A prefix of assignments/redirects with no command name: + * `FOO=bar`, `A=1 B=2`, `> out`, `> out FOO=bar`. */ + private assembleNamelessPrefix(prefix: Frame[]): Frame { + const assignments = prefix.filter((f) => f.type === 'variable_assignment'); + if (assignments.length === prefix.length) { + if (prefix.length === 1) return prefix[0]!; + return this.frame('variable_assignments', prefix[0]!.start, prefix.at(-1)!.end, prefix); + } + return this.frame('redirected_statement', prefix[0]!.start, prefix.at(-1)!.end, prefix); + } + + /** True when a subscripted assignment (`arr[0]=x`, `a[i+1]+=2`) starts at + * the peeked word token. */ + private isSubscriptAssignmentAhead(token: Token): boolean { + if (!IDENTIFIER_RE.test(this.tokenText(token))) return false; + return SUBSCRIPT_ASSIGNMENT_RE.test(this.source.slice(token.start)); + } + + /** variable_assignment with a subscript name: `arr[0]=x`, `a[i+1]+=2`. + * The tokens are the name, `[`, index pieces, `]` and a word starting + * with the operator; ranges come from a regex over the source. */ + private parseSubscriptAssignment(): Frame { + const start = this.lexer.peek().start; + const match = SUBSCRIPT_ASSIGNMENT_RE.exec(this.source.slice(start))!; + const nameEnd = start + match[1]!.length; + const indexStart = nameEnd + 1; + const indexEnd = indexStart + match[2]!.length; + const opStart = indexEnd + 1; + const opEnd = opStart + match[3]!.length; + let valueEnd = opEnd; + // Consume tokens covering [start, opEnd) — the last one may extend past + // the operator into the value (`=x"y"` is one word token). + for (;;) { + const token = this.lexer.peek(); + if (token.start >= opEnd) break; + this.lexer.next(); + valueEnd = Math.max(valueEnd, token.end); + } + // Merge following adjacent word tokens into the value. + for (;;) { + const token = this.lexer.peek(); + if (token.type !== 'word' || token.start !== valueEnd) break; + this.lexer.next(); + valueEnd = token.end; + } + const subscriptKids: Frame[] = [this.frame('variable_name', start, nameEnd), this.anon('[', nameEnd, indexStart)]; + if (indexEnd > indexStart) { + subscriptKids.push(this.parseLiteral(indexStart, indexEnd)); + } else { + this.hasError = true; // empty subscript index + } + subscriptKids.push(this.anon(']', indexEnd, opStart)); + const kids: Frame[] = [ + this.frame('subscript', start, opStart, subscriptKids), + this.anon(match[3]!, opStart, opEnd), + ]; + if (valueEnd > opEnd) { + kids.push(this.parseLiteral(opEnd, valueEnd)); + } + return this.frame('variable_assignment', start, valueEnd, kids); + } + + /** variable_assignment: NAME ( = | += ) value — the word token is split + * into variable_name, the operator, and a value parsed from the rest of + * the token plus any ADJACENT word tokens (the lexer splits { } [ ] out + * as single-character tokens, so `x={1..5}` and `x=a{b}` are single + * assignments). An empty value followed immediately by `(` is an array + * literal (`arr=(a b)`). */ + private parseVariableAssignment(token: Token): Frame { + const text = this.tokenText(token); + const match = ASSIGNMENT_SPLIT_RE.exec(text)!; + const nameEnd = token.start + match[1]!.length; + const opEnd = nameEnd + match[2]!.length; + const kids: Frame[] = [ + this.frame('variable_name', token.start, nameEnd), + this.anon(match[2]!, nameEnd, opEnd), + ]; + let valueEnd = token.end; + for (;;) { + const next = this.lexer.peek(); + if (next.type !== 'word' || next.start !== valueEnd) break; + this.lexer.next(); + valueEnd = next.end; + } + if (valueEnd > opEnd) { + kids.push(this.parseLiteral(opEnd, valueEnd)); + return this.frame('variable_assignment', token.start, valueEnd, kids); + } + const next = this.lexer.peek(); + if (next.type === 'op' && this.tokenText(next) === '(' && next.start === token.end) { + const array = this.parseArray(); + kids.push(array); + return this.frame('variable_assignment', token.start, array.end, kids); + } + return this.frame('variable_assignment', token.start, token.end, kids); + } + + /** array: `(` literal* `)` — the elements of an array assignment. */ + private parseArray(): Frame { + const open = this.lexer.next(); + const kids: Frame[] = [this.anon('(', open.start, open.end)]; + for (;;) { + const token = this.lexer.peek(); + if (token.type === 'word') { + kids.push(this.parseWordArgument()); + continue; + } + if (token.type === 'newline') { + this.lexer.next(); + this.completeHeredocs(token.heredocBodies); + continue; + } + if (token.type === 'comment') { + this.lexer.next(); + kids.push(this.frame('comment', token.start, token.end)); + continue; + } + if (token.type === 'op' && this.tokenText(token) === ')') { + this.lexer.next(); + kids.push(this.anon(')', token.start, token.end)); + return this.frame('array', open.start, token.end, kids); + } + this.hasError = true; // unterminated array + return this.frame('array', open.start, this.endOf(kids, open.end), kids); + } + } + + // --------------------------------------------------------------- redirect + + /** Parse one redirect with an optional io_number descriptor prefix already + * peeked. Dispatches to file_redirect / herestring / heredoc handling. + * `maxDestinations` limits greedy destination consumption (command-prefix + * redirects take exactly one). */ + private parseRedirect(maxDestinations = Number.POSITIVE_INFINITY): Frame { + let descriptor: Frame | null = null; + let token = this.lexer.peek(); + if (token.type === 'io_number') { + this.lexer.next(); + descriptor = this.frame('file_descriptor', token.start, token.end); + token = this.lexer.peek(); + } + if (token.type !== 'op') { + // io_number not followed by an operator — recover. + this.hasError = true; + const stray = descriptor ?? this.frame('ERROR', token.start, token.end); + return this.frame('ERROR', stray.start, stray.end, descriptor === null ? [] : [stray]); + } + const op = this.tokenText(token); + if (op === '<<' || op === '<<-') { + return this.noHeredoc ? this.parseBrokenHeredoc(descriptor) : this.parseHeredocRedirect(descriptor); + } + if (op === '<<<') return this.parseHerestringRedirect(descriptor); + if (isFileRedirectOp(op)) return this.parseFileRedirect(descriptor, maxDestinations); + this.hasError = true; + const stray = descriptor ?? this.anon(op, token.start, token.end); + return this.frame('ERROR', stray.start, stray.end, [stray]); + } + + /** file_redirect: [fd] op destination* — `>&-`/`<&-` take no destination. + * In command-prefix position only one destination is taken (`> a cmd x` + * makes `cmd` the command_name); after the command name the redirect is + * greedy, matching tree-sitter-bash (`cmd > out arg` puts both words in + * the redirect). */ + private parseFileRedirect(descriptor: Frame | null, maxDestinations = Number.POSITIVE_INFINITY): Frame { + const opToken = this.lexer.next(); + const op = this.tokenText(opToken); + const kids: Frame[] = []; + if (descriptor !== null) kids.push(descriptor); + kids.push(this.anon(op, opToken.start, opToken.end)); + let end = opToken.end; + if (op !== '>&-' && op !== '<&-') { + let destinations = 0; + while (destinations < maxDestinations && this.lexer.peek().type === 'word') { + const destination = this.parseWordArgument(); + kids.push(destination); + end = destination.end; + destinations++; + } + if (destinations === 0) this.hasError = true; // missing destination + } + return this.frame('file_redirect', descriptor?.start ?? opToken.start, end, kids); + } + + /** herestring_redirect: [fd] `<<<` literal */ + private parseHerestringRedirect(descriptor: Frame | null): Frame { + const opToken = this.lexer.next(); + const kids: Frame[] = []; + if (descriptor !== null) kids.push(descriptor); + kids.push(this.anon('<<<', opToken.start, opToken.end)); + let end = opToken.end; + if (this.lexer.peek().type === 'word') { + const value = this.parseWordArgument(); + kids.push(value); + end = value.end; + } else { + this.hasError = true; + } + return this.frame('herestring_redirect', descriptor?.start ?? opToken.start, end, kids); + } + + /** + * heredoc_redirect: [fd] (`<<` | `<<-`) heredoc_start, then the rest of + * the line (arguments, redirects, a pipeline or &&/|| tail, and `;`/`&` + * separated follow-up statements) absorbed into the redirect — matching + * tree-sitter-bash's grammar, which swallows all of these into + * heredoc_redirect. * * Only ONE heredoc may be registered per line region: a second `<<` * (or a `<<` inside a swallowed follow-up statement) cannot be @@ -736,22 +1643,16 @@ export class Parser { * splits { } [ ] out as single-character tokens) merge back into a single * literal, so `a{b}` is one concatenation argument. */ private parseWordArgument(): Frame { - const first = this.lexer.next(); - let end = first.end; - for (;;) { - const token = this.lexer.peek(); - if (token.type !== 'word' || token.start !== end) break; - this.lexer.next(); - end = token.end; - } - return this.parseLiteral(first.start, end); + const [start, end] = this.consumeWordRun(); + return this.parseLiteral(start, end); } /** * _literal over a source range: a single word/number/string/expansion/…, * or a concatenation of adjacent pieces. The single characters { } [ ] * become their own word pieces, matching tree-sitter-bash's - * _special_character alias. + * _special_character alias — except `{N..M}` (digits only), which is a + * brace_expression, and a leading `((…))`, which is an arithmetic command. * * Guarded by literalDepth: parseLiteral ↔ parseString / parseExpansion * recurse through parseDollar, and unlike command substitutions (which @@ -797,12 +1698,24 @@ export class Parser { continue; } if (ch === '$') { + // $"…" at the START of a literal is a translated_string (the + // reference produces it everywhere except command-argument + // position, where parseCommandArgument splits the bare `$` and the + // string into two arguments). Mid-literal it stays a bare `$` plus + // a string piece inside the concatenation. + if (this.source[i + 1] === '"' && i === start) { + const [translated, next] = this.parseString(i + 1, end); + pieces.push(this.frame('translated_string', i, next, [this.anon('$', i, i + 1), translated])); + i = next; + continue; + } const dollar = this.parseDollar(i, end); if (dollar !== null) { pieces.push(dollar[0]); i = dollar[1]; } else { - // Bare `$` (not followed by anything expandable). + // Bare `$` (not followed by anything expandable), e.g. the `$` of + // a mid-literal $"…" (see above). pieces.push(this.anon('$', i, i + 1)); i++; } @@ -814,7 +1727,36 @@ export class Parser { i = next; continue; } - if (ch === '{' || ch === '}' || ch === '[' || ch === ']') { + if (ch === '(' && this.source[i + 1] === '(' && i + 1 < end) { + const [piece, next] = this.parseParenArithmetic(i, end); + pieces.push(piece); + i = next; + continue; + } + if (ch === '{') { + const brace = BRACE_EXPRESSION_RE.exec(this.source.slice(i, end)); + if (brace !== null) { + const [full, low, high] = brace; + const lowStart = i + 1; + const highStart = lowStart + low!.length + 2; + const close = highStart + high!.length; + pieces.push( + this.frame('brace_expression', i, i + full.length, [ + this.anon('{', i, lowStart), + this.frame('number', lowStart, lowStart + low!.length), + this.anon('..', lowStart + low!.length, highStart), + this.frame('number', highStart, close), + this.anon('}', close, close + 1), + ]), + ); + i += full.length; + continue; + } + pieces.push(this.frame('word', i, i + 1)); + i++; + continue; + } + if (ch === '}' || ch === '[' || ch === ']') { pieces.push(this.frame('word', i, i + 1)); i++; continue; @@ -823,7 +1765,7 @@ export class Parser { let j = i; let sinceTick = 0; while (j < end) { - if (++sinceTick >= 2048) { + if (++sinceTick >= SCAN_TICK_INTERVAL) { this.budget.progress(); sinceTick = 0; } @@ -831,6 +1773,7 @@ export class Parser { if (next === '"' || next === "'" || next === '`' || next === '$') break; if (next === '{' || next === '}' || next === '[' || next === ']') break; if ((next === '<' || next === '>') && this.source[j + 1] === '(' && j + 1 < end) break; + if (next === '(' && this.source[j + 1] === '(' && j + 1 < end) break; if (next === '\\' && j + 1 < end) { j += 2; continue; @@ -838,7 +1781,10 @@ export class Parser { j++; } if (j === i) j++; // defensive: never stall - pieces.push(this.frame('word', i, j)); + // A bare run that is exactly a number is a number node even as one + // piece of a concatenation (`a[2]b` → word + number + word). + const numeric = NUMBER_RE.test(this.text(i, j)); + pieces.push(this.frame(numeric ? 'number' : 'word', i, j)); i = j; } if (pieces.length === 1) { @@ -863,7 +1809,7 @@ export class Parser { }; let sinceTick = 0; while (i < rangeEnd) { - if (++sinceTick >= 2048) { + if (++sinceTick >= SCAN_TICK_INTERVAL) { this.budget.progress(); sinceTick = 0; } @@ -908,8 +1854,9 @@ export class Parser { } /** Dispatch a $-construct at `i`: simple_expansion, expansion, - * command_substitution, or the arithmetic placeholder. Returns null for a - * bare `$` (including the M2 forms $"…" and $'…'). */ + * command_substitution, arithmetic_expansion, or ansi_c_string. Returns + * null for a bare `$` (including the `$` of a mid-literal $"…" — see + * parseLiteralPieces for the translated_string rule). */ private parseDollar(i: number, rangeEnd: number): [Frame, number] | null { const next = this.source[i + 1]; if (next === '(' && i + 1 < rangeEnd) { @@ -917,9 +1864,20 @@ export class Parser { return this.parseCommandSubstitution(i, rangeEnd); } if (next === '{' && i + 1 < rangeEnd) return this.parseExpansion(i, rangeEnd); + if (next === '[' && i + 1 < rangeEnd) return this.parseBracketArithmetic(i, rangeEnd); + if (next === "'" && i + 1 < rangeEnd) return this.parseAnsiCString(i, rangeEnd); if (next !== undefined && /[\w]/.test(next)) { let j = i + 1; while (j < rangeEnd && /[\w]/.test(this.source[j]!)) j++; + // `$0` is a special_variable_name; other digits are variable_name + // (`$1`, and `${10}` below) — matching the reference grammar, whose + // \w+ rule wins for everything except the standalone 0. + if (j === i + 2 && next === '0') { + return [ + this.frame('simple_expansion', i, j, [this.anon('$', i, i + 1), this.frame('special_variable_name', i + 1, j)]), + j, + ]; + } const expansion = this.frame('simple_expansion', i, j, [ this.anon('$', i, i + 1), this.frame('variable_name', i + 1, j), @@ -936,10 +1894,60 @@ export class Parser { return null; } - /** expansion: ${ … } with optional prefix operators (! #), a variable_name - * / special_variable_name / subscript, an optional infix operator - * (:- ## % / …), and a value parsed as a literal. */ + /** ansi_c_string: $' … ' — one leaf node, \' is an escaped quote. */ + private parseAnsiCString(i: number, rangeEnd: number): [Frame, number] { + let j = i + 2; + let sinceTick = 0; + while (j < rangeEnd) { + if (++sinceTick >= SCAN_TICK_INTERVAL) { + this.budget.progress(); + sinceTick = 0; + } + const ch = this.source[j]!; + if (ch === '\\') { + j += 2; + continue; + } + if (ch === "'") { + return [this.frame('ansi_c_string', i, j + 1), j + 1]; + } + j++; + } + this.hasError = true; // unterminated ANSI-C string + return [this.frame('ansi_c_string', i, rangeEnd), rangeEnd]; + } + + /** + * expansion: ${ … } with optional prefix operators (! #), a variable_name + * / special_variable_name / subscript, and an optional infix operator + * whose value layout follows the reference: + * # ## % %% removal — the pattern is a regex node (quotes become + * string/raw_string pieces) + * / // /# /% replacement — regex pattern, optional `/` + literal + * ^ ^^ , ,, case modification — optional regex + * @X transformation — two anonymous operator nodes + * : max length — arithmetic values around an optional `:` + * :- := :+ :? - = + ? default/assign/… — a word/concatenation value + * + * Guarded by literalDepth directly (not only via parseLiteral): the + * pattern → string → $ → expansion recursion cycle bypasses parseLiteral + * and would otherwise overflow the stack on pathological nesting. + */ private parseExpansion(i: number, rangeEnd: number): [Frame, number] { + if (this.literalDepth >= MAX_PARSE_DEPTH) { + this.hasError = true; + const scan = this.scanBalanced(i + 1, rangeEnd, '{', '}'); + return [this.frame('ERROR', i, scan.end), scan.end]; + } + this.literalDepth++; + try { + return this.parseExpansionInner(i, rangeEnd); + } finally { + this.literalDepth--; + } + } + + private parseExpansionInner(i: number, rangeEnd: number): [Frame, number] { const scan = this.scanBalanced(i + 1, rangeEnd, '{', '}'); const close = scan.end; const terminated = scan.balanced; @@ -948,9 +1956,11 @@ export class Parser { const expansion = this.frame('expansion', i, close, [this.anon('${', i, i + 2)]); let j = i + 2; // Prefix operators: ${#v} (length), ${!v} (indirect). + let bangPrefix = false; while (j < innerEnd && (this.source[j] === '#' || this.source[j] === '!')) { const after = this.source[j + 1]; if (after === undefined || j + 1 >= innerEnd || !/[\w@*#?$!-]/.test(after)) break; + if (this.source[j] === '!') bangPrefix = true; this.addKid(expansion, this.anon(this.source[j]!, j, j + 1)); j++; } @@ -959,7 +1969,9 @@ export class Parser { if (j < innerEnd && /[\w]/.test(this.source[j]!)) { let nameEnd = j; while (nameEnd < innerEnd && /[\w]/.test(this.source[nameEnd]!)) nameEnd++; - this.addKid(expansion, this.frame('variable_name', j, nameEnd)); + // ${0} is a special_variable_name; other digits are variable_name. + const special = nameEnd === j + 1 && this.source[j] === '0'; + this.addKid(expansion, this.frame(special ? 'special_variable_name' : 'variable_name', j, nameEnd)); j = nameEnd; hasName = true; } else if (j < innerEnd && SPECIAL_VARIABLE_CHARS.includes(this.source[j]!)) { @@ -983,30 +1995,210 @@ export class Parser { } else { this.hasError = true; } - if (sub.balanced) this.addKid(subscript, this.anon(']', subEnd - 1, subEnd)); - this.addKid(expansion, subscript); - j = subEnd; + if (sub.balanced) this.addKid(subscript, this.anon(']', subEnd - 1, subEnd)); + this.addKid(expansion, subscript); + j = subEnd; + } + // ${!prefix*} / ${!name@}: a trailing * or @ after an indirect name. + if (bangPrefix && j < innerEnd && (this.source[j] === '*' || this.source[j] === '@')) { + this.addKid(expansion, this.anon(this.source[j]!, j, j + 1)); + j++; + } + // Infix operator plus an optional value. + if (j < innerEnd) { + j = this.parseExpansionInfix(expansion, j, innerEnd); + } + if (terminated) this.addKid(expansion, this.anon('}', close - 1, close)); + return [expansion, close]; + } + + /** The infix part of an expansion (see parseExpansion). Returns the new + * scan position. */ + private parseExpansionInfix(expansion: Frame, j: number, innerEnd: number): number { + const one = this.source[j]!; + const two = this.source.slice(j, j + 2); + if (two === '##' || two === '%%' || one === '#' || one === '%') { + const operator = two === '##' || two === '%%' ? two : one; + this.addKid(expansion, this.anon(operator, j, j + operator.length)); + return this.parseExpansionPattern(expansion, j + operator.length, innerEnd); + } + if (two === '//' || two === '/#' || two === '/%' || one === '/') { + const operator = two === '//' || two === '/#' || two === '/%' ? two : one; + this.addKid(expansion, this.anon(operator, j, j + operator.length)); + let pos = this.parseExpansionPattern(expansion, j + operator.length, innerEnd, '/'); + if (pos < innerEnd && this.source[pos] === '/') { + this.addKid(expansion, this.anon('/', pos, pos + 1)); + pos++; + if (pos < innerEnd) { + this.addKid(expansion, this.parseExpansionValue(pos, innerEnd)); + pos = innerEnd; + } + } + return pos; + } + if (two === '^^' || two === ',,' || one === '^' || one === ',') { + const operator = two === '^^' || two === ',,' ? two : one; + this.addKid(expansion, this.anon(operator, j, j + operator.length)); + return this.parseExpansionPattern(expansion, j + operator.length, innerEnd); + } + if (one === '@' && j + 1 < innerEnd) { + this.addKid(expansion, this.anon('@', j, j + 1)); + this.addKid(expansion, this.anon(this.source[j + 1]!, j + 1, j + 2)); + return j + 2; + } + if (one === ':' && two !== ':-' && two !== ':=' && two !== ':+' && two !== ':?') { + // Max length: ${v:offset:length} — arithmetic values. + this.addKid(expansion, this.anon(':', j, j + 1)); + let pos = this.parseMaxLengthValue(expansion, j + 1, innerEnd); + if (pos < innerEnd && this.source[pos] === ':') { + this.addKid(expansion, this.anon(':', pos, pos + 1)); + pos = this.parseMaxLengthValue(expansion, pos + 1, innerEnd); + } + if (pos < innerEnd) { + this.hasError = true; // extra content after ${v:o:l} + this.addKid(expansion, this.parseLiteral(pos, innerEnd)); + pos = innerEnd; + } + return pos; + } + for (const operator of [':-', ':=', ':+', ':?', '-', '=', '+', '?']) { + if (this.source.startsWith(operator, j) && j + operator.length <= innerEnd) { + this.addKid(expansion, this.anon(operator, j, j + operator.length)); + const pos = j + operator.length; + if (pos < innerEnd) { + this.addKid(expansion, this.parseExpansionValue(pos, innerEnd)); + } + return innerEnd; + } + } + // Unknown infix content: keep it as a literal (and flag the error). + this.hasError = true; + this.addKid(expansion, this.parseLiteral(j, innerEnd)); + return innerEnd; + } + + /** A removal/replacement pattern inside ${…}: regex chunks (raw text, + * spaces included) interleaved with string/raw_string pieces. `stop` + * optionally ends the pattern (the replacement separator `/`). */ + private parseExpansionPattern(expansion: Frame, j: number, innerEnd: number, stop?: string): number { + let pos = j; + while (pos < innerEnd) { + this.budget.progress(); + const ch = this.source[pos]!; + if (ch === '"') { + const [piece, next] = this.parseString(pos, innerEnd); + this.addKid(expansion, piece); + pos = next; + continue; + } + if (ch === "'") { + const close = skipSingleQuoted(this.source, this.budget, pos, innerEnd); + this.addKid(expansion, this.frame('raw_string', pos, close)); + pos = close; + continue; + } + let k = pos; + let sinceTick = 0; + while (k < innerEnd && this.source[k] !== '"' && this.source[k] !== "'" && this.source[k] !== stop) { + if (++sinceTick >= SCAN_TICK_INTERVAL) { + this.budget.progress(); + sinceTick = 0; + } + if (this.source[k] === '\\' && k + 1 < innerEnd) { + k += 2; + continue; + } + k++; + } + if (k === pos) break; // defensive: only the stop character remains + this.addKid(expansion, this.frame('regex', pos, k)); + pos = k; + } + return pos; + } + + /** A default/assign value inside ${…} (`${v:-word}`): bare text is a word + * (never a number — `${v:-1}` is a word in the reference); a bare value + * containing a space splits into two word pieces inside a concatenation + * (`${v:-d e f}` → word "d" + word " e f", matching the reference's + * _expansion_word scanner). */ + private parseExpansionValue(start: number, end: number): Frame { + const raw = this.text(start, end); + if (!/["'$`\\]/.test(raw)) { + const space = raw.indexOf(' '); + if (space > 0) { + return this.frame('concatenation', start, end, [ + this.frame('word', start, start + space), + this.frame('word', start + space, end), + ]); + } + return this.frame('word', start, end); + } + return this.parseLiteral(start, end); + } + + /** One arithmetic value of ${v:offset:length} (up to `:` or innerEnd). */ + private parseMaxLengthValue(expansion: Frame, j: number, innerEnd: number): number { + let end = j; + let sinceTick = 0; + while (end < innerEnd && this.source[end] !== ':') { + if (++sinceTick >= SCAN_TICK_INTERVAL) { + this.budget.progress(); + sinceTick = 0; + } + const ch = this.source[end]!; + if (ch === '\\') { + end += 2; + continue; + } + if (ch === '$') { + end = this.skipDollarConstruct(end, innerEnd); + continue; + } + if (ch === '(') { + end = this.scanBalanced(end, innerEnd, '(', ')').end; + continue; + } + end++; } - // Infix operator plus an optional value. - if (j < innerEnd) { - for (const operator of EXPANSION_OPERATORS) { - if (j + operator.length <= innerEnd && this.source.startsWith(operator, j)) { - this.addKid(expansion, this.anon(operator, j, j + operator.length)); - j += operator.length; - break; - } + if (end > j) { + // In max-length context a negative offset is a number literal, not a + // unary expression (`${x: -5}` → number "-5" in the reference). + const negative = /^\s*(-\d+)\s*$/.exec(this.text(j, end)); + if (negative !== null) { + const start = j + this.text(j, end).indexOf('-'); + this.addKid(expansion, this.frame('number', start, start + negative[1]!.length)); + return end; } - if (j < innerEnd) { - this.addKid(expansion, this.parseLiteral(j, innerEnd)); + const st = this.newExprState(j, end, 'arith'); + const value = this.parseExpression(st, 0); + if (value !== null) { + this.addKid(expansion, value); } + const leftover = this.exprLeftover(st); + if (leftover !== null) this.addKid(expansion, leftover); } - if (terminated) this.addKid(expansion, this.anon('}', close - 1, close)); - return [expansion, close]; + return end; + } + + /** Skip a $-construct without building nodes (max-length scanning). */ + private skipDollarConstruct(i: number, end: number): number { + const next = this.source[i + 1]; + if (next === '(') return this.scanBalanced(i + 1, end, '(', ')').end; + if (next === '{') return this.scanBalanced(i + 1, end, '{', '}').end; + if (next !== undefined && /[\w]/.test(next)) { + let j = i + 1; + while (j < end && /[\w]/.test(this.source[j]!)) j++; + return j; + } + return i + 1 < end ? i + 2 : i + 1; } - /** command_substitution: $( _statements ) */ + /** command_substitution: $( _statements ). The close-paren scan is + * case-aware (see scanBalancedStatements): a case_item pattern `)` must + * not close the substitution early. */ private parseCommandSubstitution(i: number, rangeEnd: number): [Frame, number] { - const scan = this.scanBalanced(i + 1, rangeEnd, '(', ')'); + const scan = scanBalancedStatements(this.source, this.budget, i + 1, rangeEnd); const close = scan.end; if (!scan.balanced) this.hasError = true; const innerEnd = scan.balanced ? close - 1 : close; @@ -1034,9 +2226,10 @@ export class Parser { return [substitution, close]; } - /** process_substitution: ( <( | >( ) _statements ) */ + /** process_substitution: ( <( | >( ) _statements ) — case-aware close + * scan, like command_substitution. */ private parseProcessSubstitution(i: number, rangeEnd: number): [Frame, number] { - const scan = this.scanBalanced(i + 1, rangeEnd, '(', ')'); + const scan = scanBalancedStatements(this.source, this.budget, i + 1, rangeEnd); const close = scan.end; if (!scan.balanced) this.hasError = true; const innerEnd = scan.balanced ? close - 1 : close; @@ -1049,31 +2242,836 @@ export class Parser { return [substitution, close]; } - /** - * TODO(M2): real arithmetic expansion parsing ($(( … )) with - * binary_expression / number / … children). For M1 the whole construct is - * surfaced as an arithmetic_expansion whose raw inner text sits in a - * single word child, so the node type and range are already correct and - * downstream consumers can rely on both. - */ + // ------------------------------------------------------ expression engine + + private newExprState(start: number, end: number, mode: ExprMode): ExprState { + return { pos: start, end, mode, lookahead: null, parenDepth: 0, expectOperator: false }; + } + + private exprPeek(st: ExprState): ExprToken { + st.lookahead ??= this.scanExprToken(st); + return st.lookahead; + } + + /** When unconsumed input remains in the expression state, wrap it in an + * ERROR node (covering the rest of the region, so no source text + * silently drops out of the tree) and flag the error. */ + private exprLeftover(st: ExprState): Frame | null { + const token = this.exprPeek(st); + if (token.kind === 'end') return null; + this.hasError = true; + return this.frame('ERROR', token.start, st.end); + } + + private exprNext(st: ExprState): ExprToken { + const token = this.exprPeek(st); + st.lookahead = null; + return token; + } + + /** arithmetic_expansion: $(( expr (, expr)* )) */ private parseArithmeticExpansion(i: number, rangeEnd: number): [Frame, number] { const scan = this.scanBalanced(i + 1, rangeEnd, '(', ')'); const close = scan.end; - if (!scan.balanced) this.hasError = true; // The content sits between `$((` and the final `))`. const closed = scan.balanced && close - 2 >= i + 3 && this.source[close - 2] === ')'; + if (!closed) this.hasError = true; const innerStart = Math.min(i + 3, close); const innerEnd = closed ? close - 2 : close; const expansion = this.frame('arithmetic_expansion', i, close, [this.anon('$((', i, innerStart)]); - if (innerEnd > innerStart) { - this.addKid(expansion, this.frame('word', innerStart, innerEnd)); + this.addArithmeticChildren(expansion, innerStart, innerEnd); + if (closed) { + this.addKid(expansion, this.anon('))', innerEnd, close)); } + return [expansion, close]; + } + + /** arithmetic_expansion without the `$`: (( expr )) at word position. */ + private parseParenArithmetic(i: number, rangeEnd: number): [Frame, number] { + const scan = this.scanBalanced(i, rangeEnd, '(', ')'); + const close = scan.end; + const closed = scan.balanced && close - 2 >= i + 2 && this.source[close - 2] === ')'; + if (!closed) this.hasError = true; + const innerStart = Math.min(i + 2, close); + const innerEnd = closed ? close - 2 : close; + const expansion = this.frame('arithmetic_expansion', i, close, [this.anon('((', i, innerStart)]); + this.addArithmeticChildren(expansion, innerStart, innerEnd); if (closed) { this.addKid(expansion, this.anon('))', innerEnd, close)); } return [expansion, close]; } + /** arithmetic_expansion: $[ expr ] (legacy form). */ + private parseBracketArithmetic(i: number, rangeEnd: number): [Frame, number] { + const scan = this.scanBalanced(i + 1, rangeEnd, '[', ']'); + const close = scan.end; + if (!scan.balanced) this.hasError = true; + const innerEnd = scan.balanced ? close - 1 : close; + const expansion = this.frame('arithmetic_expansion', i, close, [this.anon('$[', i, i + 2)]); + this.addArithmeticChildren(expansion, i + 2, innerEnd); + if (scan.balanced) { + this.addKid(expansion, this.anon(']', close - 1, close)); + } + return [expansion, close]; + } + + /** The comma-separated expression list inside an arithmetic_expansion. */ + private addArithmeticChildren(expansion: Frame, start: number, end: number): void { + const st = this.newExprState(start, end, 'arith'); + for (;;) { + const expression = this.parseExpression(st, 0); + if (expression !== null) this.addKid(expansion, expression); + const token = this.exprPeek(st); + if (token.kind === 'op' && token.text === ',') { + this.exprNext(st); + this.addKid(expansion, this.anon(',', token.start, token.end)); + continue; + } + break; + } + const leftover = this.exprLeftover(st); + if (leftover !== null) this.addKid(expansion, leftover); + } + + /** + * Pratt parser for arithmetic / c-style-for / test expressions. Mirrors + * grammar.js: left-associative binaries parse their right side at + * level+1, `**` is right-associative only in test mode (left in + * arithmetic), prefix -/+/~/! take their operand at level 11 (so + * `-x + ~y` is -(x + ~y)) and prefix ++/-- at level 17, postfix ++/-- sit + * at level 18, ternary at level 2. + */ + private parseExpression(st: ExprState, minPrecedence: number): Frame | null { + let left: Frame | null; + const head = this.exprPeek(st); + // `;` and `,` are separators (c-style for parts, comma lists), handled + // by the caller — never operands. + if (head.kind === 'op' && (head.text === ';' || head.text === ',')) return null; + if (head.kind === 'op' && (head.text === '++' || head.text === '--')) { + this.exprNext(st); + const kids: Frame[] = [this.anon(head.text, head.start, head.end)]; + const operand = this.parseExpression(st, PREC_PREFIX); + if (operand === null) this.hasError = true; + else kids.push(operand); + left = this.frame('unary_expression', head.start, this.endOf(kids, head.end), kids); + } else if (head.kind === 'op' && (head.text === '!' || head.text === '~' || head.text === '+' || head.text === '-')) { + this.exprNext(st); + const kids: Frame[] = [this.anon(head.text, head.start, head.end)]; + const operand = this.parseExpression(st, PREC_UNARY); + if (operand === null) this.hasError = true; + else kids.push(operand); + left = this.frame('unary_expression', head.start, this.endOf(kids, head.end), kids); + } else if (head.kind === 'testop') { + this.exprNext(st); + // A test_operator with no usable operand ahead is just a word + // (`[[ $a == -foo ]]` → word "-foo", `[[ -foo == x ]]` → word, + // `[[ -f ]]` → word — the reference's fallback when the operator + // reading fails). + const after = this.exprPeek(st); + const demote = + after.kind === 'end' || + after.kind === 'rparen' || + (after.kind === 'op' && + (after.text === '=' || after.text === '==' || after.text === '!=' || after.text === '=~' || after.text === '&&' || after.text === '||')); + if (demote) { + left = this.frame('word', head.start, head.end); + } else { + const kids: Frame[] = [this.frame('test_operator', head.start, head.end)]; + const operand = this.parseExpression(st, PREC_TEST); + if (operand === null) this.hasError = true; + else kids.push(operand); + left = this.frame('unary_expression', head.start, this.endOf(kids, head.end), kids); + } + } else { + left = this.parseExprPrimary(st); + } + if (left === null) return null; + // A complete operand: what follows is operator position (test mode). + st.expectOperator = true; + for (;;) { + const token = this.exprPeek(st); + if (token.kind === 'op' && (token.text === '++' || token.text === '--') && PREC_POSTFIX >= minPrecedence) { + this.exprNext(st); + left = this.frame('postfix_expression', left.start, token.end, [ + left, + this.anon(token.text, token.start, token.end), + ]); + continue; + } + if (token.kind === 'op' && token.text === '?' && PREC_TERNARY >= minPrecedence) { + this.exprNext(st); + st.expectOperator = false; + const kids: Frame[] = [left, this.anon('?', token.start, token.end)]; + const consequence = this.parseExpression(st, 0); + if (consequence === null) this.hasError = true; + else kids.push(consequence); + const colon = this.exprPeek(st); + if (colon.kind === 'op' && colon.text === ':') { + this.exprNext(st); + st.expectOperator = false; + kids.push(this.anon(':', colon.start, colon.end)); + } else { + this.hasError = true; // missing : in ternary + } + const alternative = this.parseExpression(st, PREC_TERNARY + 1); + if (alternative === null) this.hasError = true; + else kids.push(alternative); + left = this.frame('ternary_expression', left.start, this.endOf(kids, token.end), kids); + st.expectOperator = true; + continue; + } + const isTestOp = token.kind === 'testop'; + const precedence = isTestOp ? PREC_TEST : token.kind === 'op' ? EXPRESSION_PRECEDENCE[token.text] : undefined; + if (precedence === undefined || precedence < minPrecedence) break; + this.exprNext(st); + st.expectOperator = false; + // `=~` in a test command takes a raw regex (or string/expansion) as + // its right side instead of a full expression. + if (st.mode === 'test' && token.kind === 'op' && token.text === '=~') { + const right = this.parseTestRegex(st); + const kids: Frame[] = [left, this.anon('=~', token.start, token.end)]; + if (right === null) this.hasError = true; + else kids.push(right); + left = this.frame('binary_expression', left.start, this.endOf(kids, token.end), kids); + st.expectOperator = true; + continue; + } + const rightPrecedence = token.text === '**' && st.mode === 'test' ? precedence : precedence + 1; + let right: Frame | null; + // An extglob group pattern after ==/!= is one extglob_pattern node + // (`[[ $a == +(!a) ]]` in the reference). + if (st.mode === 'test' && (token.text === '==' || token.text === '!=')) { + right = this.tryParseExtglobGroup(st); + if (right !== null) { + const kids: Frame[] = [left, this.anon(token.text, token.start, token.end), right]; + left = this.frame('binary_expression', left.start, right.end, kids); + st.expectOperator = true; + continue; + } + } + right = this.parseExpression(st, rightPrecedence); + if (right === null) { + this.hasError = true; // missing right operand + } else if (st.mode === 'test' && token.kind === 'op') { + right = this.convertTestRightSide(token.text, right, st); + } + const operator = + token.kind === 'testop' + ? this.frame('test_operator', token.start, token.end) + : this.anon(token.text, token.start, token.end); + const kids: Frame[] = right === null ? [left, operator] : [left, operator, right]; + left = this.frame('binary_expression', left.start, this.endOf(kids, token.end), kids); + st.expectOperator = true; + } + return left; + } + + private parseExprPrimary(st: ExprState): Frame | null { + const token = this.exprPeek(st); + switch (token.kind) { + case 'end': + case 'rparen': + return null; + case 'number': + this.exprNext(st); + return this.frame('number', token.start, token.end); + case 'ident': { + this.exprNext(st); + if (st.mode === 'c') { + // `name = value` in a c-style for header is a variable_assignment + // (all other assignment operators are binary_expressions). + const next = this.exprPeek(st); + if (next.kind === 'op' && next.text === '=') { + this.exprNext(st); + const kids: Frame[] = [ + this.frame('variable_name', token.start, token.end), + this.anon('=', next.start, next.end), + ]; + const value = this.parseExpression(st, 0); + if (value === null) this.hasError = true; + else kids.push(value); + return this.frame('variable_assignment', token.start, this.endOf(kids, next.end), kids); + } + return this.frame('word', token.start, token.end); + } + return this.frame('variable_name', token.start, token.end); + } + case 'word': { + this.exprNext(st); + // In a test command a `-digits` operand is unary minus, not a + // number literal (`[[ $a == -1 ]]` → unary(-, number 1) — the + // opposite of the max-length rule in ${v: -1}). + if (st.mode === 'test' && /^-\d+$/.test(token.text)) { + return this.frame('unary_expression', token.start, token.end, [ + this.anon('-', token.start, token.start + 1), + this.frame('number', token.start + 1, token.end), + ]); + } + return this.parseLiteral(token.start, token.end); + } + case 'subst': + case 'string': + this.exprNext(st); + return token.frame!; + case 'lparen': { + this.exprNext(st); + if (this.exprDepth >= MAX_PARSE_DEPTH) { + this.hasError = true; + st.pos = st.end; + st.lookahead = null; + return this.frame('ERROR', token.start, st.end); + } + this.exprDepth++; + st.parenDepth++; + const kids: Frame[] = [this.anon('(', token.start, token.end)]; + const inner = this.parseExpression(st, 0); + if (inner !== null) kids.push(inner); + if (st.mode === 'c') { + // _c_parenthesized_expression: comma-separated expressions. + for (;;) { + const comma = this.exprPeek(st); + if (comma.kind !== 'op' || comma.text !== ',') break; + this.exprNext(st); + kids.push(this.anon(',', comma.start, comma.end)); + const next = this.parseExpression(st, 0); + if (next === null) { + this.hasError = true; + break; + } + kids.push(next); + } + } + let end = this.endOf(kids, token.end); + const close = this.exprPeek(st); + if (close.kind === 'rparen') { + this.exprNext(st); + kids.push(this.anon(')', close.start, close.end)); + end = close.end; + } else { + this.hasError = true; // unterminated parenthesized expression + } + st.parenDepth--; + this.exprDepth--; + return this.frame('parenthesized_expression', token.start, end, kids); + } + case 'op': + case 'testop': + case 'unknown': { + // An operator where an operand was expected: recover. + this.exprNext(st); + this.hasError = true; + return this.frame('ERROR', token.start, token.end); + } + } + } + + /** The right side of `=~` in a test command: a quoted string, an + * expansion, or a raw regex run (up to whitespace). */ + private parseTestRegex(st: ExprState): Frame | null { + let i = st.pos; + while (i < st.end && (this.source[i] === ' ' || this.source[i] === '\t' || this.source[i] === '\r')) i++; + st.pos = i; + st.lookahead = null; + if (i >= st.end) return null; + const ch = this.source[i]!; + if (ch === '"') { + const [piece, next] = this.parseString(i, st.end); + st.pos = next; + return piece; + } + if (ch === "'") { + const close = skipSingleQuoted(this.source, this.budget, i, st.end); + st.pos = close; + return this.frame('raw_string', i, close); + } + if (ch === '$') { + // An expansion holds the pattern: parse a normal operand. + return this.parseExpression(st, PREC_TEST + 1); + } + let j = i; + let sinceTick = 0; + while (j < st.end) { + if (++sinceTick >= SCAN_TICK_INTERVAL) { + this.budget.progress(); + sinceTick = 0; + } + const c = this.source[j]!; + if (c === ' ' || c === '\t' || c === '\r' || c === '\n') break; + if (c === '\\') { + j += 2; + continue; + } + j++; + } + st.pos = j; + return this.frame('regex', i, j); + } + + /** An extglob group pattern (`?(…)`, `*(…)`, `+(…)`, `@(…)`, `!(…)`) + * after `==`/`!=` in a test command: one raw extglob_pattern node over + * the operator and its balanced group. Null (no state touched) when the + * upcoming text is not a group. */ + private tryParseExtglobGroup(st: ExprState): Frame | null { + let i = st.pos; + while (i < st.end && (this.source[i] === ' ' || this.source[i] === '\t' || this.source[i] === '\r')) i++; + const ch = this.source[i]; + if ((ch === '?' || ch === '*' || ch === '+' || ch === '@' || ch === '!') && this.source[i + 1] === '(') { + const scan = this.scanBalanced(i + 1, st.end, '(', ')'); + if (!scan.balanced) return null; + st.pos = scan.end; + st.lookahead = null; + return this.frame('extglob_pattern', i, scan.end); + } + return null; + } + + /** Reference-shaped right-hand sides for test comparisons: + * - after `=`, a bare word is a regex when it contains a glob character + * or `=` — but only outside parentheses (`[[ a = b*c ]]` → regex, + * `[[ a = b=c ]]` → regex, `[[ ( $a = x* ) ]]` → word); + * - after `==`/`!=`, a bare word follows the scanner.c extglob_pattern + * rule — see isTestExtglob. + * The right side may arrive as a concatenation of word/number fragments + * (the lexer splits [ ] into their own pieces, so `[0-9]*` is not one + * word); bare fragment runs are judged — and converted — as a whole. */ + private convertTestRightSide(operator: string, right: Frame, st: ExprState): Frame { + if (right.type !== 'word' && !(right.type === 'concatenation' && this.isBarePieces(right))) { + return right; + } + const text = this.text(right.start, right.end); + if (operator === '=' && st.parenDepth === 0 && /[*?[\]=]/.test(text)) { + return this.frame('regex', right.start, right.end); + } + if ((operator === '==' || operator === '!=') && this.isTestExtglob(right)) { + return this.frame('extglob_pattern', right.start, right.end); + } + return right; + } + + /** True when a concatenation consists only of word/number fragments, so + * it can be treated as one bare word in test comparisons. */ + private isBarePieces(frame: Frame): boolean { + return frame.children.every((child) => child.type === 'word' || child.type === 'number'); + } + + /** + * The extglob_pattern scan rule from tree-sitter-bash's scanner.c + * (the `extglob_pattern:` label), reduced to what a bare test word can + * reach (test words never contain blanks, quotes, `$`, `|` or parens): + * - the first character must be a letter or one of `?*+@!-)\.[`; + * - a leading `-letter` is just a word (`[[ $a == -foo ]]` → word); + * - a single character becomes a glob only when directly followed by + * whitespace (`[[ $a == b ]]` → glob, `[[ (a == b) ]]` → word); + * - a `-letters` second run followed by `)`, `\` or `.` stays a word; + * - otherwise the second character must be alphanumeric or one of + * `[?/\_*` (skipped when the word starts with `[`), and the word is + * a glob iff any character is neither a letter nor `.` — escapes + * count only the character after the backslash when it is not a + * space/quote (`foo\ bar` → word, `b\*c` → glob). + */ + private isTestExtglob(right: Frame): boolean { + const text = this.text(right.start, right.end); + const nextChar = this.source[right.end]; + const isSpace = (ch: string | undefined): boolean => ch === ' ' || ch === '\t' || ch === '\r'; + const isAlpha = (ch: string): boolean => /[A-Za-z]/.test(ch); + const isAlnum = (ch: string): boolean => /[A-Za-z0-9]/.test(ch); + const c0 = text[0]!; + if (!isAlpha(c0) && !'?*+@!-)\\.['.includes(c0)) return false; + let sawNonAlphaDot = !isAlpha(c0); + let i = 1; + if (i >= text.length) { + if (isSpace(nextChar) || nextChar === '|') return true; + if ((c0 === ')' || c0 === '*') && nextChar === ')' && isSpace(this.source[right.end + 1])) { + return sawNonAlphaDot; + } + return false; + } + // "-\w" is just a word after ==/!= in the reference. + if (c0 === '-' && isAlpha(text[i]!)) return false; + if (text[i] === '-') { + // "-\w" in second position: just a word, unless something special + // follows the run. + i++; + while (i < text.length && isAlnum(text[i]!)) i++; + const after = i < text.length ? text[i]! : nextChar; + if (after === ')' || after === '\\' || after === '.') return false; + if (i >= text.length) return true; + } + // The second-character check is skipped for a leading `[` (the scanner + // does not advance past it before checking). + if (c0 !== '[' && !isAlnum(text[i]!) && !'[?/\\_*'.includes(text[i]!)) return false; + for (; i < text.length; i++) { + const ch = text[i]!; + if (ch === '\\') { + // A backslash never counts itself; an escaped space/quote is + // consumed silently (anything else is judged on its own). + const after = text[i + 1]; + if (after === ' ' || after === '\t' || after === '\r' || after === '"') i++; + continue; + } + if (!isAlpha(ch) && ch !== '.') sawNonAlphaDot = true; + } + return sawNonAlphaDot; + } + + /** Character-level tokenizer for the expression engine over [st.pos, + * st.end). Produces one token per call and advances st.pos. */ + private scanExprToken(st: ExprState): ExprToken { + const end = st.end; + let i = st.pos; + let sinceTick = 0; + for (;;) { + if (++sinceTick >= SCAN_TICK_INTERVAL) { + this.budget.progress(); + sinceTick = 0; + } + if (i >= end) break; + const ch = this.source[i]!; + if (ch === ' ' || ch === '\t' || ch === '\r' || ch === '\n') { + i++; + continue; + } + if (ch === '\\' && this.source[i + 1] === '\n') { + i += 2; // line continuation + continue; + } + break; + } + st.pos = i; + if (i >= end) return { kind: 'end', start: i, end: i, text: '' }; + const ch = this.source[i]!; + if (st.mode === 'test') return this.scanTestToken(st, i, ch); + + // Arithmetic / c-style-for tokens. + if (ch >= '0' && ch <= '9') { + let j = i; + if (ch === '0' && (this.source[i + 1] === 'x' || this.source[i + 1] === 'X')) { + j = i + 2; + while (j < end && /[0-9a-fA-F]/.test(this.source[j]!)) j++; + } else { + while (j < end && this.source[j]! >= '0' && this.source[j]! <= '9') j++; + if (this.source[j] === '#') { + j++; + while (j < end && /[0-9A-Za-z@_]/.test(this.source[j]!)) j++; + } + } + st.pos = j; + return { kind: 'number', start: i, end: j, text: this.text(i, j) }; + } + if (/[A-Za-z_]/.test(ch)) { + let j = i + 1; + while (j < end && /\w/.test(this.source[j]!)) j++; + if (this.source[j] === '[' && j < end) { + // Subscript: name[index] — the index is a literal. + const sub = this.scanBalanced(j, end, '[', ']'); + if (!sub.balanced) this.hasError = true; + const indexEnd = sub.balanced ? sub.end - 1 : sub.end; + const kids: Frame[] = [this.frame('variable_name', i, j), this.anon('[', j, j + 1)]; + if (indexEnd > j + 1) { + kids.push(this.parseLiteral(j + 1, indexEnd)); + } else { + this.hasError = true; // empty subscript index + } + if (sub.balanced) kids.push(this.anon(']', sub.end - 1, sub.end)); + st.pos = sub.end; + return { kind: 'subst', start: i, end: sub.end, text: this.text(i, sub.end), frame: this.frame('subscript', i, sub.end, kids) }; + } + st.pos = j; + return { kind: 'ident', start: i, end: j, text: this.text(i, j) }; + } + if (ch === '$') { + const dollar = this.parseDollar(i, end); + if (dollar !== null) { + st.pos = dollar[1]; + return { kind: 'subst', start: i, end: dollar[1], text: this.text(i, dollar[1]), frame: dollar[0] }; + } + this.hasError = true; // bare $ inside arithmetic + st.pos = i + 1; + return { kind: 'subst', start: i, end: i + 1, text: '$', frame: this.frame('ERROR', i, i + 1) }; + } + if (ch === '"') { + const [piece, next] = this.parseString(i, end); + st.pos = next; + return { kind: 'string', start: i, end: next, text: this.text(i, next), frame: piece }; + } + if (ch === '`') { + const [piece, next] = this.parseBacktickSubstitution(i, end); + st.pos = next; + return { kind: 'subst', start: i, end: next, text: this.text(i, next), frame: piece }; + } + if (ch === '(') { + st.pos = i + 1; + return { kind: 'lparen', start: i, end: i + 1, text: '(' }; + } + if (ch === ')') { + st.pos = i + 1; + return { kind: 'rparen', start: i, end: i + 1, text: ')' }; + } + for (const operator of EXPRESSION_OPERATORS) { + if (i + operator.length <= end && this.source.startsWith(operator, i)) { + st.pos = i + operator.length; + return { kind: 'op', start: i, end: st.pos, text: operator }; + } + } + // Unknown character: recover one char at a time. + st.pos = i + 1; + return { kind: 'unknown', start: i, end: i + 1, text: ch }; + } + + /** Tokenizer for test-command expressions ([[ … ]] / [ … ]). Words stop + * at whitespace and ( ) < > & | only — `=` and `!` are word characters + * unless they start a token (`a=b` is one word in the reference, while + * spaced `=`/`==`/`!=`/`=~` are operators). */ + private scanTestToken(st: ExprState, i: number, ch: string): ExprToken { + const end = st.end; + if (ch === '(') { + // `((…))` inside a test is an arithmetic_expansion when the balanced + // region closes with `))` (`[[ ((a)) == x ]]` in the reference); + // otherwise the parens nest as parenthesized_expressions. + if (this.source[i + 1] === '(') { + const scan = this.scanBalanced(i, end, '(', ')'); + if (scan.balanced && scan.end - 2 > i + 1 && this.source[scan.end - 2] === ')') { + const [piece, next] = this.parseParenArithmetic(i, end); + st.pos = next; + return { kind: 'subst', start: i, end: next, text: this.text(i, next), frame: piece }; + } + } + st.pos = i + 1; + return { kind: 'lparen', start: i, end: i + 1, text: '(' }; + } + if (ch === ')') { + st.pos = i + 1; + return { kind: 'rparen', start: i, end: i + 1, text: ')' }; + } + if (ch === '&' && this.source[i + 1] === '&') { + st.pos = i + 2; + return { kind: 'op', start: i, end: i + 2, text: '&&' }; + } + if (ch === '|' && this.source[i + 1] === '|') { + st.pos = i + 2; + return { kind: 'op', start: i, end: i + 2, text: '||' }; + } + if (ch === '<' || ch === '>') { + const wide = this.source[i + 1] === '=' ? 2 : 1; + st.pos = i + wide; + return { kind: 'op', start: i, end: i + wide, text: this.text(i, i + wide) }; + } + if (ch === '!' || ch === '=') { + if (st.expectOperator) { + // After an operand these are comparison operators even when the + // next word is attached (`=~^x`, `==b`, `=b` all split). + const two = this.source.slice(i, i + 2); + const wide = two === '==' || two === '=~' || two === '!=' ? 2 : 1; + st.pos = i + wide; + return { kind: 'op', start: i, end: i + wide, text: this.text(i, i + wide) }; + } + // At operand position only a standalone `!` is the prefix operator; + // everything else (=b, !x, !=b) is word material in the reference. + const after = this.source[i + 1]; + if (ch === '!' && (after === undefined || after === ' ' || after === '\t' || after === '\r' || i + 1 >= end)) { + st.pos = i + 1; + return { kind: 'op', start: i, end: i + 1, text: '!' }; + } + // Fall through to the word region below. + } + if (ch === '-' && /[A-Za-z]/.test(this.source[i + 1] ?? '')) { + // test_operator: -letters followed by whitespace AND by a real + // operand (scanner.c's rule: -\w followed by an operator, the closer + // or nothing is just a word — `[[ -foo == x ]]`, `[[ $a == -foo ]]`, + // `[[ -f ]]` all keep `-foo`/`-f` as words in the reference). + let j = i + 1; + while (j < end && /[A-Za-z]/.test(this.source[j]!)) j++; + const after = this.source[j]; + if (j < end && (after === ' ' || after === '\t' || after === '\r')) { + let k = j; + while (k < end && (this.source[k] === ' ' || this.source[k] === '\t' || this.source[k] === '\r')) k++; + const next = this.source[k]; + if (k < end && next !== '=' && next !== ']' && next !== '&' && next !== '|' && next !== ')') { + st.pos = j; + return { kind: 'testop', start: i, end: j, text: this.text(i, j) }; + } + } + } + // A word region: quote- and substitution-aware, stopping at blanks and + // the operator characters above. + let j = i; + let sinceTick = 0; + while (j < end) { + if (++sinceTick >= SCAN_TICK_INTERVAL) { + this.budget.progress(); + sinceTick = 0; + } + const c = this.source[j]!; + if (c === ' ' || c === '\t' || c === '\r' || c === '\n') break; + if (c === '(' || c === ')' || c === '<' || c === '>' || c === '&' || c === '|') break; + if (c === '"') { + j = skipDoubleQuoted(this.source, this.budget, j, end); + continue; + } + if (c === "'") { + j = skipSingleQuoted(this.source, this.budget, j, end); + continue; + } + if (c === '`') { + j = skipBacktick(this.source, this.budget, j, end); + continue; + } + if (c === '$') { + j = this.skipDollarConstruct(j, end); + continue; + } + if (c === '\\') { + j += 2; + continue; + } + j++; + } + if (j === i) j++; // defensive: never stall + st.pos = j; + // A lone arithmetic operator character is an operator, not a word. + if (j === i + 1 && (ch === '+' || ch === '*' || ch === '/' || ch === '%')) { + return { kind: 'op', start: i, end: j, text: ch }; + } + return { kind: 'word', start: i, end: j, text: this.text(i, j) }; + } + + // ------------------------------------------------------------ test command + + /** + * test_command in the `(( expression ))` form — only reached for + * statement-position `((word++…))` / `((word--…))` (see PAREN_TEST_RE); + * every other `((…))` is an arithmetic_expansion. The header arrived as + * one word token (see the lexer), so no repositioning is needed. + */ + private parseParenTestCommand(): Frame { + const token = this.lexer.next(); + const closed = + token.end - token.start >= 4 && this.source[token.end - 2] === ')' && this.source[token.end - 1] === ')'; + if (!closed) this.hasError = true; + const kids: Frame[] = [this.anon('((', token.start, token.start + 2)]; + const innerEnd = closed ? token.end - 2 : token.end; + if (innerEnd > token.start + 2) { + const st = this.newExprState(token.start + 2, innerEnd, 'test'); + const expression = this.parseExpression(st, 0); + if (expression !== null) kids.push(expression); + const leftover = this.exprLeftover(st); + if (leftover !== null) kids.push(leftover); + } + if (closed) { + kids.push(this.anon('))', token.end - 2, token.end)); + } + return this.frame('test_command', token.start, this.endOf(kids, token.end), kids); + } + + /** + * test_command: `[[ expression ]]` or `[ expression ]`. The peeked token + * is the single-character word `[`; `[[` is detected by the adjacent + * second `[` in the source. The expression range is scanned + * character-wise (quote/substitution aware) up to the closer or the end + * of the line, parsed by the expression engine in test mode, and the main + * lexer is then repositioned past the closer. + */ + private parseTestCommand(): Frame { + const openToken = this.lexer.next(); + const double = this.source[openToken.end] === '['; + const opener = double ? '[[' : '['; + const closer = double ? ']]' : ']'; + const exprStart = openToken.end + (double ? 1 : 0); + const kids: Frame[] = [this.anon(opener, openToken.start, exprStart)]; + const scan = this.scanTestCloser(exprStart, double); + if (scan.closerStart > exprStart) { + const st = this.newExprState(exprStart, scan.closerStart, 'test'); + const expression = this.parseExpression(st, 0); + if (expression !== null) kids.push(expression); + const leftover = this.exprLeftover(st); + if (leftover !== null) kids.push(leftover); + } + let end = scan.closerStart; + if (scan.found) { + if (double && kids.length === 1) { + // `[[ ]]` (empty expression): the reference parses the closer text + // as two word pieces inside a concatenation and inserts a + // zero-width `]]` after them. + this.hasError = true; + kids.push( + this.frame('concatenation', scan.closerStart, scan.afterCloser, [ + this.frame('word', scan.closerStart, scan.closerStart + 1), + this.frame('word', scan.closerStart + 1, scan.afterCloser), + ]), + ); + kids.push(this.anon(closer, scan.afterCloser, scan.afterCloser)); + } else { + kids.push(this.anon(closer, scan.closerStart, scan.afterCloser)); + } + end = scan.afterCloser; + } else { + // Unterminated: keep a zero-width closer and flag the error (our + // documented recovery policy); trailing blanks stay outside the node, + // as in the reference. The reference inserts a zero-width closer only + // for an unterminated `[` or for `[[` ending in a complete unary + // test — other unterminated `[[` forms become an ERROR node there. + this.hasError = true; + let closeAt = scan.closerStart; + while (closeAt > exprStart && (this.source[closeAt - 1] === ' ' || this.source[closeAt - 1] === '\t' || this.source[closeAt - 1] === '\r')) { + closeAt--; + } + kids.push(this.anon(closer, closeAt, closeAt)); + end = closeAt; + } + this.lexer.reposition(end); + return this.frame('test_command', openToken.start, end, kids); + } + + /** Find the closer of a test command, skipping quotes and substitutions. + * Bounded by the end of the line and the lexer's range. */ + private scanTestCloser(start: number, double: boolean): { closerStart: number; afterCloser: number; found: boolean } { + const end = this.lexer.rangeEnd; + let j = start; + let sinceTick = 0; + while (j < end) { + if (++sinceTick >= SCAN_TICK_INTERVAL) { + this.budget.progress(); + sinceTick = 0; + } + const ch = this.source[j]!; + if (ch === '\n') break; + if (ch === '\\') { + j += 2; + continue; + } + if (ch === '"') { + j = skipDoubleQuoted(this.source, this.budget, j, end); + continue; + } + if (ch === "'") { + j = skipSingleQuoted(this.source, this.budget, j, end); + continue; + } + if (ch === '`') { + j = skipBacktick(this.source, this.budget, j, end); + continue; + } + if (ch === '$') { + j = this.skipDollarConstruct(j, end); + continue; + } + if ((ch === '<' || ch === '>') && this.source[j + 1] === '(') { + j = this.scanBalanced(j + 1, end, '(', ')').end; + continue; + } + if (ch === ']') { + if (double) { + if (this.source[j + 1] === ']') return { closerStart: j, afterCloser: j + 2, found: true }; + } else { + return { closerStart: j, afterCloser: j + 1, found: true }; + } + } + j++; + } + return { closerStart: j, afterCloser: j, found: false }; + } + /** heredoc_body content: heredoc_content chunks interleaved with * expansions and command substitutions (unquoted delimiters only). * Unlike tree-sitter-bash's scanner, every plain chunk — including the @@ -1089,7 +3087,7 @@ export class Parser { }; let sinceTick = 0; while (i < end) { - if (++sinceTick >= 2048) { + if (++sinceTick >= SCAN_TICK_INTERVAL) { this.budget.progress(); sinceTick = 0; } diff --git a/packages/tree-sitter-bash/test/parse.test.ts b/packages/tree-sitter-bash/test/parse.test.ts index 9a8b25f8a8..bb322741f4 100644 --- a/packages/tree-sitter-bash/test/parse.test.ts +++ b/packages/tree-sitter-bash/test/parse.test.ts @@ -114,7 +114,7 @@ describe('simple commands', () => { it('attaches comments as named children', () => { expectTree( '# a comment\necho hi # trailing', - `(program (comment "# a comment") "\\n" (command (command_name (word "echo")) (word "hi")) (comment "# trailing"))`, + `(program (comment "# a comment") (command (command_name (word "echo")) (word "hi")) (comment "# trailing"))`, ); }); @@ -192,10 +192,10 @@ describe('expansions and substitutions', () => { ); }); - it('parses $((...)) as an arithmetic_expansion placeholder (TODO(M2))', () => { + it('parses $((...)) as a real arithmetic expression', () => { expectTree( 'echo $((1<<2))', - `(program (command (command_name (word "echo")) (arithmetic_expansion "$((" (word "1<<2") "))")))`, + `(program (command (command_name (word "echo")) (arithmetic_expansion "$((" (binary_expression (number "1") "<<" (number "2")) "))")))`, ); }); }); @@ -321,7 +321,7 @@ describe('heredocs', () => { it('degrades a second heredoc on the same line to ERROR (matches tree-sitter-bash)', () => { expectTree( 'cat < { it('mixes semicolons and newlines as terminators', () => { expectTree( 'ls; echo a\necho b', - `(program (command (command_name (word "ls"))) ";" (command (command_name (word "echo")) (word "a")) "\\n" (command (command_name (word "echo")) (word "b")))`, + `(program (command (command_name (word "ls"))) ";" (command (command_name (word "echo")) (word "a")) (command (command_name (word "echo")) (word "b")))`, ); }); diff --git a/packages/tree-sitter-bash/test/parser-compound.test.ts b/packages/tree-sitter-bash/test/parser-compound.test.ts new file mode 100644 index 0000000000..a411fb4236 --- /dev/null +++ b/packages/tree-sitter-bash/test/parser-compound.test.ts @@ -0,0 +1,1232 @@ +import { describe, expect, it } from 'vitest'; + +import type { SyntaxNode } from '#/node'; +import { descendantsOfType } from '#/node'; +import { parse } from '#/parse'; + +/** Compact S-expression of the tree: named leaves are (type "text"), + * anonymous leaves are just their quoted type, inner nodes nest. */ +function sexp(node: SyntaxNode): string { + if (node.children.length === 0) { + return node.isNamed ? `(${node.type} ${JSON.stringify(node.text)})` : JSON.stringify(node.type); + } + const label = node.isNamed ? node.type : JSON.stringify(node.type); + return `(${label} ${node.children.map(sexp).join(' ')})`; +} + +function parseOk(source: string): { rootNode: SyntaxNode; hasError: boolean } { + const result = parse(source); + expect(result.ok).toBe(true); + if (!result.ok) throw new Error('unreachable'); + return result; +} + +/** Parse and assert the exact tree shape plus the hasError flag. */ +function expectTree(source: string, expected: string, hasError = false): SyntaxNode { + const { rootNode, hasError: actual } = parseOk(source); + expect(actual).toBe(hasError); + expect(sexp(rootNode)).toBe(expected); + return rootNode; +} + +describe('if / while / until', () => { + it('parses a one-line if statement', () => { + expectTree( + 'if true; then echo yes; fi', + `(program (if_statement "if" (command (command_name (word "true"))) ";" "then" (command (command_name (word "echo")) (word "yes")) ";" "fi"))`, + ); + }); + + it('parses elif and else clauses', () => { + expectTree( + 'if a; then b; elif c; then d; else e; fi', + `(program (if_statement "if" (command (command_name (word "a"))) ";" "then" (command (command_name (word "b"))) ";" (elif_clause "elif" (command (command_name (word "c"))) ";" "then" (command (command_name (word "d"))) ";") (else_clause "else" (command (command_name (word "e"))) ";") "fi"))`, + ); + }); + + it('parses a multiline if without emitting newline nodes', () => { + expectTree( + 'if a\nthen\n b\nfi', + `(program (if_statement "if" (command (command_name (word "a"))) "then" (command (command_name (word "b"))) "fi"))`, + ); + }); + + it('parses a while loop with a trailing redirect', () => { + expectTree( + 'while read -r line; do echo "$line"; done < file', + `(program (redirected_statement (while_statement "while" (command (command_name (word "read")) (word "-r") (word "line")) ";" (do_group "do" (command (command_name (word "echo")) (string "\\"" (simple_expansion "$" (variable_name "line")) "\\"")) ";" "done")) (file_redirect "<" (word "file"))))`, + ); + }); + + it('parses until as a while_statement', () => { + expectTree( + 'until x; do y; done', + `(program (while_statement "until" (command (command_name (word "x"))) ";" (do_group "do" (command (command_name (word "y"))) ";" "done")))`, + ); + }); + + it('parses an empty do_group', () => { + expectTree( + 'while x; do done', + `(program (while_statement "while" (command (command_name (word "x"))) ";" (do_group "do" "done")))`, + ); + }); + + it('only treats keywords as keywords in statement position', () => { + expectTree( + 'echo if for while', + `(program (command (command_name (word "echo")) (word "if") (word "for") (word "while")))`, + ); + expectTree('fi', `(program (command (command_name (word "fi"))))`); + expectTree('done', `(program (command (command_name (word "done"))))`); + expectTree('}', `(program (command (command_name (word "}"))))`); + expectTree('time ls -la', `(program (command (command_name (word "time")) (word "ls") (word "-la")))`); + // Reserved words are plain arguments inside conditions and bodies. + expectTree( + 'if echo then; then x; fi', + `(program (if_statement "if" (command (command_name (word "echo")) (word "then")) ";" "then" (command (command_name (word "x"))) ";" "fi"))`, + ); + // A prefix assignment disables the keyword reading. + expectTree( + 'x=1 if a; then b; fi', + `(program (command (variable_assignment (variable_name "x") "=" (number "1")) (command_name (word "if")) (word "a")) ";" (command (command_name (word "then")) (word "b")) ";" (command (command_name (word "fi"))))`, + ); + }); +}); + +describe('for / select', () => { + it('parses a for-in loop', () => { + expectTree( + 'for f in a b c; do echo "$f"; done', + `(program (for_statement "for" (variable_name "f") "in" (word "a") (word "b") (word "c") ";" (do_group "do" (command (command_name (word "echo")) (string "\\"" (simple_expansion "$" (variable_name "f")) "\\"")) ";" "done")))`, + ); + }); + + it('parses a for loop without in', () => { + expectTree( + 'for f; do x; done', + `(program (for_statement "for" (variable_name "f") ";" (do_group "do" (command (command_name (word "x"))) ";" "done")))`, + ); + }); + + it('parses select as a for_statement', () => { + expectTree( + 'select opt in a b; do echo $opt; done', + `(program (for_statement "select" (variable_name "opt") "in" (word "a") (word "b") ";" (do_group "do" (command (command_name (word "echo")) (simple_expansion "$" (variable_name "opt"))) ";" "done")))`, + ); + }); + + it('parses newline-separated for headers', () => { + expectTree( + 'for f in a b\ndo x\ndone', + `(program (for_statement "for" (variable_name "f") "in" (word "a") (word "b") (do_group "do" (command (command_name (word "x"))) "done")))`, + ); + }); + + it('flags a bare `in` with no values as an error', () => { + expectTree( + 'select x in; do y; done', + `(program (for_statement "select" (variable_name "x") (ERROR "in") ";" (do_group "do" (command (command_name (word "y"))) ";" "done")))`, + true, + ); + }); + + it('parses a c-style for loop', () => { + expectTree( + 'for ((i=0;i<3;i++)); do echo $i; done', + `(program (c_style_for_statement "for" "((" (variable_assignment (variable_name "i") "=" (number "0")) ";" (binary_expression (word "i") "<" (number "3")) ";" (postfix_expression (word "i") "++") "))" ";" (do_group "do" (command (command_name (word "echo")) (simple_expansion "$" (variable_name "i"))) ";" "done")))`, + ); + }); + + it('parses a spaced c-style for with an update assignment operator', () => { + expectTree( + 'for (( i = 0; i < 10; i += 2 )); do x; done', + `(program (c_style_for_statement "for" "((" (variable_assignment (variable_name "i") "=" (number "0")) ";" (binary_expression (word "i") "<" (number "10")) ";" (binary_expression (word "i") "+=" (number "2")) "))" ";" (do_group "do" (command (command_name (word "x"))) ";" "done")))`, + ); + }); + + it('parses an empty c-style for header', () => { + expectTree( + 'for ((;;)); do x; done', + `(program (c_style_for_statement "for" "((" ";" ";" "))" ";" (do_group "do" (command (command_name (word "x"))) ";" "done")))`, + ); + }); + + it('parses comma-separated c-style for parts', () => { + expectTree( + 'for ((i=0,j=1; i<3; i++,j--)); do x; done', + `(program (c_style_for_statement "for" "((" (variable_assignment (variable_name "i") "=" (number "0")) "," (variable_assignment (variable_name "j") "=" (number "1")) ";" (binary_expression (word "i") "<" (number "3")) ";" (postfix_expression (word "i") "++") "," (postfix_expression (word "j") "--") "))" ";" (do_group "do" (command (command_name (word "x"))) ";" "done")))`, + ); + }); + + it('parses a compound_statement as the c-style for body', () => { + expectTree( + 'for ((i=0;i<3;i++)) { echo $i; }', + `(program (c_style_for_statement "for" "((" (variable_assignment (variable_name "i") "=" (number "0")) ";" (binary_expression (word "i") "<" (number "3")) ";" (postfix_expression (word "i") "++") "))" (compound_statement "{" (command (command_name (word "echo")) (simple_expansion "$" (variable_name "i"))) ";" "}")))`, + ); + }); +}); + +describe('case', () => { + it('parses a multi-item case with all termination forms', () => { + expectTree( + 'case $x in\n a) echo A ;;\n b|c) echo BC ;&\n *) echo other ;;\nesac', + `(program (case_statement "case" (simple_expansion "$" (variable_name "x")) "in" (case_item (word "a") ")" (command (command_name (word "echo")) (word "A")) ";;") (case_item (extglob_pattern "b") "|" (word "c") ")" (command (command_name (word "echo")) (word "BC")) ";&") (case_item (extglob_pattern "*") ")" (command (command_name (word "echo")) (word "other")) ";;") "esac"))`, + ); + }); + + it('parses optional parens and empty item bodies', () => { + expectTree( + 'case $x in (a) x ;; esac', + `(program (case_statement "case" (simple_expansion "$" (variable_name "x")) "in" (case_item "(" (word "a") ")" (command (command_name (word "x"))) ";;") "esac"))`, + ); + expectTree( + 'case x in a) ;; esac', + `(program (case_statement "case" (word "x") "in" (case_item (word "a") ")" ";;") "esac"))`, + ); + expectTree( + 'case $x in esac', + `(program (case_statement "case" (simple_expansion "$" (variable_name "x")) "in" "esac"))`, + ); + }); + + it('parses pattern forms: globs, quotes and expansions', () => { + expectTree( + 'case $x in [a-z]) x ;; esac', + `(program (case_statement "case" (simple_expansion "$" (variable_name "x")) "in" (case_item (extglob_pattern "[a-z]") ")" (command (command_name (word "x"))) ";;") "esac"))`, + ); + expectTree( + 'case $x in "a") x ;; esac', + `(program (case_statement "case" (simple_expansion "$" (variable_name "x")) "in" (case_item (string "\\"" (string_content "a") "\\"") ")" (command (command_name (word "x"))) ";;") "esac"))`, + ); + expectTree( + 'case $x in $v|${w}) x ;; esac', + `(program (case_statement "case" (simple_expansion "$" (variable_name "x")) "in" (case_item (simple_expansion "$" (variable_name "v")) "|" (expansion "\${" (variable_name "w") "}") ")" (command (command_name (word "x"))) ";;") "esac"))`, + ); + }); + + it('ends the last case_item at esac even without ;;', () => { + expectTree( + 'case $x in a) x esac', + `(program (case_statement "case" (simple_expansion "$" (variable_name "x")) "in" (case_item (word "a") ")" (command (command_name (word "x")))) "esac"))`, + ); + }); + + it('flags a fallthrough terminator on the last case_item', () => { + const { hasError } = parseOk('case $x in a) x ;;& esac'); + expect(hasError).toBe(true); + }); +}); + +describe('functions and compound statements', () => { + it('parses the name() form', () => { + expectTree( + 'foo() { echo hi; }', + `(program (function_definition (word "foo") "(" ")" (compound_statement "{" (command (command_name (word "echo")) (word "hi")) ";" "}")))`, + ); + }); + + it('parses the function keyword forms', () => { + expectTree( + 'function bar { echo hi; }', + `(program (function_definition "function" (word "bar") (compound_statement "{" (command (command_name (word "echo")) (word "hi")) ";" "}")))`, + ); + expectTree( + 'function baz() { echo hi; return 0; }', + `(program (function_definition "function" (word "baz") "(" ")" (compound_statement "{" (command (command_name (word "echo")) (word "hi")) ";" (command (command_name (word "return")) (number "0")) ";" "}")))`, + ); + expectTree( + 'f () { x; }', + `(program (function_definition (word "f") "(" ")" (compound_statement "{" (command (command_name (word "x"))) ";" "}")))`, + ); + }); + + it('parses subshell, test and if bodies', () => { + expectTree( + 'foo() ( echo sub )', + `(program (function_definition (word "foo") "(" ")" (subshell "(" (command (command_name (word "echo")) (word "sub")) ")")))`, + ); + expectTree( + 'foo() [[ -f x ]]', + `(program (function_definition (word "foo") "(" ")" (test_command "[[" (unary_expression (test_operator "-f") (word "x")) "]]")))`, + ); + expectTree( + 'foo() if a; then b; fi', + `(program (function_definition (word "foo") "(" ")" (if_statement "if" (command (command_name (word "a"))) ";" "then" (command (command_name (word "b"))) ";" "fi")))`, + ); + }); + + it('keeps a trailing redirect inside the function_definition', () => { + expectTree( + 'foo() { x; } > out', + `(program (function_definition (word "foo") "(" ")" (compound_statement "{" (command (command_name (word "x"))) ";" "}") (file_redirect ">" (word "out"))))`, + ); + }); + + it('parses a standalone compound statement', () => { + expectTree( + '{ echo a; echo b; }', + `(program (compound_statement "{" (command (command_name (word "echo")) (word "a")) ";" (command (command_name (word "echo")) (word "b")) ";" "}"))`, + ); + expectTree('{ls;}', `(program (compound_statement "{" (command (command_name (word "ls"))) ";" "}"))`); + }); + + it('parses coproc as an ordinary command (like the reference)', () => { + expectTree( + 'coproc myjob { echo hi; }', + `(program (command (command_name (word "coproc")) (word "myjob") (word "{") (word "echo") (word "hi")) ";" (command (command_name (word "}"))))`, + ); + }); +}); + +describe('test commands', () => { + it('parses unary test operators', () => { + expectTree( + '[[ -f file.txt ]]', + `(program (test_command "[[" (unary_expression (test_operator "-f") (word "file.txt")) "]]"))`, + ); + expectTree( + '[[ -z $s ]]', + `(program (test_command "[[" (unary_expression (test_operator "-z") (simple_expansion "$" (variable_name "s"))) "]]"))`, + ); + }); + + it('parses string comparisons', () => { + expectTree( + '[[ $x == "foo" ]]', + `(program (test_command "[[" (binary_expression (simple_expansion "$" (variable_name "x")) "==" (string "\\"" (string_content "foo") "\\"")) "]]"))`, + ); + expectTree( + '[ "$a" = "b" ]', + `(program (test_command "[" (binary_expression (string "\\"" (simple_expansion "$" (variable_name "a")) "\\"") "=" (string "\\"" (string_content "b") "\\"")) "]"))`, + ); + }); + + it('parses =~ with a regex right side', () => { + expectTree( + '[[ $x =~ ^ab+c$ ]]', + `(program (test_command "[[" (binary_expression (simple_expansion "$" (variable_name "x")) "=~" (regex "^ab+c$")) "]]"))`, + ); + expectTree( + '[[ $x =~ a|b ]]', + `(program (test_command "[[" (binary_expression (simple_expansion "$" (variable_name "x")) "=~" (regex "a|b")) "]]"))`, + ); + expectTree( + '[[ $x =~ $re ]]', + `(program (test_command "[[" (binary_expression (simple_expansion "$" (variable_name "x")) "=~" (simple_expansion "$" (variable_name "re"))) "]]"))`, + ); + }); + + it('parses pattern right sides: regex after =, extglob_pattern after ==/!=', () => { + expectTree( + '[[ a = b*c ]]', + `(program (test_command "[[" (binary_expression (word "a") "=" (regex "b*c")) "]]"))`, + ); + expectTree( + '[[ a = bc ]]', + `(program (test_command "[[" (binary_expression (word "a") "=" (word "bc")) "]]"))`, + ); + expectTree( + '[[ $x == b*c ]]', + `(program (test_command "[[" (binary_expression (simple_expansion "$" (variable_name "x")) "==" (extglob_pattern "b*c")) "]]"))`, + ); + expectTree( + '[[ $x == 123 ]]', + `(program (test_command "[[" (binary_expression (simple_expansion "$" (variable_name "x")) "==" (number "123")) "]]"))`, + ); + }); + + it('parses && / || / ! combinations', () => { + expectTree( + '[[ -n $s && -d /tmp ]]', + `(program (test_command "[[" (binary_expression (unary_expression (test_operator "-n") (simple_expansion "$" (variable_name "s"))) "&&" (unary_expression (test_operator "-d") (word "/tmp"))) "]]"))`, + ); + expectTree( + '[[ a < b || ! -e f ]]', + `(program (test_command "[[" (binary_expression (binary_expression (word "a") "<" (word "b")) "||" (unary_expression "!" (unary_expression (test_operator "-e") (word "f")))) "]]"))`, + ); + }); + + it('parses binary test operators and parentheses', () => { + expectTree( + '[[ $a -eq 3 ]]', + `(program (test_command "[[" (binary_expression (simple_expansion "$" (variable_name "a")) (test_operator "-eq") (number "3")) "]]"))`, + ); + expectTree( + '[[ $x -eq 1 && $y -ne 2 ]]', + `(program (test_command "[[" (binary_expression (binary_expression (simple_expansion "$" (variable_name "x")) (test_operator "-eq") (number "1")) "&&" (binary_expression (simple_expansion "$" (variable_name "y")) (test_operator "-ne") (number "2"))) "]]"))`, + ); + // Inside parentheses the == right side is a word, not extglob_pattern. + expectTree( + '[[ (a == b) && c ]]', + `(program (test_command "[[" (binary_expression (parenthesized_expression "(" (binary_expression (word "a") "==" (word "b")) ")") "&&" (word "c")) "]]"))`, + ); + }); + + it('parses the single-bracket form and adjacent =', () => { + expectTree('[ -f file ]', `(program (test_command "[" (unary_expression (test_operator "-f") (word "file")) "]"))`); + expectTree('[ x ]', `(program (test_command "[" (word "x") "]"))`); + expectTree('[[ a=b ]]', `(program (test_command "[[" (word "a=b") "]]"))`); + }); + + it('parses negated test commands', () => { + expectTree( + '! [[ -f x ]]', + `(program (negated_command "!" (test_command "[[" (unary_expression (test_operator "-f") (word "x")) "]]")))`, + ); + expectTree( + '! [ -f x ]', + `(program (negated_command "!" (test_command "[" (unary_expression (test_operator "-f") (word "x")) "]")))`, + ); + }); + + it('keeps [ in argument position as plain words', () => { + expectTree('echo [ x ]', `(program (command (command_name (word "echo")) (word "[") (word "x") (word "]")))`); + }); + + it('recovers an unterminated test command with a zero-width closer', () => { + expectTree('[ -f file', `(program (test_command "[" (unary_expression (test_operator "-f") (word "file")) "]"))`, true); + }); +}); + +describe('arithmetic', () => { + it('parses operator precedence levels', () => { + expectTree( + 'echo $((1 + 2 * 3))', + `(program (command (command_name (word "echo")) (arithmetic_expansion "$((" (binary_expression (number "1") "+" (binary_expression (number "2") "*" (number "3"))) "))")))`, + ); + expectTree( + 'echo $((x << 2 | 1))', + `(program (command (command_name (word "echo")) (arithmetic_expansion "$((" (binary_expression (binary_expression (variable_name "x") "<<" (number "2")) "|" (number "1")) "))")))`, + ); + expectTree( + 'echo $((n % 2 == 0))', + `(program (command (command_name (word "echo")) (arithmetic_expansion "$((" (binary_expression (binary_expression (variable_name "n") "%" (number "2")) "==" (number "0")) "))")))`, + ); + }); + + it('parses ternary, postfix, prefix and unary expressions', () => { + expectTree( + 'echo $((a > b ? a : b))', + `(program (command (command_name (word "echo")) (arithmetic_expansion "$((" (ternary_expression (binary_expression (variable_name "a") ">" (variable_name "b")) "?" (variable_name "a") ":" (variable_name "b")) "))")))`, + ); + expectTree( + 'echo $((i++ + --j))', + `(program (command (command_name (word "echo")) (arithmetic_expansion "$((" (binary_expression (postfix_expression (variable_name "i") "++") "+" (unary_expression "--" (variable_name "j"))) "))")))`, + ); + // Prefix operators grab the whole tighter-precedence expression, like + // the reference: -(x + ~y). + expectTree( + 'echo $((-x + ~y))', + `(program (command (command_name (word "echo")) (arithmetic_expansion "$((" (unary_expression "-" (binary_expression (variable_name "x") "+" (unary_expression "~" (variable_name "y")))) "))")))`, + ); + }); + + it('parses parentheses and exponent (left-associative in arithmetic)', () => { + expectTree( + 'echo $(( (1+2) ** 3 ))', + `(program (command (command_name (word "echo")) (arithmetic_expansion "$((" (binary_expression (parenthesized_expression "(" (binary_expression (number "1") "+" (number "2")) ")") "**" (number "3")) "))")))`, + ); + expectTree( + 'echo $((2 ** 3 ** 2))', + `(program (command (command_name (word "echo")) (arithmetic_expansion "$((" (binary_expression (binary_expression (number "2") "**" (number "3")) "**" (number "2")) "))")))`, + ); + }); + + it('parses assignment operators, subscripts and comma lists', () => { + expectTree( + 'echo $((x += 5))', + `(program (command (command_name (word "echo")) (arithmetic_expansion "$((" (binary_expression (variable_name "x") "+=" (number "5")) "))")))`, + ); + expectTree( + 'echo $((x = y = 1))', + `(program (command (command_name (word "echo")) (arithmetic_expansion "$((" (binary_expression (binary_expression (variable_name "x") "=" (variable_name "y")) "=" (number "1")) "))")))`, + ); + expectTree( + 'echo $((arr[0] + 1))', + `(program (command (command_name (word "echo")) (arithmetic_expansion "$((" (binary_expression (subscript (variable_name "arr") "[" (number "0") "]") "+" (number "1")) "))")))`, + ); + expectTree( + 'echo $((1, 2, 3))', + `(program (command (command_name (word "echo")) (arithmetic_expansion "$((" (number "1") "," (number "2") "," (number "3") "))")))`, + ); + }); + + it('parses the ((…)) command and legacy $[…] form', () => { + expectTree( + '((x = y + 1))', + `(program (command (command_name (arithmetic_expansion "((" (binary_expression (variable_name "x") "=" (binary_expression (variable_name "y") "+" (number "1"))) "))"))))`, + ); + expectTree( + '(( ls ))', + `(program (command (command_name (arithmetic_expansion "((" (variable_name "ls") "))"))))`, + ); + expectTree( + 'echo $[1+2]', + `(program (command (command_name (word "echo")) (arithmetic_expansion "$[" (binary_expression (number "1") "+" (number "2")) "]")))`, + ); + }); + + it('parses arithmetic across newlines and nested arithmetic', () => { + expectTree( + 'echo $((\n1 +\n2\n))', + `(program (command (command_name (word "echo")) (arithmetic_expansion "$((" (binary_expression (number "1") "+" (number "2")) "))")))`, + ); + expectTree( + 'for ((i = $((1 + 1)); i < 3; i++)); do x; done', + `(program (c_style_for_statement "for" "((" (variable_assignment (variable_name "i") "=" (arithmetic_expansion "$((" (binary_expression (number "1") "+" (number "1")) "))")) ";" (binary_expression (word "i") "<" (number "3")) ";" (postfix_expression (word "i") "++") "))" ";" (do_group "do" (command (command_name (word "x"))) ";" "done")))`, + ); + }); +}); + +describe('arrays and subscripts', () => { + it('parses array assignments', () => { + expectTree( + 'arr=(a b "c d")', + `(program (variable_assignment (variable_name "arr") "=" (array "(" (word "a") (word "b") (string "\\"" (string_content "c d") "\\"") ")")))`, + ); + expectTree( + 'x+=(c d)', + `(program (variable_assignment (variable_name "x") "+=" (array "(" (word "c") (word "d") ")")))`, + ); + }); + + it('parses subscript assignments', () => { + expectTree( + 'arr[0]=x', + `(program (variable_assignment (subscript (variable_name "arr") "[" (number "0") "]") "=" (word "x")))`, + ); + expectTree( + 'a[i+1]=$y', + `(program (variable_assignment (subscript (variable_name "a") "[" (word "i+1") "]") "=" (simple_expansion "$" (variable_name "y"))))`, + ); + expectTree( + 'x[1]+=2', + `(program (variable_assignment (subscript (variable_name "x") "[" (number "1") "]") "+=" (number "2")))`, + ); + }); + + it('parses subscript expansions including @ and $ indexes', () => { + expectTree( + 'echo ${arr[@]}', + `(program (command (command_name (word "echo")) (expansion "\${" (subscript (variable_name "arr") "[" (word "@") "]") "}")))`, + ); + expectTree( + 'echo ${a[$i]}', + `(program (command (command_name (word "echo")) (expansion "\${" (subscript (variable_name "a") "[" (simple_expansion "$" (variable_name "i")) "]") "}")))`, + ); + }); +}); + +describe('declaration and unset commands', () => { + it('parses export/declare/local/readonly', () => { + expectTree( + 'export FOO=bar BAZ', + `(program (declaration_command "export" (variable_assignment (variable_name "FOO") "=" (word "bar")) (variable_name "BAZ")))`, + ); + expectTree( + 'declare -r x=1', + `(program (declaration_command "declare" (word "-r") (variable_assignment (variable_name "x") "=" (number "1"))))`, + ); + expectTree('local y', `(program (declaration_command "local" (variable_name "y")))`); + expectTree( + 'readonly z=2', + `(program (declaration_command "readonly" (variable_assignment (variable_name "z") "=" (number "2"))))`, + ); + }); + + it('parses unset', () => { + expectTree('unset a b', `(program (unset_command "unset" (variable_name "a") (variable_name "b")))`); + }); +}); + +describe('strings and brace expressions', () => { + it('parses ANSI-C strings as a single node', () => { + expectTree( + `echo $'a\\nb\\t'`, + `(program (command (command_name (word "echo")) (ansi_c_string "$'a\\\\nb\\\\t'")))`, + ); + }); + + it('parses $"…" as an anonymous $ plus a string (no translated_string)', () => { + expectTree( + 'echo $"hello $USER"', + `(program (command (command_name (word "echo")) "$" (string "\\"" (string_content "hello ") (simple_expansion "$" (variable_name "USER")) "\\"")))`, + ); + }); + + it('parses $"…" as translated_string outside argument position', () => { + expectTree( + 'x=$"v"', + `(program (variable_assignment (variable_name "x") "=" (translated_string "$" (string "\\"" (string_content "v") "\\""))))`, + ); + expectTree( + 'cat > $"out"', + `(program (redirected_statement (command (command_name (word "cat"))) (file_redirect ">" (translated_string "$" (string "\\"" (string_content "out") "\\"")))))`, + ); + expectTree( + 'for f in $"a"; do x; done', + `(program (for_statement "for" (variable_name "f") "in" (translated_string "$" (string "\\"" (string_content "a") "\\"")) ";" (do_group "do" (command (command_name (word "x"))) ";" "done")))`, + ); + // Mid-concatenation the bare $ and the string stay separate pieces. + expectTree( + 'echo a$"b"', + `(program (command (command_name (word "echo")) (concatenation (word "a") "$" (string "\\"" (string_content "b") "\\""))))`, + ); + }); + + it('parses {N..M} brace expressions', () => { + expectTree( + 'echo {1..10}', + `(program (command (command_name (word "echo")) (brace_expression "{" (number "1") ".." (number "10") "}")))`, + ); + expectTree( + 'echo a{1..3}b', + `(program (command (command_name (word "echo")) (concatenation (word "a") (brace_expression "{" (number "1") ".." (number "3") "}") (word "b"))))`, + ); + expectTree( + 'x={1..5}', + `(program (variable_assignment (variable_name "x") "=" (brace_expression "{" (number "1") ".." (number "5") "}")))`, + ); + expectTree( + '{1..3}', + `(program (command (command_name (brace_expression "{" (number "1") ".." (number "3") "}"))))`, + ); + }); + + it('parses other brace forms as plain concatenations (like the reference)', () => { + expectTree( + 'echo {1..10..2}', + `(program (command (command_name (word "echo")) (concatenation (word "{") (word "1..10..2") (word "}"))))`, + ); + expectTree( + 'echo {a..z}', + `(program (command (command_name (word "echo")) (concatenation (word "{") (word "a..z") (word "}"))))`, + ); + expectTree( + 'echo {-5..5}', + `(program (command (command_name (word "echo")) (concatenation (word "{") (word "-5..5") (word "}"))))`, + ); + expectTree( + 'echo a{b,c}d', + `(program (command (command_name (word "echo")) (concatenation (word "a") (word "{") (word "b,c") (word "}") (word "d"))))`, + ); + }); +}); + +describe('expansion operators', () => { + it('parses removal patterns as regex nodes', () => { + expectTree( + 'echo ${x##*/} ${x%%.*}', + `(program (command (command_name (word "echo")) (expansion "\${" (variable_name "x") "##" (regex "*/") "}") (expansion "\${" (variable_name "x") "%%" (regex ".*") "}")))`, + ); + expectTree( + 'echo ${x##a b}', + `(program (command (command_name (word "echo")) (expansion "\${" (variable_name "x") "##" (regex "a b") "}")))`, + ); + expectTree( + 'echo ${x#"pat"}', + `(program (command (command_name (word "echo")) (expansion "\${" (variable_name "x") "#" (string "\\"" (string_content "pat") "\\"") "}")))`, + ); + }); + + it('parses replacements with regex pattern and literal replacement', () => { + expectTree( + 'echo ${x/y/z} ${x//a/b}', + `(program (command (command_name (word "echo")) (expansion "\${" (variable_name "x") "/" (regex "y") "/" (word "z") "}") (expansion "\${" (variable_name "x") "//" (regex "a") "/" (word "b") "}")))`, + ); + expectTree( + 'echo ${x/a b/c}', + `(program (command (command_name (word "echo")) (expansion "\${" (variable_name "x") "/" (regex "a b") "/" (word "c") "}")))`, + ); + expectTree( + 'echo ${x/a/$v}', + `(program (command (command_name (word "echo")) (expansion "\${" (variable_name "x") "/" (regex "a") "/" (simple_expansion "$" (variable_name "v")) "}")))`, + ); + expectTree( + 'echo ${x/pat/rep/extra}', + `(program (command (command_name (word "echo")) (expansion "\${" (variable_name "x") "/" (regex "pat") "/" (word "rep/extra") "}")))`, + ); + }); + + it('parses case modification and transformation operators', () => { + expectTree( + 'echo ${x^} ${x,,}', + `(program (command (command_name (word "echo")) (expansion "\${" (variable_name "x") "^" "}") (expansion "\${" (variable_name "x") ",," "}")))`, + ); + expectTree( + 'echo ${x@Q}', + `(program (command (command_name (word "echo")) (expansion "\${" (variable_name "x") "@" "Q" "}")))`, + ); + }); + + it('parses max-length expansions with arithmetic values', () => { + expectTree( + 'echo ${v:1:2}', + `(program (command (command_name (word "echo")) (expansion "\${" (variable_name "v") ":" (number "1") ":" (number "2") "}")))`, + ); + expectTree( + 'echo ${arr[@]:1:2}', + `(program (command (command_name (word "echo")) (expansion "\${" (subscript (variable_name "arr") "[" (word "@") "]") ":" (number "1") ":" (number "2") "}")))`, + ); + }); + + it('parses default values as words, splitting bare values with spaces', () => { + expectTree( + 'echo ${v:-1} ${v:=2x}', + `(program (command (command_name (word "echo")) (expansion "\${" (variable_name "v") ":-" (word "1") "}") (expansion "\${" (variable_name "v") ":=" (word "2x") "}")))`, + ); + expectTree( + 'echo ${x:-d e f}', + `(program (command (command_name (word "echo")) (expansion "\${" (variable_name "x") ":-" (concatenation (word "d") (word " e f")) "}")))`, + ); + expectTree( + 'echo ${v:-"d e"}', + `(program (command (command_name (word "echo")) (expansion "\${" (variable_name "v") ":-" (string "\\"" (string_content "d e") "\\"") "}")))`, + ); + }); + + it('parses indirect expansions with trailing * and @', () => { + expectTree( + 'echo ${!prefix*} ${!name@}', + `(program (command (command_name (word "echo")) (expansion "\${" "!" (variable_name "prefix") "*" "}") (expansion "\${" "!" (variable_name "name") "@" "}")))`, + ); + }); +}); + +describe('combinations', () => { + it('nests if inside for inside case', () => { + expectTree( + 'case $1 in start) for s in a b; do if [[ -n $s ]]; then echo "$s"; fi; done ;; *) echo usage ;; esac', + `(program (case_statement "case" (simple_expansion "$" (variable_name "1")) "in" (case_item (word "start") ")" (for_statement "for" (variable_name "s") "in" (word "a") (word "b") ";" (do_group "do" (if_statement "if" (test_command "[[" (unary_expression (test_operator "-n") (simple_expansion "$" (variable_name "s"))) "]]") ";" "then" (command (command_name (word "echo")) (string "\\"" (simple_expansion "$" (variable_name "s")) "\\"")) ";" "fi") ";" "done")) ";;") (case_item (extglob_pattern "*") ")" (command (command_name (word "echo")) (word "usage")) ";;") "esac"))`, + ); + }); + + it('parses a heredoc inside a function body', () => { + expectTree( + 'foo() { cat < { + expectTree( + 'if a; then b; fi | grep x', + `(program (pipeline (if_statement "if" (command (command_name (word "a"))) ";" "then" (command (command_name (word "b"))) ";" "fi") "|" (command (command_name (word "grep")) (word "x"))))`, + ); + expectTree( + 'if a; then b; fi && c', + `(program (list (if_statement "if" (command (command_name (word "a"))) ";" "then" (command (command_name (word "b"))) ";" "fi") "&&" (command (command_name (word "c")))))`, + ); + expectTree( + 'echo hi | if a; then b; fi', + `(program (pipeline (command (command_name (word "echo")) (word "hi")) "|" (if_statement "if" (command (command_name (word "a"))) ";" "then" (command (command_name (word "b"))) ";" "fi")))`, + ); + }); + + it('parses redirects on compound statements', () => { + expectTree( + 'if a; then b; fi > out', + `(program (redirected_statement (if_statement "if" (command (command_name (word "a"))) ";" "then" (command (command_name (word "b"))) ";" "fi") (file_redirect ">" (word "out"))))`, + ); + }); + + it('parses test commands as loop conditions', () => { + expectTree( + 'while [[ -n $s ]]; do s=; done', + `(program (while_statement "while" (test_command "[[" (unary_expression (test_operator "-n") (simple_expansion "$" (variable_name "s"))) "]]") ";" (do_group "do" (variable_assignment (variable_name "s") "=") ";" "done")))`, + ); + expectTree( + 'if [ -f f ]; then x; fi', + `(program (if_statement "if" (test_command "[" (unary_expression (test_operator "-f") (word "f")) "]") ";" "then" (command (command_name (word "x"))) ";" "fi"))`, + ); + }); + + it('parses negation, lists and nested ifs in conditions', () => { + expectTree( + 'if ! grep -q x f; then echo missing; fi', + `(program (if_statement "if" (negated_command "!" (command (command_name (word "grep")) (word "-q") (word "x") (word "f"))) ";" "then" (command (command_name (word "echo")) (word "missing")) ";" "fi"))`, + ); + expectTree( + 'if a && b; then c; fi', + `(program (if_statement "if" (list (command (command_name (word "a"))) "&&" (command (command_name (word "b")))) ";" "then" (command (command_name (word "c"))) ";" "fi"))`, + ); + expectTree( + 'if if a; then b; fi; then c; fi', + `(program (if_statement "if" (if_statement "if" (command (command_name (word "a"))) ";" "then" (command (command_name (word "b"))) ";" "fi") ";" "then" (command (command_name (word "c"))) ";" "fi"))`, + ); + }); + + it('parses arrays and subscript expansions inside functions', () => { + expectTree( + 'f() { arr=(1 2); echo ${arr[0]}; }', + `(program (function_definition (word "f") "(" ")" (compound_statement "{" (variable_assignment (variable_name "arr") "=" (array "(" (number "1") (number "2") ")")) ";" (command (command_name (word "echo")) (expansion "\${" (subscript (variable_name "arr") "[" (number "0") "]") "}")) ";" "}")))`, + ); + }); + + it('parses a command subshell argument and command substitutions in for values', () => { + expectTree( + 'foo (ls)', + `(program (command (command_name (word "foo")) (subshell "(" (command (command_name (word "ls"))) ")")))`, + ); + expectTree( + 'for i in $(seq 3); do echo $i; done', + `(program (for_statement "for" (variable_name "i") "in" (command_substitution "$(" (command (command_name (word "seq")) (number "3")) ")") ";" (do_group "do" (command (command_name (word "echo")) (simple_expansion "$" (variable_name "i"))) ";" "done")))`, + ); + }); +}); + +describe('recovery and adversarial input', () => { + it('recovers unterminated compound commands without throwing', () => { + for (const source of [ + 'if a; then b', + 'if a', + 'while x; do y', + 'for f in a b; do x', + 'case $x in a) x', + 'case $x', + 'foo() { x;', + '{ x;', + 'function f', + 'for ((i=0;i<3', + 'echo $((1 + ', + '[[ -f x', + 'arr=(1 2', + ]) { + const result = parse(source); + expect(result.ok).toBe(true); + if (result.ok) expect(result.hasError).toBe(true); + } + }); + + it('recovers stray case terminators at top level', () => { + for (const source of ['a ;& b', 'a ;;& b', ';;&']) { + const result = parse(source); + expect(result.ok).toBe(true); + if (result.ok) expect(result.hasError).toBe(true); + } + }); + + it('flags an empty subshell', () => { + const result = parse('()'); + expect(result.ok).toBe(true); + if (result.ok) expect(result.hasError).toBe(true); + }); + + it('handles deeply nested compound commands within the depth cap', () => { + const source = 'if a; then '.repeat(600) + 'b' + ' fi'.repeat(600); + const result = parse(source, { timeoutMs: 10_000 }); + expect(result.ok).toBe(true); + if (result.ok) expect(result.hasError).toBe(true); + }); + + it('handles deeply nested arithmetic parentheses within the depth cap', () => { + const source = 'echo $(( ' + '('.repeat(600) + '1' + ')'.repeat(600) + ' ))'; + const result = parse(source, { timeoutMs: 10_000 }); + expect(result.ok).toBe(true); + if (result.ok) expect(result.hasError).toBe(true); + }); + + it('handles deeply nested case statements within the depth cap', () => { + const source = 'case x in a) '.repeat(600) + 'y' + ' ;; esac'.repeat(600); + const result = parse(source, { timeoutMs: 10_000 }); + expect(result.ok).toBe(true); + if (result.ok) expect(result.hasError).toBe(true); + }); + + it('parses a pathological test command under budget', () => { + const result = parse('[[ ' + 'a && '.repeat(2000) + 'a ]]'); + // Either parses (with bounded work) or the budget cuts it off; never throws. + if (result.ok) { + expect(result.rootNode.type).toBe('program'); + } else { + expect(result.reason).toBe('aborted'); + } + }); +}); + +describe('M2 review regressions', () => { + it('keeps a full case_statement inside $() despite pattern parens', () => { + expectTree( + 'x=$(case y in a) 1;; esac)', + `(program (variable_assignment (variable_name "x") "=" (command_substitution "$(" (case_statement "case" (word "y") "in" (case_item (word "a") ")" (command (command_name (number "1"))) ";;") "esac") ")")))`, + ); + // Optional-paren item form stays balanced too. + expectTree( + 'x=$(case y in (b) 2;; esac)', + `(program (variable_assignment (variable_name "x") "=" (command_substitution "$(" (case_statement "case" (word "y") "in" (case_item "(" (word "b") ")" (command (command_name (number "2"))) ";;") "esac") ")")))`, + ); + expectTree( + 'diff <(case y in a) 1;; esac) other', + `(program (command (command_name (word "diff")) (process_substitution "<(" (case_statement "case" (word "y") "in" (case_item (word "a") ")" (command (command_name (number "1"))) ";;") "esac") ")") (word "other")))`, + ); + }); + + it('does not mistake an argument named case for a case_statement in $()', () => { + expectTree( + '$(echo case; ls)', + `(program (command (command_name (command_substitution "$(" (command (command_name (word "echo")) (word "case")) ";" (command (command_name (word "ls"))) ")"))))`, + ); + }); + + it('converts test right-hand sides by operator, glob and paren depth', () => { + expectTree( + '[[ ( $a == x* ) ]]', + `(program (test_command "[[" (parenthesized_expression "(" (binary_expression (simple_expansion "$" (variable_name "a")) "==" (extglob_pattern "x*")) ")") "]]"))`, + ); + expectTree( + '[[ ( $a = x* ) ]]', + `(program (test_command "[[" (parenthesized_expression "(" (binary_expression (simple_expansion "$" (variable_name "a")) "=" (word "x*")) ")") "]]"))`, + ); + expectTree( + '[[ $x == b=c ]]', + `(program (test_command "[[" (binary_expression (simple_expansion "$" (variable_name "x")) "==" (word "b=c")) "]]"))`, + ); + expectTree( + '[[ a = b=c ]]', + `(program (test_command "[[" (binary_expression (word "a") "=" (regex "b=c")) "]]"))`, + ); + expectTree( + '[[ a = =b ]]', + `(program (test_command "[[" (binary_expression (word "a") "=" (regex "=b")) "]]"))`, + ); + }); + + it('parses statement-position ((word++…)) as a test_command', () => { + expectTree('((a++))', `(program (test_command "((" (word "a++") "))"))`); + expectTree('((a--))', `(program (test_command "((" (word "a--") "))"))`); + expectTree( + '((a++ + b))', + `(program (test_command "((" (binary_expression (word "a++") "+" (word "b")) "))"))`, + ); + // Other ((…)) forms and non-statement positions stay arithmetic. + expectTree( + '((b + a++))', + `(program (command (command_name (arithmetic_expansion "((" (binary_expression (variable_name "b") "+" (postfix_expression (variable_name "a") "++")) "))"))))`, + ); + expectTree( + 'echo ((a++))', + `(program (command (command_name (word "echo")) (arithmetic_expansion "((" (postfix_expression (variable_name "a") "++") "))")))`, + ); + expectTree( + '! ((a++))', + `(program (negated_command "!" (test_command "((" (word "a++") "))")))`, + ); + }); + + it('keeps unconsumed expression input in ERROR nodes', () => { + const arithmetic = parseOk('echo $((x[1][2]))'); + expect(arithmetic.hasError).toBe(true); + expect(sexp(arithmetic.rootNode)).toBe( + `(program (command (command_name (word "echo")) (arithmetic_expansion "$((" (subscript (variable_name "x") "[" (number "1") "]") (ERROR "[2]") "))")))`, + ); + const test = parseOk('[[ $a =~ a b ]]'); + expect(test.hasError).toBe(true); + const errors = descendantsOfType(test.rootNode, 'ERROR'); + expect(errors).toHaveLength(1); + expect(errors[0]!.text).toContain('b'); + }); + + it('parses negative offsets in max-length expansions as numbers', () => { + expectTree( + 'echo ${x: -5}', + `(program (command (command_name (word "echo")) (expansion "\${" (variable_name "x") ":" (number "-5") "}")))`, + ); + expectTree( + 'echo ${x: -5:2}', + `(program (command (command_name (word "echo")) (expansion "\${" (variable_name "x") ":" (number "-5") ":" (number "2") "}")))`, + ); + }); + + it('parses $0 and ${0} as special_variable_name', () => { + expectTree( + 'echo $0 ${0} $1 ${10}', + `(program (command (command_name (word "echo")) (simple_expansion "$" (special_variable_name "0")) (expansion "\${" (special_variable_name "0") "}") (simple_expansion "$" (variable_name "1")) (expansion "\${" (variable_name "10") "}")))`, + ); + }); + + it('degrades locally on deep pattern→string→expansion nesting', () => { + let inner = '${a#""}'; + for (let k = 0; k < 650; k++) inner = `\${a#"${inner}"}`; + const result = parse(`echo ${inner}`, { timeoutMs: 10_000 }); + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.hasError).toBe(true); + // Local degradation: the tree keeps the outer expansions instead of + // collapsing into a single ERROR child under program. + expect(result.rootNode.children[0]!.type).not.toBe('ERROR'); + expect(descendantsOfType(result.rootNode, 'expansion').length).toBeGreaterThan(100); + }); + + it('treats = and ! as word characters at operand position in tests', () => { + expectTree('[[ a!=b ]]', `(program (test_command "[[" (word "a!=b") "]]"))`); + expectTree('[[ a=b ]]', `(program (test_command "[[" (word "a=b") "]]"))`); + expectTree( + '[[ !x = y ]]', + `(program (test_command "[[" (binary_expression (word "!x") "=" (word "y")) "]]"))`, + ); + expectTree( + '[[ $a==b* ]]', + `(program (test_command "[[" (concatenation (simple_expansion "$" (variable_name "a")) (word "==b*")) "]]"))`, + ); + // But they are operators at operator position, even when attached. + expectTree( + '[[ a ==b ]]', + `(program (test_command "[[" (binary_expression (word "a") "==" (extglob_pattern "b")) "]]"))`, + ); + expectTree( + '[[ a =b ]]', + `(program (test_command "[[" (binary_expression (word "a") "=" (word "b")) "]]"))`, + ); + expectTree( + '[[ $a =~^x ]]', + `(program (test_command "[[" (binary_expression (simple_expansion "$" (variable_name "a")) "=~" (regex "^x")) "]]"))`, + ); + }); + + it('parses [[ ]] as a concatenation plus a zero-width closer (reference quirk)', () => { + expectTree( + '[[ ]]', + `(program (test_command "[[" (concatenation (word "]") (word "]")) "]]"))`, + true, + ); + }); + + it('parses bare numbers inside square brackets as number pieces', () => { + expectTree( + 'echo [1] a[2]b', + `(program (command (command_name (word "echo")) (concatenation (word "[") (number "1") (word "]")) (concatenation (word "a") (word "[") (number "2") (word "]") (word "b"))))`, + ); + expectTree( + 'echo a[-5]b', + `(program (command (command_name (word "echo")) (concatenation (word "a") (word "[") (number "-5") (word "]") (word "b"))))`, + ); + }); +}); + +describe('M2 review round 2 regressions', () => { + it('applies the scanner.c extglob_pattern rule to ==/!= right sides', () => { + // Single letter followed by whitespace → glob; followed by `)` → word. + expectTree( + '[[ $a == b ]]', + `(program (test_command "[[" (binary_expression (simple_expansion "$" (variable_name "a")) "==" (extglob_pattern "b")) "]]"))`, + ); + expectTree( + '[[ (a == b) && c ]]', + `(program (test_command "[[" (binary_expression (parenthesized_expression "(" (binary_expression (word "a") "==" (word "b")) ")") "&&" (word "c")) "]]"))`, + ); + expectTree( + '[[ ( $a == x ) ]]', + `(program (test_command "[[" (parenthesized_expression "(" (binary_expression (simple_expansion "$" (variable_name "a")) "==" (extglob_pattern "x")) ")") "]]"))`, + ); + // Multi-character all-letter words stay words, dots don't count. + expectTree( + '[[ $a == foo ]]', + `(program (test_command "[[" (binary_expression (simple_expansion "$" (variable_name "a")) "==" (word "foo")) "]]"))`, + ); + expectTree( + '[[ $a == x.y ]]', + `(program (test_command "[[" (binary_expression (simple_expansion "$" (variable_name "a")) "==" (word "x.y")) "]]"))`, + ); + // Any non-letter non-dot character makes it a glob. + for (const word of ['x1', 'foo-', 'x_', 'b*c=d', 'bc=d', 'b?=e', '.x', 'a-b']) { + const result = parseOk(`[[ $a == ${word} ]]`); + const globs = descendantsOfType(result.rootNode, 'extglob_pattern'); + expect(globs.map((g) => g.text)).toEqual([word]); + } + // `=` vetoes only as the second character; a leading `=` vetoes too. + expectTree( + '[[ $x == b=c ]]', + `(program (test_command "[[" (binary_expression (simple_expansion "$" (variable_name "x")) "==" (word "b=c")) "]]"))`, + ); + expectTree( + '[[ a == =b ]]', + `(program (test_command "[[" (binary_expression (word "a") "==" (word "=b")) "]]"))`, + ); + // Round-1 fixed behavior must not regress. + expectTree( + '[[ $x == b*c ]]', + `(program (test_command "[[" (binary_expression (simple_expansion "$" (variable_name "x")) "==" (extglob_pattern "b*c")) "]]"))`, + ); + expectTree( + '[[ $x == 123 ]]', + `(program (test_command "[[" (binary_expression (simple_expansion "$" (variable_name "x")) "==" (number "123")) "]]"))`, + ); + expectTree( + '[[ a ==b ]]', + `(program (test_command "[[" (binary_expression (word "a") "==" (extglob_pattern "b")) "]]"))`, + ); + }); + + it('keeps the = right-side regex rule independent of the glob rule', () => { + expectTree( + '[[ a = b*c ]]', + `(program (test_command "[[" (binary_expression (word "a") "=" (regex "b*c")) "]]"))`, + ); + expectTree( + '[[ a = bc ]]', + `(program (test_command "[[" (binary_expression (word "a") "=" (word "bc")) "]]"))`, + ); + expectTree( + '[[ a = b=c ]]', + `(program (test_command "[[" (binary_expression (word "a") "=" (regex "b=c")) "]]"))`, + ); + expectTree( + '[[ ( $a = x* ) ]]', + `(program (test_command "[[" (parenthesized_expression "(" (binary_expression (simple_expansion "$" (variable_name "a")) "=" (word "x*")) ")") "]]"))`, + ); + }); + + it('survives 2500 levels of lexer-side pattern nesting', () => { + let inner = '${a#""}'; + for (let k = 0; k < 2500; k++) inner = `\${a#"${inner}"}`; + const result = parse(`echo ${inner}`, { timeoutMs: 30_000 }); + // No RangeError, no abort: the scan depth cap degrades locally. + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.hasError).toBe(true); + expect(result.rootNode.children[0]!.type).not.toBe('ERROR'); + }); + + it('parses ((…)) inside a test as an arithmetic_expansion', () => { + expectTree( + '[[ ((a)) == x ]]', + `(program (test_command "[[" (binary_expression (arithmetic_expansion "((" (variable_name "a") "))") "==" (extglob_pattern "x")) "]]"))`, + ); + // Without the closing )), parens still nest as parenthesized_expressions. + expectTree( + '[[ ((a) == x) && y ]]', + `(program (test_command "[[" (binary_expression (parenthesized_expression "(" (binary_expression (parenthesized_expression "(" (word "a") ")") "==" (word "x")) ")") "&&" (word "y")) "]]"))`, + ); + }); + + it('parses ((ab-cd++)) as a test_command', () => { + expectTree('((ab-cd++))', `(program (test_command "((" (word "ab-cd++") "))"))`); + }); + + it('flags a missing separator before a compound keyword', () => { + for (const source of [ + 'if a; then case x in b) 1;; esac elif c; then d; fi', + 'if a; then b fi', + '{ ls }', + 'while x do y done', + 'for f in a b; do x done', + ]) { + const result = parse(source); + expect(result.ok).toBe(true); + if (result.ok) expect(result.hasError).toBe(true); + } + // Valid inputs stay clean: case items need no terminator before esac, + // subshells none before ), compounds with proper terminators. + for (const source of ['if a; then b; fi', '{ ls; }', '(ls)', 'case $x in a) x esac', 'while x; do y; done']) { + const result = parse(source); + expect(result.ok).toBe(true); + if (result.ok) expect(result.hasError).toBe(false); + } + }); +}); + +describe('M2 review round 3 regressions', () => { + it('converts […]-containing glob right sides as a whole', () => { + expectTree( + '[[ $ver == [0-9]* ]]', + `(program (test_command "[[" (binary_expression (simple_expansion "$" (variable_name "ver")) "==" (extglob_pattern "[0-9]*")) "]]"))`, + ); + expectTree( + '[[ $a == [abc] ]]', + `(program (test_command "[[" (binary_expression (simple_expansion "$" (variable_name "a")) "==" (extglob_pattern "[abc]")) "]]"))`, + ); + expectTree( + '[[ ( $a == [xyz] ) ]]', + `(program (test_command "[[" (parenthesized_expression "(" (binary_expression (simple_expansion "$" (variable_name "a")) "==" (extglob_pattern "[xyz]")) ")") "]]"))`, + ); + expectTree( + '[[ $a == [!a]* ]]', + `(program (test_command "[[" (binary_expression (simple_expansion "$" (variable_name "a")) "==" (extglob_pattern "[!a]*")) "]]"))`, + ); + expectTree( + '[[ $a = [abc] ]]', + `(program (test_command "[[" (binary_expression (simple_expansion "$" (variable_name "a")) "=" (regex "[abc]")) "]]"))`, + ); + // Bracket fragments outside test commands are untouched. + expectTree( + 'echo a[2]b', + `(program (command (command_name (word "echo")) (concatenation (word "a") (word "[") (number "2") (word "]") (word "b"))))`, + ); + }); + + it('keeps -word as a word when no operand follows it', () => { + expectTree( + '[[ $a == -foo ]]', + `(program (test_command "[[" (binary_expression (simple_expansion "$" (variable_name "a")) "==" (word "-foo")) "]]"))`, + ); + expectTree( + '[[ -foo == x ]]', + `(program (test_command "[[" (binary_expression (word "-foo") "==" (extglob_pattern "x")) "]]"))`, + ); + expectTree('[[ -f ]]', `(program (test_command "[[" (word "-f") "]]"))`); + // A real operand still makes it a test_operator. + expectTree( + '[[ -f file.txt ]]', + `(program (test_command "[[" (unary_expression (test_operator "-f") (word "file.txt")) "]]"))`, + ); + }); + + it('parses extglob group patterns after ==/!=', () => { + expectTree( + '[[ $a == +(!a) ]]', + `(program (test_command "[[" (binary_expression (simple_expansion "$" (variable_name "a")) "==" (extglob_pattern "+(!a)")) "]]"))`, + ); + expectTree( + '[[ $a == ?(a|b) ]]', + `(program (test_command "[[" (binary_expression (simple_expansion "$" (variable_name "a")) "==" (extglob_pattern "?(a|b)")) "]]"))`, + ); + expectTree( + '[[ $a == !(x) ]]', + `(program (test_command "[[" (binary_expression (simple_expansion "$" (variable_name "a")) "==" (extglob_pattern "!(x)")) "]]"))`, + ); + }); + + it('parses a negative number operand in tests as unary minus', () => { + expectTree( + '[[ $a == -1 ]]', + `(program (test_command "[[" (binary_expression (simple_expansion "$" (variable_name "a")) "==" (unary_expression "-" (number "1"))) "]]"))`, + ); + expectTree( + '[[ $a -eq -1 ]]', + `(program (test_command "[[" (binary_expression (simple_expansion "$" (variable_name "a")) (test_operator "-eq") (unary_expression "-" (number "1"))) "]]"))`, + ); + }); + + it('handles escaped characters in glob right sides', () => { + expectTree( + '[[ $a == foo\\ bar ]]', + `(program (test_command "[[" (binary_expression (simple_expansion "$" (variable_name "a")) "==" (word "foo\\\\ bar")) "]]"))`, + ); + expectTree( + '[[ $a == b\\*c ]]', + `(program (test_command "[[" (binary_expression (simple_expansion "$" (variable_name "a")) "==" (extglob_pattern "b\\\\*c")) "]]"))`, + ); + }); + + it('extends elif/else clause ranges over the trailing newline', () => { + const { rootNode, hasError } = parseOk('if a; then b; elif c; then d\nelse e\nfi'); + expect(hasError).toBe(false); + const elifClause = descendantsOfType(rootNode, 'elif_clause')[0]!; + const elseClause = descendantsOfType(rootNode, 'else_clause')[0]!; + expect(elifClause.text).toBe('elif c; then d\n'); + expect(elseClause.text).toBe('else e\n'); + }); + + it('degrades 400-level command substitution nesting locally', () => { + const source = 'echo ' + '$('.repeat(400) + 'x' + ')'.repeat(400); + const result = parse(source, { timeoutMs: 10_000 }); + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.hasError).toBe(true); + // Local degradation (MAX_SUBSTITUTION_DEPTH), not a last-resort + // single-ERROR root. + expect(result.rootNode.children[0]!.type).not.toBe('ERROR'); + expect(descendantsOfType(result.rootNode, 'command_substitution').length).toBeGreaterThan(100); + }); +}); From 83b963678a6b6d8afd8ec51c89db1701dcff8e1a Mon Sep 17 00:00:00 2001 From: "haozhe.yang" Date: Tue, 21 Jul 2026 20:49:09 +0800 Subject: [PATCH 4/7] test(tree-sitter-bash): add differential fixtures, corpus, fuzz and perf suites Turn the ad-hoc wasm comparison work into permanent infrastructure: a differential helper pinning reference-equivalent and known-difference samples against the real tree-sitter-bash wasm (478 match / 79 known-diff fixtures plus the official v0.25.0 corpus), a three-way consistency check between fixtures, the known-difference registry and the README, deterministic seeded fuzz with tree-integrity assertions, and performance smoke tests. Converges 15 further divergence groups found by systematic probing and documents the rest; the full suite is 853 tests in ~4s. --- AGENTS.md | 1 + packages/tree-sitter-bash/README.md | 195 +- packages/tree-sitter-bash/src/lexer.ts | 27 +- packages/tree-sitter-bash/src/parser.ts | 774 ++++++- .../test/differential.test.ts | 169 ++ .../tree-sitter-bash/test/fixtures/README.md | 42 + .../test/fixtures/corpus/README.md | 21 + .../test/fixtures/corpus/commands.txt | 725 ++++++ .../test/fixtures/corpus/crlf.txt | 13 + .../test/fixtures/corpus/known-diffs.txt | 2009 +++++++++++++++++ .../test/fixtures/corpus/literals.txt | 1371 +++++++++++ .../test/fixtures/corpus/programs.txt | 108 + .../test/fixtures/corpus/statements.txt | 1622 +++++++++++++ .../test/fixtures/differential/arithmetic.txt | 236 ++ .../test/fixtures/differential/case.txt | 436 ++++ .../test/fixtures/differential/expansions.txt | 599 +++++ .../test/fixtures/differential/heredoc.txt | 597 +++++ .../test/fixtures/differential/recovery.txt | 112 + .../test/fixtures/differential/redirects.txt | 120 + .../test/fixtures/differential/statements.txt | 391 ++++ .../fixtures/differential/test-command.txt | 650 ++++++ packages/tree-sitter-bash/test/fuzz.test.ts | 165 ++ .../test/helpers/differential.ts | 271 +++ .../test/helpers/known-differences.ts | 179 ++ .../tree-sitter-bash/test/performance.test.ts | 95 + 25 files changed, 10812 insertions(+), 116 deletions(-) create mode 100644 packages/tree-sitter-bash/test/differential.test.ts create mode 100644 packages/tree-sitter-bash/test/fixtures/README.md create mode 100644 packages/tree-sitter-bash/test/fixtures/corpus/README.md create mode 100644 packages/tree-sitter-bash/test/fixtures/corpus/commands.txt create mode 100644 packages/tree-sitter-bash/test/fixtures/corpus/crlf.txt create mode 100644 packages/tree-sitter-bash/test/fixtures/corpus/known-diffs.txt create mode 100644 packages/tree-sitter-bash/test/fixtures/corpus/literals.txt create mode 100644 packages/tree-sitter-bash/test/fixtures/corpus/programs.txt create mode 100644 packages/tree-sitter-bash/test/fixtures/corpus/statements.txt create mode 100644 packages/tree-sitter-bash/test/fixtures/differential/arithmetic.txt create mode 100644 packages/tree-sitter-bash/test/fixtures/differential/case.txt create mode 100644 packages/tree-sitter-bash/test/fixtures/differential/expansions.txt create mode 100644 packages/tree-sitter-bash/test/fixtures/differential/heredoc.txt create mode 100644 packages/tree-sitter-bash/test/fixtures/differential/recovery.txt create mode 100644 packages/tree-sitter-bash/test/fixtures/differential/redirects.txt create mode 100644 packages/tree-sitter-bash/test/fixtures/differential/statements.txt create mode 100644 packages/tree-sitter-bash/test/fixtures/differential/test-command.txt create mode 100644 packages/tree-sitter-bash/test/fuzz.test.ts create mode 100644 packages/tree-sitter-bash/test/helpers/differential.ts create mode 100644 packages/tree-sitter-bash/test/helpers/known-differences.ts create mode 100644 packages/tree-sitter-bash/test/performance.test.ts diff --git a/AGENTS.md b/AGENTS.md index 2349080c17..26bf9dcdba 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -28,6 +28,7 @@ This is a TypeScript monorepo built for agent-assisted development. Keep the roo - `packages/kap-server`: the Kimi Code server, backed by the DI × Scope agent engine (`@moonshot-ai/agent-core-v2`). Exposes sessions over REST + WebSocket (`/api/v1` + `/api/v1/ws`); bootstrapped from `src/start.ts` and consumed by `apps/kimi-code`. The RPC surface is `/api/v1/debug/*` — a reflection dispatcher over the ENTIRE scoped DI registry (every Service callable, no whitelist; `src/transport/registerDebugRoutes.ts` + `serviceDispatcherRoutes.ts`), mounted only with `--debug-endpoints` on a loopback bind and gated by the global bearer auth; repo dev scripts pass the flag. - `packages/klient`: the client SDK — a contract-driven facade over agent-core-v2 with aggregated `global.*` / `session(id).*` / `agent(id).*` methods, zod validation on every call, and klient-level typed event forwarding. Transport is chosen once at creation via subpath entry (`@moonshot-ai/klient/ipc|memory`); both return the same `Klient`. The package also hosts the e2e suites: the legacy `/api/v1` live suites (`test/e2e/legacy/`) and the docker e2e runner (`pnpm --filter @moonshot-ai/klient docker:e2e`). See `packages/klient/AGENTS.md`. - `packages/server-e2e`: live e2e tests and scenarios against a running server (`KIMI_SERVER_URL`, default `http://127.0.0.1:58627`). See `packages/server-e2e/AGENTS.md`. +- `packages/tree-sitter-bash`: a pure-TypeScript bash parser (no runtime deps, no wasm) that produces a syntax tree with tree-sitter-bash 0.25.0 named-node type names and UTF-16 code-unit offsets. `parse(source, { timeoutMs, maxNodes })` runs under a deterministic budget (default 50 ms / 50k nodes, plus per-chain recursion depth caps) and returns a discriminated `ParseResult` (`{ ok, rootNode, hasError }` or `{ ok: false, reason: 'aborted' }`) — callers must treat aborted/hasError trees as "cannot analyze" and degrade. Parser only, no safety judgments; consumers (e.g. Bash tool permission matching) live elsewhere. Known deviations from the reference are tracked in the package README's "Known differences" section, pinned by differential fixtures tested against the real `tree-sitter-bash` wasm (dev-only). ## Environment Requirements diff --git a/packages/tree-sitter-bash/README.md b/packages/tree-sitter-bash/README.md index c677c35ee5..8ef128d62f 100644 --- a/packages/tree-sitter-bash/README.md +++ b/packages/tree-sitter-bash/README.md @@ -2,34 +2,21 @@ A pure-TypeScript bash parser that produces a syntax tree whose named node types match [tree-sitter-bash](https://github.com/tree-sitter/tree-sitter-bash) -one-to-one, built for agent-side command permission analysis. +0.25.0 one-to-one, built for agent-side command permission analysis. - No native addons; offsets are UTF-16 code units (`node.text` is always a direct `source.slice(startIndex, endIndex)`). -- Parsing runs under a hard budget (default 50 ms / 50 000 nodes); exceeding - it returns `{ ok: false, reason: 'aborted' }` instead of throwing. The - budget caps total work, not input size: a multi-hundred-KB string or - heredoc body parses fine (it produces only a handful of nodes), but a - source whose tree would exceed 50 000 nodes — a megabyte-long `case` - statement, tens of thousands of arithmetic expressions — is aborted - under the defaults. Pass `timeoutMs` / `maxNodes` to raise the limits. -- Malformed input never throws either. Unterminated constructs (quotes, - expansions, substitutions, heredocs, compound commands) are kept as - partial nodes and flagged with `hasError: true`; tokens that cannot start - or continue a statement (stray `)`, a leading `&&`, …) are wrapped in - `ERROR` nodes and parsing continues. -- Newlines never appear in the tree: statement-terminating `\n`s produce no - nodes at all (matching tree-sitter-bash, whose scanner only emits `\n` - tokens in heredoc position), while `;` / `&` terminators are anonymous - children. -- Recursion is depth-capped so pathological nesting degrades locally - (ERROR nodes, `hasError: true`) instead of overflowing the stack: - `MAX_SUBSTITUTION_DEPTH = 150` for the $( … ) / `…` / <( … ) sub-parser - chain (the deepest per level, ~13 frames — measured stack overflow at - ~380–500 levels on a default Node stack, so the cap keeps a ≥2.5× - margin), `MAX_PARSE_DEPTH = 500` for subshell/compound/`${…}`/expression - nesting (verified safe at those caps), and `MAX_SCAN_DEPTH = 1024` for - the lexer's own scan recursion. +- Parsing runs under a hard budget and never throws: budget exhaustion + returns `{ ok: false, reason: 'aborted' }`, malformed input returns a + degraded tree with `hasError: true`. +- Correctness is enforced by a differential test suite that parses hundreds + of samples — plus the complete official tree-sitter-bash 0.25.0 corpus — + with both this parser and the real tree-sitter-bash (wasm) and compares + the trees byte-for-byte (see `test/fixtures/`). The only remaining + deviations are the documented ones listed under + [Known differences](#known-differences-from-tree-sitter-bash) below. + +## API ```ts import { parse } from '@moonshot-ai/tree-sitter-bash'; @@ -40,11 +27,59 @@ if (result.ok) { } ``` +- `parse(source: string, options?: ParseOptions): ParseResult` +- `ParseResult` is one of: + - `{ ok: true; rootNode: SyntaxNode; hasError: boolean }` + - `{ ok: false; reason: 'aborted' }` +- `ParseOptions`: `{ timeoutMs?: number; maxNodes?: number }` (defaults + 50 ms / 50 000 nodes; `timeoutMs: Infinity` disables the time check). +- `SyntaxNode`: `{ type, text, startIndex, endIndex, isNamed, parent, + children, namedChildren }`. Named node types come from tree-sitter-bash's + `node-types.json`; `children` includes the anonymous (punctuation / + keyword token) nodes as well, `namedChildren` only the named ones, and + `descendantsOfType(root, ...types)` walks named descendants. +- Depth caps (pathological nesting degrades locally — ERROR nodes, + `hasError: true` — instead of overflowing the stack): + - `MAX_SUBSTITUTION_DEPTH = 150` for the $( … ) / `…` / <( … ) + sub-parser chain (the deepest per level, ~13 frames — measured stack + overflow at ~380–500 levels on a default Node stack, so the cap keeps + a ≥2.5× margin); + - `MAX_PARSE_DEPTH = 500` for subshell/compound/`${…}`/expression + nesting (verified safe at those caps); + - `MAX_SCAN_DEPTH = 1024` for the lexer's own scan recursion. + +## Budget semantics + +The budget caps total work, not input size: a multi-hundred-KB string or +heredoc body parses fine (it produces only a handful of nodes), but a +source whose tree would exceed 50 000 nodes — a megabyte-long `case` +statement, tens of thousands of arithmetic expressions — is aborted under +the defaults. Every created node ticks the budget; long character-level +scan loops re-check the deadline periodically. When either limit is hit, +`parse` returns `{ ok: false, reason: 'aborted' }` and the abort is prompt +(see Performance). Pass `timeoutMs` / `maxNodes` to raise the limits. + +## Error recovery + +Malformed input never throws either. Unterminated constructs (quotes, +expansions, substitutions, heredocs, compound commands) are kept as +partial nodes and flagged with `hasError: true`; tokens that cannot start +or continue a statement (stray `)`, a leading `&&`, …) are wrapped in +`ERROR` nodes and parsing continues. Newlines never appear in the tree: +statement-terminating `\n`s produce no nodes at all (matching +tree-sitter-bash, whose scanner only emits `\n` tokens in heredoc +position), while `;` / `&` terminators are anonymous children. As a +last-resort guard against parser bugs, any unexpected internal exception +degrades to a `program` root with a single `ERROR` child spanning the +source and `hasError: true` — callers still get a usable tree. + ## Known differences from tree-sitter-bash Named node types always come from tree-sitter-bash's `node-types.json`, but for the following constructs the tree shape deliberately deviates from what -tree-sitter-bash 0.25.0 produces (verified against the real parser): +tree-sitter-bash 0.25.0 produces (each verified against the real parser, +and pinned by a stored expectation in `test/fixtures/` — see +`test/helpers/known-differences.ts`): - `<>` (read-write redirect) is parsed as a normal `file_redirect` operator; tree-sitter-bash 0.25.0 fails to parse it. @@ -63,6 +98,10 @@ tree-sitter-bash 0.25.0 produces (verified against the real parser): substitutions are parsed as `command_substitution` children. tree-sitter-bash's scanner hides the leading chunk and swallows backticks into the content. +- A `$'` sequence inside an unquoted heredoc body is plain text here + (bash does not expand ANSI-C strings in heredocs; the reference does the + same mid-line). Quirk: when a body line STARTS with `$'…'`, the + reference's scanner errors and drops the whole `heredoc_redirect`. - A heredoc redirect at statement start (`<`) is a + `word` in the reference even when its content would make it a glob + (`cmake_modules` — a scanner-state quirk); this parser classifies + patterns by content, so such a pattern is an `extglob_pattern` here. A + continuation INSIDE a pattern (`cont\ued)`) is an error in the + reference (word "cont" + `ERROR` "ued", `hasError: true`), and a + continuation fused with a digit (`a\1)`) is an + `extglob_pattern` there; this parser keeps the continuation and parses + the whole pattern cleanly as one `word` — which is also how bash itself + reads it. +- A comparison right side with TWO substitutions or quotes + (`*${x}*${y}`, `*"s"*"t"`) does not fit the reference's one-construct + pattern rule either; it recovers with a nested `binary_expression` + there, while this parser keeps a single `concatenation` right side. - A few `[[ … ]]` comparison right-hand sides deviate: - - An extglob group in the MIDDLE of a pattern (`foo*(.txt|.log)`, - `*foo*(a)`, `[a-z]*(x)`) is one `extglob_pattern` node in the - reference; this parser ends the pattern at the group paren and flags - `hasError: true` with an `ERROR` tail. A group at the START of the - right side (`+(a|b)`) matches the reference exactly, and a group - directly after a pure literal (`x@(y|z)w`) is an error in the - reference too (an `ERROR` node there). - A negative decimal operand (`[[ $n == -0.5 ]]`) is a `unary_expression` (`-` over word "0.5") in the reference; this parser produces a single `extglob_pattern` "-0.5" (the unary-minus reading @@ -123,10 +192,40 @@ tree-sitter-bash 0.25.0 produces (verified against the real parser): expression in the reference — `binary_expression(==, extglob "foo\")` then `|` then word "bar" (a reference quirk); this parser keeps one `extglob_pattern` "foo\|bar". +- `${!# }` and `${!## }` (and `${!##/}`) are pathological expansion forms: + the reference recovers with zero-width operator tokens, this parser with + a flagged partial `expansion`; the shapes differ. (The sane forms + `${#}`, `${!#}`, `${!##}`, `${#!}` all match exactly.) +- A base prefix fused with an expansion (`10#${x}`) is one `number` node + spanning the expansion in the reference (a quirk); this parser produces + a `concatenation` of word "10#" and the expansion. +- `${v:-(default)}` (a parenthesized default value) is a plain `word` + here; the reference parses it as an `array` node (a quirk). +- `${=1}` (zsh-style word splitting flag) produces a `variable_name` "1" + in the reference; this parser produces a `word`. +- An escaped space or tab between arguments is dropped by the reference + as invisible extras (its tree has gaps: `echo 1 \ 2` produces number + "1" and number "2", and `echo a\b` splits into word "a" and word + "b"); this parser keeps the escape inside the word. (An escaped space + INSIDE a word, `a\ b`, is kept by both parsers and matches.) +- A `$` directly fused with a backtick substitution (`` $`echo x` ``) is + one `command_substitution` with a `$`+backtick token in the reference; + this parser produces a `($)` token followed by the substitution. +- A non-ASCII “identifier” in assignment position (`变量=值`, `é=1`) is a + `variable_assignment` flagged `hasError` in the reference; this parser + keeps it a plain command word — which is also how bash itself treats it + (variable names are ASCII-only, so the text runs as a command). +- A negative literal after a compound assignment in a c-style for header + (`for (( … j *= -1, … ))`) is folded into a `number` "-1" by the + reference; this parser produces a `unary_expression` (`-` over number + "1") — matching what the reference itself produces everywhere else. +- A double backslash in a replacement value (`${x// /\\|}`) loses its + first backslash in the reference's tree (a quirk — the `word` starts + one character later); this parser keeps both. - `string_content` is not split at newlines (tree-sitter-bash's scanner splits it). -For completeness, two constructs that LOOK like deviations but are not — +For completeness, a few constructs that LOOK like deviations but are not — the reference behaves the same way (verified): - `coproc` has no grammar support in tree-sitter-bash 0.25.0 and parses as @@ -152,3 +251,23 @@ the reference behaves the same way (verified): `cmd` the `command_name`), while redirects after the command name consume every following word (`cmd > out arg` puts both words in the redirect) — this matches tree-sitter-bash's actual disambiguation. + +## Performance + +Measured on an Apple-silicon MacBook (Node 24, default 50 ms / 50 000-node +budget): + +- A typical one-line command (`git status && rm -rf /`): ~4 µs per parse. +- A 100 KB realistic deployment script (functions, loops, redirects, + expansions, heredocs): tens of milliseconds per parse (~20–60 ms across + runs and machines) — the same order of magnitude as the default 50 ms + budget, so such inputs may return `ok: true` or a prompt abort depending + on the machine; raise `timeoutMs` when parsing whole scripts. +- A 500 KB heredoc body: parses within the default budget (the tree has + only a handful of nodes). +- The abort path is prompt: a 400 KB node-budget bomb aborts in ~20 ms + (hard guarantee asserted by the test suite: < 100 ms). + +These numbers are smoke-tested as orders of magnitude in +`test/performance.test.ts` to guard against accidental quadratic +complexity; absolute timings vary by machine. diff --git a/packages/tree-sitter-bash/src/lexer.ts b/packages/tree-sitter-bash/src/lexer.ts index 200db43ab7..807deb7ee9 100644 --- a/packages/tree-sitter-bash/src/lexer.ts +++ b/packages/tree-sitter-bash/src/lexer.ts @@ -352,12 +352,35 @@ export function scanBalancedStatements( } /** Skip a $-construct starting at `i` (which points at the `$`). Handles - * $(...), $((...)), ${...}, $name and the single-character specials. A `$` - * followed by anything else (including a quote) consumes just the `$`. */ + * $(...), $((...)), ${...}, $'...' (escape-aware: \' does not close), + * $name and the single-character specials. A `$` followed by anything else + * (including a double quote) consumes just the `$`. */ export function skipDollar(source: string, budget: ParseBudget, i: number, end: number): number { const next = source[i + 1]; if (next === '(') return scanBalancedStatements(source, budget, i + 1, end).end; if (next === '{') return scanBalanced(source, budget, i + 1, end, '{', '}').end; + // $[...] legacy arithmetic (only when `[` directly follows the `$`; `$a[0]` + // takes the word-character branch above). + if (next === '[') return scanBalanced(source, budget, i + 1, end, '[', ']').end; + if (next === "'") { + // ANSI-C string: \' is an escaped quote and does not terminate it. + let j = i + 2; + let sinceTick = 0; + while (j < end) { + if (++sinceTick >= SCAN_TICK_INTERVAL) { + budget.progress(); + sinceTick = 0; + } + const ch = source[j]!; + if (ch === '\\') { + j += 2; + continue; + } + if (ch === "'") return j + 1; + j++; + } + return end; + } if (isWordChar(next)) { let j = i + 1; while (j < end && isWordChar(source[j])) j++; diff --git a/packages/tree-sitter-bash/src/parser.ts b/packages/tree-sitter-bash/src/parser.ts index 83f5917984..be37427c4b 100644 --- a/packages/tree-sitter-bash/src/parser.ts +++ b/packages/tree-sitter-bash/src/parser.ts @@ -55,7 +55,7 @@ import { SPECIAL_VARIABLE_CHARS, UNSET_COMMAND_KEYWORDS, } from '#/grammar'; -import { Lexer, scanBalanced, scanBalancedStatements, skipBacktick, skipDoubleQuoted, skipSingleQuoted } from '#/lexer'; +import { Lexer, scanBalanced, scanBalancedStatements, skipBacktick, skipDollar, skipDoubleQuoted, skipSingleQuoted } from '#/lexer'; import type { BalancedScan, HeredocBody, HeredocSpec, Token } from '#/lexer'; import { SyntaxNodeBuilder } from '#/node'; @@ -87,6 +87,9 @@ const ASSIGNMENT_RE = /^[A-Za-z_][A-Za-z0-9_]*\+?=/; const ASSIGNMENT_SPLIT_RE = /^([A-Za-z_][A-Za-z0-9_]*)(\+?=)/; const SUBSCRIPT_ASSIGNMENT_RE = /^(\w+)\[([^\]\n]*)\](\+?=)/; const IDENTIFIER_RE = /^[A-Za-z_]\w*$/; +/** Function names additionally allow `:` (`foo::bar() { … }` — the + * reference accepts any such word as a function name). */ +const FUNCTION_NAME_RE = /^[A-Za-z_][\w:]*$/; const BRACE_EXPRESSION_RE = /^\{(\d+)\.\.(\d+)\}/; /** Statement-position `((…))` that tree-sitter-bash parses as a * test_command: inner text starting with an identifier-ish token @@ -956,6 +959,13 @@ export class Parser { * (* ? [) or that are followed by `|` — those are extglob_pattern nodes, * matching the reference scanner. */ + /** case_item: `( pattern (| pattern)* ) statements? terminator?`. + * Patterns are scanned character-wise (they may hold extglob groups with + * parens the token stream cannot represent) and classified following the + * reference scanner: a bare literal is a word (or concatenation pieces), + * glob material is one extglob_pattern, and a pattern mixing glob text + * with a quote/expansion splits into extglob_pattern pieces around the + * construct (the grammar's _extglob_blob). */ private parseCaseItem(): Frame { const kids: Frame[] = []; let token = this.lexer.peek(); @@ -964,21 +974,17 @@ export class Parser { kids.push(this.anon('(', token.start, token.end)); } // Patterns. + let firstAlternative = true; for (;;) { token = this.lexer.peek(); if (token.type !== 'word') { this.hasError = true; // missing pattern break; } - const [start, end] = this.consumeWordRun(); - const raw = this.text(start, end); - const bare = !/["'$`\\]/.test(raw); - const followedByAlternate = this.lexer.peek().type === 'op' && this.tokenText(this.lexer.peek()) === '|'; - if (bare && (/[*?[\]]/.test(raw) || followedByAlternate)) { - kids.push(this.frame('extglob_pattern', start, end)); - } else { - kids.push(this.parseLiteral(start, end)); - } + const alt = this.scanCasePatternEnd(token.start); + kids.push(...this.parseCasePattern(token.start, alt.end, firstAlternative)); + firstAlternative = false; + this.lexer.reposition(alt.end); token = this.lexer.peek(); if (token.type === 'op' && this.tokenText(token) === '|') { this.lexer.next(); @@ -1010,6 +1016,265 @@ export class Parser { return this.frame('case_item', start, end, kids); } + /** End of one case pattern alternative: the position of the top-level `)` + * or `|` (extglob group parens nest). Quote/escape/substitution aware. */ + private scanCasePatternEnd(i: number): { end: number } { + const end = this.lexer.rangeEnd; + let depth = 0; + let j = i; + let sinceTick = 0; + while (j < end) { + if (++sinceTick >= SCAN_TICK_INTERVAL) { + this.budget.progress(); + sinceTick = 0; + } + const ch = this.source[j]!; + if (ch === '\n' || ch === ';') break; // defensive: malformed item + if (ch === '\\') { + j += 2; + continue; + } + if (ch === '"') { + j = skipDoubleQuoted(this.source, this.budget, j, end); + continue; + } + if (ch === "'") { + j = skipSingleQuoted(this.source, this.budget, j, end); + continue; + } + if (ch === '`') { + j = skipBacktick(this.source, this.budget, j, end); + continue; + } + if (ch === '$') { + j = this.skipDollarConstruct(j, end); + continue; + } + if (ch === '(') { + depth++; + j++; + continue; + } + if (ch === ')') { + if (depth === 0) return { end: j }; + depth--; + j++; + continue; + } + if (ch === '|' && depth === 0) return { end: j }; + if ((ch === ' ' || ch === '\t' || ch === '\r') && depth === 0) return { end: j }; + j++; + } + return { end: j }; + } + + /** Build the frames for one case pattern alternative. The extglob_blob + * piece split (glob text around a quote/expansion) only applies to the + * FIRST alternative — after `|` the reference keeps such patterns as + * plain concatenations. */ + private parseCasePattern(start: number, end: number, allowBlob: boolean): Frame[] { + const raw = this.text(start, end); + // Extglob group present: accepted groups make the whole alternative one + // extglob_pattern; rejected ones degrade with hasError (the reference + // reparses the group as a new item — a recovery shape we do not mirror). + // `.(` is not a sigil, but the reference rejects it the same way. + const group = /[?*+@!.]\(/.exec(raw); + if (group !== null) { + if (this.extglobGroupAccepted(start, end)) return [this.frame('extglob_pattern', start, end)]; + this.hasError = true; + return [this.frame('ERROR', start, end)]; + } + if (!/["'$`]/.test(raw)) { + // Bare pattern. + if (this.isBareGlobPattern(start, end)) return [this.frame('extglob_pattern', start, end)]; + return [this.parseLiteral(start, end)]; + } + // Glob text mixed with a quote/expansion: extglob_pattern pieces around + // a single construct when the leading run qualifies as a glob. + if (allowBlob) { + const pieces = this.parseExtglobBlob(start, end); + if (pieces !== null) return pieces; + } + return [this.parseLiteral(start, end)]; + } + + /** + * Whether the extglob scanner accepts a group-containing pattern + * (scanner.c's first/second character rules): the first character must be + * a letter or one of `?*+@!-)\.[` (a leading escape consumes the escaped + * space/quote with it, plus one more character), and the character at the + * scanner's check position must be alphanumeric or one of `([?/\_*`. + */ + private extglobGroupAccepted(start: number, end: number): boolean { + const isAlpha = (ch: string): boolean => /[A-Za-z]/.test(ch); + const isAlnum = (ch: string): boolean => /[A-Za-z0-9]/.test(ch); + const c0 = this.source[start]!; + if (!isAlpha(c0) && !'?*+@!-)\\.['.includes(c0)) return false; + if (c0 === '[') return true; // the scanner skips the second-char check + let check = start + 1; + if (c0 === '\\') { + const after = this.source[start + 1]; + if (after === undefined || !(after === ' ' || after === '\t' || after === '\r' || after === '"')) return false; + check = start + 3; // escape pair + one unconditionally consumed char + } + if (check >= end) return false; + const ch = this.source[check]!; + return isAlnum(ch) || '([?/\\_*'.includes(ch); + } + + /** + * Whether a bare case pattern (no quotes, substitutions or groups) is an + * extglob_pattern in the reference, derived from scanner.c's + * extglob_pattern scanner (the same source as isTestExtglob for `[[ ]]` + * right sides) and verified against the wasm build on an ~90-pattern + * matrix (foo_bar/word1/abc1d/v2.0/a1/a_ → glob; abc/a.b/1a/é/a-b → word): + * 1. the first character must be a (Unicode) letter or one of + * `?*+@!-)\.[`, else the pattern is a literal; + * 2. a leading single dash makes the pattern a word when the rest is + * plain alphanumerics (the scanner's special-variable branch eats + * the `-` before the glob scan: `-o`, `-xy`, `-abc` → word, `-1` → + * number); `--anything` is a glob, and so is a single dash with an + * inner dash run (`-x-`, `-x-y`, `-x-y-z` → glob) unless the inner + * dashes are doubled (`-x--` → word); + * 3. single-character patterns use the scanner's early exits: a letter + * before `)` is a word, anything else (and anything before `|` or + * whitespace) is a glob; + * 4. a dash in second position is consumed together with its + * alphanumeric run BEFORE the validity check; the pattern is a word + * when the run ends the pattern directly before `)` (`a-b)`, `z-)`) + * or is followed by `.` / a character that is not valid pattern + * material (`a-b-c`). Before `|` or whitespace the glob survives + * (`a-b|`, `a-b `); + * 5. the second character must be alphanumeric or one of `([?/\_*` + * (`*.txt`, `a.b`, `q+w`, `a:b` → word; skipped for a leading `[`); + * 6. ending quirk: a pattern directly followed by `|` is always a glob; + * 7. otherwise the pattern is a glob iff some character counts as glob + * material: the first character when it is not a letter, or any later + * character that is neither a letter nor `.` (digits and `_` count — + * `foo_bar`, `abc1d` are globs; `abc`, `foo.txt` are words). + */ + private isBareGlobPattern(start: number, end: number): boolean { + const raw = this.text(start, end); + if (raw.includes('\\')) return false; + if (NUMBER_RE.test(raw)) return false; // 12, -1, 0x1F → number + const isAlpha = (ch: string): boolean => /^\p{L}$/u.test(ch); + const isAlnum = (ch: string): boolean => /^[\p{L}\p{N}]$/u.test(ch); + const next = this.source[end] ?? ''; + const c0 = raw[0]!; + // (1) first character + if (!isAlpha(c0) && !'?*+@!-)\\.['.includes(c0)) return false; + // (2) leading dash: the scanner's special-variable branch eats a + // leading `-` before the glob scan. A single dash followed by plain + // alphanumerics is a word/number (`-o`, `-xy`, `-x1` → word, `-1` → + // number); `--anything` is a glob; so is a single dash with exactly + // one INNER dash run (`-x-`, `-x-y`, `-x-y-z` → glob) — but a doubled + // inner dash falls back to a word (`-x--`). + if (c0 === '-') { + if (raw.length > 1 && raw[1] === '-') return true; + let innerDash = -1; + for (let i = 1; i < raw.length; i++) { + if (raw[i] === '-') { + innerDash = i; + break; + } + } + if (innerDash === -1) return false; + return raw[innerDash + 1] !== '-'; + } + // (3) single-character patterns: the scanner's early exits — but only + // at a real pattern END (`)`, `|` or whitespace; when a quote or + // substitution follows, the run continues as a blob piece and the + // glob-material check below decides). + if (raw.length === 1) { + if (next === ')') return !isAlpha(c0); + if (next === '|' || next === ' ' || next === '\t' || next === '\r' || next === '\n' || next === '') return true; + } + // (4) a dash in second position is consumed together with its + // alphanumeric run BEFORE the validity check; the pattern is a word + // when the run ends the pattern directly before `)` (a-b), z-)) or is + // followed by `.` / a character that is not valid pattern material + // (a-b-c). Before `|` or whitespace the glob survives (a-b|, a-b␣). + if (raw.length > 1) { + let p = 1; + if (raw[1] === '-') { + let i = 2; + while (i < raw.length && isAlnum(raw[i]!)) i++; + if (i >= raw.length) { + if (next === ')') return false; + return true; + } + const after = raw[i]!; + if (after === '.' || after === '\\') return false; + if (!isAlnum(after) && !'([?/\\_*'.includes(after)) return false; + p = i; + } + // (5) second-character validity (skipped for a leading `[`) + if (c0 !== '[' && p === 1 && !isAlnum(raw[1]!) && !'([?/\\_*'.includes(raw[1]!)) return false; + } + // (6) ending quirk: directly before `|` the scanner accepts unconditionally + if (next === '|') return true; + // (7) glob material? + if (!isAlpha(c0)) return true; + for (let i = 1; i < raw.length; i++) { + const ch = raw[i]!; + if (ch !== '.' && !isAlpha(ch)) return true; + } + return false; + } + + /** + * The _extglob_blob shape: extglob_pattern pieces around ONE construct + * (string / raw_string / expansion / command_substitution). Returns null + * when the leading bare run does not qualify as a glob (the pattern is a + * plain concatenation in the reference) or the shape does not fit. + */ + private parseExtglobBlob(start: number, end: number, leadValidator?: (s: number, e: number) => boolean): Frame[] | null { + // Find the first construct. + let i = start; + while (i < end) { + const ch = this.source[i]!; + if (ch === '\\') { + i += 2; + continue; + } + if (ch === '"' || ch === "'" || ch === '`' || ch === '$') break; + i++; + } + if (i >= end || i === start) return null; + // The leading bare run must qualify as a glob. Runs with escapes count + // when they contain an explicit glob character (`*\ ${x}` → extglob + // "*\ "); unescaped runs go through the bare-pattern rule. A custom + // validator applies for group-containing runs (test patterns). + const leadText = this.text(start, i); + const leadIsGlob = + leadValidator !== undefined + ? leadValidator(start, i) + : leadText.includes('\\') + ? /[*?[]/.test(leadText) + : this.isBareGlobPattern(start, i); + if (!leadIsGlob) return null; + const lead = this.frame('extglob_pattern', start, i); + // The construct. + let construct: Frame; + let next: number; + const ch = this.source[i]!; + if (ch === '"') { + [construct, next] = this.parseString(i, end); + } else if (ch === "'") { + next = skipSingleQuoted(this.source, this.budget, i, end); + construct = this.frame('raw_string', i, next); + } else if (ch === '`') { + [construct, next] = this.parseBacktickSubstitution(i, end); + } else { + const dollar = this.parseDollar(i, end); + if (dollar === null) return null; + [construct, next] = dollar; + } + if (next >= end) return [lead, construct]; + const trail = this.frame('extglob_pattern', next, end); + return [lead, construct, trail]; + } + /** case_item statements with the esac-argument guard active (see * caseItemDepth). */ private parseCaseItemStatements(): Frame[] { @@ -1101,7 +1366,7 @@ export class Parser { private isFunctionDefinitionAhead(): boolean { const name = this.lexer.peek(); const text = this.tokenText(name); - if (!IDENTIFIER_RE.test(text) || RESERVED_WORD_SET.has(text)) return false; + if (!FUNCTION_NAME_RE.test(text) || RESERVED_WORD_SET.has(text)) return false; const open = this.lexer.peekAt(1); if (open.type !== 'op' || this.tokenText(open) !== '(') return false; const close = this.lexer.peekAt(2); @@ -1433,7 +1698,15 @@ export class Parser { if (descriptor !== null) kids.push(descriptor); kids.push(this.anon(op, opToken.start, opToken.end)); let end = opToken.end; - if (op !== '>&-' && op !== '<&-') { + if (op === '>&-' || op === '<&-') { + // Close-fd operators take an OPTIONAL single destination in the + // reference (`exec {a} <&- {b}>&-` puts `{b}` in the `<&-` redirect). + if (this.lexer.peek().type === 'word') { + const destination = this.parseWordArgument(); + kids.push(destination); + end = destination.end; + } + } else { let destinations = 0; while (destinations < maxDestinations && this.lexer.peek().type === 'word') { const destination = this.parseWordArgument(); @@ -1807,6 +2080,22 @@ export class Parser { this.addKid(string, this.frame('string_content', chunkStart, upto)); } }; + /** A whitespace-only chunk is absorbed into the next construct's opener + * (` " ${x}"` → (${) token " ${"), or into the closing quote after an + * expansion (`"$x "` → (") token " \"") — the reference scanner never + * emits a whitespace-only string_content at those positions. Leaf + * constructs without an opener token (ansi_c_string) cannot absorb. */ + const whitespaceOnlyChunk = (upto: number): boolean => { + if (upto <= chunkStart) return false; + return /^[ \t\r]+$/.test(this.source.slice(chunkStart, upto)); + }; + const absorbInto = (piece: Frame): boolean => { + const first = piece.children[0]; + if (first === undefined) return false; + piece.start = chunkStart; + first.start = chunkStart; + return true; + }; let sinceTick = 0; while (i < rangeEnd) { if (++sinceTick >= SCAN_TICK_INTERVAL) { @@ -1815,8 +2104,14 @@ export class Parser { } const ch = this.source[i]!; if (ch === '"') { - flushChunk(i); - this.addKid(string, this.anon('"', i, i + 1)); + const last = string.children.at(-1); + if (whitespaceOnlyChunk(i) && last !== undefined && last.isNamed && last.type !== 'string_content') { + // Absorbed into the closing quote (see whitespaceOnlyChunk). + this.addKid(string, this.anon('"', chunkStart, i + 1)); + } else { + flushChunk(i); + this.addKid(string, this.anon('"', i, i + 1)); + } string.end = i + 1; return [string, i + 1]; } @@ -1827,13 +2122,20 @@ export class Parser { if (ch === '$') { const dollar = this.parseDollar(i, rangeEnd); if (dollar !== null) { - flushChunk(i); + if (!(whitespaceOnlyChunk(i) && absorbInto(dollar[0]))) { + flushChunk(i); + } this.addKid(string, dollar[0]); i = dollar[1]; chunkStart = i; continue; } + // A `$` that starts no construct is an anonymous ($) token, splitting + // the content (`"s$"` → string_content "s", ($) "$"). + flushChunk(i); + this.addKid(string, this.anon('$', i, i + 1)); i++; + chunkStart = i; continue; } if (ch === '`') { @@ -1921,8 +2223,8 @@ export class Parser { * expansion: ${ … } with optional prefix operators (! #), a variable_name * / special_variable_name / subscript, and an optional infix operator * whose value layout follows the reference: - * # ## % %% removal — the pattern is a regex node (quotes become - * string/raw_string pieces) + * # ## % %% removal — the pattern is a regex node (single-quoted + * segments fold into it; double quotes stay strings) * / // /# /% replacement — regex pattern, optional `/` + literal * ^ ^^ , ,, case modification — optional regex * @X transformation — two anonymous operator nodes @@ -1947,6 +2249,53 @@ export class Parser { } } + /** + * Whether a ${a[…]} subscript index parses as an expression in the + * reference (binary/unary/parenthesized) rather than as a plain literal: + * true when the index contains a top-level arithmetic operator ADJACENT + * TO WHITESPACE (`${a[1 + 2]}` — a fused `i+1` stays a word) or starts + * with ++/-- (`${w[++counter]}`). Operators inside parens, brackets, + * quotes and $-constructs do not count. + */ + private isArithmeticSubscriptIndex(start: number, end: number): boolean { + if (this.source.startsWith('++', start) || this.source.startsWith('--', start)) return true; + const isBlank = (ch: string | undefined): boolean => ch === ' ' || ch === '\t' || ch === '\r'; + let i = start; + while (i < end) { + const ch = this.source[i]!; + if (ch === '\\') { + i += 2; + continue; + } + if (ch === '"' || ch === "'" || ch === '`') { + i++; + continue; + } + if (ch === '$') { + i = this.skipDollarConstruct(i, end); + continue; + } + if (ch === '(') { + i = this.scanBalanced(i, end, '(', ')').end; + continue; + } + if (ch === '[') { + i = this.scanBalanced(i, end, '[', ']').end; + continue; + } + if ('+-*/%<>=!&|^~?,'.includes(ch) && (isBlank(this.source[i - 1]) || isBlank(this.source[i + 1]))) return true; + i++; + } + return false; + } + + /** Rename variable_name leaves to word (the reference uses plain _literal + * operands — word, not variable_name — in subscript index expressions). */ + private remapVariableNamesToWords(frame: Frame): void { + if (frame.type === 'variable_name') frame.type = 'word'; + for (const child of frame.children) this.remapVariableNamesToWords(child); + } + private parseExpansionInner(i: number, rangeEnd: number): [Frame, number] { const scan = this.scanBalanced(i + 1, rangeEnd, '{', '}'); const close = scan.end; @@ -1974,12 +2323,21 @@ export class Parser { this.addKid(expansion, this.frame(special ? 'special_variable_name' : 'variable_name', j, nameEnd)); j = nameEnd; hasName = true; + } else if (j < innerEnd && (this.source[j] === '#' || this.source[j] === '!') && j + 1 === innerEnd) { + // A lone # / ! in variable position is an anonymous token in the + // reference (`${#}`, `${!#}`, `${!##}`, `${#!}` — the length/indirect + // operator with no name), not a special_variable_name. + this.addKid(expansion, this.anon(this.source[j]!, j, j + 1)); + j++; } else if (j < innerEnd && SPECIAL_VARIABLE_CHARS.includes(this.source[j]!)) { this.addKid(expansion, this.frame('special_variable_name', j, j + 1)); j++; hasName = true; } - // Subscript: ${a[0]} — replaces the bare variable_name child. + // Subscript: ${a[0]} — replaces the bare variable_name child. A simple + // index is a literal; an index with a top-level arithmetic operator or a + // ++/-- prefix is an expression in the reference (`${a[1 + 2]}` → + // binary_expression, `${w[++c]}` → unary_expression). if (j < innerEnd && this.source[j] === '[' && hasName) { const sub = this.scanBalanced(j, rangeEnd, '[', ']'); const subEnd = sub.end; @@ -1991,7 +2349,22 @@ export class Parser { this.anon('[', j, j + 1), ]); if (indexEnd > j + 1) { - this.addKid(subscript, this.parseLiteral(j + 1, indexEnd)); + const indexText = this.text(j + 1, indexEnd); + if (indexText === '*' || indexText === '@') { + // ${a[*]} / ${a[@]} — all-indices subscripts are plain words. + this.addKid(subscript, this.parseLiteral(j + 1, indexEnd)); + } else if (this.isArithmeticSubscriptIndex(j + 1, indexEnd)) { + const st = this.newExprState(j + 1, indexEnd, 'arith'); + const index = this.parseExpression(st, 0); + if (index !== null) { + this.remapVariableNamesToWords(index); + this.addKid(subscript, index); + } + const leftover = this.exprLeftover(st); + if (leftover !== null) this.addKid(subscript, leftover); + } else { + this.addKid(subscript, this.parseLiteral(j + 1, indexEnd)); + } } else { this.hasError = true; } @@ -2030,7 +2403,7 @@ export class Parser { this.addKid(expansion, this.anon('/', pos, pos + 1)); pos++; if (pos < innerEnd) { - this.addKid(expansion, this.parseExpansionValue(pos, innerEnd)); + for (const piece of this.parseExpansionValue(pos, innerEnd)) this.addKid(expansion, piece); pos = innerEnd; } } @@ -2066,7 +2439,7 @@ export class Parser { this.addKid(expansion, this.anon(operator, j, j + operator.length)); const pos = j + operator.length; if (pos < innerEnd) { - this.addKid(expansion, this.parseExpansionValue(pos, innerEnd)); + for (const piece of this.parseExpansionValue(pos, innerEnd)) this.addKid(expansion, piece); } return innerEnd; } @@ -2077,11 +2450,29 @@ export class Parser { return innerEnd; } - /** A removal/replacement pattern inside ${…}: regex chunks (raw text, - * spaces included) interleaved with string/raw_string pieces. `stop` - * optionally ends the pattern (the replacement separator `/`). */ + /** A removal/replacement pattern inside ${…}: regex chunks interleaved + * with string pieces. Rules from the reference scanner: + * - leading whitespace after the operator drops out of the tree when + * more pattern follows (${G%% *} → regex "*"); an all-whitespace + * pattern stays a regex (${B[0]# } → regex " "); + * - in the REPLACEMENT pattern (before the separator `/`), single-quoted + * segments fold INTO the surrounding regex chunk (${M/'N'X/O} → + * regex "'N'X"); in removal patterns (# ## % %% ^ ^ , ,,) they stay + * separate raw_string nodes, and double quotes always stay strings; + * - the replacement separator `/` ends the pattern even when escaped + * (a pattern ending in backslash-slash → regex keeps the backslash, + * then the `/` operator). */ private parseExpansionPattern(expansion: Frame, j: number, innerEnd: number, stop?: string): number { let pos = j; + while (pos < innerEnd && (this.source[pos] === ' ' || this.source[pos] === '\t' || this.source[pos] === '\r')) { + pos++; + } + if (pos > j && (pos >= innerEnd || (stop !== undefined && this.source[pos] === stop))) { + // An all-whitespace pattern stays a regex node. + this.addKid(expansion, this.frame('regex', j, pos)); + return pos; + } + const mergeQuotes = stop === '/'; while (pos < innerEnd) { this.budget.progress(); const ch = this.source[pos]!; @@ -2091,20 +2482,32 @@ export class Parser { pos = next; continue; } - if (ch === "'") { + if (ch === "'" && !mergeQuotes) { const close = skipSingleQuoted(this.source, this.budget, pos, innerEnd); this.addKid(expansion, this.frame('raw_string', pos, close)); pos = close; continue; } + // One regex chunk: bare text, plus single-quoted segments when they + // fold into the regex (replacement patterns). let k = pos; let sinceTick = 0; - while (k < innerEnd && this.source[k] !== '"' && this.source[k] !== "'" && this.source[k] !== stop) { + while (k < innerEnd && this.source[k] !== '"' && this.source[k] !== stop) { if (++sinceTick >= SCAN_TICK_INTERVAL) { this.budget.progress(); sinceTick = 0; } - if (this.source[k] === '\\' && k + 1 < innerEnd) { + const ck = this.source[k]!; + if (ck === "'") { + if (!mergeQuotes) break; + k = skipSingleQuoted(this.source, this.budget, k, innerEnd); + continue; + } + if (ck === '\\' && k + 1 < innerEnd) { + if (stop !== undefined && this.source[k + 1] === stop) { + k++; // the escape backslash stays in the chunk; the stop char ends it + break; + } k += 2; continue; } @@ -2117,24 +2520,73 @@ export class Parser { return pos; } - /** A default/assign value inside ${…} (`${v:-word}`): bare text is a word - * (never a number — `${v:-1}` is a word in the reference); a bare value - * containing a space splits into two word pieces inside a concatenation - * (`${v:-d e f}` → word "d" + word " e f", matching the reference's - * _expansion_word scanner). */ - private parseExpansionValue(start: number, end: number): Frame { + /** A default/assign value inside ${…} (`${v:-word}`), as a list of frames + * (usually one — the reference sometimes attaches the value as several + * direct children of the expansion). Layout rules from the reference: + * - a bare value is a word (never a number — `${v:-1}` is a word). An + * interior space splits it into word + word (the second keeps the + * leading space: `${v:-d e f}` → word "d" + word " e f"); trailing-only + * whitespace stays inside the single word (`${v:-) }` → word ") "), and + * a leading `? ` (Gentoo atom syntax) keeps the whole value as one word; + * - bare word pieces directly before a quote/construct are right-trimmed + * (`${v:-command '$1' x}` → word "command", raw_string, word " x") and + * a whitespace-only piece before a construct drops out entirely + * (`${2+ ${2}}` → just the expansion); + * - a command_substitution followed by a bare remainder attaches as two + * children without a concatenation wrapper. */ + private parseExpansionValue(start: number, end: number): Frame[] { const raw = this.text(start, end); if (!/["'$`\\]/.test(raw)) { - const space = raw.indexOf(' '); - if (space > 0) { - return this.frame('concatenation', start, end, [ - this.frame('word', start, start + space), - this.frame('word', start + space, end), - ]); + // Bare value. Rules from the reference's _expansion_word scanner: + // - an interior space splits it into word + word (the second keeps + // the leading space: `${v:-d e f}` → word "d" + word " e f"); + // - leading or trailing whitespace stays inside the single word + // (`${v:- -quiet}` → word " -quiet", `${v:-) }` → word ") "), and a + // leading `? ` (Gentoo atom syntax) keeps the whole value as one + // word. Whitespace only drops out directly before a quote or a + // construct (handled by the piece path below). + if (this.source.startsWith('? ', start)) return [this.frame('word', start, end)]; + for (let k = start + 1; k < end; k++) { + const ch = this.source[k]!; + if (ch !== ' ' && ch !== '\t' && ch !== '\r') continue; + // Split at the first interior space (non-space text follows it). + if (/[^ \t\r]/.test(this.source.slice(k + 1, end))) { + return [ + this.frame('concatenation', start, end, [ + this.frame('word', start, k), + this.frame('word', k, end), + ]), + ]; + } + break; + } + return [this.frame('word', start, end)]; + } + const value = this.parseLiteral(start, end); + const pieces = value.type === 'concatenation' ? [...value.children] : [value]; + // Right-trim bare word pieces that directly precede another piece (the + // whitespace drops out of the tree); drop pieces that become empty. + // Pieces starting with `(` keep their spaces (the reference's + // expansion_word scanner treats them via its paren branch). + const trimmed: Frame[] = []; + for (let p = 0; p < pieces.length; p++) { + const piece = pieces[p]!; + if (piece.type === 'word' && p + 1 < pieces.length && this.source[piece.start] !== '(') { + let newEnd = piece.end; + while (newEnd > piece.start && (this.source[newEnd - 1] === ' ' || this.source[newEnd - 1] === '\t' || this.source[newEnd - 1] === '\r')) { + newEnd--; + } + if (newEnd === piece.start) continue; + trimmed.push(newEnd === piece.end ? piece : this.frame('word', piece.start, newEnd)); + } else { + trimmed.push(piece); } - return this.frame('word', start, end); } - return this.parseLiteral(start, end); + // command_substitution + bare remainder: two direct children. + if (trimmed.length === 2 && trimmed[0]!.type === 'command_substitution') return trimmed; + if (trimmed.length === 1) return trimmed; + if (trimmed.length === 0) return []; + return [this.frame('concatenation', start, end, trimmed)]; } /** One arithmetic value of ${v:offset:length} (up to `:` or innerEnd). */ @@ -2194,7 +2646,9 @@ export class Parser { return i + 1 < end ? i + 2 : i + 1; } - /** command_substitution: $( _statements ). The close-paren scan is + /** command_substitution: $( _statements ) — or the special $( redirect ) + * form (`$(< file)`), which holds a bare file_redirect without the + * redirected_statement wrapper in the reference. The close-paren scan is * case-aware (see scanBalancedStatements): a case_item pattern `)` must * not close the substitution early. */ private parseCommandSubstitution(i: number, rangeEnd: number): [Frame, number] { @@ -2203,7 +2657,16 @@ export class Parser { if (!scan.balanced) this.hasError = true; const innerEnd = scan.balanced ? close - 1 : close; const substitution = this.frame('command_substitution', i, close, [this.anon('$(', i, i + 2)]); - for (const child of this.parseScopedStatements(i + 2, innerEnd)) { + let children = this.parseScopedStatements(i + 2, innerEnd); + if ( + children.length === 1 && + children[0]!.type === 'redirected_statement' && + children[0]!.children.length > 0 && + children[0]!.children.every((child) => child.type === 'file_redirect' || child.type === 'herestring_redirect') + ) { + children = children[0]!.children; + } + for (const child of children) { this.addKid(substitution, child); } if (scan.balanced) this.addKid(substitution, this.anon(')', close - 1, close)); @@ -2432,20 +2895,20 @@ export class Parser { const right = this.parseTestRegex(st); const kids: Frame[] = [left, this.anon('=~', token.start, token.end)]; if (right === null) this.hasError = true; - else kids.push(right); + else kids.push(...right); left = this.frame('binary_expression', left.start, this.endOf(kids, token.end), kids); st.expectOperator = true; continue; } const rightPrecedence = token.text === '**' && st.mode === 'test' ? precedence : precedence + 1; let right: Frame | null; - // An extglob group pattern after ==/!= is one extglob_pattern node - // (`[[ $a == +(!a) ]]` in the reference). + // A pattern-shaped right side after ==/!= (extglob group, or glob + // text mixed with a quote/expansion) — see tryParseTestPattern. if (st.mode === 'test' && (token.text === '==' || token.text === '!=')) { - right = this.tryParseExtglobGroup(st); - if (right !== null) { - const kids: Frame[] = [left, this.anon(token.text, token.start, token.end), right]; - left = this.frame('binary_expression', left.start, right.end, kids); + const pattern = this.tryParseTestPattern(st); + if (pattern !== null) { + const kids: Frame[] = [left, this.anon(token.text, token.start, token.end), ...pattern.frames]; + left = this.frame('binary_expression', left.start, pattern.end, kids); st.expectOperator = true; continue; } @@ -2566,9 +3029,14 @@ export class Parser { } } - /** The right side of `=~` in a test command: a quoted string, an - * expansion, or a raw regex run (up to whitespace). */ - private parseTestRegex(st: ExprState): Frame | null { + /** The right side of `=~` in a test command, as a list of frames. Quote + * rules from the reference scanner: a regex STARTING with a single quote + * folds the quoted segments into the regex token (`' boop '(.*)$` is one + * regex) unless the whole thing is just a quoted string plus plain word + * characters (`'a'b` → raw_string + word); a quote appearing mid-run + * makes the whole thing a plain concatenation (`a'b'c`). Whitespace + * inside (…) stays inside the regex token. */ + private parseTestRegex(st: ExprState): Frame[] | null { let i = st.pos; while (i < st.end && (this.source[i] === ' ' || this.source[i] === '\t' || this.source[i] === '\r')) i++; st.pos = i; @@ -2578,18 +3046,22 @@ export class Parser { if (ch === '"') { const [piece, next] = this.parseString(i, st.end); st.pos = next; - return piece; - } - if (ch === "'") { - const close = skipSingleQuoted(this.source, this.budget, i, st.end); - st.pos = close; - return this.frame('raw_string', i, close); + return [piece]; } if (ch === '$') { // An expansion holds the pattern: parse a normal operand. - return this.parseExpression(st, PREC_TEST + 1); - } + const operand = this.parseExpression(st, PREC_TEST + 1); + return operand === null ? null : [operand]; + } + // Scan the run as one regex token: it ends at whitespace that is not + // protected by an open paren (the reference scanner includes whitespace + // inside (…) in the token), at an unbalanced closer, or at the region + // end. Single quotes toggle an in-quote state; `$`-constructs are raw + // text (the scanner does not split them out here). let j = i; + let inQuote = false; + let hasQuote = false; + let parenDepth = 0; let sinceTick = 0; while (j < st.end) { if (++sinceTick >= SCAN_TICK_INTERVAL) { @@ -2597,33 +3069,121 @@ export class Parser { sinceTick = 0; } const c = this.source[j]!; - if (c === ' ' || c === '\t' || c === '\r' || c === '\n') break; + if (!inQuote && parenDepth === 0 && (c === ' ' || c === '\t' || c === '\r' || c === '\n')) break; + if (c === "'") { + hasQuote = true; + inQuote = !inQuote; + j++; + continue; + } if (c === '\\') { j += 2; continue; } + if (!inQuote) { + if (c === '(') parenDepth++; + else if (c === ')') { + if (parenDepth === 0) break; + parenDepth--; + } + } j++; } st.pos = j; - return this.frame('regex', i, j); + if (ch === "'") { + // Quoted string plus plain word characters → a plain literal. + if (/^'[^']*'\w*$/.test(this.text(i, j))) return [this.parseLiteral(i, j)]; + return [this.frame('regex', i, j)]; + } + if (hasQuote) return [this.parseLiteral(i, j)]; + return [this.frame('regex', i, j)]; } - /** An extglob group pattern (`?(…)`, `*(…)`, `+(…)`, `@(…)`, `!(…)`) - * after `==`/`!=` in a test command: one raw extglob_pattern node over - * the operator and its balanced group. Null (no state touched) when the - * upcoming text is not a group. */ - private tryParseExtglobGroup(st: ExprState): Frame | null { + /** + * Pattern-shaped right side of ==/!= in a test command (the grammar's + * _extglob_blob). Handles what a plain expression parse cannot: + * - a whole extglob group pattern (+(a|b), X/@(default).vim with X a + * glob, *([0-9])([0-9])) as ONE extglob_pattern, when the scanner's + * first/second character rules accept it; + * - glob text mixed with one quote/expansion (*${var2}*, *"7f 4c"*), + * split into extglob_pattern pieces around the construct. + * Returns null (no state touched) for plain operands — those go through + * the normal expression parse plus convertTestRightSide. + */ + private tryParseTestPattern(st: ExprState): { frames: Frame[]; end: number } | null { let i = st.pos; while (i < st.end && (this.source[i] === ' ' || this.source[i] === '\t' || this.source[i] === '\r')) i++; - const ch = this.source[i]; - if ((ch === '?' || ch === '*' || ch === '+' || ch === '@' || ch === '!') && this.source[i + 1] === '(') { - const scan = this.scanBalanced(i + 1, st.end, '(', ')'); - if (!scan.balanced) return null; - st.pos = scan.end; - st.lookahead = null; - return this.frame('extglob_pattern', i, scan.end); + if (i >= st.end) return null; + // Find the end of the pattern: unquoted whitespace at paren depth 0. + let depth = 0; + let j = i; + let sawGroup = false; + let sawConstruct = false; + while (j < st.end) { + const ch = this.source[j]!; + if (ch === '\\') { + j += 2; + continue; + } + if (ch === '"') { + sawConstruct = true; + j = skipDoubleQuoted(this.source, this.budget, j, st.end); + continue; + } + if (ch === "'") { + sawConstruct = true; + j = skipSingleQuoted(this.source, this.budget, j, st.end); + continue; + } + if (ch === '`') { + sawConstruct = true; + j = skipBacktick(this.source, this.budget, j, st.end); + continue; + } + if (ch === '$') { + sawConstruct = true; + j = skipDollar(this.source, this.budget, j, st.end); + continue; + } + if ((ch === '?' || ch === '*' || ch === '+' || ch === '@' || ch === '!') && this.source[j + 1] === '(') { + sawGroup = true; + } + if (ch === '(') { + depth++; + j++; + continue; + } + if (ch === ')') { + if (depth > 0) { + depth--; + j++; + continue; + } + break; + } + if (depth === 0 && (ch === ' ' || ch === '\t' || ch === '\r' || ch === '\n')) break; + j++; } - return null; + const hasGroup = sawGroup; + const hasConstruct = sawConstruct; + // A pattern containing an extglob group but no construct: one + // extglob_pattern node when the scanner accepts it; rejected groups + // keep the normal (error) path. + if (hasGroup && !hasConstruct) { + if (!this.extglobGroupAccepted(i, j)) return null; + st.pos = j; + st.lookahead = null; + return { frames: [this.frame('extglob_pattern', i, j)], end: j }; + } + if (!hasConstruct) return null; + // Glob text (possibly with a group) mixed with one quote/expansion: + // extglob_pattern pieces around the construct (`@(*${x}.tar|*.sig)` → + // extglob "@(*", expansion, extglob ".tar|*.sig)"). + const pieces = this.parseExtglobBlob(i, j, hasGroup ? (s, e) => this.extglobGroupAccepted(s, e) : undefined); + if (pieces === null) return null; + st.pos = j; + st.lookahead = null; + return { frames: pieces, end: j }; } /** Reference-shaped right-hand sides for test comparisons: @@ -2980,6 +3540,23 @@ export class Parser { const exprStart = openToken.end + (double ? 1 : 0); const kids: Frame[] = [this.anon(opener, openToken.start, exprStart)]; const scan = this.scanTestCloser(exprStart, double); + // A single-bracket test whose body contains a top-level redirect is a + // redirected statement in the reference, not a test expression + // (`[ ! command -v go &>/dev/null ]` → redirected_statement with a + // negated_command inside the test_command). + if (!double && scan.closerStart > exprStart && this.rangeHasTopLevelRedirect(exprStart, scan.closerStart)) { + const statements = this.parseScopedStatements(exprStart, scan.closerStart); + kids.push(...statements); + let end = scan.closerStart; + if (scan.found) { + kids.push(this.anon(closer, scan.closerStart, scan.afterCloser)); + end = scan.afterCloser; + } else { + this.hasError = true; + } + this.lexer.reposition(end); + return this.frame('test_command', openToken.start, end, kids); + } if (scan.closerStart > exprStart) { const st = this.newExprState(exprStart, scan.closerStart, 'test'); const expression = this.parseExpression(st, 0); @@ -3023,6 +3600,44 @@ export class Parser { return this.frame('test_command', openToken.start, end, kids); } + /** Whether [start, end) contains a top-level redirect operator (`<`/`>` + * outside quotes, substitutions and `<(`/`>(`). Used to recognize the + * redirected-statement form of a single-bracket test command. */ + private rangeHasTopLevelRedirect(start: number, end: number): boolean { + let j = start; + let sinceTick = 0; + while (j < end) { + if (++sinceTick >= SCAN_TICK_INTERVAL) { + this.budget.progress(); + sinceTick = 0; + } + const ch = this.source[j]!; + if (ch === '\\') { + j += 2; + continue; + } + if (ch === '"') { + j = skipDoubleQuoted(this.source, this.budget, j, end); + continue; + } + if (ch === "'") { + j = skipSingleQuoted(this.source, this.budget, j, end); + continue; + } + if (ch === '`') { + j = skipBacktick(this.source, this.budget, j, end); + continue; + } + if (ch === '$') { + j = this.skipDollarConstruct(j, end); + continue; + } + if ((ch === '<' || ch === '>') && this.source[j + 1] !== '(') return true; + j++; + } + return false; + } + /** Find the closer of a test command, skipping quotes and substitutions. * Bounded by the end of the line and the lexer's range. */ private scanTestCloser(start: number, double: boolean): { closerStart: number; afterCloser: number; found: boolean } { @@ -3097,6 +3712,13 @@ export class Parser { continue; } if (ch === '$') { + // `$'` is NOT an ANSI-C string inside a heredoc body — bash does not + // expand those there and the reference keeps the text as plain + // content too (except for its line-start quirk, see README). + if (this.source[i + 1] === "'") { + i++; + continue; + } const dollar = this.parseDollar(i, end); if (dollar !== null) { flush(i); diff --git a/packages/tree-sitter-bash/test/differential.test.ts b/packages/tree-sitter-bash/test/differential.test.ts new file mode 100644 index 0000000000..0c185e5125 --- /dev/null +++ b/packages/tree-sitter-bash/test/differential.test.ts @@ -0,0 +1,169 @@ +// test/differential.test.ts +// +// Differential suite: every sample in test/fixtures/differential/ and every +// official tree-sitter-bash 0.25.0 corpus case in test/fixtures/corpus/ is +// parsed by BOTH this package and the reference (web-tree-sitter + the +// official wasm build) and the normalized trees are compared byte-for-byte. +// +// - @match samples must produce a tree identical to the reference; +// - @known-diff samples must (a) still differ from the reference and +// (b) match the stored dump of our parser exactly — so a documented +// deviation cannot silently drift, and a fixed deviation fails loudly +// ("remove it from the known list"). +// +// The known-difference registry (test/helpers/known-differences.ts) is the +// single source of truth and is cross-checked against README.md's Known +// differences section here. + +import { readFileSync, readdirSync } from 'node:fs'; +import path from 'node:path'; + +import { beforeAll, describe, expect, it } from 'vitest'; + +import { compareSource, diffDumps, ourDump, parseCorpusFile, parseFixtureFile, referenceDump } from './helpers/differential'; +import type { FixtureSample } from './helpers/differential'; +import { KNOWN_DIFFERENCES, isKnownDifferenceId } from './helpers/known-differences'; + +const PKG_ROOT = path.resolve(import.meta.dirname, '..'); +const DIFF_DIR = path.join(PKG_ROOT, 'test/fixtures/differential'); +const CORPUS_DIR = path.join(PKG_ROOT, 'test/fixtures/corpus'); + +interface LocatedSample extends FixtureSample { + file: string; +} + +function loadDifferentialFixtures(): LocatedSample[] { + const out: LocatedSample[] = []; + for (const file of readdirSync(DIFF_DIR).filter((f) => f.endsWith('.txt')).toSorted()) { + const content = readFileSync(path.join(DIFF_DIR, file), 'utf8'); + for (const sample of parseFixtureFile(`${DIFF_DIR}/${file}`, content)) { + out.push({ ...sample, file }); + } + } + return out; +} + +function loadCorpusKnownDiffs(): Map { + const file = 'known-diffs.txt'; + const content = readFileSync(path.join(CORPUS_DIR, file), 'utf8'); + const map = new Map(); + for (const sample of parseFixtureFile(`${CORPUS_DIR}/${file}`, content)) { + // description: "corpus ::" + const key = sample.description.replace(/^corpus /, ''); + map.set(key, { ...sample, file }); + } + return map; +} + +function label(sample: LocatedSample): string { + return `${sample.file}:${sample.line} ${sample.kind === 'match' ? 'match' : `known-diff ${sample.id}`} — ${sample.description.slice(0, 60)}`; +} + +const fixtures = loadDifferentialFixtures(); +const corpusKnownDiffs = loadCorpusKnownDiffs(); +const corpusCases = readdirSync(CORPUS_DIR) + .filter((f) => f.endsWith('.txt') && f !== 'known-diffs.txt') + .toSorted() + .flatMap((file) => parseCorpusFile(readFileSync(path.join(CORPUS_DIR, file), 'utf8')).map((c) => ({ ...c, file }))); + +beforeAll(async () => { + // Fails with a descriptive error when the wasm reference cannot be loaded. + await compareSource('echo sanity'); +}); + +describe('differential fixtures', () => { + for (const sample of fixtures) registerFixtureTest(sample); +}); + +/** Register one fixture test (lookup table instead of conditionals — the + * vitest lint rules reject conditionals around `it`). */ +function registerFixtureTest(sample: LocatedSample): void { + const register: Record void> = { + match: () => { + it(label(sample), async () => { + const cmp = await compareSource(sample.source); + expect(cmp.diff).toBe(''); + }); + }, + 'known-diff': () => { + it(label(sample), async () => { + const [ours, reference] = [ourDump(sample.source), await referenceDump(sample.source)]; + expect( + ours !== reference, + 'this sample now matches the reference — remove it from the known-difference list (fixture + registry + README)', + ).toBe(true); + expect(ours === sample.expectedOurs, `deviation shape drifted:\n${diffDumps(ours, sample.expectedOurs!)}`).toBe( + true, + ); + }); + }, + }; + register[sample.kind](); +} + +describe('official tree-sitter-bash 0.25.0 corpus', () => { + for (const corpusCase of corpusCases) registerCorpusTest(corpusCase); +}); + +/** Register one corpus case test. */ +function registerCorpusTest(corpusCase: { file: string; name: string; input: string }): void { + const key = `${corpusCase.file}::${corpusCase.name}`; + const known = corpusKnownDiffs.get(key) ?? null; + const register: Record<'match' | 'known-diff', () => void> = { + match: () => { + it(`${key} (match)`, async () => { + const cmp = await compareSource(corpusCase.input); + expect(cmp.diff).toBe(''); + }); + }, + 'known-diff': () => { + it(`${key} (known-diff ${known!.id})`, async () => { + const [ours, reference] = [ourDump(corpusCase.input), await referenceDump(corpusCase.input)]; + expect( + ours !== reference, + 'this corpus case now matches the reference — remove it from known-diffs.txt, the registry and the README', + ).toBe(true); + expect(ours === known!.expectedOurs, `deviation shape drifted:\n${diffDumps(ours, known!.expectedOurs!)}`).toBe( + true, + ); + }); + }, + }; + register[known === null ? 'match' : 'known-diff'](); +} + +describe('known-difference registry consistency', () => { + const usedIds = new Set(); + for (const sample of fixtures) { + if (sample.id !== undefined) usedIds.add(sample.id); + } + for (const sample of corpusKnownDiffs.values()) { + if (sample.id !== undefined) usedIds.add(sample.id); + } + + it('every @known-diff id used in fixtures exists in the registry', () => { + for (const id of usedIds) { + expect(isKnownDifferenceId(id), `fixture uses unregistered id '${id}'`).toBe(true); + } + }); + + it('every registry id is exercised by at least one fixture or corpus case', () => { + for (const entry of KNOWN_DIFFERENCES) { + expect(usedIds.has(entry.id), `registry id '${entry.id}' has no fixture sample`).toBe(true); + } + }); + + it('every registry entry is anchored in README.md Known differences', () => { + const readme = readFileSync(path.join(PKG_ROOT, 'README.md'), 'utf8'); + const sectionMatch = /## Known differences[^\n]*\n([\s\S]*?)(?=\n## |\s*$)/.exec(readme); + expect(sectionMatch, 'README.md has no "## Known differences" section').not.toBeNull(); + // Whitespace-normalized so anchors may span line breaks in the README. + const section = sectionMatch![1]!.replaceAll(/\s+/g, ' '); + for (const entry of KNOWN_DIFFERENCES) { + expect( + section.includes(entry.readmeAnchor.replaceAll(/\s+/g, ' ')), + `README.md Known differences does not contain the anchor for '${entry.id}': ${entry.readmeAnchor}`, + ).toBe(true); + } + }); +}); diff --git a/packages/tree-sitter-bash/test/fixtures/README.md b/packages/tree-sitter-bash/test/fixtures/README.md new file mode 100644 index 0000000000..bcb4403b06 --- /dev/null +++ b/packages/tree-sitter-bash/test/fixtures/README.md @@ -0,0 +1,42 @@ +# Differential fixtures + +Fixtures for the differential test suite (`test/differential.test.ts`): +every sample is parsed by this package and by the reference tree-sitter-bash +0.25.0 parser (web-tree-sitter + wasm), and the normalized trees are +compared. + +## Format + +Blocks are separated by a line containing exactly `===`. Each block starts +with a directive line: + +``` +@match: + +=== +@known-diff : + +--- + +=== +``` + +- `@match`: the two trees must be identical. +- `@known-diff `: the sample documents a known deviation. `` must + exist in `test/helpers/known-differences.ts` (the registry is + cross-checked against the README's Known differences section). The block + must still deviate from the reference, and our tree must match the stored + dump exactly — so deviations cannot silently drift, and a fixed deviation + fails the test with a reminder to remove it from the list. + +`differential/` holds curated samples grouped by theme (statements, +redirects, heredoc, expansions, test-command, arithmetic, case, recovery). +`corpus/` holds the official tree-sitter-bash corpus (see its README); +corpus cases that deviate are listed in `corpus/known-diffs.txt` in the same +format. + +The normalized dump is a pre-order walk of every node (named and +anonymous): one line per node — `type [start,end] "text preview"`, +anonymous types in parentheses — preceded by a `hasError:` line. Offsets +are UTF-16 code units on both sides (web-tree-sitter reports UTF-16 +offsets for string input, same as this parser). diff --git a/packages/tree-sitter-bash/test/fixtures/corpus/README.md b/packages/tree-sitter-bash/test/fixtures/corpus/README.md new file mode 100644 index 0000000000..12524df4ed --- /dev/null +++ b/packages/tree-sitter-bash/test/fixtures/corpus/README.md @@ -0,0 +1,21 @@ +# Official tree-sitter-bash corpus + +The five `*.txt` files in this directory are an unmodified copy of the +official tree-sitter-bash test corpus: + +- Source: https://github.com/tree-sitter/tree-sitter-bash/tree/v0.25.0/test/corpus +- Version: tag `v0.25.0` +- Retrieved: 2026-07-21 (via the GitHub release tarball; only `test/corpus/` + was taken) + +They are inputs to `test/differential.test.ts`: every corpus case is parsed +by both this package and the live reference parser (web-tree-sitter + +tree-sitter-bash.wasm v0.25.0) and the trees are compared byte-for-byte. +The expected S-expressions inside the corpus files are not used — the live +wasm build is the comparison target. + +`known-diffs.txt` lists the corpus cases whose trees deliberately deviate +(see the "Known differences" section of the package README), in the fixture +format described in `test/fixtures/README.md`. A corpus case listed there +must still deviate from the reference AND keep the exact stored deviation +shape; every other corpus case must match the reference byte-for-byte. diff --git a/packages/tree-sitter-bash/test/fixtures/corpus/commands.txt b/packages/tree-sitter-bash/test/fixtures/corpus/commands.txt new file mode 100644 index 0000000000..63e95ce2d9 --- /dev/null +++ b/packages/tree-sitter-bash/test/fixtures/corpus/commands.txt @@ -0,0 +1,725 @@ +=============================== +Commands +=============================== + +whoami + +--- + +(program + (command (command_name (word)))) + +=============================== +Commands with arguments +=============================== + +cat file1.txt +git diff --word-diff=color -- file1.txt file2.txt +echo $sing\ +levar + +--- + +(program + (command (command_name (word)) (word)) + (command (command_name (word)) (word) (word) (word) (word) (word)) + (command (command_name (word)) (simple_expansion (variable_name)) (word))) + +=============================== +Quoted command names +=============================== + +"$a/$b" c + +--- + +(program + (command + (command_name (string (simple_expansion (variable_name)) (string_content) (simple_expansion (variable_name)))) + (word))) + +=============================== +Commands with numeric arguments +=============================== + +exit 1 + +--- + +(program + (command (command_name (word)) (number))) + +=================================== +Commands with environment variables +=================================== + +VAR1=1 ./script/test +VAR1=a VAR2="ok" git diff --word-diff=color + +--- + +(program + (command + (variable_assignment (variable_name) (number)) + (command_name (word))) + (command + (variable_assignment (variable_name) (word)) + (variable_assignment (variable_name) (string (string_content))) + (command_name (word)) + (word) + (word))) + +=================================== +Empty environment variables +=================================== + +VAR1= +VAR2= echo + +--- + +(program + (variable_assignment (variable_name)) + (command (variable_assignment (variable_name)) (command_name (word)))) + +=============================== +File redirects +=============================== + +whoami > /dev/null +cat a b > /dev/null +2>&1 whoami +echo "foobar" >&2 +[ ! command -v go &>/dev/null ] && return + +if [ ]; then + >aa >bb +fi + +exec {VIRTWL[0]} {VIRTWL[1]} <&- >&- +exec {VIRTWL[0]}<&- {VIRTWL[1]}>&- + +grep 2>/dev/null -q "^/usr/bin/scponly$" /etc/shells + +x | /dev/null +cat a b >| /dev/null + +--- + +(program + (redirected_statement + (command (command_name (word))) + (file_redirect (word))) + (redirected_statement + (command (command_name (word)) (word) (word)) + (file_redirect (word)))) + +=============================== +Heredoc redirects +=============================== + +node < $tmpfile +a $B ${C} +EOF + +wc -l $tmpfile + +--- + +(program + (redirected_statement + (command + (command_name + (word))) + (heredoc_redirect + (heredoc_start) + (file_redirect + (simple_expansion + (variable_name))) + (heredoc_body + (simple_expansion + (variable_name)) + (heredoc_content) + (expansion + (variable_name)) + (heredoc_content)) + (heredoc_end))) + (command + (command_name + (word)) + (word) + (simple_expansion + (variable_name)))) + +================================= +Heredocs with many file redirects +================================= + +FOO=bar echo < err.txt > hello.txt +hello +EOF + +--- + +(program + (redirected_statement + body: (command + (variable_assignment + name: (variable_name) + value: (word)) + name: (command_name + (word))) + redirect: (heredoc_redirect + (heredoc_start) + redirect: (file_redirect + descriptor: (file_descriptor) + destination: (word)) + redirect: (file_redirect + destination: (word)) + (heredoc_body) + (heredoc_end)))) + +================================= +Heredocs with pipes +================================= + +one < temp +EOF + +--- + +(program + (redirected_statement + body: (command + name: (command_name + (word))) + redirect: (heredoc_redirect + (heredoc_start) + (heredoc_body) + (heredoc_end))) + (redirected_statement + body: (command + name: (command_name + (word))) + redirect: (heredoc_redirect + (heredoc_start) + (heredoc_body) + (heredoc_end))) + (function_definition + name: (word) + body: (compound_statement + (redirected_statement + body: (command + name: (command_name + (word))) + redirect: (heredoc_redirect + (heredoc_start) + (heredoc_body) + (heredoc_end))))) + (redirected_statement + body: (command + name: (command_name + (word))) + redirect: (heredoc_redirect + (heredoc_start) + redirect: (file_redirect + destination: (word)) + (heredoc_body) + (heredoc_end)))) + +========================================== +Heredocs with weird characters +========================================== + +node <<_DELIMITER_WITH_UNDERSCORES_ +Hello. +_DELIMITER_WITH_UNDERSCORES_ + +node <<'```' +Hello. +``` + +node < /dev/null; + +<< $tmpfile +a $B ${C} +EOF + +wc -l $tmpfile +--- +hasError: false +program [0,50] "cat < $tmpfile\na $B ${C}\nEOF\n\nw..." + redirected_statement [0,34] "cat < $tmpfile\na $B ${C}\nEOF" + command [0,3] "cat" + command_name [0,3] "cat" + word [0,3] "cat" + heredoc_redirect [4,34] "< $tmpfile\na $B ${C}\nEOF" + (<<) [4,6] "<<" + heredoc_start [6,9] "EOF" + file_redirect [10,20] "> $tmpfile" + (>) [10,11] ">" + simple_expansion [12,20] "$tmpfile" + ($) [12,13] "$" + variable_name [13,20] "tmpfile" + heredoc_body [21,31] "a $B ${C}\n" + heredoc_content [21,23] "a " + simple_expansion [23,25] "$B" + ($) [23,24] "$" + variable_name [24,25] "B" + heredoc_content [25,26] " " + expansion [26,30] "${C}" + (${) [26,28] "${" + variable_name [28,29] "C" + (}) [29,30] "}" + heredoc_content [30,31] "\n" + heredoc_end [31,34] "EOF" + command [36,50] "wc -l $tmpfile" + command_name [36,38] "wc" + word [36,38] "wc" + word [39,41] "-l" + simple_expansion [42,50] "$tmpfile" + ($) [42,43] "$" + variable_name [43,50] "tmpfile" +=== +=== +@known-diff heredoc-content-chunks: corpus commands.txt::Heredocs with many file redirects +FOO=bar echo < err.txt > hello.txt +hello +EOF +--- +hasError: false +program [0,51] "FOO=bar echo < err.txt > hello..." + redirected_statement [0,51] "FOO=bar echo < err.txt > hello..." + command [0,12] "FOO=bar echo" + variable_assignment [0,7] "FOO=bar" + variable_name [0,3] "FOO" + (=) [3,4] "=" + word [4,7] "bar" + command_name [8,12] "echo" + word [8,12] "echo" + heredoc_redirect [13,51] "< err.txt > hello.txt\nhello\nEOF" + (<<) [13,15] "<<" + heredoc_start [15,18] "EOF" + file_redirect [19,29] "2> err.txt" + file_descriptor [19,20] "2" + (>) [20,21] ">" + word [22,29] "err.txt" + file_redirect [30,41] "> hello.txt" + (>) [30,31] ">" + word [32,41] "hello.txt" + heredoc_body [42,48] "hello\n" + heredoc_content [42,48] "hello\n" + heredoc_end [48,51] "EOF" +=== +=== +@known-diff heredoc-content-chunks: corpus commands.txt::Heredocs with pipes +one <&2; echo 0), k++ <= 10); i += j )) +do +printf "$i" +done + +echo + +( for (( i = j = k = 1; i % 9 || (j *= -1, $( ((i%9)) || printf " " >&2; echo 0), k++ <= 10); i += j )) +do +printf "$i" +done ) +--- +hasError: false +program [0,542] "for (( c=1; c<=5; c++ ))\ndo\n echo $c..." + c_style_for_statement [0,42] "for (( c=1; c<=5; c++ ))\ndo\n echo $c..." + (for) [0,3] "for" + ((() [4,6] "((" + variable_assignment [7,10] "c=1" + variable_name [7,8] "c" + (=) [8,9] "=" + number [9,10] "1" + (;) [10,11] ";" + binary_expression [12,16] "c<=5" + word [12,13] "c" + (<=) [13,15] "<=" + number [15,16] "5" + (;) [16,17] ";" + postfix_expression [18,21] "c++" + word [18,19] "c" + (++) [19,21] "++" + ())) [22,24] "))" + do_group [25,42] "do\n echo $c\ndone" + (do) [25,27] "do" + command [30,37] "echo $c" + command_name [30,34] "echo" + word [30,34] "echo" + simple_expansion [35,37] "$c" + ($) [35,36] "$" + variable_name [36,37] "c" + (done) [38,42] "done" + c_style_for_statement [44,81] "for (( c=1; c<=5; c++ )) {\n\techo $c\n}" + (for) [44,47] "for" + ((() [48,50] "((" + variable_assignment [51,54] "c=1" + variable_name [51,52] "c" + (=) [52,53] "=" + number [53,54] "1" + (;) [54,55] ";" + binary_expression [56,60] "c<=5" + word [56,57] "c" + (<=) [57,59] "<=" + number [59,60] "5" + (;) [60,61] ";" + postfix_expression [62,65] "c++" + word [62,63] "c" + (++) [63,65] "++" + ())) [66,68] "))" + compound_statement [69,81] "{\n\techo $c\n}" + ({) [69,70] "{" + command [72,79] "echo $c" + command_name [72,76] "echo" + word [72,76] "echo" + simple_expansion [77,79] "$c" + ($) [77,78] "$" + variable_name [78,79] "c" + (}) [80,81] "}" + c_style_for_statement [83,121] "for (( ; ; ))\ndo\n echo 'forever'\ndone" + (for) [83,86] "for" + ((() [87,89] "((" + (;) [90,91] ";" + (;) [92,93] ";" + ())) [94,96] "))" + do_group [97,121] "do\n echo 'forever'\ndone" + (do) [97,99] "do" + command [102,116] "echo 'forever'" + command_name [102,106] "echo" + word [102,106] "echo" + raw_string [107,116] "'forever'" + (done) [117,121] "done" + c_style_for_statement [123,190] "for ((cx = 0; c = $cx / $pf, c < $wc ..." + (for) [123,126] "for" + ((() [127,129] "((" + variable_assignment [129,135] "cx = 0" + variable_name [129,131] "cx" + (=) [132,133] "=" + number [134,135] "0" + (;) [135,136] ";" + variable_assignment [137,150] "c = $cx / $pf" + variable_name [137,138] "c" + (=) [139,140] "=" + binary_expression [141,150] "$cx / $pf" + simple_expansion [141,144] "$cx" + ($) [141,142] "$" + variable_name [142,144] "cx" + (/) [145,146] "/" + simple_expansion [147,150] "$pf" + ($) [147,148] "$" + variable_name [148,150] "pf" + (,) [150,151] "," + binary_expression [152,164] "c < $wc - $k" + word [152,153] "c" + (<) [154,155] "<" + binary_expression [156,164] "$wc - $k" + simple_expansion [156,159] "$wc" + ($) [156,157] "$" + variable_name [157,159] "wc" + (-) [160,161] "-" + simple_expansion [162,164] "$k" + ($) [162,163] "$" + variable_name [163,164] "k" + (;) [164,165] ";" + ())) [166,168] "))" + (;) [168,169] ";" + do_group [170,190] "do\n\t\techo \"$cx\"\ndone" + (do) [170,172] "do" + command [175,185] "echo \"$cx\"" + command_name [175,179] "echo" + word [175,179] "echo" + string [180,185] "\"$cx\"" + (") [180,181] "\"" + simple_expansion [181,184] "$cx" + ($) [181,182] "$" + variable_name [182,184] "cx" + (") [184,185] "\"" + (done) [186,190] "done" + c_style_for_statement [192,264] "for (( i = 4;;i--)) ; do echo $i; if ..." + (for) [192,195] "for" + ((() [196,198] "((" + variable_assignment [199,204] "i = 4" + variable_name [199,200] "i" + (=) [201,202] "=" + number [203,204] "4" + (;) [204,205] ";" + (;) [205,206] ";" + postfix_expression [206,209] "i--" + word [206,207] "i" + (--) [207,209] "--" + ())) [209,211] "))" + (;) [212,213] ";" + do_group [214,264] "do echo $i; if (( $i == 0 )); then br..." + (do) [214,216] "do" + command [217,224] "echo $i" + command_name [217,221] "echo" + word [217,221] "echo" + simple_expansion [222,224] "$i" + ($) [222,223] "$" + variable_name [223,224] "i" + (;) [224,225] ";" + if_statement [226,258] "if (( $i == 0 )); then break; fi" + (if) [226,228] "if" + command [229,242] "(( $i == 0 ))" + command_name [229,242] "(( $i == 0 ))" + arithmetic_expansion [229,242] "(( $i == 0 ))" + ((() [229,231] "((" + binary_expression [232,239] "$i == 0" + simple_expansion [232,234] "$i" + ($) [232,233] "$" + variable_name [233,234] "i" + (==) [235,237] "==" + number [238,239] "0" + ())) [240,242] "))" + (;) [242,243] ";" + (then) [244,248] "then" + command [249,254] "break" + command_name [249,254] "break" + word [249,254] "break" + (;) [254,255] ";" + (fi) [256,258] "fi" + (;) [258,259] ";" + (done) [260,264] "done" + comment [266,287] "# added post-bash-4.2" + c_style_for_statement [288,409] "for (( i = j = k = 1; i % 9 || (j *= ..." + (for) [288,291] "for" + ((() [292,294] "((" + variable_assignment [295,308] "i = j = k = 1" + variable_name [295,296] "i" + (=) [297,298] "=" + variable_assignment [299,308] "j = k = 1" + variable_name [299,300] "j" + (=) [301,302] "=" + variable_assignment [303,308] "k = 1" + variable_name [303,304] "k" + (=) [305,306] "=" + number [307,308] "1" + (;) [308,309] ";" + binary_expression [310,378] "i % 9 || (j *= -1, $( ((i%9)) || prin..." + binary_expression [310,315] "i % 9" + word [310,311] "i" + (%) [312,313] "%" + number [314,315] "9" + (||) [316,318] "||" + parenthesized_expression [319,378] "(j *= -1, $( ((i%9)) || printf \" \" >&..." + (() [319,320] "(" + binary_expression [320,327] "j *= -1" + word [320,321] "j" + (*=) [322,324] "*=" + unary_expression [325,327] "-1" + (-) [325,326] "-" + number [326,327] "1" + (,) [327,328] "," + command_substitution [329,366] "$( ((i%9)) || printf \" \" >&2; echo 0)" + ($() [329,331] "$(" + list [332,357] "((i%9)) || printf \" \" >&2" + command [332,339] "((i%9))" + command_name [332,339] "((i%9))" + arithmetic_expansion [332,339] "((i%9))" + ((() [332,334] "((" + binary_expression [334,337] "i%9" + variable_name [334,335] "i" + (%) [335,336] "%" + number [336,337] "9" + ())) [337,339] "))" + (||) [340,342] "||" + redirected_statement [343,357] "printf \" \" >&2" + command [343,353] "printf \" \"" + command_name [343,349] "printf" + word [343,349] "printf" + string [350,353] "\" \"" + (") [350,351] "\"" + string_content [351,352] " " + (") [352,353] "\"" + file_redirect [354,357] ">&2" + (>&) [354,356] ">&" + number [356,357] "2" + (;) [357,358] ";" + command [359,365] "echo 0" + command_name [359,363] "echo" + word [359,363] "echo" + number [364,365] "0" + ()) [365,366] ")" + (,) [366,367] "," + binary_expression [368,377] "k++ <= 10" + postfix_expression [368,371] "k++" + word [368,369] "k" + (++) [369,371] "++" + (<=) [372,374] "<=" + number [375,377] "10" + ()) [377,378] ")" + (;) [378,379] ";" + binary_expression [380,386] "i += j" + word [380,381] "i" + (+=) [382,384] "+=" + word [385,386] "j" + ())) [387,389] "))" + do_group [390,409] "do\nprintf \"$i\"\ndone" + (do) [390,392] "do" + command [393,404] "printf \"$i\"" + command_name [393,399] "printf" + word [393,399] "printf" + string [400,404] "\"$i\"" + (") [400,401] "\"" + simple_expansion [401,403] "$i" + ($) [401,402] "$" + variable_name [402,403] "i" + (") [403,404] "\"" + (done) [405,409] "done" + command [411,415] "echo" + command_name [411,415] "echo" + word [411,415] "echo" + subshell [417,542] "( for (( i = j = k = 1; i % 9 || (j *..." + (() [417,418] "(" + c_style_for_statement [419,540] "for (( i = j = k = 1; i % 9 || (j *= ..." + (for) [419,422] "for" + ((() [423,425] "((" + variable_assignment [426,439] "i = j = k = 1" + variable_name [426,427] "i" + (=) [428,429] "=" + variable_assignment [430,439] "j = k = 1" + variable_name [430,431] "j" + (=) [432,433] "=" + variable_assignment [434,439] "k = 1" + variable_name [434,435] "k" + (=) [436,437] "=" + number [438,439] "1" + (;) [439,440] ";" + binary_expression [441,509] "i % 9 || (j *= -1, $( ((i%9)) || prin..." + binary_expression [441,446] "i % 9" + word [441,442] "i" + (%) [443,444] "%" + number [445,446] "9" + (||) [447,449] "||" + parenthesized_expression [450,509] "(j *= -1, $( ((i%9)) || printf \" \" >&..." + (() [450,451] "(" + binary_expression [451,458] "j *= -1" + word [451,452] "j" + (*=) [453,455] "*=" + unary_expression [456,458] "-1" + (-) [456,457] "-" + number [457,458] "1" + (,) [458,459] "," + command_substitution [460,497] "$( ((i%9)) || printf \" \" >&2; echo 0)" + ($() [460,462] "$(" + list [463,488] "((i%9)) || printf \" \" >&2" + command [463,470] "((i%9))" + command_name [463,470] "((i%9))" + arithmetic_expansion [463,470] "((i%9))" + ((() [463,465] "((" + binary_expression [465,468] "i%9" + variable_name [465,466] "i" + (%) [466,467] "%" + number [467,468] "9" + ())) [468,470] "))" + (||) [471,473] "||" + redirected_statement [474,488] "printf \" \" >&2" + command [474,484] "printf \" \"" + command_name [474,480] "printf" + word [474,480] "printf" + string [481,484] "\" \"" + (") [481,482] "\"" + string_content [482,483] " " + (") [483,484] "\"" + file_redirect [485,488] ">&2" + (>&) [485,487] ">&" + number [487,488] "2" + (;) [488,489] ";" + command [490,496] "echo 0" + command_name [490,494] "echo" + word [490,494] "echo" + number [495,496] "0" + ()) [496,497] ")" + (,) [497,498] "," + binary_expression [499,508] "k++ <= 10" + postfix_expression [499,502] "k++" + word [499,500] "k" + (++) [500,502] "++" + (<=) [503,505] "<=" + number [506,508] "10" + ()) [508,509] ")" + (;) [509,510] ";" + binary_expression [511,517] "i += j" + word [511,512] "i" + (+=) [513,515] "+=" + word [516,517] "j" + ())) [518,520] "))" + do_group [521,540] "do\nprintf \"$i\"\ndone" + (do) [521,523] "do" + command [524,535] "printf \"$i\"" + command_name [524,530] "printf" + word [524,530] "printf" + string [531,535] "\"$i\"" + (") [531,532] "\"" + simple_expansion [532,534] "$i" + ($) [532,533] "$" + variable_name [533,534] "i" + (") [534,535] "\"" + (done) [536,540] "done" + ()) [541,542] ")" +=== +=== +@known-diff case-short-option-pattern: corpus statements.txt::Test command paren statefulness with a case glob +[[ ${test} == @($(IFS='|'; echo "${skip[*]}")) ]] + +case ${out} in +*"not supported"*|\ +*"operation not supported"*) + ;; +esac + +case $1 in +-o) + owner=$2 + shift + ;; +-g) ;; +esac + +[[ a == \"+(?)\" ]] +--- +hasError: false +program [0,193] "[[ ${test} == @($(IFS='|'; echo \"${sk..." + test_command [0,49] "[[ ${test} == @($(IFS='|'; echo \"${sk..." + ([[) [0,2] "[[" + binary_expression [3,46] "${test} == @($(IFS='|'; echo \"${skip[..." + expansion [3,10] "${test}" + (${) [3,5] "${" + variable_name [5,9] "test" + (}) [9,10] "}" + (==) [11,13] "==" + extglob_pattern [14,16] "@(" + command_substitution [16,45] "$(IFS='|'; echo \"${skip[*]}\")" + ($() [16,18] "$(" + variable_assignment [18,25] "IFS='|'" + variable_name [18,21] "IFS" + (=) [21,22] "=" + raw_string [22,25] "'|'" + (;) [25,26] ";" + command [27,44] "echo \"${skip[*]}\"" + command_name [27,31] "echo" + word [27,31] "echo" + string [32,44] "\"${skip[*]}\"" + (") [32,33] "\"" + expansion [33,43] "${skip[*]}" + (${) [33,35] "${" + subscript [35,42] "skip[*]" + variable_name [35,39] "skip" + ([) [39,40] "[" + word [40,41] "*" + (]) [41,42] "]" + (}) [42,43] "}" + (") [43,44] "\"" + ()) [44,45] ")" + extglob_pattern [45,46] ")" + (]]) [47,49] "]]" + case_statement [51,123] "case ${out} in\n*\"not supported\"*|\\\n*\"..." + (case) [51,55] "case" + expansion [56,62] "${out}" + (${) [56,58] "${" + variable_name [58,61] "out" + (}) [61,62] "}" + (in) [63,65] "in" + case_item [66,118] "*\"not supported\"*|\\\n*\"operation not s..." + extglob_pattern [66,67] "*" + string [67,82] "\"not supported\"" + (") [67,68] "\"" + string_content [68,81] "not supported" + (") [81,82] "\"" + extglob_pattern [82,83] "*" + (|) [83,84] "|" + concatenation [86,113] "*\"operation not supported\"*" + word [86,87] "*" + string [87,112] "\"operation not supported\"" + (") [87,88] "\"" + string_content [88,111] "operation not supported" + (") [111,112] "\"" + word [112,113] "*" + ()) [113,114] ")" + (;;) [116,118] ";;" + (esac) [119,123] "esac" + case_statement [125,172] "case $1 in\n-o)\n\towner=$2\n\tshift\n\t;;\n-..." + (case) [125,129] "case" + simple_expansion [130,132] "$1" + ($) [130,131] "$" + variable_name [131,132] "1" + (in) [133,135] "in" + case_item [136,160] "-o)\n\towner=$2\n\tshift\n\t;;" + word [136,138] "-o" + ()) [138,139] ")" + variable_assignment [141,149] "owner=$2" + variable_name [141,146] "owner" + (=) [146,147] "=" + simple_expansion [147,149] "$2" + ($) [147,148] "$" + variable_name [148,149] "2" + command [151,156] "shift" + command_name [151,156] "shift" + word [151,156] "shift" + (;;) [158,160] ";;" + case_item [161,167] "-g) ;;" + word [161,163] "-g" + ()) [163,164] ")" + (;;) [165,167] ";;" + (esac) [168,172] "esac" + test_command [174,193] "[[ a == \\\"+(?)\\\" ]]" + ([[) [174,176] "[[" + binary_expression [177,190] "a == \\\"+(?)\\\"" + word [177,178] "a" + (==) [179,181] "==" + extglob_pattern [182,190] "\\\"+(?)\\\"" + (]]) [191,193] "]]" + +=== +@known-diff dollar-backtick-substitution: corpus statements.txt::Command substution with $ and backticks +$(eval echo $`echo ${foo}`) +--- +hasError: false +program [0,27] "$(eval echo $`echo ${foo}`)" + command [0,27] "$(eval echo $`echo ${foo}`)" + command_name [0,27] "$(eval echo $`echo ${foo}`)" + command_substitution [0,27] "$(eval echo $`echo ${foo}`)" + ($() [0,2] "$(" + command [2,26] "eval echo $`echo ${foo}`" + command_name [2,6] "eval" + word [2,6] "eval" + word [7,11] "echo" + concatenation [12,26] "$`echo ${foo}`" + ($) [12,13] "$" + command_substitution [13,26] "`echo ${foo}`" + (`) [13,14] "`" + command [14,25] "echo ${foo}" + command_name [14,18] "echo" + word [14,18] "echo" + expansion [19,25] "${foo}" + (${) [19,21] "${" + variable_name [21,24] "foo" + (}) [24,25] "}" + (`) [25,26] "`" + ()) [26,27] ")" +=== +=== diff --git a/packages/tree-sitter-bash/test/fixtures/corpus/literals.txt b/packages/tree-sitter-bash/test/fixtures/corpus/literals.txt new file mode 100644 index 0000000000..61543ca3b4 --- /dev/null +++ b/packages/tree-sitter-bash/test/fixtures/corpus/literals.txt @@ -0,0 +1,1371 @@ +================================================================================ +Literal words +================================================================================ + +echo a +echo a b + +-------------------------------------------------------------------------------- + +(program + (command + (command_name + (word)) + (word)) + (command + (command_name + (word)) + (word) + (word))) + +================================================================================ +Words with special characters +================================================================================ + +echo {o[k]} +echo }}} +echo ]]] === + +-------------------------------------------------------------------------------- + +(program + (command + (command_name + (word)) + (concatenation + (word) + (word) + (word) + (word) + (word) + (word))) + (command + (command_name + (word)) + (concatenation + (word) + (word) + (word))) + (command + (command_name + (word)) + (concatenation + (word) + (word) + (word)) + (word))) + +================================================================================ +Simple variable expansions +================================================================================ + +echo $abc + +-------------------------------------------------------------------------------- + +(program + (command + (command_name + (word)) + (simple_expansion + (variable_name)))) + +================================================================================ +Special variable expansions +================================================================================ + +echo $# $* $@ $! + +-------------------------------------------------------------------------------- + +(program + (command + (command_name + (word)) + (simple_expansion + (special_variable_name)) + (simple_expansion + (special_variable_name)) + (simple_expansion + (special_variable_name)) + (simple_expansion + (special_variable_name)))) + +================================================================================ +Variable expansions +================================================================================ + +echo ${} +echo ${#} +echo ${var1#*#} +echo ${!abc} +echo ${abc} +echo ${abc:-def} +echo ${abc:+ghi} +echo ${abc:- } +echo ${abc: +} +echo ${abc,?} +echo ${abc^^b} +echo ${abc@U} +echo ${abc:- -quiet} + +-------------------------------------------------------------------------------- + +(program + (command (command_name (word)) (expansion)) + (command (command_name (word)) (expansion)) + (command (command_name (word)) (expansion (variable_name) (regex))) + (command (command_name (word)) (expansion (variable_name))) + (command (command_name (word)) (expansion (variable_name))) + (command (command_name (word)) (expansion (variable_name) (word))) + (command (command_name (word)) (expansion (variable_name) (word))) + (command (command_name (word)) (expansion (variable_name) (word))) + (command (command_name (word)) (expansion (variable_name))) + (command (command_name (word)) (expansion (variable_name) (regex))) + (command (command_name (word)) (expansion (variable_name) (regex))) + (command (command_name (word)) (expansion (variable_name))) + (command (command_name (word)) (expansion (variable_name) (word)))) + +================================================================================ +Variable expansions with operators +================================================================================ + +A="${B[0]# }" +C="${D/#* -E /}" +F="${G%% *}" +H="${I#*;}" +J="${K##*;}" +L="${M%;*}" +N="${O%%;*}" +P="${Q%|*}" +R="${S%()}" +T="${U%(}" +V="${W%)}" +X="${Y%<}" +Z="${A#*}" +C="${D%*}" +F="${#!}" +G=${H,,[I]} +J=${K^^[L]} +L="${M/'N'*/O}" + +-------------------------------------------------------------------------------- + +(program + (variable_assignment + (variable_name) + (string + (expansion + (subscript + (variable_name) + (number)) + (regex)))) + (variable_assignment + (variable_name) + (string + (expansion + (variable_name) + (regex)))) + (variable_assignment + (variable_name) + (string + (expansion + (variable_name) + (regex)))) + (variable_assignment + (variable_name) + (string + (expansion + (variable_name) + (regex)))) + (variable_assignment + (variable_name) + (string + (expansion + (variable_name) + (regex)))) + (variable_assignment + (variable_name) + (string + (expansion + (variable_name) + (regex)))) + (variable_assignment + (variable_name) + (string + (expansion + (variable_name) + (regex)))) + (variable_assignment + (variable_name) + (string + (expansion + (variable_name) + (regex)))) + (variable_assignment + (variable_name) + (string + (expansion + (variable_name) + (regex)))) + (variable_assignment + (variable_name) + (string + (expansion + (variable_name) + (regex)))) + (variable_assignment + (variable_name) + (string + (expansion + (variable_name) + (regex)))) + (variable_assignment + (variable_name) + (string + (expansion + (variable_name) + (regex)))) + (variable_assignment + (variable_name) + (string + (expansion + (variable_name) + (regex)))) + (variable_assignment + (variable_name) + (string + (expansion + (variable_name) + (regex)))) + (variable_assignment + (variable_name) + (string + (expansion))) + (variable_assignment + (variable_name) + (expansion + (variable_name) + (regex))) + (variable_assignment + (variable_name) + (expansion + (variable_name) + (regex))) + (variable_assignment + (variable_name) + (string + (expansion + (variable_name) + (regex) + (word))))) + +================================================================================ +More Variable expansions with operators +================================================================================ + +${parameter-default} +${parameter:-default} +${parameter=default} +${parameter:=default} +${parameter+alt_value} +${parameter:+alt_value} +${parameter?err_msg} +${parameter:?err_msg} +${var%Pattern} +${var%%Pattern} +${var:pos} +${var:pos:len} +${MATRIX:$(($RANDOM%${#MATRIX})):1} +${PKG_CONFIG_LIBDIR:-${ESYSROOT}/usr/$(get_libdir)/pkgconfig} +${ver_str::${#ver_str}-${#not_match}} +${value#\{sd.cicd.} + +-------------------------------------------------------------------------------- + +(program + (command + (command_name + (expansion + (variable_name) + (word)))) + (command + (command_name + (expansion + (variable_name) + (word)))) + (command + (command_name + (expansion + (variable_name) + (word)))) + (command + (command_name + (expansion + (variable_name) + (word)))) + (command + (command_name + (expansion + (variable_name) + (word)))) + (command + (command_name + (expansion + (variable_name) + (word)))) + (command + (command_name + (expansion + (variable_name) + (word)))) + (command + (command_name + (expansion + (variable_name) + (word)))) + (command + (command_name + (expansion + (variable_name) + (regex)))) + (command + (command_name + (expansion + (variable_name) + (regex)))) + (command + (command_name + (expansion + (variable_name) + (variable_name)))) + (command + (command_name + (expansion + (variable_name) + (variable_name) + (variable_name)))) + (command + (command_name + (expansion + (variable_name) + (arithmetic_expansion + (binary_expression + (simple_expansion + (variable_name)) + (expansion + (variable_name)))) + (number)))) + (command + (command_name + (expansion + (variable_name) + (concatenation + (expansion (variable_name)) + (word) + (command_substitution + (command (command_name (word)))) + (word))))) + (command + (command_name + (expansion + (variable_name) + (binary_expression + (expansion (variable_name)) + (expansion (variable_name)))))) + (command + (command_name + (expansion + (variable_name) + (regex))))) + +================================================================================ +Variable expansions in strings +================================================================================ + +A="${A:-$B/c}" +A="${b=$c/$d}" +MY_PV="${PV/_pre/$'\x7e'pre}" + +-------------------------------------------------------------------------------- + +(program + (variable_assignment + (variable_name) + (string + (expansion + (variable_name) + (concatenation + (simple_expansion + (variable_name)) + (word))))) + (variable_assignment + (variable_name) + (string + (expansion + (variable_name) + (concatenation + (simple_expansion + (variable_name)) + (word) + (simple_expansion + (variable_name)))))) + (variable_assignment + (variable_name) + (string + (expansion + (variable_name) + (regex) + (concatenation (ansi_c_string) (word)))))) + +================================================================================ +Variable expansions with regexes +================================================================================ + +A=${B//:;;/$'\n'} + +# escaped space +C=${D/;\ *;|} +MOFILES=${LINGUAS// /.po }.po +MY_P="${PN/aspell/aspell"${ASPELL_VERSION}"}" +pyc=${pyc//*\/} +${pv/\.} +${new_test_cp//"${old_ver_cp}"/} +${tests_to_run//"${classes}"\/} +${allarchives// /\\|} + +-------------------------------------------------------------------------------- + +(program + (variable_assignment + (variable_name) + (expansion + (variable_name) + (regex) + (ansi_c_string))) + (comment) + (variable_assignment + (variable_name) + (expansion (variable_name) (regex))) + (variable_assignment + (variable_name) + (concatenation + (expansion (variable_name) (regex) (word)) + (word))) + (variable_assignment + (variable_name) + (string + (expansion + (variable_name) + (regex) + (concatenation + (word) + (string (expansion (variable_name))))))) + (variable_assignment + (variable_name) + (expansion (variable_name) (regex))) + (command (command_name (expansion (variable_name) (regex)))) + (command + (command_name + (expansion (variable_name) (string (expansion (variable_name)))))) + (command + (command_name + (expansion (variable_name) (string (expansion (variable_name))) (regex)))) + (command (command_name (expansion (variable_name) (regex) (word))))) + +================================================================================ +Other variable expansion operators +================================================================================ + +cat ${BAR} ${ABC=def} ${GHI:?jkl} +[ "$a" != "${a#[Bc]}" ] + +-------------------------------------------------------------------------------- + +(program + (command + (command_name + (word)) + (expansion + (variable_name)) + (expansion + (variable_name) + (word)) + (expansion + (variable_name) + (word))) + (test_command + (binary_expression + (string + (simple_expansion + (variable_name))) + (string + (expansion + (variable_name) + (regex)))))) + +================================================================================ +Variable Expansions: Length +================================================================================ + +${parameter:-1} + +${parameter: -1} + +${parameter:(-1)} + +${matrix:$(($random%${#matrix})):1} + +"${_component_to_single:${len}:2}" + +"${PN::-1}" + +${trarr:$(ver_cut 2):1} + +${comp[@]:start:end*2-start} + +-------------------------------------------------------------------------------- + +(program + (command (command_name (expansion (variable_name) (word)))) + (command (command_name (expansion (variable_name) (number)))) + (command + (command_name + (expansion (variable_name) (parenthesized_expression (unary_expression (number)))))) + (command + (command_name + (expansion + (variable_name) + (arithmetic_expansion + (binary_expression (simple_expansion (variable_name)) (expansion (variable_name)))) + (number)))) + (command (command_name (string (expansion (variable_name) (expansion (variable_name)) (number))))) + (command (command_name (string (expansion (variable_name) (number))))) + (command + (command_name + (expansion (variable_name) (command_substitution (command (command_name (word)) (number))) (number)))) + (command + (command_name + (expansion + (subscript (variable_name) (word)) + (variable_name) + (binary_expression (binary_expression (variable_name) (number)) (variable_name)))))) + +================================================================================ +Variable Expansions with operators +================================================================================ + +${parameter-default} +${parameter- default} +${!varprefix*} +${!varprefix@} +${parameter@U} + +-------------------------------------------------------------------------------- + +(program + (command (command_name (expansion (variable_name) (word)))) + (command (command_name (expansion (variable_name) (word)))) + (command (command_name (expansion (variable_name)))) + (command (command_name (expansion (variable_name)))) + (command (command_name (expansion (variable_name))))) + +================================================================================ +Variable Expansions: Bizarre Cases +================================================================================ + +${!#} +${!# } +${!##} +${!## } +${!##/} +# here be dragons +echo "${kw}? ( ${cond:+${cond}? (} ${baseuri}-${ver}-${kw}.${suff} ${cond:+) })" + +-------------------------------------------------------------------------------- + +(program + (command (command_name (expansion))) + (command (command_name (expansion))) + (command (command_name (expansion))) + (command (command_name (expansion))) + (command (command_name (expansion (special_variable_name) (regex)))) + (comment) + (command + (command_name (word)) + (string + (expansion (variable_name)) + (string_content) + (expansion + (variable_name) + (concatenation (expansion (variable_name)) (word))) + (expansion (variable_name)) + (string_content) + (expansion (variable_name)) + (string_content) + (expansion (variable_name)) + (string_content) + (expansion (variable_name)) + (expansion (variable_name) (word)) + (string_content)))) + +================================================================================ +Variable Expansions: Weird Cases +================================================================================ + +${completions[*]} +${=1} +${2?} +${p_key#*=} +${abc:- } +${B[0]# } +${to_enables[0]##*/} +exec "${0#-}" --rcfile "${BASH_IT_BASHRC:-${HOME?}/.bashrc}" +recho "TDEFAULTS = ${selvecs:+-DSELECT_VECS=\"$selvecs\"}" +local msg="${2:-command '$1' does not exist}" +${cdir:+#} +${dict_langs:+;} +${UTIL_LINUX_LIBC[@]/%/? ( sys-apps/util-linux )} +${id}${2+ ${2}} +${BRANDING_GCC_PKGVERSION/(/(Gentoo ${PVR}${extvers}, } # look at that parenthesis! +some-command ${foo:+--arg <(printf '%s\n' "$foo")} + +-------------------------------------------------------------------------------- + +(program + (command (command_name (expansion (subscript (variable_name) (word))))) + (command (command_name (expansion (variable_name)))) + (command (command_name (expansion (variable_name)))) + (command (command_name (expansion (variable_name) (regex)))) + (command (command_name (expansion (variable_name) (word)))) + (command (command_name (expansion (subscript (variable_name) (number)) (regex)))) + (command (command_name (expansion (subscript (variable_name) (number)) (regex)))) + (command + (command_name (word)) + (string (expansion (special_variable_name) (regex))) + (word) + (string (expansion (variable_name) (concatenation (expansion (variable_name)) (word))))) + (command + (command_name (word)) + (string + (string_content) + (expansion (variable_name) (concatenation (word) (simple_expansion (variable_name)) (word))))) + (declaration_command + (variable_assignment + (variable_name) + (string (expansion (variable_name) (concatenation (word) (raw_string) (word)))))) + (command (command_name (expansion (variable_name) (word)))) + (command (command_name (expansion (variable_name) (word)))) + (command (command_name (expansion (subscript (variable_name) (word)) (word)))) + (command + (command_name + (concatenation (expansion (variable_name)) (expansion (variable_name) (expansion (variable_name)))))) + (command + (command_name + (expansion + (variable_name) + (regex) + (concatenation (word) (expansion (variable_name)) (expansion (variable_name)) (word))))) + (comment) + (command + (command_name (word)) + (expansion + (variable_name) + (concatenation + (word) + (process_substitution + (command + (command_name (word)) + (raw_string) + (string (simple_expansion (variable_name))))))))) + +================================================================================ +Variable Expansions: Regex +================================================================================ + +A=${B//:;;/$'\n'} +C="${D/#* -E /}" +BASH_IT_GIT_URL="${BASH_IT_GIT_URL/git@/https://}" +10#${command_start##*.} +echo ${LIB_DEPEND//\[static-libs(+)]} +${ALL_LLVM_TARGETS[@]/%/(-)?} +filterdiff -p1 ${paths[@]/#/-i } +${cflags//-O? /$(get-flag O) } +curf="${f%'-roff2html'*}.html" +reff="${f/'-roff2html'*/'-ref'}.html" + +-------------------------------------------------------------------------------- + +(program + (variable_assignment (variable_name) (expansion (variable_name) (regex) (ansi_c_string))) + (variable_assignment (variable_name) (string (expansion (variable_name) (regex)))) + (variable_assignment (variable_name) (string (expansion (variable_name) (regex) (word)))) + (command (command_name (number (expansion (variable_name) (regex))))) + (command (command_name (word)) (expansion (variable_name) (regex))) + (command (command_name (expansion (subscript (variable_name) (word)) (word)))) + (command (command_name (word)) (word) (expansion (subscript (variable_name) (word)) (word))) + (command + (command_name + (expansion (variable_name) (regex) (command_substitution (command (command_name (word)) (word))) (word)))) + (variable_assignment + (variable_name) + (string (expansion (variable_name) (raw_string) (regex)) (string_content))) + (variable_assignment + (variable_name) + (string (expansion (variable_name) (regex) (raw_string)) (string_content)))) + +================================================================================ +Words ending with '$' +================================================================================ + +grep ^${var}$ + +-------------------------------------------------------------------------------- + +(program + (command + (command_name + (word)) + (concatenation + (word) + (expansion + (variable_name))))) + +================================================================================ +Command substitutions +================================================================================ + +echo `echo hi` +echo `echo hi; echo there` +echo $(echo $(echo hi)) +echo $(< some-file) + +# both of these are concatenations! +echo `echo otherword`word +echo word`echo otherword` + +-------------------------------------------------------------------------------- + +(program + (command + (command_name + (word)) + (command_substitution + (command + (command_name + (word)) + (word)))) + (command + (command_name + (word)) + (command_substitution + (command + (command_name + (word)) + (word)) + (command + (command_name + (word)) + (word)))) + (command + (command_name + (word)) + (command_substitution + (command + (command_name + (word)) + (command_substitution + (command + (command_name + (word)) + (word)))))) + (command + (command_name + (word)) + (command_substitution + (file_redirect + (word)))) + (comment) + (command + (command_name + (word)) + (concatenation + (command_substitution + (command + (command_name + (word)) + (word))) + (word))) + (command + (command_name + (word)) + (concatenation + (word) + (command_substitution + (command + (command_name + (word)) + (word)))))) + +================================================================================ +Process substitutions +================================================================================ + +wc -c <(echo abc && echo def) +wc -c <(echo abc; echo def) +echo abc > >(wc -c) + +-------------------------------------------------------------------------------- + +(program + (command + (command_name + (word)) + (word) + (process_substitution + (list + (command + (command_name + (word)) + (word)) + (command + (command_name + (word)) + (word))))) + (command + (command_name + (word)) + (word) + (process_substitution + (command + (command_name + (word)) + (word)) + (command + (command_name + (word)) + (word)))) + (redirected_statement + (command + (command_name + (word)) + (word)) + (file_redirect + (process_substitution + (command + (command_name + (word)) + (word)))))) + +================================================================================ +Single quoted strings +================================================================================ + +echo 'a b' 'c d' + +-------------------------------------------------------------------------------- + +(program + (command + (command_name + (word)) + (raw_string) + (raw_string))) + +================================================================================ +Double quoted strings +================================================================================ + +echo "a" "b" +echo "a ${b} c" "d $e" + +-------------------------------------------------------------------------------- + +(program + (command + (command_name + (word)) + (string (string_content)) + (string (string_content))) + (command + (command_name + (word)) + (string + (string_content) + (expansion + (variable_name)) + (string_content)) + (string + (string_content) + (simple_expansion + (variable_name))))) + +================================================================================ +Strings containing command substitutions +================================================================================ + +find "`dirname $file`" -name "$base"'*' + +-------------------------------------------------------------------------------- + +(program + (command + (command_name + (word)) + (string + (command_substitution + (command + (command_name + (word)) + (simple_expansion + (variable_name))))) + (word) + (concatenation + (string + (simple_expansion + (variable_name))) + (raw_string)))) + +================================================================================ +Strings containing escape sequence +================================================================================ + +echo "\"The great escape\`\${var}" + +-------------------------------------------------------------------------------- + +(program + (command + (command_name + (word)) + (string (string_content)))) + +================================================================================ +Strings containing special characters +================================================================================ + +echo "s/$/'/" +echo "#" +echo "s$" + +-------------------------------------------------------------------------------- + +(program + (command + (command_name + (word)) + (string (string_content) (string_content))) + (command + (command_name + (word)) + (string (string_content))) + (command + (command_name + (word)) + (string (string_content)))) + +================================================================================ +Strings with ANSI-C quoting +================================================================================ + +echo $'Here\'s Johnny!\r\n' + +-------------------------------------------------------------------------------- + +(program + (command + (command_name + (word)) + (ansi_c_string))) + +================================================================================ +Arrays and array expansions +================================================================================ + +a=() +b=(1 2 3) + +echo ${a[@]} +echo ${#b[@]} + +a[$i]=50 +a+=(foo "bar" $(baz)) + +printf " %-9s" "${seq0:-(default)}" + +-------------------------------------------------------------------------------- + +(program + (variable_assignment + (variable_name) + (array)) + (variable_assignment + (variable_name) + (array + (number) + (number) + (number))) + (command + (command_name + (word)) + (expansion + (subscript + (variable_name) + (word)))) + (command + (command_name + (word)) + (expansion + (subscript + (variable_name) + (word)))) + (variable_assignment + (subscript + (variable_name) + (simple_expansion + (variable_name))) + (number)) + (variable_assignment + (variable_name) + (array + (word) + (string (string_content)) + (command_substitution + (command + (command_name + (word)))))) + (command + (command_name + (word)) + (string (string_content)) + (string + (expansion + (variable_name) + (array + (word)))))) + +================================================================================ +Escaped characters in strings +================================================================================ + +echo -ne "\033k$1\033\\" > /dev/stderr + +-------------------------------------------------------------------------------- + +(program + (redirected_statement + (command + (command_name + (word)) + (word) + (string + (string_content) + (simple_expansion + (variable_name)) + (string_content))) + (file_redirect + (word)))) + +================================================================================ +Words containing bare '#' +================================================================================ + +curl -# localhost #comment without space +nix build nixpkgs#hello -v # comment with space + +-------------------------------------------------------------------------------- + +(program + (command + (command_name + (word)) + (word) + (word)) + (comment) + (command + (command_name + (word)) + (word) + (word) + (word)) + (comment)) + +================================================================================ +Words containing # that are not comments +================================================================================ + +echo 'word'#not-comment # a legit comment +echo $(uname -a)#not-comment # a legit comment +echo `uname -a`#not-comment # a legit comment +echo $hey#not-comment # a legit comment +var=#not-comment # a legit comment +echo "'$var'" # -> '#not-comment' + +-------------------------------------------------------------------------------- + +(program + (command + (command_name + (word)) + (concatenation + (raw_string) + (word))) + (comment) + (command + (command_name + (word)) + (concatenation + (command_substitution + (command + (command_name + (word)) + (word))) + (word))) + (comment) + (command + (command_name + (word)) + (concatenation + (command_substitution + (command + (command_name + (word)) + (word))) + (word))) + (comment) + (command + (command_name + (word)) + (concatenation + (simple_expansion + (variable_name)) + (word))) + (comment) + (variable_assignment + (variable_name) + (word)) + (comment) + (command + (command_name + (word)) + (string + (string_content) + (simple_expansion + (variable_name)) + (string_content))) + (comment)) + +================================================================================ +Variable assignments immediately followed by a terminator +================================================================================ + +loop=; variables=& here=;; + +-------------------------------------------------------------------------------- + +(program + (variable_assignment + (variable_name)) + (variable_assignment + (variable_name)) + (variable_assignment + (variable_name))) + +================================================================================ +Multiple variable assignments +================================================================================ + +component_type="${1}" item_name="${2?}" + +-------------------------------------------------------------------------------- + +(program + (variable_assignments + (variable_assignment + (variable_name) + (string + (expansion + (variable_name)))) + (variable_assignment + (variable_name) + (string + (expansion + (variable_name)))))) + +================================================================================ +Arithmetic expansions +================================================================================ + +echo $((1 + 2 - 3 * 4 / 5)) +a=$((6 % 7 ** 8 << 9 >> 10 & 11 | 12 ^ 13)) +$(((${1:-${SECONDS}} % 12) + 144)) +((foo=0)) +echo $((bar=1)) +echo $((-1, 1)) +echo $((! -a || ~ +b || ++c || --d)) +echo $((foo-- || bar++)) +(("${MULTIBUILD_VARIANTS}" > 1)) +$(("$(stat --printf '%05a' "${save_file}")" & 07177)) +soft_errors_count=$[soft_errors_count + 1] + +-------------------------------------------------------------------------------- + +(program + (command + (command_name (word)) + (arithmetic_expansion + (binary_expression + (binary_expression (number) (number)) + (binary_expression + (binary_expression (number) (number)) + (number))))) + (variable_assignment + (variable_name) + (arithmetic_expansion + (binary_expression + (binary_expression + (binary_expression + (binary_expression + (binary_expression + (number) + (binary_expression (number) (number))) + (number)) + (number)) + (number)) + (binary_expression (number) (number))))) + (command + (command_name + (arithmetic_expansion + (binary_expression + (parenthesized_expression + (binary_expression + (expansion + (variable_name) + (expansion (variable_name))) + (number))) + (number))))) + (command + (command_name + (arithmetic_expansion + (binary_expression (variable_name) (number))))) + (command + (command_name (word)) + (arithmetic_expansion + (binary_expression (variable_name) (number)))) + (command + (command_name (word)) + (arithmetic_expansion (unary_expression (number)) (number))) + (command + (command_name (word)) + (arithmetic_expansion + (binary_expression + (binary_expression + (binary_expression + (unary_expression (unary_expression (variable_name))) + (unary_expression (unary_expression (variable_name)))) + (unary_expression (variable_name))) + (unary_expression (variable_name))))) + (command + (command_name (word)) + (arithmetic_expansion + (binary_expression + (postfix_expression + (variable_name)) + (postfix_expression + (variable_name))))) + (command + (command_name + (arithmetic_expansion + (binary_expression + (string + (expansion + (variable_name))) + (number))))) + (command + (command_name + (arithmetic_expansion + (binary_expression + (string + (command_substitution + (command + (command_name + (word)) + (word) + (raw_string) + (string + (expansion + (variable_name)))))) + (number))))) + (variable_assignment + (variable_name) + (arithmetic_expansion + (binary_expression + (variable_name) + (number))))) + +================================================================================ +Concatenation with double backticks +================================================================================ + +main() { + local foo="asd"` + `"fgh" +} + +--- + +(program + (function_definition + (word) + (compound_statement + (declaration_command + (variable_assignment + (variable_name) + (concatenation + (string (string_content)) + (string (string_content)))))))) + +================================================================================ +Brace expressions and lookalikes +================================================================================ + +echo {1..2} +echo {0..5} +echo {0..2 # not a brace expression +echo }{0..2} +echo {0..n} # not a brace expression +echo {0..n..2} # not a brace expression +echo {0..2}{1..2} + +--- + +(program + (command + (command_name (word)) + (brace_expression (number) (number))) + (command + (command_name (word)) + (brace_expression (number) (number))) + (command + (command_name (word)) + (concatenation (word) (word))) + (comment) + (command + (command_name (word)) + (concatenation + (word) + (brace_expression (number) (number)))) + (command + (command_name (word)) + (concatenation (word) (word) (word))) + (comment) + (command + (command_name (word)) + (concatenation (word) (word) (word))) + (comment) + (command + (command_name (word)) + (concatenation + (brace_expression (number) (number)) + (brace_expression (number) (number))))) diff --git a/packages/tree-sitter-bash/test/fixtures/corpus/programs.txt b/packages/tree-sitter-bash/test/fixtures/corpus/programs.txt new file mode 100644 index 0000000000..cae772ea91 --- /dev/null +++ b/packages/tree-sitter-bash/test/fixtures/corpus/programs.txt @@ -0,0 +1,108 @@ +=============================== +Comments +=============================== + +#!/bin/bash +# hi + +--- + +(program + (comment) + (comment)) + +=============================== +Escaped newlines +=============================== + +abc \ + d \ + e + +f=g \ + h=i \ + j \ + --k + +--- + +(program + (command + (command_name + (word)) + (word) + (word)) + (command + (variable_assignment + (variable_name) + (word)) + (variable_assignment + (variable_name) + (word)) + (command_name + (word)) + (word))) + +============================= +escaped newline immediately after a char +============================= + +echo a \ + b + +echo a\ + b + +echo a\ + b\ + c + + +----------------------------- + +(program + (command + (command_name + (word)) + (word) + (word)) + (command + (command_name + (word)) + (word) + (word)) + (command + (command_name + (word)) + (word) + (word) + (word))) + +============================= +Escaped whitespace +============================= + +echo 1 \ 2 \ 3 + +--- + +(program + (command + (command_name + (word)) + (number) + (number) + (number))) + +==================================== +Files without trailing terminators +==================================== + +echo hi +--- + +(program + (command + (command_name + (word)) + (word))) diff --git a/packages/tree-sitter-bash/test/fixtures/corpus/statements.txt b/packages/tree-sitter-bash/test/fixtures/corpus/statements.txt new file mode 100644 index 0000000000..b530fd4e24 --- /dev/null +++ b/packages/tree-sitter-bash/test/fixtures/corpus/statements.txt @@ -0,0 +1,1622 @@ +================================================================================ +Pipelines +================================================================================ + +whoami | cat +cat foo | grep -v bar +cat baz | head -n 1 + +-------------------------------------------------------------------------------- + +(program + (pipeline + (command + name: (command_name + (word))) + (command + name: (command_name + (word)))) + (pipeline + (command + name: (command_name + (word)) + argument: (word)) + (command + name: (command_name + (word)) + argument: (word) + argument: (word))) + (pipeline + (command + name: (command_name + (word)) + argument: (word)) + (command + name: (command_name + (word)) + argument: (word) + argument: (number)))) + + +================================================================================ +Lists +================================================================================ + +a | b && c && d; d e f || e g + +-------------------------------------------------------------------------------- + +(program + (list + (list + (pipeline + (command + (command_name + (word))) + (command + (command_name + (word)))) + (command + (command_name + (word)))) + (command + (command_name + (word)))) + (list + (command + (command_name + (word)) + (word) + (word)) + (command + (command_name + (word)) + (word)))) + +================================================================================ +While statements +================================================================================ + +while something happens; do + echo a + echo b +done + +while local name="$1" val="$2"; shift 2; do + printf "%s (%s)\n" "$val" "$name" +done + +-------------------------------------------------------------------------------- + +(program + (while_statement + condition: (command + name: (command_name + (word)) + argument: (word)) + body: (do_group + (command + name: (command_name + (word)) + argument: (word)) + (command + name: (command_name + (word)) + argument: (word)))) + (while_statement + condition: (declaration_command + (variable_assignment + name: (variable_name) + value: (string + (simple_expansion + (variable_name)))) + (variable_assignment + name: (variable_name) + value: (string + (simple_expansion + (variable_name))))) + condition: (command + name: (command_name + (word)) + argument: (number)) + body: (do_group + (command + name: (command_name + (word)) + argument: (string (string_content)) + argument: (string + (simple_expansion + (variable_name))) + argument: (string + (simple_expansion + (variable_name))))))) + +================================================================================ +Until statements +================================================================================ + +until something happens; do + echo a + echo b +done + +-------------------------------------------------------------------------------- + +(program + (while_statement + condition: (command + name: (command_name + (word)) + argument: (word)) + body: (do_group + (command + name: (command_name + (word)) + argument: (word)) + (command + name: (command_name + (word)) + argument: (word))))) + +================================================================================ +While statements with IO redirects +================================================================================ + +while read line; do + echo $line +done < <(cat file) + +-------------------------------------------------------------------------------- + +(program + (redirected_statement + body: (while_statement + condition: (command + name: (command_name + (word)) + argument: (word)) + body: (do_group + (command + name: (command_name + (word)) + argument: (simple_expansion + (variable_name))))) + redirect: (file_redirect + destination: (process_substitution + (command + name: (command_name + (word)) + argument: (word)))))) + +================================================================================ +For statements +================================================================================ + +for a in 1 2 $(seq 5 10); do + echo $a +done + +for ARG; do + echo $ARG + ARG='' +done + +for c in ${=1}; do + echo c +done + +-------------------------------------------------------------------------------- + +(program + (for_statement + variable: (variable_name) + value: (number) + value: (number) + value: (command_substitution + (command + name: (command_name + (word)) + argument: (number) + argument: (number))) + body: (do_group + (command + name: (command_name + (word)) + argument: (simple_expansion + (variable_name))))) + (for_statement + variable: (variable_name) + body: (do_group + (command + name: (command_name + (word)) + argument: (simple_expansion + (variable_name))) + (variable_assignment + name: (variable_name) + value: (raw_string)))) + (for_statement + variable: (variable_name) + value: (expansion + (variable_name)) + body: (do_group + (command + name: (command_name + (word)) + argument: (word))))) + +================================================================================ +Select statements +================================================================================ + +select choice in X Y $(ls); do + echo $choice + break +done + +select ARG; do + echo $ARG + ARG='' +done + +-------------------------------------------------------------------------------- + +(program + (for_statement + (variable_name) + (word) + (word) + (command_substitution + (command + (command_name + (word)))) + (do_group + (command + (command_name + (word)) + (simple_expansion + (variable_name))) + (command + (command_name + (word))))) + (for_statement + (variable_name) + (do_group + (command + (command_name + (word)) + (simple_expansion + (variable_name))) + (variable_assignment + (variable_name) + (raw_string))))) + +================================================================================ +C-style for statements +================================================================================ + +for (( c=1; c<=5; c++ )) +do + echo $c +done + +for (( c=1; c<=5; c++ )) { + echo $c +} + +for (( ; ; )) +do + echo 'forever' +done + +for ((cx = 0; c = $cx / $pf, c < $wc - $k; )); do + echo "$cx" +done + +for (( i = 4;;i--)) ; do echo $i; if (( $i == 0 )); then break; fi; done + +# added post-bash-4.2 +for (( i = j = k = 1; i % 9 || (j *= -1, $( ((i%9)) || printf " " >&2; echo 0), k++ <= 10); i += j )) +do +printf "$i" +done + +echo + +( for (( i = j = k = 1; i % 9 || (j *= -1, $( ((i%9)) || printf " " >&2; echo 0), k++ <= 10); i += j )) +do +printf "$i" +done ) + +-------------------------------------------------------------------------------- + +(program + (c_style_for_statement + (variable_assignment + (variable_name) + (number)) + (binary_expression + (word) + (number)) + (postfix_expression + (word)) + (do_group + (command + (command_name + (word)) + (simple_expansion + (variable_name))))) + (c_style_for_statement + (variable_assignment + (variable_name) + (number)) + (binary_expression + (word) + (number)) + (postfix_expression + (word)) + (compound_statement + (command + (command_name + (word)) + (simple_expansion + (variable_name))))) + (c_style_for_statement + (do_group + (command + (command_name + (word)) + (raw_string)))) + (c_style_for_statement + (variable_assignment + (variable_name) + (number)) + (variable_assignment + (variable_name) + (binary_expression + (simple_expansion + (variable_name)) + (simple_expansion + (variable_name)))) + (binary_expression + (word) + (binary_expression + (simple_expansion + (variable_name)) + (simple_expansion + (variable_name)))) + (do_group + (command + (command_name + (word)) + (string + (simple_expansion + (variable_name)))))) + (c_style_for_statement + (variable_assignment + (variable_name) + (number)) + (postfix_expression + (word)) + (do_group + (command + (command_name + (word)) + (simple_expansion + (variable_name))) + (if_statement + (command + (command_name + (arithmetic_expansion + (binary_expression + (simple_expansion + (variable_name)) + (number))))) + (command + (command_name + (word)))))) + (comment) + (c_style_for_statement + (variable_assignment + (variable_name) + (variable_assignment + (variable_name) + (variable_assignment + (variable_name) + (number)))) + (binary_expression + (binary_expression + (word) + (number)) + (parenthesized_expression + (binary_expression + (word) + (number)) + (command_substitution + (redirected_statement + (list + (command + (command_name + (arithmetic_expansion + (binary_expression + (variable_name) + (number))))) + (command + (command_name + (word)) + (string))) + (file_redirect + (number))) + (command + (command_name + (word)) + (number))) + (binary_expression + (postfix_expression + (word)) + (number)))) + (binary_expression + (word) + (word)) + (do_group + (command + (command_name + (word)) + (string + (simple_expansion + (variable_name)))))) + (command + (command_name + (word))) + (subshell + (c_style_for_statement + (variable_assignment + (variable_name) + (variable_assignment + (variable_name) + (variable_assignment + (variable_name) + (number)))) + (binary_expression + (binary_expression + (word) + (number)) + (parenthesized_expression + (binary_expression + (word) + (number)) + (command_substitution + (redirected_statement + (list + (command + (command_name + (arithmetic_expansion + (binary_expression + (variable_name) + (number))))) + (command + (command_name + (word)) + (string))) + (file_redirect + (number))) + (command + (command_name + (word)) + (number))) + (binary_expression + (postfix_expression + (word)) + (number)))) + (binary_expression + (word) + (word)) + (do_group + (command + (command_name + (word)) + (string + (simple_expansion + (variable_name)))))))) + +================================================================================ +If statements +================================================================================ + +if cat some_file | grep -v ok; then + echo one +elif cat other_file | grep -v ok; then + echo two +else + exit +fi + +-------------------------------------------------------------------------------- + +(program + (if_statement + (pipeline + (command + (command_name + (word)) + (word)) + (command + (command_name + (word)) + (word) + (word))) + (command + (command_name + (word)) + (word)) + (elif_clause + (pipeline + (command + (command_name + (word)) + (word)) + (command + (command_name + (word)) + (word) + (word))) + (command + (command_name + (word)) + (word))) + (else_clause + (command + (command_name + (word)))))) + +================================================================================ +If statements with conditional expressions +================================================================================ + +if [ "$(uname)" == 'Darwin' ]; then + echo one +fi + +if [ a = -d ]; then + echo two +fi + +[[ abc == +(a|b|c) ]] && echo 1 +[[ abc != +(a|b|c) ]] && echo 2 + +-------------------------------------------------------------------------------- + +(program + (if_statement + (test_command + (binary_expression + (string + (command_substitution + (command + (command_name + (word))))) + (raw_string))) + (command + (command_name + (word)) + (word))) + (if_statement + (test_command + (binary_expression + (word) + (word))) + (command + (command_name + (word)) + (word))) + (list + (test_command + (binary_expression + (word) + (extglob_pattern))) + (command + (command_name + (word)) + (number))) + (list + (test_command + (binary_expression + (word) + (extglob_pattern))) + (command + (command_name + (word)) + (number)))) + +================================================================================ +If statements with negated command +================================================================================ + +if ! command -v echo; then + echo 'hello' +fi + +-------------------------------------------------------------------------------- + +(program + (if_statement + condition: (negated_command + (command + name: (command_name + (word)) + argument: (word) + argument: (word))) + (command + name: (command_name + (word)) + argument: (raw_string)))) + +================================================================================ +If statements with command +================================================================================ + +if command -v echo; then + echo 'hello' +fi + +-------------------------------------------------------------------------------- + +(program + (if_statement + condition: (command + name: (command_name + (word)) + argument: (word) + argument: (word)) + (command + name: (command_name + (word)) + argument: (raw_string)))) + +================================================================================ +If statements with variable assignment by command substitution +================================================================================ + +if result=$(echo 'hello'); then + echo 'hello' +fi + +-------------------------------------------------------------------------------- + +(program + (if_statement + condition: (variable_assignment + name: (variable_name) + value: (command_substitution + (command + name: (command_name + (word)) + argument: (raw_string)))) + (command + name: (command_name + (word)) + argument: (raw_string)))) + +================================================================================ +If statements with negated variable assignment by command substitution +================================================================================ + +if ! result=$(echo 'hello'); then + echo 'hello' +fi + +-------------------------------------------------------------------------------- + +(program + (if_statement + condition: (negated_command + (variable_assignment + name: (variable_name) + value: (command_substitution + (command + name: (command_name + (word)) + argument: (raw_string))))) + (command + name: (command_name + (word)) + argument: (raw_string)))) + +================================================================================ +If statements with variable assignment +================================================================================ + +if foo=1; then + echo 'hello' +fi + +-------------------------------------------------------------------------------- + +(program + (if_statement + condition: (variable_assignment + name: (variable_name) + value: (number)) + (command + name: (command_name + (word)) + argument: (raw_string)))) + +================================================================================ +If statements with negated variable assignment +================================================================================ + +if ! foo=1; then + echo 'hello' +fi + +-------------------------------------------------------------------------------- + +(program + (if_statement + condition: (negated_command + (variable_assignment + name: (variable_name) + value: (number))) + (command + name: (command_name + (word)) + argument: (raw_string)))) + +================================================================================ +Case statements +================================================================================ + +case "opt" in + a) + echo a + ;; + + b) + echo b + ;& + + c) + echo c;; +esac + +case "opt" in + (a) + echo a + ;; + + (b) + echo b + ;& + + (c) + echo c;; +esac + +case "$Z" in + ab*|cd*) ef +esac + +case $dest in + *.[1357]) + exit $? + ;; +esac + +case x in x) echo meow ;; esac + +case foo in + bar\ baz) : ;; +esac + +case "$arg" in + *([0-9])([0-9])) echo "$arg" +esac + +case ${lang} in +CMakeLists.txt | \ + cmake_modules | \ + ${PN}.pot) ;; +*) rm -r ${lang} || die ;; +esac + +-------------------------------------------------------------------------------- + +(program + (case_statement + (string (string_content)) + (case_item (word) (command (command_name (word)) (word))) + (case_item (word) (command (command_name (word)) (word))) + (case_item (word) (command (command_name (word)) (word)))) + (case_statement + (string (string_content)) + (case_item (word) (command (command_name (word)) (word))) + (case_item (word) (command (command_name (word)) (word))) + (case_item (word) (command (command_name (word)) (word)))) + (case_statement + (string (simple_expansion (variable_name))) + (case_item (extglob_pattern) (extglob_pattern) (command (command_name (word))))) + (case_statement + (simple_expansion (variable_name)) + (case_item + (concatenation (word) (word) (number) (word)) + (command (command_name (word)) (simple_expansion (special_variable_name))))) + (case_statement (word) (case_item (word) (command (command_name (word)) (word)))) + (case_statement (word) (case_item (word) (command (command_name (word))))) + (case_statement + (string (simple_expansion (variable_name))) + (case_item + (extglob_pattern) + (command (command_name (word)) (string (simple_expansion (variable_name)))))) + (case_statement + (expansion (variable_name)) + (case_item (word) (word) (concatenation (expansion (variable_name)) (word))) + (case_item + (extglob_pattern) + (list + (command (command_name (word)) (word) (expansion (variable_name))) + (command (command_name (word))))))) + +================================================================================ +Test commands +================================================================================ + +if [[ "$lsb_dist" != 'Ubuntu' || $(ver_to_int "$lsb_release") < $(ver_to_int '14.04') ]]; then + return 1 +fi + +[[ ${PV} != $(sed -n -e 's/^Version: //p' "${ED}/usr/$(get_libdir)/pkgconfig/tss2-tcti-tabrmd.pc" || die) ]] + +[[ ${f} != */@(default).vim ]] + +[[ "${MY_LOCALES}" != *en_US* || a != 2 ]] + +[[ $(LC_ALL=C $(tc-getCC) ${LDFLAGS} -Wl,--version 2>/dev/null) != @(LLD|GNU\ ld)* ]] + +[[ -f "${EROOT}/usr/share/php/.packagexml/${MY_P}.xml" && \ + -x "${EROOT}/usr/bin/peardev" ]] + +[[ ${test} == @($(IFS='|'; echo "${skip[*]}")) ]] + +[[ ${SRC_URI} == */${a}* ]] + +[[ a == *_@(LIB|SYMLINK) ]] + +[[ ${1} =~ \.(lisp|lsp|cl)$ ]] + +[[ a == - ]] + +-------------------------------------------------------------------------------- + +(program + (if_statement + (test_command + (binary_expression + (binary_expression (string (simple_expansion (variable_name))) (raw_string)) + (binary_expression + (command_substitution + (command (command_name (word)) (string (simple_expansion (variable_name))))) + (command_substitution (command (command_name (word)) (raw_string)))))) + (command (command_name (word)) (number))) + (test_command + (binary_expression + (expansion (variable_name)) + (command_substitution + (list + (command + (command_name (word)) + (word) + (word) + (raw_string) + (string + (expansion (variable_name)) + (string_content) + (command_substitution (command (command_name (word)))) + (string_content))) + (command (command_name (word))))))) + (test_command (binary_expression (expansion (variable_name)) (extglob_pattern))) + (test_command + (binary_expression + (binary_expression (string (expansion (variable_name))) (extglob_pattern)) + (binary_expression (word) (number)))) + (test_command + (binary_expression + (command_substitution + (redirected_statement + (command + (variable_assignment (variable_name) (word)) + (command_name (command_substitution (command (command_name (word))))) + (expansion (variable_name)) + (word)) + (file_redirect (file_descriptor) (word)))) + (extglob_pattern))) + (test_command + (binary_expression + (unary_expression + (test_operator) + (string + (expansion (variable_name)) + (string_content) + (expansion (variable_name)) + (string_content))) + (unary_expression + (test_operator) + (string (expansion (variable_name)) (string_content))))) + (test_command + (binary_expression + (expansion (variable_name)) + (extglob_pattern) + (command_substitution + (variable_assignment (variable_name) (raw_string)) + (command + (command_name (word)) + (string (expansion (subscript (variable_name) (word)))))) + (extglob_pattern))) + (test_command + (binary_expression + (expansion (variable_name)) + (extglob_pattern) + (expansion (variable_name)) + (extglob_pattern))) + (test_command (binary_expression (word) (extglob_pattern))) + (test_command (binary_expression (expansion (variable_name)) (regex))) + (test_command (binary_expression (word) (extglob_pattern)))) + +================================================================================ +Test commands with ternary +================================================================================ + +if (( 1 < 2 ? 1 : 2 )); then + return 1 +fi + +-------------------------------------------------------------------------------- + +(program + (if_statement + (command + (command_name + (arithmetic_expansion + (ternary_expression + (binary_expression + (number) + (number)) + (number) + (number))))) + (command + (command_name + (word)) + (number)))) + +================================================================================ +Ternary expressions +================================================================================ + +$((n < 10 ? n : 10)) +$(($n < 10 ? $n : 10)) +$((${n} < 10 ? ${n} : 10)) + +-------------------------------------------------------------------------------- + +(program + (command + (command_name + (arithmetic_expansion + (ternary_expression + (binary_expression + (variable_name) + (number)) + (variable_name) + (number))))) + (command + (command_name + (arithmetic_expansion + (ternary_expression + (binary_expression + (simple_expansion + (variable_name)) + (number)) + (simple_expansion + (variable_name)) + (number))))) + (command + (command_name + (arithmetic_expansion + (ternary_expression + (binary_expression + (expansion + (variable_name)) + (number)) + (expansion + (variable_name)) + (number)))))) + +================================================================================ +Test commands with regexes +================================================================================ + +[[ "35d8b" =~ ^[0-9a-fA-F] ]] +[[ $CMD =~ (^|;)update_terminal_cwd($|;) ]] +[[ ! " ${completions[*]} " =~ " $alias_cmd " ]] +! [[ "$a" =~ ^a|b\ *c|d$ ]] +[[ "$1" =~ ^${var}${var}*=..* ]] +[[ "$1" =~ ^\-${var}+ ]] +[[ ${var1} == *${var2}* ]] +[[ "$server" =~ [0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3} ]] +[[ "$primary_wins" =~ ([0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}) ]] +[[ -f ${x} && $(od -t x1 -N 4 "${x}") == *"7f 45 4c 46"* ]] +[[ ${PV} =~ 99999999$ ]] +[[ $1 == -- ]] +[[ " ${REPLACING_VERSIONS} " != *\ ${PVR}\ * ]] +[[ ${file} == @(*${GENTOO_PATCH_NAME}.tar.xz|*.asc|*.sig) ]] +[[ $RUBY_TARGETS != *$( eselect ruby show | awk 'NR==2' | tr -d ' ' )* ]] +[[ " ${m[0]##*/}" =~ ^(\ ${skip_files[*]/%/.*|\\} ) ]] +[[ ' boop xyz' =~ ' boop '(.*)$ ]] +[[ $b = foo* ]] + +-------------------------------------------------------------------------------- + +(program + (test_command (binary_expression (string (string_content)) (regex))) + (test_command (binary_expression (simple_expansion (variable_name)) (regex))) + (test_command + (binary_expression + (unary_expression + (string (expansion (subscript (variable_name) (word))))) + (string (simple_expansion (variable_name))))) + (negated_command (test_command (binary_expression (string (simple_expansion (variable_name))) (regex)))) + (test_command (binary_expression (string (simple_expansion (variable_name))) (regex))) + (test_command (binary_expression (string (simple_expansion (variable_name))) (regex))) + (test_command + (binary_expression + (expansion (variable_name)) + (extglob_pattern) + (expansion (variable_name)) + (extglob_pattern))) + (test_command (binary_expression (string (simple_expansion (variable_name))) (regex))) + (test_command (binary_expression (string (simple_expansion (variable_name))) (regex))) + (test_command + (binary_expression + (unary_expression (test_operator) (expansion (variable_name))) + (binary_expression + (command_substitution + (command + (command_name (word)) + (word) + (word) + (word) + (number) + (string (expansion (variable_name))))) + (extglob_pattern) + (string (string_content)) + (extglob_pattern)))) + (test_command (binary_expression (expansion (variable_name)) (regex))) + (test_command (binary_expression (simple_expansion (variable_name)) (extglob_pattern))) + (test_command + (binary_expression + (string (expansion (variable_name))) + (extglob_pattern) + (expansion (variable_name)) + (extglob_pattern))) + (test_command + (binary_expression + (expansion (variable_name)) + (extglob_pattern) + (expansion (variable_name)) + (extglob_pattern))) + (test_command + (binary_expression + (simple_expansion (variable_name)) + (extglob_pattern) + (command_substitution + (pipeline + (command (command_name (word)) (word) (word)) + (command (command_name (word)) (raw_string)) + (command (command_name (word)) (word) (raw_string)))) + (extglob_pattern))) + (test_command + (binary_expression (string (expansion (subscript (variable_name) (number)) (regex))) (regex))) + (test_command (binary_expression (raw_string) (regex))) + (test_command (binary_expression (simple_expansion (variable_name)) (regex)))) + +================================================================================ +Test command paren statefulness with a case glob +================================================================================ + +[[ ${test} == @($(IFS='|'; echo "${skip[*]}")) ]] + +case ${out} in +*"not supported"*|\ +*"operation not supported"*) + ;; +esac + +case $1 in +-o) + owner=$2 + shift + ;; +-g) ;; +esac + +[[ a == \"+(?)\" ]] + +--- + +(program + (test_command + (binary_expression + (expansion (variable_name)) + (extglob_pattern) + (command_substitution + (variable_assignment (variable_name) (raw_string)) + (command + (command_name (word)) + (string (expansion (subscript (variable_name) (word)))))) + (extglob_pattern))) + (case_statement + (expansion (variable_name)) + (case_item + (extglob_pattern) + (string (string_content)) + (extglob_pattern) + (concatenation (word) (string (string_content)) (word)))) + (case_statement + (simple_expansion (variable_name)) + (case_item + (extglob_pattern) + (variable_assignment (variable_name) (simple_expansion (variable_name))) + (command (command_name (word)))) + (case_item (word))) + (test_command (binary_expression (word) (extglob_pattern)))) + +================================================================================ +Subshells +================================================================================ + +( + ./start-server --port=80 +) & + +time ( cd tests && sh run-tests.sh ) + +-------------------------------------------------------------------------------- + +(program + (subshell (command (command_name (word)) (word))) + (command + (command_name (word)) + (subshell + (list + (command (command_name (word)) (word)) + (command (command_name (word)) (word)))))) + +================================================================================ +Function definitions +================================================================================ + +do_something() { + echo ok +} + +run_subshell_command() ( + true +) + +run_test_command() [[ -e foo ]] + +function do_something_else() { + a | xargs -I{} find xml/{} -type f +} + +function do_yet_another_thing { + echo ok +} 2>&1 + +do_nothing() { return 0; } + +foo::bar() { + echo what +} + +foo::baz() { + echo how +} + +assert() + if ! $1; then + return 1 + fi + +-------------------------------------------------------------------------------- + +(program + (function_definition + (word) + (compound_statement + (command + (command_name + (word)) + (word)))) + (function_definition + (word) + (subshell + (command + (command_name + (word))))) + (function_definition + (word) + (test_command + (unary_expression + (test_operator) + (word)))) + (function_definition + (word) + (compound_statement + (pipeline + (command + (command_name + (word))) + (command + (command_name + (word)) + (concatenation + (word) + (word) + (word)) + (word) + (concatenation + (word) + (word) + (word)) + (word) + (word))))) + (function_definition + (word) + (compound_statement + (command + (command_name + (word)) + (word))) + (file_redirect + (file_descriptor) + (number))) + (function_definition + (word) + (compound_statement + (command + (command_name + (word)) + (number)))) + (function_definition + (word) + (compound_statement + (command + (command_name + (word)) + (word)))) + (function_definition + (word) + (compound_statement + (command + (command_name + (word)) + (word)))) + (function_definition + (word) + (if_statement + (negated_command + (command + (command_name + (simple_expansion + (variable_name))))) + (command + (command_name + (word)) + (number))))) + +================================================================================ +Variable declaration: declare & typeset +================================================================================ + +declare var1 +typeset -i -r var2=42 var3=10 + +-------------------------------------------------------------------------------- + +(program + (declaration_command + (variable_name)) + (declaration_command + (word) + (word) + (variable_assignment + (variable_name) + (number)) + (variable_assignment + (variable_name) + (number)))) + +================================================================================ +Variable declaration: readonly +================================================================================ + +readonly var1 +readonly var2=42 + +-------------------------------------------------------------------------------- + +(program + (declaration_command + (variable_name)) + (declaration_command + (variable_assignment + (variable_name) + (number)))) + +================================================================================ +Variable declaration: local +================================================================================ + +local a=42 b +local -r c +local var=word1\ word2 + +-------------------------------------------------------------------------------- + +(program + (declaration_command + (variable_assignment + (variable_name) + (number)) + (variable_name)) + (declaration_command + (word) + (variable_name)) + (declaration_command + (variable_assignment + (variable_name) + (word)))) + +================================================================================ +Variable declaration: export +================================================================================ + +export PATH +export FOOBAR PATH="$PATH:/usr/foobar/bin" +export $FOO:$BAR + +-------------------------------------------------------------------------------- + +(program + (declaration_command + (variable_name)) + (declaration_command + (variable_name) + (variable_assignment + (variable_name) + (string + (simple_expansion + (variable_name)) + (string_content)))) + (declaration_command + (concatenation + (simple_expansion + (variable_name)) + (word) + (simple_expansion + (variable_name))))) + +================================================================================ +Variable declaration: command substitution with semi-colon +================================================================================ + +_path=$( + while statement; do + cd ".." + done; + echo $PWD +) + +-------------------------------------------------------------------------------- + +(program + (variable_assignment + (variable_name) + (command_substitution + (while_statement + (command + (command_name + (word))) + (do_group + (command + (command_name + (word)) + (string (string_content))))) + (command + (command_name + (word)) + (simple_expansion + (variable_name)))))) + +=========================================================== +Command substution with $ and backticks +=========================================================== + +$(eval echo $`echo ${foo}`) + +--- + +(program + (command + (command_name + (command_substitution + (command + (command_name + (word)) + (word) + (command_substitution + (command + (command_name + (word)) + (expansion + (variable_name))))))))) + +================================================================================ +Expressions passed to declaration commands +================================================================================ + +export "$(echo ${key} | tr [:lower:] [:upper:])=${p_key#*=}" + +-------------------------------------------------------------------------------- + +(program + (declaration_command + (string + (command_substitution + (pipeline + (command + (command_name + (word)) + (expansion + (variable_name))) + (command + (command_name + (word)) + (concatenation + (word) + (word) + (word)) + (concatenation + (word) + (word) + (word))))) + (string_content) + (expansion + (variable_name) + (regex))))) + +================================================================================ +Unset commands +================================================================================ + +unset A +unset "$variable_name" +unsetenv -f ONE TWO + +-------------------------------------------------------------------------------- + +(program + (unset_command + (variable_name)) + (unset_command + (string + (simple_expansion + (variable_name)))) + (unset_command + (word) + (variable_name) + (variable_name))) + +================================================================================ +Compound statements +================================================================================ + +a () { + ls || { echo "b"; return 0; } + echo c +} + +{ echo "a" + echo "b" +} >&2 + +-------------------------------------------------------------------------------- + +(program + (function_definition + (word) + (compound_statement + (list + (command + (command_name + (word))) + (compound_statement + (command + (command_name + (word)) + (string (string_content))) + (command + (command_name + (word)) + (number)))) + (command + (command_name + (word)) + (word)))) + (redirected_statement + (compound_statement + (command + (command_name + (word)) + (string (string_content))) + (command + (command_name + (word)) + (string (string_content)))) + (file_redirect + (number)))) + +================================================================================ +If condition with subshell +================================================================================ + +if (echo $BASHPID; true); then echo $BASHPID; fi + +-------------------------------------------------------------------------------- + +(program + (if_statement + (subshell + (command + (command_name + (word)) + (simple_expansion + (variable_name))) + (command + (command_name + (word)))) + (command + (command_name + (word)) + (simple_expansion + (variable_name))))) + +================================================================================ +While condition with subshell +================================================================================ + +while (echo $BASHPID; true); do echo $BASHPID; break; done + +-------------------------------------------------------------------------------- + +(program + (while_statement + (subshell + (command + (command_name + (word)) + (simple_expansion + (variable_name))) + (command + (command_name + (word)))) + (do_group + (command + (command_name + (word)) + (simple_expansion + (variable_name))) + (command + (command_name + (word)))))) diff --git a/packages/tree-sitter-bash/test/fixtures/differential/arithmetic.txt b/packages/tree-sitter-bash/test/fixtures/differential/arithmetic.txt new file mode 100644 index 0000000000..f2583d371a --- /dev/null +++ b/packages/tree-sitter-bash/test/fixtures/differential/arithmetic.txt @@ -0,0 +1,236 @@ +@match: for ((i=0;i<3;i++)); do echo $i; done +for ((i=0;i<3;i++)); do echo $i; done +=== +@match: for (( i = 0; i < 10; i += 2 )); do x; done +for (( i = 0; i < 10; i += 2 )); do x; done +=== +@match: echo $((1 + 2 * 3)) +echo $((1 + 2 * 3)) +=== +@match: echo $((a > b ? a : b)) +echo $((a > b ? a : b)) +=== +@match: echo $(( (1+2) ** 3 )) +echo $(( (1+2) ** 3 )) +=== +@match: echo $((i++ + --j)) +echo $((i++ + --j)) +=== +@match: echo $((-x + ~y)) +echo $((-x + ~y)) +=== +@match: echo $((arr[0] + 1)) +echo $((arr[0] + 1)) +=== +@match: echo $((x += 5)) +echo $((x += 5)) +=== +@match: echo $((1, 2, 3)) +echo $((1, 2, 3)) +=== +@match: ((x = y + 1)) +((x = y + 1)) +=== +@match: echo $((n % 2 == 0)) +echo $((n % 2 == 0)) +=== +@match: echo $((2 ** 3 ** 2)) +echo $((2 ** 3 ** 2)) +=== +@match: echo $((x = y = 1)) +echo $((x = y = 1)) +=== +@match: echo $(( +echo $(( +1 + +2 +)) +=== +@match: (( ls )) +(( ls )) +=== +@match: ((x=1)); echo $x +((x=1)); echo $x +=== +@match: for ((;;)); do x; done +for ((;;)); do x; done +=== +@match: for ((i=0,j=1; i<3; i++,j--)); do x; done +for ((i=0,j=1; i<3; i++,j--)); do x; done +=== +@match: for ((i=0;i<3;i++)) { echo $i; } +for ((i=0;i<3;i++)) { echo $i; } +=== +@match: echo $[1+2] +echo $[1+2] +=== +@match: echo $((x[1]+y[z])) +echo $((x[1]+y[z])) +=== +@match: ((a++)) +((a++)) +=== +@match: ((a--)) +((a--)) +=== +@match: ((a++ + b)) +((a++ + b)) +=== +@match: ((b + a++)) +((b + a++)) +=== +@known-diff arithmetic-hex: echo $((0x1F)) +echo $((0x1F)) +--- +hasError: false +program [0,14] "echo $((0x1F))" + command [0,14] "echo $((0x1F))" + command_name [0,4] "echo" + word [0,4] "echo" + arithmetic_expansion [5,14] "$((0x1F))" + ($(() [5,8] "$((" + number [8,12] "0x1F" + ())) [12,14] "))" +=== +@match: echo ((a++)) +echo ((a++)) +=== +@match: ((a[i]++)) +((a[i]++)) +=== +@match: ! ((a++)) +! ((a++)) +=== +@match: ((ab-cd++)) +((ab-cd++)) +=== +@match: echo $((1+2)) more +echo $((1+2)) more +=== +@match: $(((${1:-${SECONDS}} % 12) + 144)) +$(((${1:-${SECONDS}} % 12) + 144)) +=== +@match: echo $((-1, 1)) +echo $((-1, 1)) +=== +@match: echo $((! -a || ~ +b || ++c || --d)) +echo $((! -a || ~ +b || ++c || --d)) +=== +@match: (("${MULTIBUILD_VARIANTS}" > 1)) +(("${MULTIBUILD_VARIANTS}" > 1)) +=== +@match: $(("$(stat --printf '%05a' "${save_file}")" & 07177)) +$(("$(stat --printf '%05a' "${save_file}")" & 07177)) +=== +@match: soft_errors_count=$[soft_errors_count + 1] +soft_errors_count=$[soft_errors_count + 1] +=== +@known-diff cfor-compound-assign-negative: for (( i = j = k = 1; i % 9 || (j *= -1, $( ((i%9)) || pr... +for (( i = j = k = 1; i % 9 || (j *= -1, $( ((i%9)) || printf " " >&2; echo 0), k++ <= 10); i += j )) +do +printf "$i" +done +--- +hasError: false +program [0,121] "for (( i = j = k = 1; i % 9 || (j *= ..." + c_style_for_statement [0,121] "for (( i = j = k = 1; i % 9 || (j *= ..." + (for) [0,3] "for" + ((() [4,6] "((" + variable_assignment [7,20] "i = j = k = 1" + variable_name [7,8] "i" + (=) [9,10] "=" + variable_assignment [11,20] "j = k = 1" + variable_name [11,12] "j" + (=) [13,14] "=" + variable_assignment [15,20] "k = 1" + variable_name [15,16] "k" + (=) [17,18] "=" + number [19,20] "1" + (;) [20,21] ";" + binary_expression [22,90] "i % 9 || (j *= -1, $( ((i%9)) || prin..." + binary_expression [22,27] "i % 9" + word [22,23] "i" + (%) [24,25] "%" + number [26,27] "9" + (||) [28,30] "||" + parenthesized_expression [31,90] "(j *= -1, $( ((i%9)) || printf \" \" >&..." + (() [31,32] "(" + binary_expression [32,39] "j *= -1" + word [32,33] "j" + (*=) [34,36] "*=" + unary_expression [37,39] "-1" + (-) [37,38] "-" + number [38,39] "1" + (,) [39,40] "," + command_substitution [41,78] "$( ((i%9)) || printf \" \" >&2; echo 0)" + ($() [41,43] "$(" + list [44,69] "((i%9)) || printf \" \" >&2" + command [44,51] "((i%9))" + command_name [44,51] "((i%9))" + arithmetic_expansion [44,51] "((i%9))" + ((() [44,46] "((" + binary_expression [46,49] "i%9" + variable_name [46,47] "i" + (%) [47,48] "%" + number [48,49] "9" + ())) [49,51] "))" + (||) [52,54] "||" + redirected_statement [55,69] "printf \" \" >&2" + command [55,65] "printf \" \"" + command_name [55,61] "printf" + word [55,61] "printf" + string [62,65] "\" \"" + (") [62,63] "\"" + string_content [63,64] " " + (") [64,65] "\"" + file_redirect [66,69] ">&2" + (>&) [66,68] ">&" + number [68,69] "2" + (;) [69,70] ";" + command [71,77] "echo 0" + command_name [71,75] "echo" + word [71,75] "echo" + number [76,77] "0" + ()) [77,78] ")" + (,) [78,79] "," + binary_expression [80,89] "k++ <= 10" + postfix_expression [80,83] "k++" + word [80,81] "k" + (++) [81,83] "++" + (<=) [84,86] "<=" + number [87,89] "10" + ()) [89,90] ")" + (;) [90,91] ";" + binary_expression [92,98] "i += j" + word [92,93] "i" + (+=) [94,96] "+=" + word [97,98] "j" + ())) [99,101] "))" + do_group [102,121] "do\nprintf \"$i\"\ndone" + (do) [102,104] "do" + command [105,116] "printf \"$i\"" + command_name [105,111] "printf" + word [105,111] "printf" + string [112,116] "\"$i\"" + (") [112,113] "\"" + simple_expansion [113,115] "$i" + ($) [113,114] "$" + variable_name [114,115] "i" + (") [115,116] "\"" + (done) [117,121] "done" +=== +@match: ((x = -1)) +((x = -1)) +=== +@match: ((x += -1)) +((x += -1)) +=== +@match: echo $((-1)) +echo $((-1)) +=== +@match: echo $((x * -1)) +echo $((x * -1)) +=== +@match: echo $((x=-1)) +echo $((x=-1)) +=== diff --git a/packages/tree-sitter-bash/test/fixtures/differential/case.txt b/packages/tree-sitter-bash/test/fixtures/differential/case.txt new file mode 100644 index 0000000000..bbb781340e --- /dev/null +++ b/packages/tree-sitter-bash/test/fixtures/differential/case.txt @@ -0,0 +1,436 @@ +@match: case $x in +case $x in + a) echo A ;; + b|c) echo BC ;& + *) echo other ;; +esac +=== +@match: case $x in (a) x ;; esac +case $x in (a) x ;; esac +=== +@match: case $x in a|b) x ;; esac +case $x in a|b) x ;; esac +=== +@match: case $x in foo*|bar) x ;; esac +case $x in foo*|bar) x ;; esac +=== +@match: case $x in [a-z]) x ;; esac +case $x in [a-z]) x ;; esac +=== +@match: case $x in "a") x ;; esac +case $x in "a") x ;; esac +=== +@match: case $x in $v|${w}) x ;; esac +case $x in $v|${w}) x ;; esac +=== +@known-diff recovery-partial-nodes: case $x in a) x ;;& esac +case $x in a) x ;;& esac +--- +hasError: true +program [0,24] "case $x in a) x ;;& esac" + case_statement [0,24] "case $x in a) x ;;& esac" + (case) [0,4] "case" + simple_expansion [5,7] "$x" + ($) [5,6] "$" + variable_name [6,7] "x" + (in) [8,10] "in" + case_item [11,19] "a) x ;;&" + word [11,12] "a" + ()) [12,13] ")" + command [14,15] "x" + command_name [14,15] "x" + word [14,15] "x" + (;;&) [16,19] ";;&" + (esac) [20,24] "esac" +=== +@match: case $x in a) x esac +case $x in a) x esac +=== +@match: case $x in esac +case $x in esac +=== +@match: case x in a) ;; esac +case x in a) ;; esac +=== +@match: case $x in +case $x in +a) y ;; +esac +=== +@match: case $x in a) echo esac ;; esac +case $x in a) echo esac ;; esac +=== +@match: case $x in a) x +case $x in a) x +esac +=== +@match: x=$(case y in a) 1;; esac) +x=$(case y in a) 1;; esac) +=== +@match: x=$(case y in (b) 2;; esac) +x=$(case y in (b) 2;; esac) +=== +@match: diff <(case y in a) 1;; esac) other +diff <(case y in a) 1;; esac) other +=== +@match: case $x in [1]) x ;; esac +case $x in [1]) x ;; esac +=== +@known-diff recovery-partial-nodes: if a; then case x in b) 1;; esac elif c; then d; fi +if a; then case x in b) 1;; esac elif c; then d; fi +--- +hasError: true +program [0,51] "if a; then case x in b) 1;; esac elif..." + if_statement [0,51] "if a; then case x in b) 1;; esac elif..." + (if) [0,2] "if" + command [3,4] "a" + command_name [3,4] "a" + word [3,4] "a" + (;) [4,5] ";" + (then) [6,10] "then" + case_statement [11,32] "case x in b) 1;; esac" + (case) [11,15] "case" + word [16,17] "x" + (in) [18,20] "in" + case_item [21,27] "b) 1;;" + word [21,22] "b" + ()) [22,23] ")" + command [24,25] "1" + command_name [24,25] "1" + number [24,25] "1" + (;;) [25,27] ";;" + (esac) [28,32] "esac" + elif_clause [33,48] "elif c; then d;" + (elif) [33,37] "elif" + command [38,39] "c" + command_name [38,39] "c" + word [38,39] "c" + (;) [39,40] ";" + (then) [41,45] "then" + command [46,47] "d" + command_name [46,47] "d" + word [46,47] "d" + (;) [47,48] ";" + (fi) [49,51] "fi" +=== +@match: case $dest in +case $dest in + *.[1357]) + exit $? + ;; +esac +=== +@match: case "$arg" in +case "$arg" in + *([0-9])([0-9])) echo "$arg" +esac +=== +@known-diff case-pattern-after-continuation: case ${lang} in +case ${lang} in +CMakeLists.txt | \ + cmake_modules | \ + ${PN}.pot) ;; +*) rm -r ${lang} || die ;; +esac +--- +hasError: false +program [0,100] "case ${lang} in\nCMakeLists.txt | \\\n\tc..." + case_statement [0,100] "case ${lang} in\nCMakeLists.txt | \\\n\tc..." + (case) [0,4] "case" + expansion [5,12] "${lang}" + (${) [5,7] "${" + variable_name [7,11] "lang" + (}) [11,12] "}" + (in) [13,15] "in" + case_item [16,68] "CMakeLists.txt | \\\n\tcmake_modules | \\..." + word [16,30] "CMakeLists.txt" + (|) [31,32] "|" + extglob_pattern [36,49] "cmake_modules" + (|) [50,51] "|" + concatenation [55,64] "${PN}.pot" + expansion [55,60] "${PN}" + (${) [55,57] "${" + variable_name [57,59] "PN" + (}) [59,60] "}" + word [60,64] ".pot" + ()) [64,65] ")" + (;;) [66,68] ";;" + case_item [69,95] "*) rm -r ${lang} || die ;;" + extglob_pattern [69,70] "*" + ()) [70,71] ")" + list [72,92] "rm -r ${lang} || die" + command [72,85] "rm -r ${lang}" + command_name [72,74] "rm" + word [72,74] "rm" + word [75,77] "-r" + expansion [78,85] "${lang}" + (${) [78,80] "${" + variable_name [80,84] "lang" + (}) [84,85] "}" + (||) [86,88] "||" + command [89,92] "die" + command_name [89,92] "die" + word [89,92] "die" + (;;) [93,95] ";;" + (esac) [96,100] "esac" +=== +@match: case ${out} in +case ${out} in +*"not supported"*|\ +*"operation not supported"*) + ;; +esac +=== +@known-diff case-short-option-pattern: case $1 in +case $1 in +-o) + owner=$2 + shift + ;; +-g) ;; +esac +--- +hasError: false +program [0,47] "case $1 in\n-o)\n\towner=$2\n\tshift\n\t;;\n-..." + case_statement [0,47] "case $1 in\n-o)\n\towner=$2\n\tshift\n\t;;\n-..." + (case) [0,4] "case" + simple_expansion [5,7] "$1" + ($) [5,6] "$" + variable_name [6,7] "1" + (in) [8,10] "in" + case_item [11,35] "-o)\n\towner=$2\n\tshift\n\t;;" + word [11,13] "-o" + ()) [13,14] ")" + variable_assignment [16,24] "owner=$2" + variable_name [16,21] "owner" + (=) [21,22] "=" + simple_expansion [22,24] "$2" + ($) [22,23] "$" + variable_name [23,24] "2" + command [26,31] "shift" + command_name [26,31] "shift" + word [26,31] "shift" + (;;) [33,35] ";;" + case_item [36,42] "-g) ;;" + word [36,38] "-g" + ()) [38,39] ")" + (;;) [40,42] ";;" + (esac) [43,47] "esac" +=== +@match: case x in *.txt) ;; esac +case x in *.txt) ;; esac +=== +@match: case x in [abc]) ;; esac +case x in [abc]) ;; esac +=== +@match: case x in a|b) ;; esac +case x in a|b) ;; esac +=== +@match: case x in -o|--output) ;; esac +case x in -o|--output) ;; esac +=== +@match: case x in x) ;; esac +case x in x) ;; esac +=== +@match: case x in foo.txt) ;; esac +case x in foo.txt) ;; esac +=== +@match: case x in *foo) ;; esac +case x in *foo) ;; esac +=== +@match: case x in foo*) ;; esac +case x in foo*) ;; esac +=== +@match: case x in *) ;; esac +case x in *) ;; esac +=== +@match: case x in ?) ;; esac +case x in ?) ;; esac +=== +@match: case x in *.*) ;; esac +case x in *.*) ;; esac +=== +@match: case x in foo|bar) ;; esac +case x in foo|bar) ;; esac +=== +@match: case x in foo.txt|*.log) ;; esac +case x in foo.txt|*.log) ;; esac +=== +@match: case x in [a-z]*) ;; esac +case x in [a-z]*) ;; esac +=== +@match: case x in *.[1357]) ;; esac +case x in *.[1357]) ;; esac +=== +@match: case x in -o) ;; esac +case x in -o) ;; esac +=== +@match: case x in --output) ;; esac +case x in --output) ;; esac +=== +@match: case x in +(a|b)) ;; esac +case x in +(a|b)) ;; esac +=== +@match: case x in *(a|b)) ;; esac +case x in *(a|b)) ;; esac +=== +@match: case x in @(a|b)) ;; esac +case x in @(a|b)) ;; esac +=== +@known-diff case-rejected-group: case x in x@(y)) ;; esac +case x in x@(y)) ;; esac +--- +hasError: true +program [0,24] "case x in x@(y)) ;; esac" + case_statement [0,24] "case x in x@(y)) ;; esac" + (case) [0,4] "case" + word [5,6] "x" + (in) [7,9] "in" + case_item [10,19] "x@(y)) ;;" + ERROR [10,15] "x@(y)" + ()) [15,16] ")" + (;;) [17,19] ";;" + (esac) [20,24] "esac" +=== +@match: case x in *([0-9])([0-9])) ;; esac +case x in *([0-9])([0-9])) ;; esac +=== +@match: case x in *"s"*) ;; esac +case x in *"s"*) ;; esac +=== +@match: case x in *${x}*) ;; esac +case x in *${x}*) ;; esac +=== +@match: case x in ${x}) ;; esac +case x in ${x}) ;; esac +=== +@match: case x in ?(a)) ;; esac +case x in ?(a)) ;; esac +=== +@match: case x in !(a)) ;; esac +case x in !(a)) ;; esac +=== +@match: case x in a\ b) ;; esac +case x in a\ b) ;; esac +=== +@match: case $x in foo_bar) ;; esac +case $x in foo_bar) ;; esac +=== +@match: case $x in word1) ;; esac +case $x in word1) ;; esac +=== +@match: case $x in abc1d) ;; esac +case $x in abc1d) ;; esac +=== +@match: case $x in v2.0) ;; esac +case $x in v2.0) ;; esac +=== +@match: case $x in a1) ;; esac +case $x in a1) ;; esac +=== +@match: case $x in a_) ;; esac +case $x in a_) ;; esac +=== +@match: case $x in A1) ;; esac +case $x in A1) ;; esac +=== +@match: case $x in a-b) ;; esac +case $x in a-b) ;; esac +=== +@match: case $x in a-bc) ;; esac +case $x in a-bc) ;; esac +=== +@match: case $x in ab-c) ;; esac +case $x in ab-c) ;; esac +=== +@match: case $x in a-b-c) ;; esac +case $x in a-b-c) ;; esac +=== +@match: case $x in cmake_modules) ;; esac +case $x in cmake_modules) ;; esac +=== +@match: case $x in a-b|y) ;; esac +case $x in a-b|y) ;; esac +=== +@match: case $x in abc|y) ;; esac +case $x in abc|y) ;; esac +=== +@match: case $x in abc | y) ;; esac +case $x in abc | y) ;; esac +=== +@match: case $x in -x-) ;; esac +case $x in -x-) ;; esac +=== +@match: case $x in -x-y) ;; esac +case $x in -x-y) ;; esac +=== +@known-diff case-rejected-group: case $x in *.(a|b)) ;; esac +case $x in *.(a|b)) ;; esac +--- +hasError: true +program [0,27] "case $x in *.(a|b)) ;; esac" + case_statement [0,27] "case $x in *.(a|b)) ;; esac" + (case) [0,4] "case" + simple_expansion [5,7] "$x" + ($) [5,6] "$" + variable_name [6,7] "x" + (in) [8,10] "in" + case_item [11,22] "*.(a|b)) ;;" + ERROR [11,18] "*.(a|b)" + ()) [18,19] ")" + (;;) [20,22] ";;" + (esac) [23,27] "esac" +=== +@known-diff case-rejected-group: case $x in x.(y)) ;; esac +case $x in x.(y)) ;; esac +--- +hasError: true +program [0,25] "case $x in x.(y)) ;; esac" + case_statement [0,25] "case $x in x.(y)) ;; esac" + (case) [0,4] "case" + simple_expansion [5,7] "$x" + ($) [5,6] "$" + variable_name [6,7] "x" + (in) [8,10] "in" + case_item [11,20] "x.(y)) ;;" + ERROR [11,16] "x.(y)" + ()) [16,17] ")" + (;;) [18,20] ";;" + (esac) [21,25] "esac" +=== +@known-diff case-pattern-after-continuation: case $x in cont\ued) ;; esac +case $x in cont\ +ued) ;; esac +--- +hasError: false +program [0,29] "case $x in cont\\\nued) ;; esac" + case_statement [0,29] "case $x in cont\\\nued) ;; esac" + (case) [0,4] "case" + simple_expansion [5,7] "$x" + ($) [5,6] "$" + variable_name [6,7] "x" + (in) [8,10] "in" + case_item [11,24] "cont\\\nued) ;;" + word [11,20] "cont\\\nued" + ()) [20,21] ")" + (;;) [22,24] ";;" + (esac) [25,29] "esac" +=== +@known-diff case-pattern-after-continuation: case $x in a\1) ;; esac +case $x in a\ +1) ;; esac +--- +hasError: false +program [0,24] "case $x in a\\\n1) ;; esac" + case_statement [0,24] "case $x in a\\\n1) ;; esac" + (case) [0,4] "case" + simple_expansion [5,7] "$x" + ($) [5,6] "$" + variable_name [6,7] "x" + (in) [8,10] "in" + case_item [11,19] "a\\\n1) ;;" + word [11,15] "a\\\n1" + ()) [15,16] ")" + (;;) [17,19] ";;" + (esac) [20,24] "esac" +=== diff --git a/packages/tree-sitter-bash/test/fixtures/differential/expansions.txt b/packages/tree-sitter-bash/test/fixtures/differential/expansions.txt new file mode 100644 index 0000000000..e7394998a7 --- /dev/null +++ b/packages/tree-sitter-bash/test/fixtures/differential/expansions.txt @@ -0,0 +1,599 @@ +@match: for f in a b c; do echo "$f"; done +for f in a b c; do echo "$f"; done +=== +@match: for f; do echo "$f"; done +for f; do echo "$f"; done +=== +@match: select opt in a b; do echo $opt; done +select opt in a b; do echo $opt; done +=== +@match: a[i+1]=$y +a[i+1]=$y +=== +@match: echo ${arr[1]} ${arr[@]} +echo ${arr[1]} ${arr[@]} +=== +@match: echo $'a\nb\t' +echo $'a\nb\t' +=== +@match: echo $"hello $USER" +echo $"hello $USER" +=== +@match: echo ${x##*/} ${x%%.*} +echo ${x##*/} ${x%%.*} +=== +@match: echo ${x/y/z} ${x//a/b} +echo ${x/y/z} ${x//a/b} +=== +@match: echo ${x^} ${x^^} ${x,} ${x,,} +echo ${x^} ${x^^} ${x,} ${x,,} +=== +@match: echo ${x@Q} +echo ${x@Q} +=== +@match: echo ${v:1:2} ${v:-default} +echo ${v:1:2} ${v:-default} +=== +@match: f() { arr=(1 2); echo ${arr[0]}; } +f() { arr=(1 2); echo ${arr[0]}; } +=== +@match: for i in $(seq 3); do echo $i; done +for i in $(seq 3); do echo $i; done +=== +@match: echo ${a[$i]} ${a[i+1]} +echo ${a[$i]} ${a[i+1]} +=== +@match: echo ${!prefix*} ${!name@} +echo ${!prefix*} ${!name@} +=== +@match: echo ${x##a b} +echo ${x##a b} +=== +@match: echo ${x#"pat"} +echo ${x#"pat"} +=== +@match: echo ${x/a b/c} +echo ${x/a b/c} +=== +@match: echo ${x/a/$v} +echo ${x/a/$v} +=== +@match: echo ${x/pat/rep/extra} +echo ${x/pat/rep/extra} +=== +@match: echo ${x:?msg} ${x:+alt} ${x:-d e f} +echo ${x:?msg} ${x:+alt} ${x:-d e f} +=== +@match: echo ${v:-1} ${v:=2x} ${v:-"d e"} +echo ${v:-1} ${v:=2x} ${v:-"d e"} +=== +@match: echo ${arr[@]:1:2} +echo ${arr[@]:1:2} +=== +@known-diff recovery-partial-nodes: echo ${s//"/x} +echo ${s//"/x} +--- +hasError: true +program [0,14] "echo ${s//\"/x}" + command [0,14] "echo ${s//\"/x}" + command_name [0,4] "echo" + word [0,4] "echo" + expansion [5,14] "${s//\"/x}" + (${) [5,7] "${" + variable_name [7,8] "s" + (//) [8,10] "//" + string [10,14] "\"/x}" + (") [10,11] "\"" + string_content [11,14] "/x}" +=== +@match: x=$"v" +x=$"v" +=== +@match: for f in $"a"; do x; done +for f in $"a"; do x; done +=== +@match: echo a$"b" +echo a$"b" +=== +@match: echo $"a""b" +echo $"a""b" +=== +@match: echo x=$"v" +echo x=$"v" +=== +@match: echo $0 ${0} $1 ${10} +echo $0 ${0} $1 ${10} +=== +@match: echo ${x: -5} +echo ${x: -5} +=== +@match: echo ${x: -5:2} +echo ${x: -5:2} +=== +@known-diff recovery-partial-nodes: ${x@} +${x@} +--- +hasError: true +program [0,5] "${x@}" + command [0,5] "${x@}" + command_name [0,5] "${x@}" + expansion [0,5] "${x@}" + (${) [0,2] "${" + variable_name [2,3] "x" + word [3,4] "@" + (}) [4,5] "}" +=== +@match: echo ${x[1]} +echo ${x[1]} +=== +@match: echo "hello $NAME" 'raw' +echo "hello $NAME" 'raw' +=== +@match: echo `hostname` $(date) +echo `hostname` $(date) +=== +@match: echo ${v:-d} ${x} $1 $@ +echo ${v:-d} ${x} $1 $@ +=== +@match: echo $HOME +echo $HOME +=== +@match: echo a"b"c'd'$e${f}g +echo a"b"c'd'$e${f}g +=== +@match: echo $v +echo $v +=== +@match: echo "${a} middle ${b}" +echo "${a} middle ${b}" +=== +@match: echo "$@" +echo "$@" +=== +@match: echo $(a $(b c)) +echo $(a $(b c)) +=== +@match: echo ${#v} +echo ${#v} +=== +@match: echo ${v:=word} +echo ${v:=word} +=== +@known-diff empty-command-substitution: echo $( ) +echo $( ) +--- +hasError: false +program [0,9] "echo $( )" + command [0,9] "echo $( )" + command_name [0,4] "echo" + word [0,4] "echo" + command_substitution [5,9] "$( )" + ($() [5,7] "$(" + ()) [8,9] ")" +=== +@known-diff empty-backtick: echo `` +echo `` +--- +hasError: false +program [0,7] "echo ``" + command [0,7] "echo ``" + command_name [0,4] "echo" + word [0,4] "echo" + command_substitution [5,7] "``" + (`) [5,6] "`" + (`) [6,7] "`" +=== +@match: echo $ +echo $ +=== +@match: echo "a\$b" +echo "a\$b" +=== +@known-diff string-content-newlines: echo "line1 +echo "line1 +line2" +--- +hasError: false +program [0,18] "echo \"line1\nline2\"" + command [0,18] "echo \"line1\nline2\"" + command_name [0,4] "echo" + word [0,4] "echo" + string [5,18] "\"line1\nline2\"" + (") [5,6] "\"" + string_content [6,17] "line1\nline2" + (") [17,18] "\"" +=== +@match: echo ${v:-$(cmd)} +echo ${v:-$(cmd)} +=== +@match: echo ${a[0]} +echo ${a[0]} +=== +@match: echo ${} +echo ${} +=== +@match: echo ${#} +echo ${#} +=== +@match: echo ${var1#*#} +echo ${var1#*#} +=== +@match: echo ${!abc} +echo ${!abc} +=== +@match: echo ${abc:- } +echo ${abc:- } +=== +@match: echo ${abc: +echo ${abc: +} +=== +@match: echo ${abc,?} +echo ${abc,?} +=== +@match: echo ${abc^^b} +echo ${abc^^b} +=== +@match: echo ${abc@U} +echo ${abc@U} +=== +@match: echo ${abc:- -quiet} +echo ${abc:- -quiet} +=== +@match: A="${B[0]# }" +A="${B[0]# }" +=== +@match: C="${D/#* -E /}" +C="${D/#* -E /}" +=== +@match: F="${G%% *}" +F="${G%% *}" +=== +@match: H="${I#*;}" +H="${I#*;}" +=== +@match: R="${S%()}" +R="${S%()}" +=== +@match: F="${#!}" +F="${#!}" +=== +@match: G=${H,,[I]} +G=${H,,[I]} +=== +@match: L="${M/'N'*/O}" +L="${M/'N'*/O}" +=== +@match: A=${B//:;;/$'\n'} +A=${B//:;;/$'\n'} +=== +@match: C=${D/;\ *;|} +C=${D/;\ *;|} +=== +@match: MOFILES=${LINGUAS// /.po }.po +MOFILES=${LINGUAS// /.po }.po +=== +@match: MY_P="${PN/aspell/aspell"${ASPELL_VERSION}"}" +MY_P="${PN/aspell/aspell"${ASPELL_VERSION}"}" +=== +@match: pyc=${pyc//*\/} +pyc=${pyc//*\/} +=== +@match: ${pv/\.} +${pv/\.} +=== +@match: ${new_test_cp//"${old_ver_cp}"/} +${new_test_cp//"${old_ver_cp}"/} +=== +@known-diff expansion-replacement-escaped-backslash: ${allarchives// /\\|} +${allarchives// /\\|} +--- +hasError: false +program [0,21] "${allarchives// /\\\\|}" + command [0,21] "${allarchives// /\\\\|}" + command_name [0,21] "${allarchives// /\\\\|}" + expansion [0,21] "${allarchives// /\\\\|}" + (${) [0,2] "${" + variable_name [2,13] "allarchives" + (//) [13,15] "//" + regex [15,16] " " + (/) [16,17] "/" + word [17,20] "\\\\|" + (}) [20,21] "}" +=== +@match: ${!#} +${!#} +=== +@known-diff expansion-bang-hash-special: ${!# } +${!# } +--- +hasError: true +program [0,6] "${!# }" + command [0,6] "${!# }" + command_name [0,6] "${!# }" + expansion [0,6] "${!# }" + (${) [0,2] "${" + (!) [2,3] "!" + special_variable_name [3,4] "#" + word [4,5] " " + (}) [5,6] "}" +=== +@match: ${!##} +${!##} +=== +@known-diff expansion-bang-hash-special: ${!## } +${!## } +--- +hasError: true +program [0,7] "${!## }" + command [0,7] "${!## }" + command_name [0,7] "${!## }" + expansion [0,7] "${!## }" + (${) [0,2] "${" + (!) [2,3] "!" + (#) [3,4] "#" + special_variable_name [4,5] "#" + word [5,6] " " + (}) [6,7] "}" +=== +@known-diff expansion-bang-hash-special: ${!##/} +${!##/} +--- +hasError: false +program [0,7] "${!##/}" + command [0,7] "${!##/}" + command_name [0,7] "${!##/}" + expansion [0,7] "${!##/}" + (${) [0,2] "${" + (!) [2,3] "!" + (#) [3,4] "#" + special_variable_name [4,5] "#" + (/) [5,6] "/" + (}) [6,7] "}" +=== +@match: echo "${kw}? ( ${cond:+${cond}? (} ${baseuri}-${ver}-${kw... +echo "${kw}? ( ${cond:+${cond}? (} ${baseuri}-${ver}-${kw}.${suff} ${cond:+) })" +=== +@match: ${completions[*]} +${completions[*]} +=== +@known-diff expansion-equals-operator: ${=1} +${=1} +--- +hasError: false +program [0,5] "${=1}" + command [0,5] "${=1}" + command_name [0,5] "${=1}" + expansion [0,5] "${=1}" + (${) [0,2] "${" + (=) [2,3] "=" + word [3,4] "1" + (}) [4,5] "}" +=== +@match: ${2?} +${2?} +=== +@match: ${p_key#*=} +${p_key#*=} +=== +@match: ${B[0]# } +${B[0]# } +=== +@match: ${to_enables[0]##*/} +${to_enables[0]##*/} +=== +@match: exec "${0#-}" --rcfile "${BASH_IT_BASHRC:-${HOME?}/.bashrc}" +exec "${0#-}" --rcfile "${BASH_IT_BASHRC:-${HOME?}/.bashrc}" +=== +@match: recho "TDEFAULTS = ${selvecs:+-DSELECT_VECS=\"$selvecs\"}" +recho "TDEFAULTS = ${selvecs:+-DSELECT_VECS=\"$selvecs\"}" +=== +@match: local msg="${2:-command '$1' does not exist}" +local msg="${2:-command '$1' does not exist}" +=== +@match: ${cdir:+#} +${cdir:+#} +=== +@match: ${dict_langs:+;} +${dict_langs:+;} +=== +@match: ${UTIL_LINUX_LIBC[@]/%/? ( sys-apps/util-linux )} +${UTIL_LINUX_LIBC[@]/%/? ( sys-apps/util-linux )} +=== +@match: ${id}${2+ ${2}} +${id}${2+ ${2}} +=== +@match: ${BRANDING_GCC_PKGVERSION/(/(Gentoo ${PVR}${extvers}, } +${BRANDING_GCC_PKGVERSION/(/(Gentoo ${PVR}${extvers}, } +=== +@match: BASH_IT_GIT_URL="${BASH_IT_GIT_URL/git@/https://}" +BASH_IT_GIT_URL="${BASH_IT_GIT_URL/git@/https://}" +=== +@known-diff number-base-with-expansion: 10#${command_start##*.} +10#${command_start##*.} +--- +hasError: false +program [0,23] "10#${command_start##*.}" + command [0,23] "10#${command_start##*.}" + command_name [0,23] "10#${command_start##*.}" + concatenation [0,23] "10#${command_start##*.}" + word [0,3] "10#" + expansion [3,23] "${command_start##*.}" + (${) [3,5] "${" + variable_name [5,18] "command_start" + (##) [18,20] "##" + regex [20,22] "*." + (}) [22,23] "}" +=== +@match: echo ${LIB_DEPEND//\[static-libs(+)]} +echo ${LIB_DEPEND//\[static-libs(+)]} +=== +@match: ${ALL_LLVM_TARGETS[@]/%/(-)?} +${ALL_LLVM_TARGETS[@]/%/(-)?} +=== +@match: filterdiff -p1 ${paths[@]/#/-i } +filterdiff -p1 ${paths[@]/#/-i } +=== +@match: ${cflags//-O? /$(get-flag O) } +${cflags//-O? /$(get-flag O) } +=== +@match: curf="${f%'-roff2html'*}.html" +curf="${f%'-roff2html'*}.html" +=== +@match: reff="${f/'-roff2html'*/'-ref'}.html" +reff="${f/'-roff2html'*/'-ref'}.html" +=== +@match: echo "s/$/'/" +echo "s/$/'/" +=== +@match: echo "s$" +echo "s$" +=== +@match: echo $'Here\'s Johnny!\r\n' +echo $'Here\'s Johnny!\r\n' +=== +@known-diff expansion-default-array: printf " %-9s" "${seq0:-(default)}" +printf " %-9s" "${seq0:-(default)}" +--- +hasError: false +program [0,36] "printf \" %-9s\" \"${seq0:-(default)}\"" + command [0,36] "printf \" %-9s\" \"${seq0:-(default)}\"" + command_name [0,6] "printf" + word [0,6] "printf" + string [7,15] "\" %-9s\"" + (") [7,8] "\"" + string_content [8,14] " %-9s" + (") [14,15] "\"" + string [16,36] "\"${seq0:-(default)}\"" + (") [16,17] "\"" + expansion [17,35] "${seq0:-(default)}" + (${) [17,19] "${" + variable_name [19,23] "seq0" + (:-) [23,25] ":-" + word [25,34] "(default)" + (}) [34,35] "}" + (") [35,36] "\"" +=== +@known-diff empty-backtick: main() { +main() { + local foo="asd"` + `"fgh" +} +--- +hasError: false +program [0,46] "main() {\n local foo=\"asd\"`\n ..." + function_definition [0,46] "main() {\n local foo=\"asd\"`\n ..." + word [0,4] "main" + (() [4,5] "(" + ()) [5,6] ")" + compound_statement [7,46] "{\n local foo=\"asd\"`\n `\"fgh\"\n}" + ({) [7,8] "{" + declaration_command [13,44] "local foo=\"asd\"`\n `\"fgh\"" + (local) [13,18] "local" + variable_assignment [19,44] "foo=\"asd\"`\n `\"fgh\"" + variable_name [19,22] "foo" + (=) [22,23] "=" + concatenation [23,44] "\"asd\"`\n `\"fgh\"" + string [23,28] "\"asd\"" + (") [23,24] "\"" + string_content [24,27] "asd" + (") [27,28] "\"" + command_substitution [28,39] "`\n `" + (`) [28,29] "`" + (`) [38,39] "`" + string [39,44] "\"fgh\"" + (") [39,40] "\"" + string_content [40,43] "fgh" + (") [43,44] "\"" + (}) [45,46] "}" +=== +@known-diff dollar-backtick-substitution: $(eval echo $`echo ${foo}`) +$(eval echo $`echo ${foo}`) +--- +hasError: false +program [0,27] "$(eval echo $`echo ${foo}`)" + command [0,27] "$(eval echo $`echo ${foo}`)" + command_name [0,27] "$(eval echo $`echo ${foo}`)" + command_substitution [0,27] "$(eval echo $`echo ${foo}`)" + ($() [0,2] "$(" + command [2,26] "eval echo $`echo ${foo}`" + command_name [2,6] "eval" + word [2,6] "eval" + word [7,11] "echo" + concatenation [12,26] "$`echo ${foo}`" + ($) [12,13] "$" + command_substitution [13,26] "`echo ${foo}`" + (`) [13,14] "`" + command [14,25] "echo ${foo}" + command_name [14,18] "echo" + word [14,18] "echo" + expansion [19,25] "${foo}" + (${) [19,21] "${" + variable_name [21,24] "foo" + (}) [24,25] "}" + (`) [25,26] "`" + ()) [26,27] ")" +=== +@match: echo `echo otherword`word +echo `echo otherword`word +=== +@match: echo word`echo otherword` +echo word`echo otherword` +=== +@match: assert() +assert() + if ! $1; then + return 1 + fi +=== +@match: echo " ${x}" +echo " ${x}" +=== +@match: echo "a ${x} b" +echo "a ${x} b" +=== +@match: echo "$x " +echo "$x " +=== +@match: echo "${x} " +echo "${x} " +=== +@match: echo ${#@} +echo ${#@} +=== +@match: echo ${#*} +echo ${#*} +=== +@match: echo ${#-} +echo ${#-} +=== +@match: echo ${#x[@]} +echo ${#x[@]} +=== +@match: echo ${x/'abc'/y} +echo ${x/'abc'/y} +=== +@match: echo ${x/"abc"/y} +echo ${x/"abc"/y} +=== +@match: echo ${x/'a'*'b'/y} +echo ${x/'a'*'b'/y} +=== +@match: echo ${x##'a'} +echo ${x##'a'} +=== +@match: echo ${x%%a b} +echo ${x%%a b} +=== +@match: echo "${x} $(y)" +echo "${x} $(y)" +=== +@match: echo " $(y)" +echo " $(y)" +=== +@match: echo " $x" +echo " $x" +=== +@match: echo "${x} $y" +echo "${x} $y" +=== diff --git a/packages/tree-sitter-bash/test/fixtures/differential/heredoc.txt b/packages/tree-sitter-bash/test/fixtures/differential/heredoc.txt new file mode 100644 index 0000000000..1ac604b146 --- /dev/null +++ b/packages/tree-sitter-bash/test/fixtures/differential/heredoc.txt @@ -0,0 +1,597 @@ +@match: echo $((x << 2 | 1)) +echo $((x << 2 | 1)) +=== +@known-diff heredoc-content-chunks: foo() { cat < out +cat < out +body +EOF +--- +hasError: false +program [0,24] "cat < out\nbody\nEOF" + redirected_statement [0,24] "cat < out\nbody\nEOF" + command [0,3] "cat" + command_name [0,3] "cat" + word [0,3] "cat" + heredoc_redirect [4,24] "< out\nbody\nEOF" + (<<) [4,6] "<<" + heredoc_start [6,9] "EOF" + file_redirect [10,15] "> out" + (>) [10,11] ">" + word [12,15] "out" + heredoc_body [16,21] "body\n" + heredoc_content [16,21] "body\n" + heredoc_end [21,24] "EOF" +=== +@known-diff heredoc-tail-statements: cat < out +cmd <<< x > out +=== +@known-diff heredoc-multiple-on-line: cat <> 10 & 11 | 12 ^ 13)) +a=$((6 % 7 ** 8 << 9 >> 10 & 11 | 12 ^ 13)) +=== +@known-diff heredoc-content-chunks: cat < out +foo() { x; } > out +=== +@match: if a; then b; fi > out +if a; then b; fi > out +=== +@match: cat > $"out" +cat > $"out" +=== +@match: diff <(cmd) >(other) +diff <(cmd) >(other) +=== +@match: cmd > out 2>&1 +cmd > out 2>&1 +=== +@match: cmd &>> log +cmd &>> log +=== +@known-diff redirect-readwrite: exec 3<>file +exec 3<>file +--- +hasError: false +program [0,12] "exec 3<>file" + redirected_statement [0,12] "exec 3<>file" + command [0,4] "exec" + command_name [0,4] "exec" + word [0,4] "exec" + file_redirect [5,12] "3<>file" + file_descriptor [5,6] "3" + (<>) [6,8] "<>" + word [8,12] "file" +=== +@match: > out +> out +=== +@match: cmd 2>/dev/null +cmd 2>/dev/null +=== +@match: cmd >&- +cmd >&- +=== +@match: cmd >| file +cmd >| file +=== +@known-diff redirect-readwrite: cmd <>file +cmd <>file +--- +hasError: false +program [0,10] "cmd <>file" + redirected_statement [0,10] "cmd <>file" + command [0,3] "cmd" + command_name [0,3] "cmd" + word [0,3] "cmd" + file_redirect [4,10] "<>file" + (<>) [4,6] "<>" + word [6,10] "file" +=== +@match: cmd < in +cmd < in +=== +@match: cmd <&0- +cmd <&0- +=== +@match: cmd > out +cmd > out +=== +@match: cmd 2>&1 >/dev/null +cmd 2>&1 >/dev/null +=== +@match: > a cmd x +> a cmd x +=== +@match: (ls) > out +(ls) > out +=== +@match: ! ls > out +! ls > out +=== +@match: cmd > out arg +cmd > out arg +=== +@match: 2>&1 whoami +2>&1 whoami +=== +@match: echo "foobar" >&2 +echo "foobar" >&2 +=== +@match: exec {VIRTWL[0]} {VIRTWL[1]} <&- >&- +exec {VIRTWL[0]} {VIRTWL[1]} <&- >&- +=== +@match: exec {VIRTWL[0]}<&- {VIRTWL[1]}>&- +exec {VIRTWL[0]}<&- {VIRTWL[1]}>&- +=== +@match: x &- +exec {a[0]}<&- {b[1]}>&- +=== +@match: exec {a[0]}<&- +exec {a[0]}<&- +=== diff --git a/packages/tree-sitter-bash/test/fixtures/differential/statements.txt b/packages/tree-sitter-bash/test/fixtures/differential/statements.txt new file mode 100644 index 0000000000..24c53c5ea2 --- /dev/null +++ b/packages/tree-sitter-bash/test/fixtures/differential/statements.txt @@ -0,0 +1,391 @@ +@match: if true; then echo yes; fi +if true; then echo yes; fi +=== +@match: if a; then b; elif c; then d; else e; fi +if a; then b; elif c; then d; else e; fi +=== +@match: if a +if a +then + b +fi +=== +@match: until x; do y; done +until x; do y; done +=== +@match: foo() { echo hi; } +foo() { echo hi; } +=== +@match: function bar { echo hi; } +function bar { echo hi; } +=== +@match: function baz() { echo hi; return 0; } +function baz() { echo hi; return 0; } +=== +@match: foo() ( echo sub ) +foo() ( echo sub ) +=== +@match: { echo a; echo b; } +{ echo a; echo b; } +=== +@match: coproc myjob { echo hi; } +coproc myjob { echo hi; } +=== +@match: echo if for while +echo if for while +=== +@match: arr=(a b "c d") +arr=(a b "c d") +=== +@match: arr[0]=x +arr[0]=x +=== +@match: export FOO=bar BAZ +export FOO=bar BAZ +=== +@match: declare -r x=1 +declare -r x=1 +=== +@match: local y +local y +=== +@match: readonly z=2 +readonly z=2 +=== +@match: unset a b +unset a b +=== +@match: echo {1..10} +echo {1..10} +=== +@match: echo {1..10..2} +echo {1..10..2} +=== +@match: echo {a..z} +echo {a..z} +=== +@match: echo a{b,c}d +echo a{b,c}d +=== +@match: time ls -la +time ls -la +=== +@match: ls +ls +echo b +=== +@match: echo [ x ] +echo [ x ] +=== +@match: if [ -f f ]; then x; fi +if [ -f f ]; then x; fi +=== +@match: foo() if a; then b; fi +foo() if a; then b; fi +=== +@match: if a; then b; fi | grep x +if a; then b; fi | grep x +=== +@match: if ! grep -q x f; then echo missing; fi +if ! grep -q x f; then echo missing; fi +=== +@match: x[1]+=2 +x[1]+=2 +=== +@match: until grep -q done log; do sleep 1; done & +until grep -q done log; do sleep 1; done & +=== +@match: ( (ls) ) +( (ls) ) +=== +@match: {ls;} +{ls;} +=== +@match: fi +fi +=== +@match: done +done +=== +@match: } +} +=== +@match: echo a{1..3}b +echo a{1..3}b +=== +@match: x={1..5} +x={1..5} +=== +@match: while x; do done +while x; do done +=== +@match: echo {a..z..2} +echo {a..z..2} +=== +@match: echo {10..1} +echo {10..1} +=== +@match: echo {-5..5} +echo {-5..5} +=== +@match: select x in; do y; done +select x in; do y; done +=== +@match: function f() if x; then y; fi +function f() if x; then y; fi +=== +@match: f () { x; } +f () { x; } +=== +@match: if a && b; then c; fi +if a && b; then c; fi +=== +@match: foo (ls) +foo (ls) +=== +@match: for f in a b +for f in a b +do x +done +=== +@match: while a; b; do x; done +while a; b; do x; done +=== +@match: if a; then b; fi && c +if a; then b; fi && c +=== +@match: if if a; then b; fi; then c; fi +if if a; then b; fi; then c; fi +=== +@match: x=1 if a; then b; fi +x=1 if a; then b; fi +=== +@match: echo hi | if a; then b; fi +echo hi | if a; then b; fi +=== +@match: ! while x; do y; done +! while x; do y; done +=== +@match: ! [ -f x ] +! [ -f x ] +=== +@match: x+=(c d) +x+=(c d) +=== +@match: declare -A m +declare -A m +=== +@match: {1..3} +{1..3} +=== +@match: echo {1..3} +echo {1..3} +=== +@match: if echo then; then x; fi +if echo then; then x; fi +=== +@match: while echo do; do x; done +while echo do; do x; done +=== +@match: for f in a; do echo done; done +for f in a; do echo done; done +=== +@match: { echo }; } +{ echo }; } +=== +@match: [1] +[1] +=== +@match: echo [1] a[2]b +echo [1] a[2]b +=== +@match: echo a[-5]b +echo a[-5]b +=== +@match: if a; then b; elif c; then d +if a; then b; elif c; then d +else e +fi +=== +@match: ls -la +ls -la +=== +@match: FOO=bar env | grep FOO && echo ok || echo fail +FOO=bar env | grep FOO && echo ok || echo fail +=== +@match: ! grep -q x file +! grep -q x file +=== +@match: # a comment +# a comment +echo hi # trailing +=== +@match: (ls) +(ls) +=== +@match: FOO=bar ls +FOO=bar ls +=== +@known-diff trailing-connector: ls && +ls && +--- +hasError: true +program [0,5] "ls &&" + list [0,5] "ls &&" + command [0,2] "ls" + command_name [0,2] "ls" + word [0,2] "ls" + (&&) [3,5] "&&" +=== +@match: a & b & +a & b & +=== +@match: echo 123 45.6 +echo 123 45.6 +=== +@match: VAR=value +VAR=value +=== +@match: ls; echo a +ls; echo a +echo b +=== +@match: FOO=1 BAR=2 cmd arg +FOO=1 BAR=2 cmd arg +=== +@match: ! a | b +! a | b +=== +@match: echo "" +echo "" +=== +@match: VAR= +VAR= +=== +@match: a|b|c +a|b|c +=== +@match: ls ; +ls ; +=== +@match: echo hi;# c +echo hi;# c +ls +=== +@match: A=1 B=2 +A=1 B=2 +=== +@match: a && b && c +a && b && c +=== +@match: echo x\ +echo x\ +y +=== +@match: echo {a,b} +echo {a,b} +=== +@known-diff trailing-connector: ls | +ls | +--- +hasError: true +program [0,4] "ls |" + pipeline [0,4] "ls |" + command [0,2] "ls" + command_name [0,2] "ls" + word [0,2] "ls" + (|) [3,4] "|" +=== +@match: time ls +time ls +=== +@match: FOO="a b" cmd +FOO="a b" cmd +=== +@match: FOO=a"b"c +FOO=a"b"c +=== +@known-diff escaped-space-argument: echo 1 \ 2 \ 3 +echo 1 \ 2 \ 3 +--- +hasError: false +program [0,14] "echo 1 \\ 2 \\ 3" + command [0,14] "echo 1 \\ 2 \\ 3" + command_name [0,4] "echo" + word [0,4] "echo" + number [5,6] "1" + word [7,10] "\\ 2" + word [11,14] "\\ 3" +=== +@known-diff expansion-equals-operator: for c in ${=1}; do +for c in ${=1}; do + echo c +done +--- +hasError: false +program [0,31] "for c in ${=1}; do\n\techo c\ndone" + for_statement [0,31] "for c in ${=1}; do\n\techo c\ndone" + (for) [0,3] "for" + variable_name [4,5] "c" + (in) [6,8] "in" + expansion [9,14] "${=1}" + (${) [9,11] "${" + (=) [11,12] "=" + word [12,13] "1" + (}) [13,14] "}" + (;) [14,15] ";" + do_group [16,31] "do\n\techo c\ndone" + (do) [16,18] "do" + command [20,26] "echo c" + command_name [20,24] "echo" + word [20,24] "echo" + word [25,26] "c" + (done) [27,31] "done" +=== +@match: foo::bar() { +foo::bar() { + echo what +} +=== +@match: echo "a " +echo "a " +=== +@match: echo 你好 && ls +echo 你好 && ls +=== +@match: echo "🎉 done" | grep 🎉 && echo ✓ +echo "🎉 done" | grep 🎉 && echo ✓ +=== +@known-diff nonascii-identifier-assignment: 变量=值 echo $变量 +变量=值 echo $变量 +--- +hasError: false +program [0,13] "变量=值 echo $变量" + command [0,13] "变量=值 echo $变量" + command_name [0,4] "变量=值" + word [0,4] "变量=值" + word [5,9] "echo" + concatenation [10,13] "$变量" + ($) [10,11] "$" + word [11,13] "变量" +=== +@known-diff escaped-space-argument: echo a\b +echo a\ b +--- +hasError: false +program [0,9] "echo a\\\tb" + command [0,9] "echo a\\\tb" + command_name [0,4] "echo" + word [0,4] "echo" + word [5,9] "a\\\tb" +=== +@known-diff escaped-space-argument: echo \x +echo \ x +--- +hasError: false +program [0,8] "echo \\\tx" + command [0,8] "echo \\\tx" + command_name [0,4] "echo" + word [0,4] "echo" + word [5,8] "\\\tx" +=== diff --git a/packages/tree-sitter-bash/test/fixtures/differential/test-command.txt b/packages/tree-sitter-bash/test/fixtures/differential/test-command.txt new file mode 100644 index 0000000000..02b22149ef --- /dev/null +++ b/packages/tree-sitter-bash/test/fixtures/differential/test-command.txt @@ -0,0 +1,650 @@ +@match: [[ -f file.txt ]] +[[ -f file.txt ]] +=== +@match: [[ $x == "foo" ]] +[[ $x == "foo" ]] +=== +@match: [[ $x =~ ^ab+c$ ]] +[[ $x =~ ^ab+c$ ]] +=== +@match: [[ -n $s && -d /tmp ]] +[[ -n $s && -d /tmp ]] +=== +@match: [[ a < b || ! -e f ]] +[[ a < b || ! -e f ]] +=== +@match: [ -f file ] +[ -f file ] +=== +@match: [ "$a" = "b" ] +[ "$a" = "b" ] +=== +@match: ! [[ -f x ]] +! [[ -f x ]] +=== +@match: [ -f file +[ -f file +=== +@match: [ x ] +[ x ] +=== +@match: while [[ -n $s ]]; do s=; done +while [[ -n $s ]]; do s=; done +=== +@match: foo() [[ -f x ]] +foo() [[ -f x ]] +=== +@match: [[ -z $s ]] +[[ -z $s ]] +=== +@match: [[ $a -eq 3 ]] +[[ $a -eq 3 ]] +=== +@match: [[ $a -nt $b ]] +[[ $a -nt $b ]] +=== +@match: [[ "x" ]] +[[ "x" ]] +=== +@match: [[ ]] +[[ ]] +=== +@match: [[ a = b*c ]] +[[ a = b*c ]] +=== +@match: [[ a = bc ]] +[[ a = bc ]] +=== +@match: [[ $x == b*c ]] +[[ $x == b*c ]] +=== +@match: [[ $x == a"b" ]] +[[ $x == a"b" ]] +=== +@match: [[ $x =~ ^a.*b$ ]] +[[ $x =~ ^a.*b$ ]] +=== +@match: [[ a != b ]] +[[ a != b ]] +=== +@match: [[ (a == b) && c ]] +[[ (a == b) && c ]] +=== +@match: [[ -v var ]] +[[ -v var ]] +=== +@match: [[ -o opt ]] +[[ -o opt ]] +=== +@match: [[ $x -eq 1 && $y -ne 2 ]] +[[ $x -eq 1 && $y -ne 2 ]] +=== +@match: [[ $x == b ]] +[[ $x == b ]] +=== +@known-diff recovery-partial-nodes: [[ $x =~ ^a b$ ]] +[[ $x =~ ^a b$ ]] +--- +hasError: true +program [0,17] "[[ $x =~ ^a b$ ]]" + test_command [0,17] "[[ $x =~ ^a b$ ]]" + ([[) [0,2] "[[" + binary_expression [3,11] "$x =~ ^a" + simple_expansion [3,5] "$x" + ($) [3,4] "$" + variable_name [4,5] "x" + (=~) [6,8] "=~" + regex [9,11] "^a" + ERROR [12,15] "b$ " + (]]) [15,17] "]]" +=== +@match: [[ $x =~ a|b ]] +[[ $x =~ a|b ]] +=== +@match: [[ $x == b && $y == c ]] +[[ $x == b && $y == c ]] +=== +@match: [[ $x == 123 ]] +[[ $x == 123 ]] +=== +@match: [[ -t 1 ]] +[[ -t 1 ]] +=== +@match: [[ $x =~ $re ]] +[[ $x =~ $re ]] +=== +@match: [[ $x =~ "a b" ]] +[[ $x =~ "a b" ]] +=== +@match: [[ a=b ]] +[[ a=b ]] +=== +@match: [[ $a==b* ]] +[[ $a==b* ]] +=== +@match: [[ ( $a == x* ) ]] +[[ ( $a == x* ) ]] +=== +@match: [[ ( $a = x* ) ]] +[[ ( $a = x* ) ]] +=== +@match: [[ $x == b=c ]] +[[ $x == b=c ]] +=== +@match: [[ a == =b ]] +[[ a == =b ]] +=== +@match: [[ a = b=c ]] +[[ a = b=c ]] +=== +@match: [[ -f file +[[ -f file +=== +@known-diff recovery-partial-nodes: [[ $a +[[ $a +--- +hasError: true +program [0,5] "[[ $a" + test_command [0,5] "[[ $a" + ([[) [0,2] "[[" + simple_expansion [3,5] "$a" + ($) [3,4] "$" + variable_name [4,5] "a" + (]]) [5,5] "" +=== +@match: [ 1 ] +[ 1 ] +=== +@match: [[ 1 ]] +[[ 1 ]] +=== +@match: [[ $f == foo*(.txt|.log) ]] +[[ $f == foo*(.txt|.log) ]] +=== +@match: [[ $f == *foo*(a) ]] +[[ $f == *foo*(a) ]] +=== +@match: [[ $f == [a-z]*(x) ]] +[[ $f == [a-z]*(x) ]] +=== +@known-diff test-extglob-rejected-group: [[ $f == x@(y|z)w ]] +[[ $f == x@(y|z)w ]] +--- +hasError: true +program [0,20] "[[ $f == x@(y|z)w ]]" + test_command [0,20] "[[ $f == x@(y|z)w ]]" + ([[) [0,2] "[[" + binary_expression [3,11] "$f == x@" + simple_expansion [3,5] "$f" + ($) [3,4] "$" + variable_name [4,5] "f" + (==) [6,8] "==" + word [9,11] "x@" + ERROR [11,18] "(y|z)w " + (]]) [18,20] "]]" +=== +@known-diff test-negative-decimal: [[ $n == -0.5 ]] +[[ $n == -0.5 ]] +--- +hasError: false +program [0,16] "[[ $n == -0.5 ]]" + test_command [0,16] "[[ $n == -0.5 ]]" + ([[) [0,2] "[[" + binary_expression [3,13] "$n == -0.5" + simple_expansion [3,5] "$n" + ($) [3,4] "$" + variable_name [4,5] "n" + (==) [6,8] "==" + extglob_pattern [9,13] "-0.5" + (]]) [14,16] "]]" +=== +@known-diff test-fused-operator-expansion: [[ -x$f && -d /tmp ]] +[[ -x$f && -d /tmp ]] +--- +hasError: false +program [0,21] "[[ -x$f && -d /tmp ]]" + test_command [0,21] "[[ -x$f && -d /tmp ]]" + ([[) [0,2] "[[" + binary_expression [3,18] "-x$f && -d /tmp" + concatenation [3,7] "-x$f" + word [3,5] "-x" + simple_expansion [5,7] "$f" + ($) [5,6] "$" + variable_name [6,7] "f" + (&&) [8,10] "&&" + unary_expression [11,18] "-d /tmp" + test_operator [11,13] "-d" + word [14,18] "/tmp" + (]]) [19,21] "]]" +=== +@known-diff test-escaped-pipe-pattern: [[ $a == foo\|bar ]] +[[ $a == foo\|bar ]] +--- +hasError: false +program [0,20] "[[ $a == foo\\|bar ]]" + test_command [0,20] "[[ $a == foo\\|bar ]]" + ([[) [0,2] "[[" + binary_expression [3,17] "$a == foo\\|bar" + simple_expansion [3,5] "$a" + ($) [3,4] "$" + variable_name [4,5] "a" + (==) [6,8] "==" + extglob_pattern [9,17] "foo\\|bar" + (]]) [18,20] "]]" +=== +@known-diff recovery-partial-nodes: [[ -f +[[ -f +--- +hasError: true +program [0,5] "[[ -f" + test_command [0,5] "[[ -f" + ([[) [0,2] "[[" + word [3,5] "-f" + (]]) [5,5] "" +=== +@known-diff recovery-partial-nodes: [[ a +[[ a +--- +hasError: true +program [0,4] "[[ a" + test_command [0,4] "[[ a" + ([[) [0,2] "[[" + word [3,4] "a" + (]]) [4,4] "" +=== +@known-diff recovery-partial-nodes: [[ a == b +[[ a == b +--- +hasError: true +program [0,9] "[[ a == b" + test_command [0,9] "[[ a == b" + ([[) [0,2] "[[" + binary_expression [3,9] "a == b" + word [3,4] "a" + (==) [5,7] "==" + word [8,9] "b" + (]]) [9,9] "" +=== +@known-diff recovery-partial-nodes: [[ a && +[[ a && +--- +hasError: true +program [0,7] "[[ a &&" + test_command [0,7] "[[ a &&" + ([[) [0,2] "[[" + binary_expression [3,7] "a &&" + word [3,4] "a" + (&&) [5,7] "&&" + (]]) [7,7] "" +=== +@match: [[ -f file +[[ -f file +=== +@match: [[ a = =b ]] +[[ a = =b ]] +=== +@match: [[ $a =~^x ]] +[[ $a =~^x ]] +=== +@match: [[ !x = y ]] +[[ !x = y ]] +=== +@match: [[ a =b ]] +[[ a =b ]] +=== +@match: [[ a!=b ]] +[[ a!=b ]] +=== +@match: [[ a ==b ]] +[[ a ==b ]] +=== +@match: [[ $a == foo ]] +[[ $a == foo ]] +=== +@match: [[ $a == x.y ]] +[[ $a == x.y ]] +=== +@match: [[ ( $a == x ) ]] +[[ ( $a == x ) ]] +=== +@match: [[ $a == x1 ]] +[[ $a == x1 ]] +=== +@match: [[ $a == foo- ]] +[[ $a == foo- ]] +=== +@match: [[ $a == x_ ]] +[[ $a == x_ ]] +=== +@match: [[ a == b*c=d ]] +[[ a == b*c=d ]] +=== +@match: [[ a == bc=d ]] +[[ a == bc=d ]] +=== +@match: [[ a == b?=e ]] +[[ a == b?=e ]] +=== +@match: [[ a == .x ]] +[[ a == .x ]] +=== +@match: [[ $a == a-b ]] +[[ $a == a-b ]] +=== +@match: [[ ((a)) == x ]] +[[ ((a)) == x ]] +=== +@match: [[ $a == -foo ]] +[[ $a == -foo ]] +=== +@match: [[ $a == -f ]] +[[ $a == -f ]] +=== +@match: [[ -foo == x ]] +[[ -foo == x ]] +=== +@match: [[ $a == -1 ]] +[[ $a == -1 ]] +=== +@match: [[ $a -eq -1 ]] +[[ $a -eq -1 ]] +=== +@match: [[ $a == +(!a) ]] +[[ $a == +(!a) ]] +=== +@match: [[ $a == ?(a|b) ]] +[[ $a == ?(a|b) ]] +=== +@match: [[ $a == !(x) ]] +[[ $a == !(x) ]] +=== +@match: [[ $a == b\*c ]] +[[ $a == b\*c ]] +=== +@match: [[ $ver == [0-9]* ]] +[[ $ver == [0-9]* ]] +=== +@match: [[ $a == [abc] ]] +[[ $a == [abc] ]] +=== +@match: [[ ( $a == [xyz] ) ]] +[[ ( $a == [xyz] ) ]] +=== +@match: [[ $a = [abc] ]] +[[ $a = [abc] ]] +=== +@match: [[ $a == [!a]* ]] +[[ $a == [!a]* ]] +=== +@match: [[ $a == foo\ bar ]] +[[ $a == foo\ bar ]] +=== +@known-diff test-paren-group-logical: [[ ((a) == x) && y ]] +[[ ((a) == x) && y ]] +--- +hasError: false +program [0,21] "[[ ((a) == x) && y ]]" + test_command [0,21] "[[ ((a) == x) && y ]]" + ([[) [0,2] "[[" + binary_expression [3,18] "((a) == x) && y" + parenthesized_expression [3,13] "((a) == x)" + (() [3,4] "(" + binary_expression [4,12] "(a) == x" + parenthesized_expression [4,7] "(a)" + (() [4,5] "(" + word [5,6] "a" + ()) [6,7] ")" + (==) [8,10] "==" + word [11,12] "x" + ()) [12,13] ")" + (&&) [14,16] "&&" + word [17,18] "y" + (]]) [19,21] "]]" +=== +@match: [ ! command -v go &>/dev/null ] && return +[ ! command -v go &>/dev/null ] && return +=== +@match: [[ ${f} != */@(default).vim ]] +[[ ${f} != */@(default).vim ]] +=== +@match: [[ $(LC_ALL=C $(tc-getCC) ${LDFLAGS} -Wl,--version 2>/dev... +[[ $(LC_ALL=C $(tc-getCC) ${LDFLAGS} -Wl,--version 2>/dev/null) != @(LLD|GNU\ ld)* ]] +=== +@match: [[ ${test} == @($(IFS='|'; echo "${skip[*]}")) ]] +[[ ${test} == @($(IFS='|'; echo "${skip[*]}")) ]] +=== +@match: [[ ${SRC_URI} == */${a}* ]] +[[ ${SRC_URI} == */${a}* ]] +=== +@match: [[ a == *_@(LIB|SYMLINK) ]] +[[ a == *_@(LIB|SYMLINK) ]] +=== +@match: [[ "35d8b" =~ ^[0-9a-fA-F] ]] +[[ "35d8b" =~ ^[0-9a-fA-F] ]] +=== +@match: [[ $CMD =~ (^|;)update_terminal_cwd($|;) ]] +[[ $CMD =~ (^|;)update_terminal_cwd($|;) ]] +=== +@match: [[ ! " ${completions[*]} " =~ " $alias_cmd " ]] +[[ ! " ${completions[*]} " =~ " $alias_cmd " ]] +=== +@match: ! [[ "$a" =~ ^a|b\ *c|d$ ]] +! [[ "$a" =~ ^a|b\ *c|d$ ]] +=== +@match: [[ "$1" =~ ^${var}${var}*=..* ]] +[[ "$1" =~ ^${var}${var}*=..* ]] +=== +@match: [[ "$1" =~ ^\-${var}+ ]] +[[ "$1" =~ ^\-${var}+ ]] +=== +@match: [[ ${var1} == *${var2}* ]] +[[ ${var1} == *${var2}* ]] +=== +@match: [[ "$server" =~ [0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]... +[[ "$server" =~ [0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3} ]] +=== +@match: [[ -f ${x} && $(od -t x1 -N 4 "${x}") == *"7f 45 4c 46"* ]] +[[ -f ${x} && $(od -t x1 -N 4 "${x}") == *"7f 45 4c 46"* ]] +=== +@match: [[ " ${REPLACING_VERSIONS} " != *\ ${PVR}\ * ]] +[[ " ${REPLACING_VERSIONS} " != *\ ${PVR}\ * ]] +=== +@match: [[ ${file} == @(*${GENTOO_PATCH_NAME}.tar.xz|*.asc|*.sig) ]] +[[ ${file} == @(*${GENTOO_PATCH_NAME}.tar.xz|*.asc|*.sig) ]] +=== +@match: [[ $RUBY_TARGETS != *$( eselect ruby show | awk 'NR==2' |... +[[ $RUBY_TARGETS != *$( eselect ruby show | awk 'NR==2' | tr -d ' ' )* ]] +=== +@match: [[ " ${m[0]##*/}" =~ ^(\ ${skip_files[*]/%/.*|\\} ) ]] +[[ " ${m[0]##*/}" =~ ^(\ ${skip_files[*]/%/.*|\\} ) ]] +=== +@match: [[ ' boop xyz' =~ ' boop '(.*)$ ]] +[[ ' boop xyz' =~ ' boop '(.*)$ ]] +=== +@match: [[ $b = foo* ]] +[[ $b = foo* ]] +=== +@match: [[ a == \"+(?)\" ]] +[[ a == \"+(?)\" ]] +=== +@match: [[ ${PV} != $(sed -n -e 's/^Version: //p' "${ED}/usr/$(ge... +[[ ${PV} != $(sed -n -e 's/^Version: //p' "${ED}/usr/$(get_libdir)/pkgconfig/tss2-tcti-tabrmd.pc" || die) ]] +=== +@match: [[ "${MY_LOCALES}" != *en_US* || a != 2 ]] +[[ "${MY_LOCALES}" != *en_US* || a != 2 ]] +=== +@match: [[ -f "${EROOT}/usr/share/php/.packagexml/${MY_P}.xml" && \ +[[ -f "${EROOT}/usr/share/php/.packagexml/${MY_P}.xml" && \ + -x "${EROOT}/usr/bin/peardev" ]] +=== +@match: [[ ${1} =~ \.(lisp|lsp|cl)$ ]] +[[ ${1} =~ \.(lisp|lsp|cl)$ ]] +=== +@match: [[ a == - ]] +[[ a == - ]] +=== +@match: run_test_command() [[ -e foo ]] +run_test_command() [[ -e foo ]] +=== +@known-diff test-extglob-rejected-group: [[ $a == *.@(txt|log) ]] +[[ $a == *.@(txt|log) ]] +--- +hasError: true +program [0,24] "[[ $a == *.@(txt|log) ]]" + test_command [0,24] "[[ $a == *.@(txt|log) ]]" + ([[) [0,2] "[[" + binary_expression [3,12] "$a == *.@" + simple_expansion [3,5] "$a" + ($) [3,4] "$" + variable_name [4,5] "a" + (==) [6,8] "==" + word [9,12] "*.@" + ERROR [12,22] "(txt|log) " + (]]) [22,24] "]]" +=== +@known-diff test-extglob-rejected-group: [[ $a == x@(y) ]] +[[ $a == x@(y) ]] +--- +hasError: true +program [0,17] "[[ $a == x@(y) ]]" + test_command [0,17] "[[ $a == x@(y) ]]" + ([[) [0,2] "[[" + binary_expression [3,11] "$a == x@" + simple_expansion [3,5] "$a" + ($) [3,4] "$" + variable_name [4,5] "a" + (==) [6,8] "==" + word [9,11] "x@" + ERROR [11,15] "(y) " + (]]) [15,17] "]]" +=== +@match: [[ $a == ?(a) ]] +[[ $a == ?(a) ]] +=== +@known-diff test-pattern-two-constructs: [[ $a == *${x}*${y} ]] +[[ $a == *${x}*${y} ]] +--- +hasError: false +program [0,22] "[[ $a == *${x}*${y} ]]" + test_command [0,22] "[[ $a == *${x}*${y} ]]" + ([[) [0,2] "[[" + binary_expression [3,19] "$a == *${x}*${y}" + simple_expansion [3,5] "$a" + ($) [3,4] "$" + variable_name [4,5] "a" + (==) [6,8] "==" + extglob_pattern [9,10] "*" + expansion [10,14] "${x}" + (${) [10,12] "${" + variable_name [12,13] "x" + (}) [13,14] "}" + extglob_pattern [14,19] "*${y}" + (]]) [20,22] "]]" +=== +@match: [[ $a == ${x}* ]] +[[ $a == ${x}* ]] +=== +@known-diff test-pattern-two-constructs: [[ $a == *"s"*"t" ]] +[[ $a == *"s"*"t" ]] +--- +hasError: false +program [0,20] "[[ $a == *\"s\"*\"t\" ]]" + test_command [0,20] "[[ $a == *\"s\"*\"t\" ]]" + ([[) [0,2] "[[" + binary_expression [3,17] "$a == *\"s\"*\"t\"" + simple_expansion [3,5] "$a" + ($) [3,4] "$" + variable_name [4,5] "a" + (==) [6,8] "==" + extglob_pattern [9,10] "*" + string [10,13] "\"s\"" + (") [10,11] "\"" + string_content [11,12] "s" + (") [12,13] "\"" + extglob_pattern [13,17] "*\"t\"" + (]]) [18,20] "]]" +=== +@match: [ -f x ] +[ -f x ] +=== +@match: [ ! -f x ] +[ ! -f x ] +=== +@match: [ a = b ] 2>/dev/null +[ a = b ] 2>/dev/null +=== +@match: [[ $a == @(y) ]] +[[ $a == @(y) ]] +=== +@known-diff test-extglob-rejected-group: [[ $a == *@(y) ]] +[[ $a == *@(y) ]] +--- +hasError: true +program [0,17] "[[ $a == *@(y) ]]" + test_command [0,17] "[[ $a == *@(y) ]]" + ([[) [0,2] "[[" + binary_expression [3,11] "$a == *@" + simple_expansion [3,5] "$a" + ($) [3,4] "$" + variable_name [4,5] "a" + (==) [6,8] "==" + word [9,11] "*@" + ERROR [11,15] "(y) " + (]]) [15,17] "]]" +=== +@match: [[ $a == */@(y) ]] +[[ $a == */@(y) ]] +=== +@known-diff test-extglob-rejected-group: [[ $a == .@(y) ]] +[[ $a == .@(y) ]] +--- +hasError: true +program [0,17] "[[ $a == .@(y) ]]" + test_command [0,17] "[[ $a == .@(y) ]]" + ([[) [0,2] "[[" + binary_expression [3,11] "$a == .@" + simple_expansion [3,5] "$a" + ($) [3,4] "$" + variable_name [4,5] "a" + (==) [6,8] "==" + word [9,11] ".@" + ERROR [11,15] "(y) " + (]]) [15,17] "]]" +=== +@known-diff test-extglob-rejected-group: [[ $a == _@(y) ]] +[[ $a == _@(y) ]] +--- +hasError: true +program [0,17] "[[ $a == _@(y) ]]" + test_command [0,17] "[[ $a == _@(y) ]]" + ([[) [0,2] "[[" + binary_expression [3,11] "$a == _@" + simple_expansion [3,5] "$a" + ($) [3,4] "$" + variable_name [4,5] "a" + (==) [6,8] "==" + word [9,11] "_@" + ERROR [11,15] "(y) " + (]]) [15,17] "]]" +=== +@match: [[ $a == ?*(y) ]] +[[ $a == ?*(y) ]] +=== +@match: [[ $a == a*(y) ]] +[[ $a == a*(y) ]] +=== +@match: [[ $a == *?(y) ]] +[[ $a == *?(y) ]] +=== +@match: [[ $a =~ 'a'b ]] +[[ $a =~ 'a'b ]] +=== +@match: [[ $a =~ a'b'c ]] +[[ $a =~ a'b'c ]] +=== +@match: [ ! command -v go ] +[ ! command -v go ] +=== +@match: [ command -v go &>/dev/null ] +[ command -v go &>/dev/null ] +=== diff --git a/packages/tree-sitter-bash/test/fuzz.test.ts b/packages/tree-sitter-bash/test/fuzz.test.ts new file mode 100644 index 0000000000..cecbbbcc12 --- /dev/null +++ b/packages/tree-sitter-bash/test/fuzz.test.ts @@ -0,0 +1,165 @@ +// test/fuzz.test.ts +// +// Deterministic fuzzing (fixed-seed PRNG — no per-run randomness; scale the +// volume with TS_BASH_FUZZ_SCALE=N for deeper local runs): +// a) token soup — random sequences of bash operators/keywords/fragments; +// b) byte mutations of the curated fixtures (replace/delete/insert, +// NUL included); +// c) nesting bombs — programmatic deep $(…) / (…) / ${…} / if nesting. +// +// Contract under test: parse never throws; within budget it returns either +// ok:true or { ok:false, reason:'aborted' }; an ok:true tree is always +// structurally sound (assertTreeIntegrity); nesting beyond the documented +// depth caps degrades locally (hasError) instead of throwing. + +import { readFileSync, readdirSync } from 'node:fs'; +import path from 'node:path'; + +import { describe, expect, it } from 'vitest'; + +import { parse } from '#/parse'; +import type { ParseOptions, ParseResult } from '#/parse'; +import { assertTreeIntegrity, parseFixtureFile } from './helpers/differential'; + +const PKG_ROOT = path.resolve(import.meta.dirname, '..'); +const SCALE = Math.max(1, Number(process.env['TS_BASH_FUZZ_SCALE'] ?? 1) || 1); + +/** Park–Miller minimal standard PRNG: small, portable, deterministic. */ +function prng(seed: number): () => number { + let state = seed % 2147483647; + if (state <= 0) state += 2147483646; + return () => { + state = (state * 16807) % 2147483647; + return (state - 1) / 2147483646; + }; +} + +function pick(rand: () => number, pool: readonly string[]): string { + return pool[Math.floor(rand() * pool.length)]!; +} + +function checkContract(source: string, options?: ParseOptions): ParseResult { + let result: ParseResult | undefined; + expect(() => { + result = parse(source, options); + }).not.toThrow(); + expect(result).toBeDefined(); + if (result!.ok) { + assertTreeIntegrity(result!.rootNode, source); + } else { + expect(result!).toEqual({ ok: false, reason: 'aborted' }); + } + return result!; +} + +const TOKEN_POOL = [ + 'echo', 'ls', 'foo', 'bar', 'x', 'A=1', 'if', 'then', 'fi', 'while', 'do', 'done', 'for', 'in', 'case', 'esac', + '&&', '||', '|', '|&', ';', '&', ';;', ';&', '(', ')', '{', '}', '>', '>>', '<', '>&1', '2>', '&>', '<(p)', '<(p)', '$#', '$?', '0x1F', '..', '<<-', '>&-', 'abc_def', +] as const; + +function fixtureSources(): string[] { + const dir = path.join(PKG_ROOT, 'test/fixtures/differential'); + const out: string[] = []; + for (const file of readdirSync(dir).filter((f) => f.endsWith('.txt')).toSorted()) { + for (const sample of parseFixtureFile(file, readFileSync(path.join(dir, file), 'utf8'))) { + out.push(sample.source); + } + } + return out; +} + +describe('fuzz: token soup', () => { + it('never throws and always yields a sound tree or a clean abort', () => { + const rand = prng(0x5eed0001); + const count = 250 * SCALE; + let parsed = 0; + for (let n = 0; n < count; n++) { + const length = 3 + Math.floor(rand() * 22); + const parts: string[] = []; + for (let k = 0; k < length; k++) parts.push(pick(rand, TOKEN_POOL)); + if (checkContract(parts.join(' ')).ok) parsed++; + } + expect(parsed).toBeGreaterThan(0); + console.log(`token soup: ${count} inputs, ${parsed} parsed / ${count - parsed} aborted`); + }); +}); + +describe('fuzz: byte mutations of fixtures', () => { + it('never throws and always yields a sound tree or a clean abort', () => { + const rand = prng(0x5eed0002); + const bases = fixtureSources(); + const count = 300 * SCALE; + let parsed = 0; + for (let n = 0; n < count; n++) { + const base = pick(rand, bases); + if (base.length === 0) continue; + const pos = Math.floor(rand() * base.length); + const mode = rand(); + let mutated: string; + if (mode < 0.4) { + // replace one code unit (may be NUL) + const code = Math.floor(rand() * 256); + mutated = base.slice(0, pos) + String.fromCodePoint(code) + base.slice(pos + 1); + } else if (mode < 0.7) { + mutated = base.slice(0, pos) + base.slice(pos + 1); + } else { + const code = Math.floor(rand() * 256); + mutated = base.slice(0, pos) + String.fromCodePoint(code) + base.slice(pos); + } + if (checkContract(mutated).ok) parsed++; + } + expect(parsed).toBeGreaterThan(0); + console.log(`byte mutations: ${count} inputs, ${parsed} parsed / ${count - parsed} aborted`); + }); +}); + +describe('fuzz: nesting bombs degrade locally per the documented depth caps', () => { + const substitution = (depth: number): string => `echo ${'$('.repeat(depth)}x${')'.repeat(depth)}`; + const subshell = (depth: number): string => `${'('.repeat(depth)}x${')'.repeat(depth)}`; + const expansion = (depth: number): string => `echo ${'${a:-'.repeat(depth)}z${'}'.repeat(depth)}`; + const ifs = (depth: number): string => `${'if x; then '.repeat(depth)}y${'; fi'.repeat(depth)}`; + + it('within the caps the trees are clean', () => { + // literalDepth ticks twice per ${…} level (parseLiteral + parseExpansion), + // so 200 levels stay under MAX_PARSE_DEPTH = 500. + for (const source of [substitution(100), subshell(400), expansion(200), ifs(400)]) { + const result = checkContract(source, { timeoutMs: 60_000 }); + expect(result.ok).toBe(true); + if (result.ok) expect(result.hasError).toBe(false); + } + }); + + it('beyond the caps the parse still succeeds and flags hasError (MAX_SUBSTITUTION_DEPTH = 150)', () => { + const result = checkContract(substitution(200)); + expect(result.ok).toBe(true); + if (result.ok) expect(result.hasError).toBe(true); + }); + + it('beyond the caps the parse still succeeds and flags hasError (MAX_PARSE_DEPTH = 500)', () => { + for (const source of [subshell(600), expansion(600), ifs(600)]) { + const result = checkContract(source, { timeoutMs: 60_000 }); + expect(result.ok).toBe(true); + if (result.ok) expect(result.hasError).toBe(true); + } + }); + + it('extreme nesting never overflows the stack', () => { + for (const source of [substitution(5000), subshell(5000), expansion(5000), ifs(5000)]) { + const result = checkContract(source, { timeoutMs: 60_000 }); + // Either a locally degraded tree (hasError) or a clean budget abort. + if (result.ok) expect(result.hasError).toBe(true); + else expect(result.reason).toBe('aborted'); + } + }); + + it('the node budget aborts huge flat programs, and a raised budget parses them', () => { + const source = 'echo a; '.repeat(20_000); + expect(parse(source)).toEqual({ ok: false, reason: 'aborted' }); + const result = parse(source, { timeoutMs: 60_000, maxNodes: 10_000_000 }); + expect(result.ok).toBe(true); + if (result.ok) assertTreeIntegrity(result.rootNode, source); + }); +}); diff --git a/packages/tree-sitter-bash/test/helpers/differential.ts b/packages/tree-sitter-bash/test/helpers/differential.ts new file mode 100644 index 0000000000..e66e5ebe82 --- /dev/null +++ b/packages/tree-sitter-bash/test/helpers/differential.ts @@ -0,0 +1,271 @@ +// test/helpers/differential.ts +// +// Differential test harness: parses the same source with both this package's +// parser and the real tree-sitter-bash (web-tree-sitter + the official wasm +// build shipped in the tree-sitter-bash npm package), normalizes both trees +// into a comparable dump, and produces structured diffs. +// +// Normalization: a pre-order dump of EVERY node (named and anonymous — the +// byte-identical trees M1/M2 verified include the anonymous token layer), +// one line per node: `type [start,end] "text preview"`, anonymous +// types wrapped in parens, plus a leading `hasError:` line. Both sides use +// UTF-16 code unit offsets directly: web-tree-sitter (wasm) reports UTF-16 +// offsets for string input, same as this parser (verified: for +// "echo 你好 && ls" the reference reports the root as [0,13] and `&&` as +// [8,10] — UTF-16 code units, not UTF-8 bytes). + +import path from 'node:path'; + +import { Language, Parser as RefParser } from 'web-tree-sitter'; +import type { Node as RefNode } from 'web-tree-sitter'; + +import type { SyntaxNode } from '#/node'; +import { parse } from '#/parse'; + +const PACKAGE_ROOT = path.resolve(import.meta.dirname, '../..'); +const WASM_PATH = path.join(PACKAGE_ROOT, 'node_modules/tree-sitter-bash/tree-sitter-bash.wasm'); + +let refParserPromise: Promise | null = null; + +/** + * Lazily load web-tree-sitter + the bash wasm exactly once per test process. + * Load failures throw a descriptive error (never silently skip): the + * differential suite is meaningless without the reference. + */ +export function loadReferenceParser(): Promise { + refParserPromise ??= (async () => { + try { + await RefParser.init(); + const language = await Language.load(WASM_PATH); + const parser = new RefParser(); + parser.setLanguage(language); + return parser; + } catch (error) { + throw new Error( + `failed to load the tree-sitter-bash wasm reference (expected at ${WASM_PATH}). ` + + 'Run `pnpm install` and make sure the tree-sitter-bash devDependency is present.', + { cause: error }, + ); + } + })(); + return refParserPromise; +} + +interface DumpLine { + depth: number; + label: string; + start: number; + end: number; + text: string; +} + +function render(lines: DumpLine[], hasError: boolean): string { + const out = [`hasError: ${hasError}`]; + for (const line of lines) { + const preview = + line.text.length <= 40 ? JSON.stringify(line.text) : JSON.stringify(`${line.text.slice(0, 37)}...`); + out.push(`${' '.repeat(line.depth)}${line.label} [${line.start},${line.end}] ${preview}`); + } + return out.join('\n'); +} + +function label(type: string, isNamed: boolean): string { + return isNamed ? type : `(${type})`; +} + +/** Normalized dump of the reference tree (UTF-16 offsets as reported). */ +export async function referenceDump(source: string): Promise { + const parser = await loadReferenceParser(); + const tree = parser.parse(source); + if (tree === null) throw new Error(`reference parser returned null for ${JSON.stringify(source)}`); + const lines: DumpLine[] = []; + const stack: Array<{ node: RefNode; depth: number }> = [{ node: tree.rootNode, depth: 0 }]; + while (stack.length > 0) { + const { node, depth } = stack.pop()!; + lines.push({ + depth, + label: label(node.type, node.isNamed), + start: node.startIndex, + end: node.endIndex, + text: source.slice(node.startIndex, node.endIndex), + }); + for (let i = node.childCount - 1; i >= 0; i--) stack.push({ node: node.child(i)!, depth: depth + 1 }); + } + return render(lines, tree.rootNode.hasError); +} + +/** + * Normalized dump of our tree. Parses with a generous budget so differential + * results are never polluted by the default 50 ms / 50 000-node budget; + * fixtures are expected to parse, so an abort here is an error. + */ +export function ourDump(source: string): string { + const result = parse(source, { timeoutMs: 60_000, maxNodes: 10_000_000 }); + if (!result.ok) throw new Error(`our parser aborted on a differential fixture: ${JSON.stringify(source)}`); + const lines: DumpLine[] = []; + const stack: Array<{ node: SyntaxNode; depth: number }> = [{ node: result.rootNode, depth: 0 }]; + while (stack.length > 0) { + const { node, depth } = stack.pop()!; + lines.push({ depth, label: label(node.type, node.isNamed), start: node.startIndex, end: node.endIndex, text: node.text }); + for (let i = node.children.length - 1; i >= 0; i--) stack.push({ node: node.children[i]!, depth: depth + 1 }); + } + return render(lines, result.hasError); +} + +/** First-difference report between two dumps; empty string when identical. */ +export function diffDumps(ours: string, reference: string): string { + if (ours === reference) return ''; + const a = ours.split('\n'); + const b = reference.split('\n'); + let i = 0; + while (i < a.length && i < b.length && a[i] === b[i]) i++; + const slice = (lines: string[], from: number): string => + lines + .slice(from, from + 8) + .map((line, j) => `${from + j === i ? '>' : ' '} ${line}`) + .join('\n'); + const at = Math.max(0, i - 2); + return ( + `trees differ at dump line ${i + 1}:\n` + + ` ours:\n${slice(a, at)}\n` + + ` reference:\n${slice(b, at)}` + ); +} + +export interface Comparison { + equal: boolean; + ours: string; + reference: string; + diff: string; +} + +/** Parse `source` with both parsers and compare the normalized dumps. */ +export async function compareSource(source: string): Promise { + const [ours, reference] = [ourDump(source), await referenceDump(source)]; + return { equal: ours === reference, ours, reference, diff: diffDumps(ours, reference) }; +} + +/** + * Tree integrity self-check (used by fuzz): every node range must lie inside + * the source, `text` must equal the corresponding slice, and children must be + * contained in their parent, in source order, without overlaps. Throws with a + * descriptive message on the first violation. + */ +export function assertTreeIntegrity(root: SyntaxNode, source: string): void { + const stack: SyntaxNode[] = [root]; + while (stack.length > 0) { + const node = stack.pop()!; + if (node.startIndex < 0 || node.endIndex < node.startIndex || node.endIndex > source.length) { + throw new Error(`node ${node.type} range [${node.startIndex}, ${node.endIndex}) escapes source length ${source.length}`); + } + if (node.text !== source.slice(node.startIndex, node.endIndex)) { + throw new Error(`node ${node.type} text does not match source.slice(${node.startIndex}, ${node.endIndex})`); + } + let previousEnd = node.startIndex; + for (const child of node.children) { + if (child.startIndex < node.startIndex || child.endIndex > node.endIndex) { + throw new Error(`child ${child.type} escapes parent ${node.type}`); + } + if (child.startIndex < previousEnd) { + throw new Error(`child ${child.type} overlaps its previous sibling in ${node.type}`); + } + previousEnd = child.endIndex; + stack.push(child); + } + } +} + +// ------------------------------------------------------------- fixture I/O + +/** A curated differential fixture sample (see test/fixtures/differential/). */ +export interface FixtureSample { + /** Directive: 'match' (must equal the reference) or 'known-diff'. */ + kind: 'match' | 'known-diff'; + /** Known-difference registry id (known-diff samples only). */ + id?: string; + /** Free-form description from the directive line. */ + description: string; + source: string; + /** Stored expected dump of OUR parser (known-diff samples only). */ + expectedOurs?: string; + /** 1-based line of the directive, for error messages. */ + line: number; +} + +const DIRECTIVE_RE = /^@(match|known-diff)(?:\s+(\S+))?:\s*(.*)$/; +const SEPARATOR_RE = /^===\s*$/m; +const DUMP_SPLIT_RE = /^---\s*$/m; + +/** One official tree-sitter-bash corpus test case (input only — the + * expected S-expression is ignored; the live wasm reference is the + * comparison target). */ +export interface CorpusCase { + name: string; + input: string; +} + +/** Parse an official corpus file (`==== name ====` / input / `----` / + * expected). The separator is any run of 3+ dashes. */ +export function parseCorpusFile(content: string): CorpusCase[] { + const out: CorpusCase[] = []; + const lines = content.split('\n'); + let i = 0; + while (i < lines.length) { + if (!/^=+\s*$/.test(lines[i]!)) { + i++; + continue; + } + i++; + const nameLines: string[] = []; + while (i < lines.length && !/^=+\s*$/.test(lines[i]!)) nameLines.push(lines[i++]!); + i++; + const inputLines: string[] = []; + while (i < lines.length && !/^-{3,}\s*$/.test(lines[i]!)) inputLines.push(lines[i++]!); + i++; + while (i < lines.length && !/^=+\s*$/.test(lines[i]!)) i++; + out.push({ name: nameLines.join(' ').trim(), input: inputLines.join('\n').replace(/^\n+/, '').replace(/\n+$/, '') }); + } + return out; +} + + +/** + * Parse a fixture file. Format: blocks separated by lines of `===`; each + * block starts with `@match: ` or `@known-diff : `, + * followed by the sample source. known-diff blocks additionally carry the + * expected dump of our parser after a `---` line (the disclosed deviation + * shape — if our output drifts from it, or starts matching the reference, + * the test fails). + */ +export function parseFixtureFile(filePath: string, content: string): FixtureSample[] { + const samples: FixtureSample[] = []; + const blocks = content.split(SEPARATOR_RE); + let line = 1; + for (const rawBlock of blocks) { + const block = rawBlock.replace(/^\n/, '').replace(/\n$/, ''); + const blockLine = line; + line += rawBlock.split('\n').length; + if (block.trim().length === 0) continue; + const nl = block.indexOf('\n'); + const directiveLine = nl === -1 ? block : block.slice(0, nl); + const body = nl === -1 ? '' : block.slice(nl + 1); + const directive = DIRECTIVE_RE.exec(directiveLine.trim()); + if (directive === null) { + throw new Error(`${filePath}:${blockLine}: expected a @match: / @known-diff : directive, got ${JSON.stringify(directiveLine)}`); + } + const [, kind, id, description = ''] = directive; + if (kind === 'match') { + samples.push({ kind, description, source: body, line: blockLine }); + } else { + if (id === undefined) throw new Error(`${filePath}:${blockLine}: @known-diff requires a registry id`); + const parts = body.split(DUMP_SPLIT_RE); + if (parts.length !== 2) { + throw new Error(`${filePath}:${blockLine}: @known-diff block must contain a --- line followed by the expected dump of our parser`); + } + const source = parts[0]!.replace(/\n$/, ''); + const expectedOurs = parts[1]!.replace(/^\n/, '').replace(/\n$/, ''); + samples.push({ kind: 'known-diff', id, description, source, expectedOurs, line: blockLine }); + } + } + return samples; +} diff --git a/packages/tree-sitter-bash/test/helpers/known-differences.ts b/packages/tree-sitter-bash/test/helpers/known-differences.ts new file mode 100644 index 0000000000..b3b526e6f6 --- /dev/null +++ b/packages/tree-sitter-bash/test/helpers/known-differences.ts @@ -0,0 +1,179 @@ +// test/helpers/known-differences.ts +// +// The single source of truth for the known, deliberately documented ways +// this parser's tree deviates from tree-sitter-bash 0.25.0. Every entry is: +// - referenced by @known-diff samples in test/fixtures/differential/ and +// test/fixtures/corpus/known-diffs.txt (the stored dump in each sample +// pins the exact deviation shape — if our output drifts, or starts +// matching the reference, the differential tests fail); +// - anchored in README.md's "Known differences" section: `readmeAnchor` +// must appear there verbatim, so the list and the README cannot +// silently drift apart (enforced by test/differential.test.ts). + +export interface KnownDifference { + /** Identifier referenced from fixture @known-diff directives. */ + readonly id: string; + /** Verbatim substring of README.md's Known differences section. */ + readonly readmeAnchor: string; + /** Short human-readable description. */ + readonly summary: string; +} + +export const KNOWN_DIFFERENCES: readonly KnownDifference[] = [ + { + id: 'redirect-readwrite', + readmeAnchor: '`<>` (read-write redirect) is parsed as a normal `file_redirect`', + summary: '<> parses as file_redirect; the reference fails to parse it.', + }, + { + id: 'heredoc-multiple-on-line', + readmeAnchor: 'Several heredocs on one line', + summary: 'cat < is a word in the reference (scanner-state quirk); a continuation INSIDE a pattern is an error there. This parser classifies by content and keeps continuations.', + }, + { + id: 'nonascii-identifier-assignment', + readmeAnchor: 'A non-ASCII “identifier” in assignment position', + summary: '变量=值 is a hasError variable_assignment in the reference; this parser keeps it a plain command word (bash itself rejects such names).', + }, + { + id: 'case-rejected-group', + readmeAnchor: 'An extglob group the reference rejects in a case pattern', + summary: 'x@(y)): the reference reparses the group as a new case item; this parser degrades the whole pattern to ERROR.', + }, + { + id: 'expansion-replacement-escaped-backslash', + readmeAnchor: 'A double backslash in a replacement value (`${x// /\\\\|}`)', + summary: 'The reference drops the first backslash (a quirk); this parser keeps both.', + }, +]; + +const KNOWN_DIFFERENCE_IDS: ReadonlySet = new Set(KNOWN_DIFFERENCES.map((entry) => entry.id)); + +export function isKnownDifferenceId(id: string): boolean { + return KNOWN_DIFFERENCE_IDS.has(id); +} diff --git a/packages/tree-sitter-bash/test/performance.test.ts b/packages/tree-sitter-bash/test/performance.test.ts new file mode 100644 index 0000000000..13b094a346 --- /dev/null +++ b/packages/tree-sitter-bash/test/performance.test.ts @@ -0,0 +1,95 @@ +// test/performance.test.ts +// +// Performance smoke tests — order-of-magnitude guards only (no absolute +// timing assertions beyond the documented abort path), meant to catch +// accidental quadratic complexity in future changes. + +import { describe, expect, it } from 'vitest'; + +import { parse } from '#/parse'; + +/** ~100 KB of realistic script-shaped text (functions, loops, redirects, + * expansions, heredocs — not a pathological single construct). */ +function realisticScript(): string { + const block = `deploy() { + local version="$1" + if [[ -z "\${version}" ]]; then + echo "usage: deploy " >&2 + return 1 + fi + for pkg in core web worker; do + tar -czf "dist/\${pkg}-\${version}.tar.gz" "build/\${pkg}" || return 2 + done + grep -q "release" CHANGELOG.md && echo "releasing \${version}" >> deploy.log + ssh deploy@example.com "mkdir -p /srv/app/releases/\${version}" < /dev/null + rsync -a --delete dist/ deploy@example.com:/srv/app/releases/\${version}/ | tee -a deploy.log +} + +while read -r name status; do + case "$status" in + ok) echo "\${name}: fine" ;; + *) echo "\${name}: \${status}" >&2 ;; + esac +done < services.txt + +cat <<'SCRIPT_EOF' +deploy finished at $(date) +SCRIPT_EOF + +`; + return block.repeat(Math.ceil(100_000 / block.length)); +} + +describe('performance smoke', () => { + it('parses a 100KB realistic script fast (default budget suffices)', () => { + const source = realisticScript(); + expect(source.length).toBeGreaterThan(100_000); + const start = performance.now(); + const result = parse(source); + const elapsed = performance.now() - start; + if (!result.ok) { + // An abort is acceptable only when it is prompt (< 100 ms). + expect(result).toEqual({ ok: false, reason: 'aborted' }); + expect(elapsed).toBeLessThan(100); + return; + } + // Completed within the default 50 ms budget; re-parse unbounded and + // assert the order of magnitude (< 1 s for 100KB) as a quadratic guard. + const restart = performance.now(); + const full = parse(source, { timeoutMs: Number.POSITIVE_INFINITY }); + const fullElapsed = performance.now() - restart; + expect(full.ok).toBe(true); + expect(fullElapsed).toBeLessThan(1000); + console.log(`100KB realistic script: default-budget parse ${elapsed.toFixed(1)}ms, unbounded ${fullElapsed.toFixed(1)}ms`); + }); + + it('the abort path is prompt (< 100 ms) on a node-budget bomb', () => { + const source = 'echo a; '.repeat(50_000); + const start = performance.now(); + const result = parse(source); + const elapsed = performance.now() - start; + expect(result).toEqual({ ok: false, reason: 'aborted' }); + expect(elapsed).toBeLessThan(100); + console.log(`abort after ${elapsed.toFixed(1)}ms on a 400KB node-budget bomb`); + }); + + it('typical one-line commands parse in well under a millisecond', () => { + // Warm up. + parse('git status && rm -rf /'); + const iterations = 200; + const start = performance.now(); + for (let i = 0; i < iterations; i++) { + parse('git status && rm -rf /'); + } + const perParse = (performance.now() - start) / iterations; + expect(perParse).toBeLessThan(5); + console.log(`typical command: ${(perParse * 1000).toFixed(0)}µs per parse`); + }); + + it('a 500KB heredoc body parses within the default budget (few nodes)', () => { + const source = `cat < Date: Tue, 21 Jul 2026 21:00:21 +0800 Subject: [PATCH 5/7] feat(agent-core-v2): add App-scope bashParser service MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wrap the pure @moonshot-ai/tree-sitter-bash package as IBashParserService (L1, no dependencies): parse(source, options) returns a wire-safe BashParseResult whose nodes drop the cyclic parent link so trees can cross the RPC boundary. Budget exhaustion surfaces as { ok: false, reason: 'aborted' }, malformed input as hasError — never a throw. --- packages/agent-core-v2/package.json | 1 + .../scripts/check-domain-layers.mjs | 5 ++ .../src/app/bashParser/bashParser.ts | 42 +++++++++++ .../src/app/bashParser/bashParserService.ts | 49 +++++++++++++ packages/agent-core-v2/src/index.ts | 2 + .../app/bashParser/bashParserService.test.ts | 72 +++++++++++++++++++ pnpm-lock.yaml | 3 + 7 files changed, 174 insertions(+) create mode 100644 packages/agent-core-v2/src/app/bashParser/bashParser.ts create mode 100644 packages/agent-core-v2/src/app/bashParser/bashParserService.ts create mode 100644 packages/agent-core-v2/test/app/bashParser/bashParserService.test.ts diff --git a/packages/agent-core-v2/package.json b/packages/agent-core-v2/package.json index caba0689d6..1eb599fdc7 100644 --- a/packages/agent-core-v2/package.json +++ b/packages/agent-core-v2/package.json @@ -63,6 +63,7 @@ "@moonshot-ai/kimi-code-oauth": "workspace:^", "@moonshot-ai/minidb": "workspace:^", "@moonshot-ai/protocol": "workspace:^", + "@moonshot-ai/tree-sitter-bash": "workspace:^", "@mozilla/readability": "^0.6.0", "ajv": "^8.18.0", "ajv-formats": "^3.0.1", diff --git a/packages/agent-core-v2/scripts/check-domain-layers.mjs b/packages/agent-core-v2/scripts/check-domain-layers.mjs index b93a586984..cb2cfc7685 100644 --- a/packages/agent-core-v2/scripts/check-domain-layers.mjs +++ b/packages/agent-core-v2/scripts/check-domain-layers.mjs @@ -102,6 +102,11 @@ const DOMAIN_LAYER = new Map([ // Depends only on `_base`; sits in L1 beside the other program-control // layer substrates. ['task', 1], + // `bashParser` is the App-scope adapter over the pure + // `@moonshot-ai/tree-sitter-bash` package (bash source → syntax tree DTO). + // It injects no services, so it sits in L1 beside the other pure + // capabilities. + ['bashParser', 1], // persistence/ and os/ — the two-level scopes. `interface` holds contracts // (same layer as the old domains they replace); `backends` holds // implementations that may depend on cross-domain services at various layers. diff --git a/packages/agent-core-v2/src/app/bashParser/bashParser.ts b/packages/agent-core-v2/src/app/bashParser/bashParser.ts new file mode 100644 index 0000000000..154c8550f2 --- /dev/null +++ b/packages/agent-core-v2/src/app/bashParser/bashParser.ts @@ -0,0 +1,42 @@ +/** + * `bashParser` domain (L1) — bash source parsing capability. + * + * Defines the `IBashParserService` that parses a bash source string into a + * syntax tree through the pure `@moonshot-ai/tree-sitter-bash` package, plus + * the wire-safe DTO types it returns: `BashSyntaxNode` drops the cyclic + * `parent` link so results can cross the RPC boundary, and offsets are + * UTF-16 code units (`text` always equals `source.slice(start, end)`). The + * parse runs under a deterministic budget — budget exhaustion yields + * `{ ok: false, reason: 'aborted' }` and malformed input yields + * `hasError: true`, never a throw; callers that cannot analyze a command + * must degrade on either signal. Bound at App scope. + */ + +import { createDecorator, type ServiceIdentifier } from '#/_base/di/instantiation'; + +export interface BashSyntaxNode { + readonly type: string; + readonly text: string; + readonly startIndex: number; + readonly endIndex: number; + readonly isNamed: boolean; + readonly children: readonly BashSyntaxNode[]; +} + +export interface BashParseOptions { + readonly timeoutMs?: number; + readonly maxNodes?: number; +} + +export type BashParseResult = + | { readonly ok: true; readonly hasError: boolean; readonly root: BashSyntaxNode } + | { readonly ok: false; readonly reason: 'aborted' }; + +export interface IBashParserService { + readonly _serviceBrand: undefined; + + parse(source: string, options?: BashParseOptions): BashParseResult; +} + +export const IBashParserService: ServiceIdentifier = + createDecorator('bashParserService'); diff --git a/packages/agent-core-v2/src/app/bashParser/bashParserService.ts b/packages/agent-core-v2/src/app/bashParser/bashParserService.ts new file mode 100644 index 0000000000..60c06bcbba --- /dev/null +++ b/packages/agent-core-v2/src/app/bashParser/bashParserService.ts @@ -0,0 +1,49 @@ +/** + * `bashParser` domain (L1) — `IBashParserService` implementation. + * + * Thin adapter over the pure `@moonshot-ai/tree-sitter-bash` package: runs + * its budgeted `parse` and snapshots the returned tree into the wire-safe + * `BashSyntaxNode` DTO (source-ordered children including anonymous tokens, + * `parent` links dropped). Owns no state and injects no services. Bound at + * App scope. + */ + +import { parse } from '@moonshot-ai/tree-sitter-bash'; +import type { SyntaxNode } from '@moonshot-ai/tree-sitter-bash'; + +import { InstantiationType } from '#/_base/di/extensions'; +import { LifecycleScope, registerScopedService } from '#/_base/di/scope'; + +import type { BashParseOptions, BashParseResult, BashSyntaxNode } from './bashParser'; +import { IBashParserService } from './bashParser'; + +function snapshot(node: SyntaxNode): BashSyntaxNode { + return { + type: node.type, + text: node.text, + startIndex: node.startIndex, + endIndex: node.endIndex, + isNamed: node.isNamed, + children: node.children.map(snapshot), + }; +} + +export class BashParserService implements IBashParserService { + declare readonly _serviceBrand: undefined; + + parse(source: string, options: BashParseOptions = {}): BashParseResult { + const result = parse(source, options); + if (!result.ok) { + return { ok: false, reason: result.reason }; + } + return { ok: true, hasError: result.hasError, root: snapshot(result.rootNode) }; + } +} + +registerScopedService( + LifecycleScope.App, + IBashParserService, + BashParserService, + InstantiationType.Delayed, + 'bashParser', +); diff --git a/packages/agent-core-v2/src/index.ts b/packages/agent-core-v2/src/index.ts index a6678675b3..99a67fda31 100644 --- a/packages/agent-core-v2/src/index.ts +++ b/packages/agent-core-v2/src/index.ts @@ -282,6 +282,8 @@ export * from '#/app/workspaceRegistry/workspacePersistence'; export * from '#/app/workspaceRegistry/fileWorkspacePersistence'; import '#/app/workspaceRegistry/workspaceQueryService'; import '#/app/git/gitService'; +export * from '#/app/bashParser/bashParser'; +import '#/app/bashParser/bashParserService'; export * from '#/session/process/processRunner'; export * from '#/session/process/processRunnerService'; export * from '#/session/sessionFs/errors'; diff --git a/packages/agent-core-v2/test/app/bashParser/bashParserService.test.ts b/packages/agent-core-v2/test/app/bashParser/bashParserService.test.ts new file mode 100644 index 0000000000..2228f50cac --- /dev/null +++ b/packages/agent-core-v2/test/app/bashParser/bashParserService.test.ts @@ -0,0 +1,72 @@ +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; + +import { DisposableStore } from '#/_base/di/lifecycle'; +import { createServices, type TestInstantiationService } from '#/_base/di/test'; +import type { BashSyntaxNode } from '#/app/bashParser/bashParser'; +import { IBashParserService } from '#/app/bashParser/bashParser'; +import { BashParserService } from '#/app/bashParser/bashParserService'; + +describe('BashParserService', () => { + let disposables: DisposableStore; + let ix: TestInstantiationService; + let service: IBashParserService; + + beforeEach(() => { + disposables = new DisposableStore(); + ix = createServices(disposables, { + additionalServices: (reg) => { + reg.define(IBashParserService, BashParserService); + }, + }); + service = ix.get(IBashParserService); + }); + + afterEach(() => { + disposables.dispose(); + }); + + it('splits a compound command into per-command nodes', () => { + const result = service.parse('git status && rm -rf /'); + if (!result.ok) { + throw new Error('expected ok'); + } + expect(result.hasError).toBe(false); + expect(result.root.type).toBe('program'); + const commands: string[] = []; + const walk = (node: BashSyntaxNode): void => { + if (node.type === 'command') { + commands.push(node.text); + } + node.children.forEach(walk); + }; + walk(result.root); + expect(commands).toEqual(['git status', 'rm -rf /']); + }); + + it('flags malformed input with hasError instead of throwing', () => { + const result = service.parse('echo "unterminated'); + if (!result.ok) { + throw new Error('expected ok'); + } + expect(result.hasError).toBe(true); + }); + + it('reports budget exhaustion as aborted', () => { + const result = service.parse('ls; '.repeat(20000), { maxNodes: 100 }); + expect(result).toEqual({ ok: false, reason: 'aborted' }); + }); + + it('returns a JSON-serializable tree with text/range fidelity', () => { + const source = 'echo "你好 🎉" | grep 你'; + const result = service.parse(source); + if (!result.ok) { + throw new Error('expected ok'); + } + const roundTripped = JSON.parse(JSON.stringify(result.root)) as BashSyntaxNode; + const check = (node: BashSyntaxNode): void => { + expect(node.text).toBe(source.slice(node.startIndex, node.endIndex)); + node.children.forEach(check); + }; + check(roundTripped); + }); +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 65a2e23fa5..b0cc621a24 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -692,6 +692,9 @@ importers: '@moonshot-ai/protocol': specifier: workspace:^ version: link:../protocol + '@moonshot-ai/tree-sitter-bash': + specifier: workspace:^ + version: link:../tree-sitter-bash '@mozilla/readability': specifier: ^0.6.0 version: 0.6.0 From a9ded4966cc145d106146fdd8adfd5170763c90a Mon Sep 17 00:00:00 2001 From: "haozhe.yang" Date: Tue, 21 Jul 2026 21:01:08 +0800 Subject: [PATCH 6/7] chore: add changeset for the bash parser service --- .changeset/bash-parser-service.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/bash-parser-service.md diff --git a/.changeset/bash-parser-service.md b/.changeset/bash-parser-service.md new file mode 100644 index 0000000000..05e5f34208 --- /dev/null +++ b/.changeset/bash-parser-service.md @@ -0,0 +1,5 @@ +--- +"@moonshot-ai/kimi-code": patch +--- + +Add an internal bash parsing capability that turns shell command strings into syntax trees, in preparation for per-command permission analysis. No user-facing behavior change yet. From 2925c0a747630bed8eef3113d5c1045d7172d8a3 Mon Sep 17 00:00:00 2001 From: "haozhe.yang" Date: Wed, 22 Jul 2026 19:11:13 +0800 Subject: [PATCH 7/7] chore: update pnpmDeps hash after adding tree-sitter-bash package --- flake.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index 15c9107f7a..5261a5474c 100644 --- a/flake.nix +++ b/flake.nix @@ -162,7 +162,7 @@ inherit (finalAttrs) pname version src pnpmWorkspaces; inherit pnpm; fetcherVersion = 3; - hash = "sha256-+pzJfoWJwVXIUU8oc56LVpfNjSY6MABID5g11Cm92xw="; + hash = "sha256-niK61WYaqCJLUUlp4AGgq2/DOF3sqst6N9Kpk9noXyE="; }; nativeBuildInputs = [