From be016a1df8f24c7be8c2639e7b9e65562d7d714e Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Fri, 7 Aug 2026 06:29:55 +0000 Subject: [PATCH] Discover Bash heredoc substitutions --- IMPLEMENTATION_PLAN.md | 20 +- SPEC.md | 17 +- TOOLING.md | 5 +- .../v0-3-structured-shell-analysis/tasks.md | 6 +- .../Internal/Bash/Lexing/BashLexer.cs | 280 ++++++++++++++++-- .../Internal/Bash/Lexing/BashToken.cs | 14 + .../Bash/Parsing/BashStructuralCoordinator.cs | 101 ++++--- ...00_v03_expanding_heredoc_substitution.json | 133 +++++++++ ...literal_heredoc_substitution_spelling.json | 64 ++++ ...03_tab_stripping_heredoc_substitution.json | 133 +++++++++ ...03_v03_heredoc_multiple_substitutions.json | 202 +++++++++++++ ...4_v03_heredoc_header_tail_unparseable.json | 9 + .../DesignCorpus/v0.3/bash.json | 2 +- .../Parsing/BashStructuralProjectionTests.cs | 150 ++++++++++ .../Parsing/ShellValueOracleTests.cs | 41 +++ tools/PwshCorpusTool/CorpusJson.cs | 3 +- tools/PwshCorpusTool/Program.cs | 31 ++ 17 files changed, 1133 insertions(+), 78 deletions(-) create mode 100644 tests/ShellSyntaxTree.Tests/Corpus/bash/200_v03_expanding_heredoc_substitution.json create mode 100644 tests/ShellSyntaxTree.Tests/Corpus/bash/201_v03_literal_heredoc_substitution_spelling.json create mode 100644 tests/ShellSyntaxTree.Tests/Corpus/bash/202_v03_tab_stripping_heredoc_substitution.json create mode 100644 tests/ShellSyntaxTree.Tests/Corpus/bash/203_v03_heredoc_multiple_substitutions.json create mode 100644 tests/ShellSyntaxTree.Tests/Corpus/bash/204_v03_heredoc_header_tail_unparseable.json diff --git a/IMPLEMENTATION_PLAN.md b/IMPLEMENTATION_PLAN.md index 455c0ea..bd60066 100644 --- a/IMPLEMENTATION_PLAN.md +++ b/IMPLEMENTATION_PLAN.md @@ -265,8 +265,18 @@ priorities. depth limits, comment-safe delimiter scanning, and fail-closed command identities, background lists, assignment prefixes, backticks, heredocs, and malformed interiors. -- [ ] Extend Bash substitution discovery to iterables and expanding heredoc - bodies, then add the corresponding Netclaw approval-matrix cases. +- [x] Extend Bash substitution discovery to expanding heredoc bodies. The + bounded slice recognizes quoted, escaped, mixed, and tab-stripping + delimiters; preserves exact body/terminator provenance; surfaces nested + substitutions in authored order with isolated state; and rejects header + tails, queued heredocs, backticks, arithmetic, continuations, incomplete + interiors, and depth overflow atomically. Direct and executable-corpus + cases pin exact syntax, command ancestry, spans, completeness, and + literal-versus-expanding behavior; real-Bash output and parse-only + oracles independently pin the bounded semantic boundary. +- [ ] Extend Bash substitution discovery to iterables with the complete + `for ... in` vertical slice, then add the Bash substitution cases to the + Netclaw approval matrix. - [ ] Complete PowerShell `$()` discovery in `foreach` expressions and add the Netclaw approval-matrix cases. The simple-command slice is delivered for ordinary, adjacent, quoted, here-string, redirect, standalone, @@ -276,9 +286,9 @@ priorities. - [ ] Deliver Bash `for ... in` and PowerShell `foreach` as the first two language-specific vertical slices, then extract only the shared analysis proven by both implementations. -- [ ] Preserve the existing Bash heredoc grammar, fix quoted-delimiter - adjacency, expose body/delimiter/expansion/completeness facts, and add a - separately tested Bash `<<<` here-string redirect slice. +- [ ] Build on the delivered bounded Bash heredoc grammar and quoted-delimiter + adjacency by exposing public body/delimiter/expansion/completeness facts, + then add a separately tested Bash `<<<` here-string redirect slice. - [ ] Near the end of v0.3 delivery, expand the Web sample with curated complex Bash and PowerShell inputs and deterministic Mermaid views of syntax, occurrences, compatibility clauses, ancestry, redirects, and fail-closed diff --git a/SPEC.md b/SPEC.md index 7f1ea01..5fd8b39 100644 --- a/SPEC.md +++ b/SPEC.md @@ -978,6 +978,17 @@ quoted_string := single-quoted | double-quoted - v0.2 recognizes heredocs (`<`, `>>`, `<`, `2>`, `2>>`, `(`, `)`, `<<`, `<<-`. - **WHITESPACE** — one or more spaces, tabs, or newlines (newlines inside - a skipped heredoc body are not tokenized). A whitespace run that + a heredoc body are not emitted as ordinary tokens; the delimiter token + retains the body's resolver fragments and authored extent). A whitespace run that contains a newline — including the newline after a heredoc terminator — is flagged as a **statement separator**; the parser retains those tokens past `FilterSignificant` and splits clauses on them per §4. A @@ -1081,6 +1093,9 @@ The lexer produces tokens consumed by the parser. Token kinds: and `\X` escapes via a shared opaque-region scanner. The parser consumes this token as `Arg{ Kind=DynamicSkip, IsPath=false, Resolved=null }` per locked interpretation #2. + Expanding-heredoc substitutions use the same opaque fragment semantics but + remain attached to the delimiter token rather than entering the ordinary + command-token stream. - **UNPARSEABLE_SENTINEL** — `$((expr))` arithmetic expansion or `${var//pat/repl}` complex parameter expansion. The lexer skips past the matching close (`))` or `}` respectively) and emits a sentinel diff --git a/TOOLING.md b/TOOLING.md index 626274d..3697d92 100644 --- a/TOOLING.md +++ b/TOOLING.md @@ -40,12 +40,15 @@ and parser slice exists. The PII audit scans both corpus trees. ### PwshCorpusTool `tools/PwshCorpusTool` is the PowerShell corpus authoring aid -(SPEC.POWERSHELL.md §13). It is a dev-only console app (not packed). +(SPEC.POWERSHELL.md §13). Its shared serializer also exposes an ad-hoc Bash +check so structural Bash entries do not need hand-authored spans. It is a +dev-only console app (not packed). | Command | Purpose | |---|---| | `dotnet run --project tools/PwshCorpusTool -- generate` | Regenerate every `Corpus/powershell/NNN_slug.json` from the curated `CorpusManifest`. Run after any parser change that shifts PowerShell AST output. | | `dotnet run --project tools/PwshCorpusTool -- check ""` | Print the parser's expected-AST JSON block for a command beside the real-`pwsh` oracle verdict — the fastest way to author or debug a binding-category entry. | +| `dotnet run --project tools/PwshCorpusTool -- check-bash ""` | Print exact Bash `clauses`, `syntax`, and `commands` expectations using the corpus resolver settings. | The curated inputs live in `tools/PwshCorpusTool/CorpusManifest.cs`; the `expected` AST is generated from `PwshParser`, and `PwshOracleTests` diff --git a/openspec/changes/v0-3-structured-shell-analysis/tasks.md b/openspec/changes/v0-3-structured-shell-analysis/tasks.md index 4dd3102..bf8d5bf 100644 --- a/openspec/changes/v0-3-structured-shell-analysis/tasks.md +++ b/openspec/changes/v0-3-structured-shell-analysis/tasks.md @@ -33,14 +33,17 @@ - [x] 3.9 Add corpus expectations for syntax shape, occurrences, roles, and completeness for existing constructs. - [ ] 3.10 Implement Bash `$()` discovery in supported argument words, redirect values, iterables, and expanding heredoc bodies; retain literal/escaped spellings and fail closed on command-name substitutions, legacy backticks, or incomplete interiors. - [x] 3.10a Implement the simple-command argument and redirect-target slice, including comment-safe boundaries and fail-closed unsupported interiors. + - [x] 3.10b Implement the bounded expanding-heredoc slice with quote-removed delimiters, literal quoted/escaped bodies, tab stripping, exact provenance, and fail-closed unsupported header/body forms. - [ ] 3.11 Implement PowerShell `$()` discovery in supported words, redirect values, foreach expressions, call-operator dynamic identities, standalone expression statements, double-quoted strings, and expandable here-strings; never invent invocation from standalone output, retain literal/escaped spellings, and fail closed on trailing command-style arguments, call-operator script blocks, or unsupported execution-bearing `@()` / `@{}` forms. - [x] 3.11a Implement words, redirect values, call-operator dynamic identities, standalone statements, expandable strings/here-strings, and parent-versus-child host payload provenance; fail closed on arbitrary expression values and unsupported execution-bearing `@()` / `@{}` forms. - [ ] 3.12 Pin substitution parentage, authored sibling indices, innermost-first ordering, Bash-isolated versus PowerShell-current-scope state, unknown-state propagation, nesting/depth limits, and incomplete dynamic identities in direct tests. - [x] 3.12a Pin the Bash argument/redirect slice, isolated cwd behavior, wrapper provenance, and the shared structural-depth budget. - [x] 3.12b Pin the PowerShell simple-command slice, current-scope exact and unknown cwd propagation, parent/child wrapper provenance, expression boundaries, and the shared structural-depth budget. + - [x] 3.12c Pin expanding-heredoc sibling/nested ordering, exact spans, isolated state, delimiter modes, escape parity, depth limits, and atomic failure. - [ ] 3.13 Promote ordinary, multiple, nested, iterator, redirect, quoted, escaped, stateful, malformed, and hidden-execution substitution cases into both executable corpora and the Netclaw approval matrix. - [x] 3.13a Promote the Bash ordinary, multiple, nested, redirect, quoted, escaped, stateful, malformed, and hidden-execution cases into its executable corpus. - [x] 3.13b Promote the PowerShell ordinary, multiple, nested, redirect, quoted, escaped, stateful, malformed, expression-boundary, and hidden-execution cases into its executable corpus. + - [x] 3.13c Promote expanding, literal, tab-stripped, multiple, and malformed Bash heredoc cases with full structural expectations into the executable corpus. ## 4. Explicit Redirect Semantics @@ -106,10 +109,11 @@ ## 10. Heredoc / Here-String Slice and Separately Gated Follow-ups - [x] 10.1 Specify heredoc delimiter adjacency and quoting, expansion mode, body provenance, substitutions, tab stripping, completeness, and Bash here-string semantics. -- [ ] 10.2 Preserve existing `<<` / `<<-` behavior and fix quoted-delimiter adjacency without regressing the v0.2 compatibility redirect. +- [x] 10.2 Preserve existing `<<` / `<<-` behavior and fix quoted-delimiter adjacency without regressing the v0.2 compatibility redirect. - [ ] 10.3 Add explicit heredoc delimiter/body/expansion/completeness facts and surface every supported substitution command. - [ ] 10.4 Add Bash `<<<` here-string tokenization, explicit redirect facts, bounded operand analysis, and trailing-newline semantics. - [ ] 10.5 Add direct, malformed, quoted/unquoted, tab-stripped, dynamic, and substitution-bearing corpus cases plus real-Bash parse-only validation. + - [x] 10.5a Add direct, executable-corpus, real-Bash output, and real-Bash parse-only coverage for the bounded substitution-discovery slice; explicit redirect facts and the full heredoc matrix remain pending. - [ ] 10.6 After stable v0.3, specify process-substitution command discovery and the unknown produced descriptor/path value before enabling it. - [ ] 10.7 After stable v0.3, specify background-list concurrency, ordering, and shell-state boundaries before enabling single `&`. - [ ] 10.8 Specify C-style loop and arithmetic hidden-execution behavior before enabling either construct. diff --git a/src/ShellSyntaxTree/Internal/Bash/Lexing/BashLexer.cs b/src/ShellSyntaxTree/Internal/Bash/Lexing/BashLexer.cs index 66e86d6..af2d285 100644 --- a/src/ShellSyntaxTree/Internal/Bash/Lexing/BashLexer.cs +++ b/src/ShellSyntaxTree/Internal/Bash/Lexing/BashLexer.cs @@ -5,6 +5,7 @@ // ----------------------------------------------------------------------- using System; using System.Collections.Generic; +using System.Text; using ShellSyntaxTree.Internal.Lexing; using ShellSyntaxTree.Internal.Resolving; @@ -30,9 +31,9 @@ namespace ShellSyntaxTree.Internal.Bash.Lexing; /// ${var//pat/repl} — emit a /// . The parser /// lifts that sentinel into ParsedCommand.IsUnparseable. -/// Heredoc bodies are dropped per SPEC §4. Only the -/// <</<<- operator and the delimiter -/// word make it into the token stream. +/// Heredoc bodies do not become ordinary tokens. The delimiter token +/// retains body resolver fragments so stable v0.3 can discover +/// executable substitutions without treating body data as argv. /// /// internal static class BashLexer @@ -1228,40 +1229,71 @@ private static int ConsumeHeredoc( return i; } - // Read the delimiter as a word (no quote/escape unwrapping handling - // beyond what ReadWord does — bash supports `<<'EOF'` for - // unexpanded bodies, but we skip the body either way so the value - // doesn't matter). Capture the delimiter text from the freshly - // appended token. - var beforeDelim = tokens.Count; - var afterDelim = ReadWord(src, i, tokens); - if (tokens.Count == beforeDelim) + if (!TryReadHeredocDelimiter( + src, + i, + out var afterDelim, + out var delim, + out var delimiterQuoted, + out var delimiterError)) { - // ReadWord didn't produce a token (delimiter started with an - // operator/quote we don't unwrap here). Treat as malformed. tokens.Add(new BashToken( BashTokenKind.UnparseableSentinel, - "", + src.Slice(i, afterDelim - i).ToString(), null, i, - 0, - "heredoc operator '" + opText + "' missing delimiter")); + afterDelim - i, + delimiterError ?? "heredoc operator '" + opText + "' missing delimiter")); return afterDelim; } - var delimToken = tokens[tokens.Count - 1]; - var delim = delimToken.Value; + var delimiterIndex = tokens.Count; + tokens.Add(new BashToken( + BashTokenKind.Word, + delim, + null, + i, + afterDelim - i, + null) + { + ResolverValue = ShellValue.Literal(delim, i, afterDelim - i), + }); // Skip the heredoc body: from the next newline to the line that // contains only `delim` (or, for `<<-`, optional leading tabs + // delim). On unterminated body, emit a sentinel and stop. var j = afterDelim; - // Find the first newline that opens the body. - while (j < src.Length && src[j] != '\n') + while (j < src.Length && src[j] is ' ' or '\t') { j++; } + if (j < src.Length && src[j] == '#') + { + while (j < src.Length && src[j] is not '\n' and not '\r') + { + j++; + } + } + + if (j < src.Length && src[j] is not '\n' and not '\r') + { + var headerEnd = j; + while (headerEnd < src.Length && src[headerEnd] is not '\n' and not '\r') + { + headerEnd++; + } + + tokens.Add(new BashToken( + BashTokenKind.UnparseableSentinel, + src.Slice(j, headerEnd - j).ToString(), + null, + j, + headerEnd - j, + "tokens after a heredoc delimiter are not supported")); + return src.Length; + } + if (j >= src.Length) { // No newline at all after the delimiter — unterminated heredoc. @@ -1275,7 +1307,15 @@ private static int ConsumeHeredoc( return src.Length; } - j++; // step past the opening newline; body now starts at j. + if (src[j] == '\r' && j + 1 < src.Length && src[j + 1] == '\n') + { + j += 2; + } + else + { + j++; + } + var bodyStart = j; var stripTabs = opText == "<<-"; @@ -1288,7 +1328,11 @@ private static int ConsumeHeredoc( j++; } - var lineEnd = j; // exclusive + var lineEnd = j; + if (lineEnd > lineStart && src[lineEnd - 1] == '\r') + { + lineEnd--; + } // For <<-, optional leading tabs are stripped before comparing. var compareStart = lineStart; @@ -1303,6 +1347,38 @@ private static int ConsumeHeredoc( var lineSlice = src.Slice(compareStart, lineEnd - compareStart); if (lineSlice.SequenceEqual(delim.AsSpan())) { + var bodyLength = lineStart - bodyStart; + ShellValue bodyValue; + if (delimiterQuoted) + { + bodyValue = ShellValue.Literal( + src.Slice(bodyStart, bodyLength).ToString(), + bodyStart, + bodyLength); + } + else if (!TryBuildExpandingHeredocBodyValue( + src, + bodyStart, + bodyLength, + out bodyValue, + out var bodyError)) + { + tokens.Add(new BashToken( + BashTokenKind.UnparseableSentinel, + src.Slice(bodyStart, bodyLength).ToString(), + null, + bodyStart, + bodyLength, + bodyError)); + return src.Length; + } + + tokens[delimiterIndex] = tokens[delimiterIndex] with + { + HeredocBodyValue = bodyValue, + HeredocSourceEnd = lineEnd, + }; + // Found terminator. Skip the body silently — no token is // emitted for the body itself — but emit a single // Whitespace token covering the terminator's trailing @@ -1341,4 +1417,164 @@ private static int ConsumeHeredoc( return j; } + + private static bool TryReadHeredocDelimiter( + ReadOnlySpan src, + int start, + out int end, + out string delimiter, + out bool quoted, + out string? error) + { + var decoded = new StringBuilder(); + quoted = false; + error = null; + var index = start; + while (index < src.Length && !char.IsWhiteSpace(src[index]) && + src[index] is not ';' and not '|' and not '&' and not '<' and not '>') + { + var character = src[index]; + if (character == '\\') + { + quoted = true; + if (index + 1 >= src.Length || src[index + 1] is '\n' or '\r') + { + end = Math.Min(index + 1, src.Length); + delimiter = decoded.ToString(); + error = "heredoc delimiter has an unsupported continuation"; + return false; + } + + decoded.Append(src[index + 1]); + index += 2; + continue; + } + + if (character is '\'' or '"') + { + quoted = true; + var quote = character; + index++; + var closed = false; + while (index < src.Length) + { + character = src[index]; + if (character == quote) + { + index++; + closed = true; + break; + } + + if (quote == '"' && character == '\\' && index + 1 < src.Length && + src[index + 1] is '$' or '`' or '"' or '\\') + { + decoded.Append(src[index + 1]); + index += 2; + continue; + } + + decoded.Append(character); + index++; + } + + if (!closed) + { + end = index; + delimiter = decoded.ToString(); + error = "unterminated quote in heredoc delimiter"; + return false; + } + + continue; + } + + decoded.Append(character); + index++; + } + + end = index; + delimiter = decoded.ToString(); + if (delimiter.Length == 0) + { + return false; + } + + return true; + } + + private static bool TryBuildExpandingHeredocBodyValue( + ReadOnlySpan src, + int bodyStart, + int bodyLength, + out ShellValue value, + out string? error) + { + var bodyEnd = bodyStart + bodyLength; + var builder = new ShellValueBuilder(); + builder.AppendBoundary(bodyStart); + var index = bodyStart; + while (index < bodyEnd) + { + var character = src[index]; + if (character == '\\' && index + 1 < bodyEnd) + { + var escaped = src[index + 1]; + if (escaped is '\n' or '\r') + { + value = builder.Build(); + error = "continuations inside expanding heredoc bodies are not supported"; + return false; + } + + if (escaped is '$' or '`' or '\\') + { + builder.AppendLiteral(escaped, index, 2); + index += 2; + continue; + } + } + + if (character == '`') + { + value = builder.Build(); + error = "legacy backtick command substitution is not supported"; + return false; + } + + if (character == '$' && index + 1 < bodyEnd && src[index + 1] == '(') + { + if (index + 2 < bodyEnd && src[index + 2] == '(') + { + value = builder.Build(); + error = "arithmetic expansion '$((…))' not supported in heredoc body"; + return false; + } + + var scan = ScanCommandSubstitution(src, index + 1); + if (!scan.Closed || scan.EndIndex >= bodyEnd) + { + value = builder.Build(); + error = scan.Error ?? "unbalanced command substitution in heredoc body"; + return false; + } + + var substitutionLength = scan.EndIndex - index + 1; + builder.AppendOpaque( + src.Slice(index, substitutionLength).ToString(), + ShellOpaqueCause.CommandSubstitution, + index, + substitutionLength); + index += substitutionLength; + continue; + } + + builder.AppendLiteral(character, index, 1); + index++; + } + + value = builder.Build(); + error = null; + return true; + } } diff --git a/src/ShellSyntaxTree/Internal/Bash/Lexing/BashToken.cs b/src/ShellSyntaxTree/Internal/Bash/Lexing/BashToken.cs index a740066..60b2a52 100644 --- a/src/ShellSyntaxTree/Internal/Bash/Lexing/BashToken.cs +++ b/src/ShellSyntaxTree/Internal/Bash/Lexing/BashToken.cs @@ -57,6 +57,20 @@ internal readonly record struct BashToken( /// public ShellValue? ResolverValue { get; init; } + /// + /// Resolver fragments from the authored heredoc body associated with a + /// delimiter token. Null for ordinary tokens and malformed heredocs. + /// + public ShellValue? HeredocBodyValue { get; init; } + + /// + /// Exclusive authored-source end of the heredoc terminator associated + /// with a delimiter token. Null for ordinary tokens and malformed + /// heredocs. This lets structural spans cover the body without exposing + /// the body as an ordinary compatibility token. + /// + public int? HeredocSourceEnd { get; init; } + /// /// True when this token contains /// a newline and therefore acts as a statement separator equivalent to diff --git a/src/ShellSyntaxTree/Internal/Bash/Parsing/BashStructuralCoordinator.cs b/src/ShellSyntaxTree/Internal/Bash/Parsing/BashStructuralCoordinator.cs index a1c58b5..89d13b5 100644 --- a/src/ShellSyntaxTree/Internal/Bash/Parsing/BashStructuralCoordinator.cs +++ b/src/ShellSyntaxTree/Internal/Bash/Parsing/BashStructuralCoordinator.cs @@ -447,14 +447,20 @@ private bool TryParseCommand( } var firstSource = segmentTokens[0]; - var lastSource = segmentTokens[segmentTokens.Count - 1]; + var sourceEnd = firstSource.SourceStart + firstSource.SourceLength; + foreach (var token in segmentTokens) + { + sourceEnd = Math.Max( + sourceEnd, + token.HeredocSourceEnd ?? token.SourceStart + token.SourceLength); + } + command = new SimpleCommandSyntax { Clause = emitted, Substitutions = substitutions, SourceStart = firstSource.SourceStart, - SourceLength = lastSource.SourceStart + lastSource.SourceLength - - firstSource.SourceStart, + SourceLength = sourceEnd - firstSource.SourceStart, }; return true; } @@ -587,56 +593,59 @@ private bool TryCollectCommandSubstitutions( foreach (var token in tokens) { - var value = token.ResolverValue; - if (value is null) - { - continue; - } - - foreach (var fragment in value.Fragments) + foreach (var value in new[] { token.ResolverValue, token.HeredocBodyValue }) { - if (fragment.Kind != ShellValueFragmentKind.Opaque || - fragment.OpaqueCause != ShellOpaqueCause.CommandSubstitution) + if (value is null) { continue; } - if (fragment.SourceStart is null || fragment.SourceLength is null || - fragment.SourceLength < 2 || - fragment.SourceStart < _sourceStart || - fragment.SourceStart + fragment.SourceLength > - _sourceStart + _sourceLength) - { - substitutions = Array.Empty(); - error = "Bash command substitution has invalid source provenance"; - return false; - } - - var raw = _source.Substring( - fragment.SourceStart.Value, - fragment.SourceLength.Value); - if (raw[0] == '`') + foreach (var fragment in value.Fragments) { - substitutions = Array.Empty(); - error = "legacy backtick command substitution is not supported"; - return false; - } - - if (!raw.StartsWith("$(", StringComparison.Ordinal) || raw[raw.Length - 1] != ')') - { - substitutions = Array.Empty(); - error = "unsupported Bash command substitution provenance"; - return false; + if (fragment.Kind != ShellValueFragmentKind.Opaque || + fragment.OpaqueCause != ShellOpaqueCause.CommandSubstitution) + { + continue; + } + + if (fragment.SourceStart is null || fragment.SourceLength is null || + fragment.SourceLength < 2 || + fragment.SourceStart < _sourceStart || + fragment.SourceStart + fragment.SourceLength > + _sourceStart + _sourceLength) + { + substitutions = Array.Empty(); + error = "Bash command substitution has invalid source provenance"; + return false; + } + + var raw = _source.Substring( + fragment.SourceStart.Value, + fragment.SourceLength.Value); + if (raw[0] == '`') + { + substitutions = Array.Empty(); + error = "legacy backtick command substitution is not supported"; + return false; + } + + if (!raw.StartsWith("$(", StringComparison.Ordinal) || + raw[raw.Length - 1] != ')') + { + substitutions = Array.Empty(); + error = "unsupported Bash command substitution provenance"; + return false; + } + + if (fragment.SourceStart < commandNameEnd) + { + substitutions = Array.Empty(); + error = "Bash command-name substitution is not supported"; + return false; + } + + discovered.Add(fragment); } - - if (fragment.SourceStart < commandNameEnd) - { - substitutions = Array.Empty(); - error = "Bash command-name substitution is not supported"; - return false; - } - - discovered.Add(fragment); } } diff --git a/tests/ShellSyntaxTree.Tests/Corpus/bash/200_v03_expanding_heredoc_substitution.json b/tests/ShellSyntaxTree.Tests/Corpus/bash/200_v03_expanding_heredoc_substitution.json new file mode 100644 index 0000000..e510b67 --- /dev/null +++ b/tests/ShellSyntaxTree.Tests/Corpus/bash/200_v03_expanding_heredoc_substitution.json @@ -0,0 +1,133 @@ +{ + "name": "v0.3 expanding heredoc command substitution", + "input": "cat <", + "isDynamicSkip": false + } + ] + } + ], + "syntax": [ + { + "kind": "Block", + "parentIndex": null, + "region": "Unknown", + "childIndex": null, + "sourceStart": 0, + "sourceLength": 19, + "clauseIndex": null, + "groupKind": null, + "listOperator": null + }, + { + "kind": "SimpleCommand", + "parentIndex": 0, + "region": "Root", + "childIndex": 0, + "sourceStart": 0, + "sourceLength": 19, + "clauseIndex": 1, + "groupKind": null, + "listOperator": null + }, + { + "kind": "CommandSubstitution", + "parentIndex": 1, + "region": "Substitution", + "childIndex": 0, + "sourceStart": 10, + "sourceLength": 5, + "clauseIndex": null, + "groupKind": null, + "listOperator": null + }, + { + "kind": "Block", + "parentIndex": 2, + "region": "Substitution", + "childIndex": 0, + "sourceStart": 12, + "sourceLength": 2, + "clauseIndex": null, + "groupKind": null, + "listOperator": null + }, + { + "kind": "SimpleCommand", + "parentIndex": 3, + "region": "Statement", + "childIndex": 0, + "sourceStart": 12, + "sourceLength": 2, + "clauseIndex": 0, + "groupKind": null, + "listOperator": null + } + ], + "commands": [ + { + "clauseIndex": 0, + "immediateRole": "Substitution", + "isComplete": true, + "ancestry": [ + { + "ancestorKind": "Block", + "region": "Root", + "childIndex": 0, + "sourceStart": 0, + "sourceLength": 19 + }, + { + "ancestorKind": "CommandSubstitution", + "region": "Substitution", + "childIndex": 0, + "sourceStart": 10, + "sourceLength": 5 + }, + { + "ancestorKind": "Block", + "region": "Statement", + "childIndex": 0, + "sourceStart": 12, + "sourceLength": 2 + } + ] + }, + { + "clauseIndex": 1, + "immediateRole": "Ordinary", + "isComplete": false, + "ancestry": [ + { + "ancestorKind": "Block", + "region": "Root", + "childIndex": 0, + "sourceStart": 0, + "sourceLength": 19 + } + ] + } + ] + }, + "notes": "An unquoted heredoc delimiter enables expansion, so the inner id command is visible before cat." +} diff --git a/tests/ShellSyntaxTree.Tests/Corpus/bash/201_v03_literal_heredoc_substitution_spelling.json b/tests/ShellSyntaxTree.Tests/Corpus/bash/201_v03_literal_heredoc_substitution_spelling.json new file mode 100644 index 0000000..c4b0254 --- /dev/null +++ b/tests/ShellSyntaxTree.Tests/Corpus/bash/201_v03_literal_heredoc_substitution_spelling.json @@ -0,0 +1,64 @@ +{ + "name": "v0.3 quoted heredoc keeps substitution spelling literal", + "input": "cat <", + "isDynamicSkip": false + } + ] + } + ], + "syntax": [ + { + "kind": "Block", + "parentIndex": null, + "region": "Unknown", + "childIndex": null, + "sourceStart": 0, + "sourceLength": 21, + "clauseIndex": null, + "groupKind": null, + "listOperator": null + }, + { + "kind": "SimpleCommand", + "parentIndex": 0, + "region": "Root", + "childIndex": 0, + "sourceStart": 0, + "sourceLength": 21, + "clauseIndex": 0, + "groupKind": null, + "listOperator": null + } + ], + "commands": [ + { + "clauseIndex": 0, + "immediateRole": "Ordinary", + "isComplete": false, + "ancestry": [ + { + "ancestorKind": "Block", + "region": "Root", + "childIndex": 0, + "sourceStart": 0, + "sourceLength": 21 + } + ] + } + ] + }, + "notes": "Quote removal yields delimiter EOF, while any quoted delimiter fragment disables body expansion." +} diff --git a/tests/ShellSyntaxTree.Tests/Corpus/bash/202_v03_tab_stripping_heredoc_substitution.json b/tests/ShellSyntaxTree.Tests/Corpus/bash/202_v03_tab_stripping_heredoc_substitution.json new file mode 100644 index 0000000..34c0a2b --- /dev/null +++ b/tests/ShellSyntaxTree.Tests/Corpus/bash/202_v03_tab_stripping_heredoc_substitution.json @@ -0,0 +1,133 @@ +{ + "name": "v0.3 tab-stripping heredoc command substitution", + "input": "cat <<-EOF\n\t$(id)\n\tEOF", + "expected": { + "isUnparseable": false, + "clauses": [ + { + "operator": "None", + "verb": [ + "id" + ], + "args": [], + "redirects": [] + }, + { + "operator": "None", + "verb": [ + "cat" + ], + "args": [], + "redirects": [ + { + "direction": "In", + "target": "<", + "isDynamicSkip": false + } + ] + } + ], + "syntax": [ + { + "kind": "Block", + "parentIndex": null, + "region": "Unknown", + "childIndex": null, + "sourceStart": 0, + "sourceLength": 22, + "clauseIndex": null, + "groupKind": null, + "listOperator": null + }, + { + "kind": "SimpleCommand", + "parentIndex": 0, + "region": "Root", + "childIndex": 0, + "sourceStart": 0, + "sourceLength": 22, + "clauseIndex": 1, + "groupKind": null, + "listOperator": null + }, + { + "kind": "CommandSubstitution", + "parentIndex": 1, + "region": "Substitution", + "childIndex": 0, + "sourceStart": 12, + "sourceLength": 5, + "clauseIndex": null, + "groupKind": null, + "listOperator": null + }, + { + "kind": "Block", + "parentIndex": 2, + "region": "Substitution", + "childIndex": 0, + "sourceStart": 14, + "sourceLength": 2, + "clauseIndex": null, + "groupKind": null, + "listOperator": null + }, + { + "kind": "SimpleCommand", + "parentIndex": 3, + "region": "Statement", + "childIndex": 0, + "sourceStart": 14, + "sourceLength": 2, + "clauseIndex": 0, + "groupKind": null, + "listOperator": null + } + ], + "commands": [ + { + "clauseIndex": 0, + "immediateRole": "Substitution", + "isComplete": true, + "ancestry": [ + { + "ancestorKind": "Block", + "region": "Root", + "childIndex": 0, + "sourceStart": 0, + "sourceLength": 22 + }, + { + "ancestorKind": "CommandSubstitution", + "region": "Substitution", + "childIndex": 0, + "sourceStart": 12, + "sourceLength": 5 + }, + { + "ancestorKind": "Block", + "region": "Statement", + "childIndex": 0, + "sourceStart": 14, + "sourceLength": 2 + } + ] + }, + { + "clauseIndex": 1, + "immediateRole": "Ordinary", + "isComplete": false, + "ancestry": [ + { + "ancestorKind": "Block", + "region": "Root", + "childIndex": 0, + "sourceStart": 0, + "sourceLength": 22 + } + ] + } + ] + }, + "notes": "The <<- delimiter accepts authored leading tabs without hiding an expanding-body substitution." +} diff --git a/tests/ShellSyntaxTree.Tests/Corpus/bash/203_v03_heredoc_multiple_substitutions.json b/tests/ShellSyntaxTree.Tests/Corpus/bash/203_v03_heredoc_multiple_substitutions.json new file mode 100644 index 0000000..3d4ecfd --- /dev/null +++ b/tests/ShellSyntaxTree.Tests/Corpus/bash/203_v03_heredoc_multiple_substitutions.json @@ -0,0 +1,202 @@ +{ + "name": "v0.3 heredoc sibling command substitutions", + "input": "cat <", + "isDynamicSkip": false + } + ] + } + ], + "syntax": [ + { + "kind": "Block", + "parentIndex": null, + "region": "Unknown", + "childIndex": null, + "sourceStart": 0, + "sourceLength": 32, + "clauseIndex": null, + "groupKind": null, + "listOperator": null + }, + { + "kind": "SimpleCommand", + "parentIndex": 0, + "region": "Root", + "childIndex": 0, + "sourceStart": 0, + "sourceLength": 32, + "clauseIndex": 2, + "groupKind": null, + "listOperator": null + }, + { + "kind": "CommandSubstitution", + "parentIndex": 1, + "region": "Substitution", + "childIndex": 0, + "sourceStart": 10, + "sourceLength": 8, + "clauseIndex": null, + "groupKind": null, + "listOperator": null + }, + { + "kind": "Block", + "parentIndex": 2, + "region": "Substitution", + "childIndex": 0, + "sourceStart": 12, + "sourceLength": 5, + "clauseIndex": null, + "groupKind": null, + "listOperator": null + }, + { + "kind": "SimpleCommand", + "parentIndex": 3, + "region": "Statement", + "childIndex": 0, + "sourceStart": 12, + "sourceLength": 5, + "clauseIndex": 0, + "groupKind": null, + "listOperator": null + }, + { + "kind": "CommandSubstitution", + "parentIndex": 1, + "region": "Substitution", + "childIndex": 1, + "sourceStart": 19, + "sourceLength": 9, + "clauseIndex": null, + "groupKind": null, + "listOperator": null + }, + { + "kind": "Block", + "parentIndex": 5, + "region": "Substitution", + "childIndex": 1, + "sourceStart": 21, + "sourceLength": 6, + "clauseIndex": null, + "groupKind": null, + "listOperator": null + }, + { + "kind": "SimpleCommand", + "parentIndex": 6, + "region": "Statement", + "childIndex": 0, + "sourceStart": 21, + "sourceLength": 6, + "clauseIndex": 1, + "groupKind": null, + "listOperator": null + } + ], + "commands": [ + { + "clauseIndex": 0, + "immediateRole": "Substitution", + "isComplete": true, + "ancestry": [ + { + "ancestorKind": "Block", + "region": "Root", + "childIndex": 0, + "sourceStart": 0, + "sourceLength": 32 + }, + { + "ancestorKind": "CommandSubstitution", + "region": "Substitution", + "childIndex": 0, + "sourceStart": 10, + "sourceLength": 8 + }, + { + "ancestorKind": "Block", + "region": "Statement", + "childIndex": 0, + "sourceStart": 12, + "sourceLength": 5 + } + ] + }, + { + "clauseIndex": 1, + "immediateRole": "Substitution", + "isComplete": true, + "ancestry": [ + { + "ancestorKind": "Block", + "region": "Root", + "childIndex": 0, + "sourceStart": 0, + "sourceLength": 32 + }, + { + "ancestorKind": "CommandSubstitution", + "region": "Substitution", + "childIndex": 1, + "sourceStart": 19, + "sourceLength": 9 + }, + { + "ancestorKind": "Block", + "region": "Statement", + "childIndex": 0, + "sourceStart": 21, + "sourceLength": 6 + } + ] + }, + { + "clauseIndex": 2, + "immediateRole": "Ordinary", + "isComplete": false, + "ancestry": [ + { + "ancestorKind": "Block", + "region": "Root", + "childIndex": 0, + "sourceStart": 0, + "sourceLength": 32 + } + ] + } + ] + }, + "notes": "Sibling substitutions retain authored order before the receiving command." +} diff --git a/tests/ShellSyntaxTree.Tests/Corpus/bash/204_v03_heredoc_header_tail_unparseable.json b/tests/ShellSyntaxTree.Tests/Corpus/bash/204_v03_heredoc_header_tail_unparseable.json new file mode 100644 index 0000000..446cfef --- /dev/null +++ b/tests/ShellSyntaxTree.Tests/Corpus/bash/204_v03_heredoc_header_tail_unparseable.json @@ -0,0 +1,9 @@ +{ + "name": "v0.3 unsupported heredoc header tail fails closed", + "input": "cat < internal static class CorpusJson diff --git a/tools/PwshCorpusTool/Program.cs b/tools/PwshCorpusTool/Program.cs index 7286e55..3a0d21e 100644 --- a/tools/PwshCorpusTool/Program.cs +++ b/tools/PwshCorpusTool/Program.cs @@ -16,6 +16,8 @@ // from the curated CorpusManifest. // check "" Print the parser's expected-AST JSON block for a // command beside the real-pwsh oracle verdict. +// check-bash "" Print Bash parser expectations using the same +// resolver settings as the executable corpus. // The corpus runner pins these resolver knobs; generation must match. var options = new PwshParserOptions @@ -24,6 +26,11 @@ WorkingDirectory = "C:/work", }; var parser = new PwshParser(options); +var bashParser = new BashParser(new BashParserOptions +{ + HomeDirectory = "/home/test", + WorkingDirectory = "/work", +}); if (args.Length == 0) { @@ -37,6 +44,8 @@ return Generate(args.Length > 1 ? args[1] : DefaultCorpusDir()); case "check": return Check(string.Join(' ', args.Skip(1))); + case "check-bash": + return CheckBash(string.Join(' ', args.Skip(1))); default: PrintUsage(); return 1; @@ -116,6 +125,27 @@ int Check(string command) return 0; } +int CheckBash(string command) +{ + if (string.IsNullOrEmpty(command)) + { + Console.Error.WriteLine("check-bash: supply a command string."); + return 1; + } + + var parsed = bashParser.Parse(command); + Console.WriteLine(CorpusJson.BuildEntry( + "check", + command, + parsed, + "ad-hoc check", + outOfScope: false, + includeElements: false, + includeStructure: true, + includeOptionalAssertions: true)); + return 0; +} + static string DefaultCorpusDir() => Path.Combine( "tests", "ShellSyntaxTree.Tests", "Corpus", "powershell"); @@ -125,4 +155,5 @@ static void PrintUsage() Console.WriteLine(); Console.WriteLine(" generate [outputDir] Regenerate the Corpus/powershell/ entries."); Console.WriteLine(" check \"\" Show the parser AST + the real-pwsh verdict."); + Console.WriteLine(" check-bash \"\" Show the Bash parser AST for corpus authoring."); }