From 73ab2931e0635baf12ee5547ce20540e0530ff43 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Fri, 7 Aug 2026 07:43:31 +0000 Subject: [PATCH] Add bounded Bash for-in analysis --- IMPLEMENTATION_PLAN.md | 18 +- SPEC.md | 18 +- .../v0-3-structured-shell-analysis/design.md | 10 + .../v0-3-structured-shell-analysis/tasks.md | 15 +- .../Internal/Bash/Lexing/BashLexer.cs | 72 ++- .../Bash/Parsing/BashCommandParser.cs | 4 +- .../Internal/Bash/Parsing/BashLoopAnalysis.cs | 567 ++++++++++++++++++ .../Bash/Parsing/BashStructuralCoordinator.cs | 547 ++++++++++++++++- .../ShellSyntaxTree.Tests/Corpus/AstAssert.cs | 84 ++- .../Corpus/CorpusRunnerTests.cs | 124 +++- ...49_control_flow_keyword_after_newline.json | 4 +- .../bash/205_v03_for_literal_finite.json | 42 ++ .../206_v03_for_iterator_substitution.json | 69 +++ .../bash/207_v03_for_static_pattern.json | 144 +++++ .../Corpus/bash/208_v03_for_dynamic_root.json | 144 +++++ .../bash/209_v03_for_option_injection.json | 147 +++++ .../bash/210_v03_for_nested_correlation.json | 196 ++++++ .../bash/211_v03_for_pipeline_ancestry.json | 228 +++++++ .../Corpus/bash/212_v03_for_missing_done.json | 9 + .../bash/213_v03_for_candidate_cap_32.json | 177 ++++++ .../214_v03_for_candidate_overflow_33.json | 144 +++++ .../215_v03_for_pattern_parent_traversal.json | 144 +++++ .../216_v03_for_dot_glob_parent_escape.json | 144 +++++ .../Corpus/bash/46_unparseable_for_loop.json | 40 +- .../DesignCorpus/v0.3/bash.json | 10 + .../Lexing/BashLexerTests.cs | 12 + .../Parsing/BashCommandParserTests.cs | 17 +- .../Parsing/BashForInStructuralTests.cs | 435 ++++++++++++++ .../Parsing/ShellValueOracleTests.cs | 135 +++++ tools/PwshCorpusTool/CorpusJson.cs | 120 +++- tools/PwshCorpusTool/Program.cs | 9 +- 31 files changed, 3721 insertions(+), 108 deletions(-) create mode 100644 src/ShellSyntaxTree/Internal/Bash/Parsing/BashLoopAnalysis.cs create mode 100644 tests/ShellSyntaxTree.Tests/Corpus/bash/205_v03_for_literal_finite.json create mode 100644 tests/ShellSyntaxTree.Tests/Corpus/bash/206_v03_for_iterator_substitution.json create mode 100644 tests/ShellSyntaxTree.Tests/Corpus/bash/207_v03_for_static_pattern.json create mode 100644 tests/ShellSyntaxTree.Tests/Corpus/bash/208_v03_for_dynamic_root.json create mode 100644 tests/ShellSyntaxTree.Tests/Corpus/bash/209_v03_for_option_injection.json create mode 100644 tests/ShellSyntaxTree.Tests/Corpus/bash/210_v03_for_nested_correlation.json create mode 100644 tests/ShellSyntaxTree.Tests/Corpus/bash/211_v03_for_pipeline_ancestry.json create mode 100644 tests/ShellSyntaxTree.Tests/Corpus/bash/212_v03_for_missing_done.json create mode 100644 tests/ShellSyntaxTree.Tests/Corpus/bash/213_v03_for_candidate_cap_32.json create mode 100644 tests/ShellSyntaxTree.Tests/Corpus/bash/214_v03_for_candidate_overflow_33.json create mode 100644 tests/ShellSyntaxTree.Tests/Corpus/bash/215_v03_for_pattern_parent_traversal.json create mode 100644 tests/ShellSyntaxTree.Tests/Corpus/bash/216_v03_for_dot_glob_parent_escape.json create mode 100644 tests/ShellSyntaxTree.Tests/Parsing/BashForInStructuralTests.cs diff --git a/IMPLEMENTATION_PLAN.md b/IMPLEMENTATION_PLAN.md index bd60066..8b379e0 100644 --- a/IMPLEMENTATION_PLAN.md +++ b/IMPLEMENTATION_PLAN.md @@ -274,9 +274,21 @@ priorities. cases pin exact syntax, command ancestry, spans, completeness, and literal-versus-expanding behavior; real-Bash output and parse-only oracles independently pin the bounded semantic boundary. -- [ ] Extend Bash substitution discovery to iterables with the complete - `for ... in` vertical slice, then add the Bash substitution cases to the - Netclaw approval matrix. +- [x] Deliver the static-value Bash `for ... in` slice: locked structural + nodes and spans, iterator `$()` discovery, condition-free body + occurrences, exact/finite/pattern/unknown value domains, quote-proved + effective arguments, nested distinct-name correlation, fixed + candidate/depth limits, + strict executable-corpus facts, and real-Bash oracles. Compatibility + leaves preserve authored dynamic operands. Loop binding and cwd + mutation fail closed, loops reached after recognized prior shell-state + mutation fail closed, and occurrence cwd remains Unknown. +- [ ] Design and implement structure-aware Bash abstract-state analysis before + enabling cwd-changing loop bodies or claiming the complete `for ... in` + vertical slice. The parse-order attribution model cannot soundly publish + occurrence cwd across pipelines, conditional lists, substitutions, and + repeated iterations. Keep OpenSpec task 6.5 open, then add the remaining + loop cases and Netclaw approval matrix after that design is reviewed. - [ ] Complete PowerShell `$()` discovery in `foreach` expressions and add the Netclaw approval-matrix cases. The simple-command slice is delivered for ordinary, adjacent, quoted, here-string, redirect, standalone, diff --git a/SPEC.md b/SPEC.md index 5fd8b39..238822c 100644 --- a/SPEC.md +++ b/SPEC.md @@ -1070,9 +1070,11 @@ discovered subset. The lexer produces tokens consumed by the parser. Token kinds: - **WORD** — sequence of non-whitespace, non-operator, non-quote chars. - Example: `git`, `/etc/foo`, `--force`, `~/path`, `$VAR`. Simple - parameter expansion `${VAR}` (no `//` slash) is absorbed into a Word - token; the resolver in §8 decides `Kind`. + Example: `git`, `/etc/foo`, `--force`, `~/path`, `$VAR`. A braced + parameter is absorbed only when its body is a simple shell identifier, + positional parameter, or special parameter. Parameter operators are + unparseable because their operands can contain hidden execution; the + resolver in §8 decides `Kind` for accepted simple forms. - **QUOTED_STRING** — single- or double-quoted string. The lexer strips the quote delimiters from the token value. Example: `"hello world"` becomes the token value `hello world`. @@ -1096,8 +1098,9 @@ The lexer produces tokens consumed by the parser. Token kinds: Expanding-heredoc substitutions use the same opaque fragment semantics but remain attached to the delimiter token rather than entering the ordinary command-token stream. -- **UNPARSEABLE_SENTINEL** — `$((expr))` arithmetic expansion or - `${var//pat/repl}` complex parameter expansion. The lexer skips past +- **UNPARSEABLE_SENTINEL** — `$((expr))` arithmetic expansion or any + operator-bearing parameter expansion such as `${var:-$(cmd)}` or + `${var//pat/repl}`. The lexer skips past the matching close (`))` or `}` respectively) and emits a sentinel whose reason names the rejected construct. The parser consumes this token by setting outer `ParsedCommand.IsUnparseable = true` (see §11). @@ -1729,8 +1732,9 @@ Conditions that produce `IsUnparseable = true`: - Process substitution (`<(cmd)`, `>(cmd)`). - Arithmetic expansion `$((expr))` (per §1 non-goal; lexer emits an UNPARSEABLE_SENTINEL token; parser sets the outer flag). -- Complex parameter expansion `${var//pat/repl}` (per §1 non-goal; same - mechanism). +- Operator-bearing parameter expansion such as `${var:-$(cmd)}` or + `${var//pat/repl}` (per §1 non-goal; same mechanism). Only simple braced + identifiers, positional parameters, and special parameters are accepted. - Recursion depth exceeded on `bash -c` chains (>5 levels). **Diagnostic precedence.** When multiple conditions could fire on a diff --git a/openspec/changes/v0-3-structured-shell-analysis/design.md b/openspec/changes/v0-3-structured-shell-analysis/design.md index c540085..a509610 100644 --- a/openspec/changes/v0-3-structured-shell-analysis/design.md +++ b/openspec/changes/v0-3-structured-shell-analysis/design.md @@ -422,6 +422,16 @@ scope-isolated groups do not leak state. Branches join their possible exit states; loops include the zero-iteration path unless shell semantics prove at least one iteration. +Implementation must run this as a structure-aware abstract-state pass over the +proved syntax tree, not by exposing the compatibility parser's mutable +parse-order cwd attribution. Parse order is not execution-state order for +pipelines or conditional lists, and one symbolic loop-body parse cannot prove +the cwd of later iterations. The compatibility attribution path remains a +v0.2 leaf-construction detail. Until the abstract pass lands, loop cwd mutation +fails closed, recognized shell-state mutation before or inside a loop fails +closed, nested reuse of an active Bash binding name fails closed, and +occurrence `WorkingDirectory` stays `Unknown`. + Bash command substitution executes in an isolated subshell state. State changes affect later commands inside that substitution but never the containing command or following outer commands. PowerShell `$()` evaluates in the current runspace diff --git a/openspec/changes/v0-3-structured-shell-analysis/tasks.md b/openspec/changes/v0-3-structured-shell-analysis/tasks.md index bf8d5bf..d07049a 100644 --- a/openspec/changes/v0-3-structured-shell-analysis/tasks.md +++ b/openspec/changes/v0-3-structured-shell-analysis/tasks.md @@ -68,11 +68,18 @@ ## 6. Bash For-In Vertical Slice -- [ ] 6.1 Parse Bash `for name in literal...; do ...; done` into the locked structural nodes. -- [ ] 6.2 Emit condition-free loop-body occurrences and conservative compatibility clauses. -- [ ] 6.3 Derive exact and finite literal binding domains within the locked candidate cap. -- [ ] 6.4 Substitute a bounded binding only where Bash quoting proves argument boundaries. +- [x] 6.1 Parse Bash `for name in literal...; do ...; done` into the locked structural nodes. +- [x] 6.2 Emit condition-free loop-body occurrences and conservative compatibility clauses. +- [x] 6.3 Derive exact and finite literal binding domains within the locked candidate cap. +- [x] 6.4 Substitute a bounded binding only where Bash quoting proves argument boundaries. - [ ] 6.5 Propagate and conservatively join cwd and supported binding state across zero-or-more loop execution. + - The first static-value slice deliberately leaves occurrence cwd Unknown + and rejects loop shell-state mutation, nested active-binding reuse, or + loops reached after recognized prior shell-state mutation. A separate + structure-aware abstract-state pass is required + before enabling cwd-changing bodies; + mutable parse-order attribution is unsound across pipelines, `&&` / `||`, + substitutions, and repeated iterations. - [ ] 6.6 Cover empty iterables, separators, multiline bodies, redirects, pipelines, nested loops, and wrapper boundaries. - [ ] 6.7 Add adversarial cases for option injection, mutation, unquoted expansion, indirect expansion, substitutions, and cap overflow. - [ ] 6.8 Add sanitized Bash corpus entries and Netclaw allow/prompt/deny integration cases. diff --git a/src/ShellSyntaxTree/Internal/Bash/Lexing/BashLexer.cs b/src/ShellSyntaxTree/Internal/Bash/Lexing/BashLexer.cs index af2d285..0b9da58 100644 --- a/src/ShellSyntaxTree/Internal/Bash/Lexing/BashLexer.cs +++ b/src/ShellSyntaxTree/Internal/Bash/Lexing/BashLexer.cs @@ -844,14 +844,9 @@ private static bool TryConsumeComplexParamExpansion( ReadOnlySpan src, int start, List tokens, out int afterBrace) { // src[start] = '$', src[start+1] = '{'. We need to find the matching - // '}' and decide: simple ${VAR} -> false (let word reader take it); - // ${...//...} or any other "complex" form -> emit UnparseableSentinel. - // - // For v0.1 we treat the presence of a slash inside the braces as the - // single signal of "complex param expansion" (per the locked - // interpretation #2 in the OpenSpec change). Other operators inside - // ${...} (like ${X-default}, ${X#prefix}) fall through to the word - // reader; a future PR can tighten this if needed. + // '}' and decide: a simple variable, positional, or special parameter + // falls through to the word reader. Operators can themselves contain + // executable substitutions, so every other body fails closed. var openBrace = start + 1; var scan = OpaqueRegionScanner.Scan(src, openBrace, '{', '}'); if (!scan.Closed) @@ -870,20 +865,9 @@ private static bool TryConsumeComplexParamExpansion( var endInclusive = scan.EndIndex; var bodyStart = openBrace + 1; var bodyEnd = endInclusive; // exclusive of '}' - var hasSlash = false; - for (var k = bodyStart; k < bodyEnd; k++) + var body = src.Slice(bodyStart, bodyEnd - bodyStart); + if (IsSimpleBracedParameterName(body)) { - if (src[k] == '/') - { - hasSlash = true; - break; - } - } - - if (!hasSlash) - { - // Simple ${VAR} (or ${X-default} etc.). Caller will fall through - // to the word reader and absorb it as part of a Word token. afterBrace = -1; return false; } @@ -895,7 +879,7 @@ private static bool TryConsumeComplexParamExpansion( null, start, length, - "complex parameter expansion '${var//pat/repl}' not supported in v0.1")); + "complex parameter expansion is not supported in v0.3")); afterBrace = start + length; return true; } @@ -1101,9 +1085,9 @@ private static bool TryAppendBashExpansion( expansionLength = scan.EndIndex - start + 1; name = src.Slice(start + 2, expansionLength - 3).ToString(); - if (name.Length == 0 || name.IndexOf('/') >= 0) + if (!IsSimpleBracedParameterName(name.AsSpan())) { - error = "complex parameter expansion '${var//pat/repl}' not supported in v0.1"; + error = "complex parameter expansion is not supported in v0.3"; index += expansionLength; return true; } @@ -1173,6 +1157,46 @@ private static bool IsAllAsciiDigits(string value) return true; } + private static bool IsSimpleBracedParameterName(ReadOnlySpan value) + { + if (value.Length == 0) + { + return false; + } + + if (value.Length == 1 && + value[0] is '?' or '$' or '#' or '-' or '!' or '@' or '*') + { + return true; + } + + var allDigits = true; + for (var index = 0; index < value.Length; index++) + { + allDigits &= value[index] is >= '0' and <= '9'; + } + + if (allDigits) + { + return true; + } + + if (!IsBashIdentifierStart(value[0])) + { + return false; + } + + for (var index = 1; index < value.Length; index++) + { + if (!IsBashIdentifierContinuation(value[index])) + { + return false; + } + } + + return true; + } + private static bool IsBashIdentifierStart(char value) => value == '_' || value is >= 'A' and <= 'Z' or >= 'a' and <= 'z'; diff --git a/src/ShellSyntaxTree/Internal/Bash/Parsing/BashCommandParser.cs b/src/ShellSyntaxTree/Internal/Bash/Parsing/BashCommandParser.cs index 7a51462..644cffa 100644 --- a/src/ShellSyntaxTree/Internal/Bash/Parsing/BashCommandParser.cs +++ b/src/ShellSyntaxTree/Internal/Bash/Parsing/BashCommandParser.cs @@ -395,7 +395,8 @@ private static bool TryDetectAnomaly(IReadOnlyList tokens, out string if (nextIsVerbSlot && t.Kind == BashTokenKind.Word - && BashVerbs.ControlFlowKeywords.Contains(t.Value)) + && BashVerbs.ControlFlowKeywords.Contains(t.Value) + && t.Value is not ("for" or "do" or "done")) { reason = $"control-flow keyword '{t.Value}' is not supported in v0.1"; return true; @@ -558,6 +559,7 @@ private static ClauseResult ParseClauseSegment( // Path evidence wins before the lexical verb heuristic. // The argument pass uses the same classifier. if (fileVerbCarveout + || BashVerbs.ControlFlowKeywords.Contains(t.Value) || BashResolver.LooksLikePath(t.Value) || !BashVerbs.IsVerbLikeToken(t)) { diff --git a/src/ShellSyntaxTree/Internal/Bash/Parsing/BashLoopAnalysis.cs b/src/ShellSyntaxTree/Internal/Bash/Parsing/BashLoopAnalysis.cs new file mode 100644 index 0000000..48309e9 --- /dev/null +++ b/src/ShellSyntaxTree/Internal/Bash/Parsing/BashLoopAnalysis.cs @@ -0,0 +1,567 @@ +// ----------------------------------------------------------------------- +// +// Copyright (C) 2026 - 2026 Aaron Stannard +// +// ----------------------------------------------------------------------- +using System; +using System.Collections.Generic; +using System.Text; +using ShellSyntaxTree.Internal.Bash.Lexing; +using ShellSyntaxTree.Internal.Resolving; + +namespace ShellSyntaxTree.Internal.Bash.Parsing; + +/// +/// Preserves bounded loop-variable proofs while the Bash structural parser +/// still owns lexer provenance. Compatibility leaves deliberately retain +/// their authored dynamic values; only occurrence facts receive these +/// effective domains. +/// +internal sealed class BashLoopBindingContext +{ + private readonly List _bindings = new(); + + internal int Count => _bindings.Count; + + internal bool Contains(string name) => FindExactBinding(name) is not null; + + internal void Push(string name, ShellValueDomain domain) => + _bindings.Add(new BindingFrame(name, domain)); + + internal void Pop() + { + if (_bindings.Count > 0) + { + _bindings.RemoveAt(_bindings.Count - 1); + } + } + + internal BashLoopBindingContext Clone() + { + var clone = new BashLoopBindingContext(); + clone._bindings.AddRange(_bindings); + return clone; + } + + internal ShellValueDomain AnalyzeIterable( + IReadOnlyList words, + BashParserOptions options, + bool workingDirectoryUnknown) + { + if (words.Count == 0) + { + return ShellValueDomain.Unknown; + } + + if (words.Count == 1 && + words[0].ResolverValue is not null && + TryBuildStaticPattern( + words[0].ResolverValue!, + options, + workingDirectoryUnknown, + out var pattern)) + { + return pattern; + } + + var values = new List(words.Count); + var distinct = new HashSet(StringComparer.Ordinal); + foreach (var word in words) + { + if (word.ResolverValue is null || HasUnmodeledBraceExpansion(word)) + { + return ShellValueDomain.Unknown; + } + + ShellValueDomain wordDomain; + if (IsEntirelyLiteral(word.ResolverValue)) + { + wordDomain = new ShellValueDomain + { + Kind = ShellValueDomainKind.Exact, + Values = new[] { word.ResolverValue.Decoded }, + }; + } + else if (!TryAnalyzeEffectiveValue(word.ResolverValue, out wordDomain) || + wordDomain.Kind is not ( + ShellValueDomainKind.Exact or ShellValueDomainKind.FiniteSet)) + { + return ShellValueDomain.Unknown; + } + + foreach (var candidate in wordDomain.Values) + { + if (!distinct.Add(candidate)) + { + continue; + } + + if (distinct.Count > ShellAnalysisLimits.MaxValueCandidates) + { + return ShellValueDomain.Unknown; + } + + values.Add(candidate); + } + } + + return CreateFiniteDomain(values); + } + + internal bool TryAnalyzeEffectiveValue( + ShellValue value, + out ShellValueDomain domain) + { + var referenced = new List(); + var dependentButUnsupported = false; + foreach (var fragment in value.Fragments) + { + if (fragment.Kind != ShellValueFragmentKind.Expansion || + fragment.Expansion is null) + { + continue; + } + + var binding = FindExactBinding(fragment.Expansion.Value); + if (binding is not null) + { + if (!referenced.Contains(binding)) + { + referenced.Add(binding); + } + + if (fragment.Cardinality != ShellValueCardinality.ExactlyOne || + (fragment.AllowedTransforms & ShellLexicalTransform.FieldSplit) != 0) + { + dependentButUnsupported = true; + } + + continue; + } + + if (ReferencesBindingThroughUnsupportedExpansion(fragment.Expansion.Value)) + { + dependentButUnsupported = true; + } + } + + if (referenced.Count == 0 && !dependentButUnsupported) + { + domain = ShellValueDomain.Unknown; + return false; + } + + if (dependentButUnsupported || ContainsUnresolvedFragment(value, referenced)) + { + domain = ShellValueDomain.Unknown; + return true; + } + + if (referenced.Count == 1 && + referenced[0].Domain.Kind == ShellValueDomainKind.Pattern) + { + domain = IsOneBindingExpansion(value, referenced[0]) + ? referenced[0].Domain + : ShellValueDomain.Unknown; + return true; + } + + foreach (var binding in referenced) + { + if (binding.Domain.Kind is not ( + ShellValueDomainKind.Exact or ShellValueDomainKind.FiniteSet)) + { + domain = ShellValueDomain.Unknown; + return true; + } + } + + var candidates = new List(); + var distinct = new HashSet(StringComparer.Ordinal); + var selected = new Dictionary(); + if (!TryComposeCandidates( + value, + referenced, + bindingIndex: 0, + selected, + candidates, + distinct)) + { + domain = ShellValueDomain.Unknown; + return true; + } + + domain = CreateFiniteDomain(candidates); + return true; + } + + private bool TryComposeCandidates( + ShellValue value, + IReadOnlyList bindings, + int bindingIndex, + Dictionary selected, + List candidates, + HashSet distinct) + { + if (bindingIndex == bindings.Count) + { + var rendered = Render(value, selected); + if (rendered is null) + { + return false; + } + + if (distinct.Add(rendered)) + { + if (distinct.Count > ShellAnalysisLimits.MaxValueCandidates) + { + return false; + } + + candidates.Add(rendered); + } + + return true; + } + + var binding = bindings[bindingIndex]; + foreach (var candidate in binding.Domain.Values) + { + selected[binding] = candidate; + if (!TryComposeCandidates( + value, + bindings, + bindingIndex + 1, + selected, + candidates, + distinct)) + { + return false; + } + } + + selected.Remove(binding); + return true; + } + + private string? Render( + ShellValue value, + IReadOnlyDictionary selected) + { + var rendered = new StringBuilder(value.Decoded.Length); + foreach (var fragment in value.Fragments) + { + if (fragment.Kind == ShellValueFragmentKind.Literal) + { + rendered.Append(fragment.Value); + continue; + } + + if (fragment.Kind != ShellValueFragmentKind.Expansion || + fragment.Expansion is null) + { + return null; + } + + var binding = FindExactBinding(fragment.Expansion.Value); + if (binding is null || !selected.TryGetValue(binding, out var candidate)) + { + return null; + } + + rendered.Append(candidate); + } + + return rendered.ToString(); + } + + private bool ContainsUnresolvedFragment( + ShellValue value, + IReadOnlyList referenced) + { + foreach (var fragment in value.Fragments) + { + if (fragment.Kind == ShellValueFragmentKind.Literal) + { + continue; + } + + if (fragment.Kind != ShellValueFragmentKind.Expansion || + fragment.Expansion is null) + { + return true; + } + + var binding = FindExactBinding(fragment.Expansion.Value); + if (binding is null || !ContainsReference(referenced, binding)) + { + return true; + } + } + + return false; + } + + private static bool ContainsReference( + IReadOnlyList bindings, + BindingFrame expected) + { + foreach (var binding in bindings) + { + if (object.ReferenceEquals(binding, expected)) + { + return true; + } + } + + return false; + } + + private BindingFrame? FindExactBinding(ShellExpansionReference expansion) + { + if (expansion.Kind != ShellExpansionKind.Variable || expansion.Name is null) + { + return null; + } + + for (var index = _bindings.Count - 1; index >= 0; index--) + { + if (string.Equals(_bindings[index].Name, expansion.Name, StringComparison.Ordinal)) + { + return _bindings[index]; + } + } + + return null; + } + + private BindingFrame? FindExactBinding(string name) + { + for (var index = _bindings.Count - 1; index >= 0; index--) + { + if (string.Equals(_bindings[index].Name, name, StringComparison.Ordinal)) + { + return _bindings[index]; + } + } + + return null; + } + + private bool ReferencesBindingThroughUnsupportedExpansion( + ShellExpansionReference expansion) + { + if (expansion.Kind != ShellExpansionKind.Variable || expansion.Name is null) + { + return false; + } + + foreach (var binding in _bindings) + { + var name = expansion.Name; + if (name.Length > binding.Name.Length && + (name[0] == '!' && + string.Equals(name.Substring(1), binding.Name, StringComparison.Ordinal) || + name.StartsWith(binding.Name, StringComparison.Ordinal) && + IsParameterOperator(name[binding.Name.Length]))) + { + return true; + } + } + + return false; + } + + private static bool IsParameterOperator(char value) => + value is '[' or ':' or '-' or '+' or '=' or '?' or '%' or '#' or '/' or '^' or ','; + + private BindingFrame? BindingFor(ShellValueFragment fragment) => + fragment.Expansion is null ? null : FindExactBinding(fragment.Expansion.Value); + + private bool IsOneBindingExpansion(ShellValue value, BindingFrame binding) + { + var expansionCount = 0; + foreach (var fragment in value.Fragments) + { + if (fragment.Kind == ShellValueFragmentKind.Literal && fragment.Value.Length == 0) + { + continue; + } + + if (fragment.Kind != ShellValueFragmentKind.Expansion || + !object.ReferenceEquals(BindingFor(fragment), binding)) + { + return false; + } + + expansionCount++; + } + + return expansionCount == 1; + } + + private static bool TryBuildStaticPattern( + ShellValue value, + BashParserOptions options, + bool workingDirectoryUnknown, + out ShellValueDomain pattern) + { + var containsGlob = false; + foreach (var fragment in value.Fragments) + { + if (fragment.Kind == ShellValueFragmentKind.Literal) + { + continue; + } + + if (fragment.Kind != ShellValueFragmentKind.Expansion || + fragment.Expansion is null || + fragment.Expansion.Value.Kind != ShellExpansionKind.Glob) + { + pattern = ShellValueDomain.Unknown; + return false; + } + + containsGlob = true; + } + + var authored = value.Decoded; + if (!containsGlob || + !BashResolver.LooksLikePath(authored) || + authored.IndexOf("://", StringComparison.Ordinal) >= 0 || + ContainsParentTraversal(authored) || + HasGlobBearingDotSegment(authored)) + { + pattern = ShellValueDomain.Unknown; + return false; + } + + var firstGlob = FirstGlobIndex(authored); + var slash = authored.LastIndexOf('/', firstGlob); + var directory = slash switch + { + < 0 => ".", + 0 => "/", + _ => authored.Substring(0, slash), + }; + var resolved = BashResolver.Resolve( + ShellValue.Literal(directory), + treatAsPath: true, + options, + workingDirectoryUnknown, + ShellResolutionConsumer.BashArgument); + if (resolved.Resolved is null) + { + pattern = ShellValueDomain.Unknown; + return false; + } + + pattern = new ShellValueDomain + { + Kind = ShellValueDomainKind.Pattern, + Pattern = authored, + CoveringDirectory = resolved.Resolved, + }; + return true; + } + + private static int FirstGlobIndex(string value) + { + for (var index = 0; index < value.Length; index++) + { + if (value[index] is '*' or '?' or '[') + { + return index; + } + } + + return value.Length; + } + + private static bool ContainsParentTraversal(string path) + { + var segments = path.Replace('\\', '/').Split('/'); + foreach (var segment in segments) + { + if (string.Equals(segment, "..", StringComparison.Ordinal)) + { + return true; + } + } + + return false; + } + + private static bool HasGlobBearingDotSegment(string path) + { + var segments = path.Replace('\\', '/').Split('/'); + foreach (var segment in segments) + { + if (segment.Length > 0 && + segment[0] == '.' && + FirstGlobIndex(segment) < segment.Length) + { + return true; + } + } + + return false; + } + + private static bool IsEntirelyLiteral(ShellValue value) + { + foreach (var fragment in value.Fragments) + { + if (fragment.Kind != ShellValueFragmentKind.Literal) + { + return false; + } + } + + return true; + } + + private static bool HasUnmodeledBraceExpansion(BashToken token) => + token.Kind == BashTokenKind.Word && + (token.Value.IndexOf('{') >= 0 || token.Value.IndexOf('}') >= 0); + + private static ShellValueDomain CreateFiniteDomain(IReadOnlyList values) => + values.Count switch + { + 1 => new ShellValueDomain + { + Kind = ShellValueDomainKind.Exact, + Values = new[] { values[0] }, + }, + >= 2 and <= 32 => new ShellValueDomain + { + Kind = ShellValueDomainKind.FiniteSet, + Values = Copy(values), + }, + _ => ShellValueDomain.Unknown, + }; + + private static string[] Copy(IReadOnlyList values) + { + var copy = new string[values.Count]; + for (var index = 0; index < values.Count; index++) + { + copy[index] = values[index]; + } + + return copy; + } + + private sealed class BindingFrame + { + internal BindingFrame(string name, ShellValueDomain domain) + { + Name = name; + Domain = domain; + } + + internal string Name { get; } + + internal ShellValueDomain Domain { get; } + } +} diff --git a/src/ShellSyntaxTree/Internal/Bash/Parsing/BashStructuralCoordinator.cs b/src/ShellSyntaxTree/Internal/Bash/Parsing/BashStructuralCoordinator.cs index 89d13b5..75d2217 100644 --- a/src/ShellSyntaxTree/Internal/Bash/Parsing/BashStructuralCoordinator.cs +++ b/src/ShellSyntaxTree/Internal/Bash/Parsing/BashStructuralCoordinator.cs @@ -5,6 +5,7 @@ // ----------------------------------------------------------------------- using System; using System.Collections.Generic; +using System.Runtime.CompilerServices; using ShellSyntaxTree.Internal.Bash.Lexing; using ShellSyntaxTree.Internal.Parsing; using ShellSyntaxTree.Internal.Resolving; @@ -37,11 +38,7 @@ private static ParsedCommand ParseStructured( if (!ShellSyntaxProjection.TryProject( syntax, - simple => new CommandOccurrenceFacts - { - IsComplete = simple.Clause.Redirects.Count == 0 && - !HasUnexpandedCommandString(simple.Clause), - }, + coordinator.GetFacts, out var projection)) { return StructuralFailure( @@ -83,8 +80,13 @@ private sealed class StructuralCoordinator private readonly int _sourceStart; private readonly int _sourceLength; private readonly CdAttributionContext _attribution = new(); + private readonly BashLoopBindingContext _bindings; + private readonly Dictionary _facts = + new(ClauseReferenceComparer.Instance); private int _position; private int _subshellDepth; + private int _loopDepth; + private bool _hasUnmodeledShellStateMutation; internal StructuralCoordinator( string source, @@ -94,7 +96,9 @@ internal StructuralCoordinator( int structuralDepth, bool markBashCWrapped, int sourceStart, - int sourceLength) + int sourceLength, + BashLoopBindingContext? bindings = null, + bool hasUnmodeledShellStateMutation = false) { _source = source; _tokens = tokens; @@ -104,8 +108,21 @@ internal StructuralCoordinator( _markBashCWrapped = markBashCWrapped; _sourceStart = sourceStart; _sourceLength = sourceLength; + _bindings = bindings ?? new BashLoopBindingContext(); + _hasUnmodeledShellStateMutation = hasUnmodeledShellStateMutation; } + internal CommandOccurrenceFacts GetFacts(SimpleCommandSyntax simple) => + _facts.TryGetValue(simple.Clause, out var facts) + ? facts + : CreateDefaultFacts(simple.Clause); + + private CommandOccurrenceFacts CreateDefaultFacts(Clause clause) => new() + { + IsComplete = clause.Redirects.Count == 0 && + !HasUnexpandedCommandString(clause), + }; + internal bool TryParse(out ShellBlockSyntax syntax, out string? error) { SkipNewlines(); @@ -122,6 +139,7 @@ internal bool TryParse(out ShellBlockSyntax syntax, out string? error) if (!TryParseList( stopAtRightParen: false, + stopWord: null, CompoundOperator.None, out var command, out error) || @@ -152,6 +170,7 @@ internal bool TryParse(out ShellBlockSyntax syntax, out string? error) private bool TryParseList( bool stopAtRightParen, + string? stopWord, CompoundOperator firstCompatibilityOperator, out ShellSyntaxNode? command, out string? error) @@ -159,7 +178,9 @@ private bool TryParseList( command = null; error = null; SkipNewlines(); - if (_position == _tokens.Count || stopAtRightParen && IsOperator(")")) + if (_position == _tokens.Count || + stopAtRightParen && IsOperator(")") || + stopWord is not null && IsWord(stopWord)) { return true; } @@ -181,6 +202,11 @@ private bool TryParseList( break; } + if (stopWord is not null && IsWord(stopWord)) + { + break; + } + if (IsOperator(")")) { error = $"unbalanced parens at position {_tokens[_position].SourceStart}"; @@ -194,7 +220,9 @@ private bool TryParseList( } SkipNewlines(); - if (_position == _tokens.Count || stopAtRightParen && IsOperator(")")) + if (_position == _tokens.Count || + stopAtRightParen && IsOperator(")") || + stopWord is not null && IsWord(stopWord)) { if (listOperator == CompoundOperator.Sequence) { @@ -302,6 +330,17 @@ private bool TryParseCommand( return TryParseSubshell(compatibilityOperator, out command, out error); } + if (IsWord("for")) + { + return TryParseForIn(out command, out error); + } + + if (IsWord("do") || IsWord("done")) + { + error = $"stray Bash control-flow keyword '{_tokens[_position].Value}'"; + return false; + } + if (IsOperator(")") || IsListOperator(_tokens[_position]) || IsOperator("|")) { error = $"unexpected operator at position {_tokens[_position].SourceStart}"; @@ -334,6 +373,7 @@ private bool TryParseCommand( if (!TryCollectCommandSubstitutions( segmentTokens, + rejectCommandNameSubstitution: true, out var substitutionFragments, out error)) { @@ -358,7 +398,7 @@ private bool TryParseCommand( innerCommand!, _options, _bashCDepth + 1, - _structuralDepth + _subshellDepth + 1, + _structuralDepth + _subshellDepth + _loopDepth + 1, markBashCWrapped: true); if (inner.IsUnparseable) { @@ -388,6 +428,12 @@ private bool TryParseCommand( SourceLength = lastToken.SourceStart + lastToken.SourceLength - firstToken.SourceStart, }; + if (!TryRegisterDecodedFacts(inner, body, out error)) + { + command = null; + return false; + } + return true; } @@ -429,6 +475,15 @@ private bool TryParseCommand( IsCommandStringWrapped = _markBashCWrapped, }; var emitted = AttachAttributionArg(clause, _attribution); + var isPotentialStateMutation = IsPotentialBindingMutation(emitted); + if (_bindings.Count > 0 && isPotentialStateMutation) + { + error = "Bash loop binding mutation is not supported for bounded analysis"; + return false; + } + + _hasUnmodeledShellStateMutation |= isPotentialStateMutation; + if (!TryParseCommandSubstitutions( substitutionFragments, effectiveOptions, @@ -455,13 +510,209 @@ private bool TryParseCommand( token.HeredocSourceEnd ?? token.SourceStart + token.SourceLength); } - command = new SimpleCommandSyntax + var simple = new SimpleCommandSyntax { Clause = emitted, Substitutions = substitutions, SourceStart = firstSource.SourceStart, SourceLength = sourceEnd - firstSource.SourceStart, }; + RegisterFacts(simple, segmentTokens); + command = simple; + return true; + } + + private bool TryParseForIn( + out ShellSyntaxNode? command, + out string? error) + { + command = null; + error = null; + if (_attribution.HasAttribution || _hasUnmodeledShellStateMutation) + { + error = "Bash for-in after prior shell-state mutation requires structure-aware state analysis"; + return false; + } + + if (_structuralDepth + _subshellDepth + _loopDepth >= + ShellAnalysisLimits.MaxStructuralNesting) + { + error = "Bash structural nesting depth exceeded (>16)"; + return false; + } + + var forToken = _tokens[_position++]; + if (_position == _tokens.Count || + _tokens[_position].Kind != BashTokenKind.Word || + !HasExactLiteralValue(_tokens[_position]) || + !string.Equals( + SourceSlice(_source, _tokens[_position]), + _tokens[_position].Value, + StringComparison.Ordinal) || + !IsBashIdentifier(_tokens[_position].Value)) + { + error = "Bash for-in loop requires a shell-identifier binding"; + return false; + } + + var bindingToken = _tokens[_position++]; + if (_bindings.Contains(bindingToken.Value)) + { + error = "nested Bash for-in binding reuse requires state propagation"; + return false; + } + + if (!IsWord("in")) + { + error = "Bash for-in loop requires the contextual 'in' keyword"; + return false; + } + + _position++; + var iterableWords = new List(); + while (_position < _tokens.Count && !IsListTerminator()) + { + var token = _tokens[_position]; + if (token.Kind is not ( + BashTokenKind.Word or + BashTokenKind.QuotedString or + BashTokenKind.OpaqueSubstitution)) + { + error = $"unsupported Bash for-in iterable token at position {token.SourceStart}"; + return false; + } + + if (iterableWords.Count > 0 && + iterableWords[iterableWords.Count - 1].SourceStart + + iterableWords[iterableWords.Count - 1].SourceLength == token.SourceStart) + { + var previous = iterableWords[iterableWords.Count - 1]; + var previousValue = previous.ResolverValue ?? + ShellValue.Literal( + previous.Value, + previous.SourceStart, + previous.SourceLength); + var currentValue = token.ResolverValue ?? + ShellValue.Literal(token.Value, token.SourceStart, token.SourceLength); + iterableWords[iterableWords.Count - 1] = new BashToken( + BashTokenKind.Word, + previous.Value + token.Value, + null, + previous.SourceStart, + token.SourceStart + token.SourceLength - previous.SourceStart, + null) + { + ResolverValue = ShellValue.Concat(new[] { previousValue, currentValue }), + }; + } + else + { + iterableWords.Add(token); + } + + _position++; + } + + if (_position == _tokens.Count) + { + error = "Bash for-in loop is missing its list terminator and 'do'"; + return false; + } + + var terminator = _tokens[_position++]; + if (terminator.Kind == BashTokenKind.Whitespace) + { + SkipNewlines(); + } + + if (!IsWord("do")) + { + error = "Bash for-in loop is missing 'do'"; + return false; + } + + var doToken = _tokens[_position++]; + var iterableStart = iterableWords.Count == 0 + ? terminator.SourceStart + : iterableWords[0].SourceStart; + var iterableEnd = iterableWords.Count == 0 + ? iterableStart + : iterableWords[iterableWords.Count - 1].SourceStart + + iterableWords[iterableWords.Count - 1].SourceLength; + var iterableDomain = _bindings.AnalyzeIterable( + iterableWords, + _options, + workingDirectoryUnknown: false); + if (!TryParseIteratorSubstitutions( + iterableWords, + CurrentOptions(), + iterableStart, + iterableEnd - iterableStart, + out var iteratorCommands, + out error)) + { + return false; + } + + _bindings.Push(bindingToken.Value, iterableDomain); + _loopDepth++; + var parsedBody = TryParseList( + stopAtRightParen: false, + stopWord: "done", + CompoundOperator.None, + out var bodyCommand, + out error); + _loopDepth--; + _bindings.Pop(); + if (!parsedBody) + { + return false; + } + + if (bodyCommand is null) + { + error = "Bash for-in loop body cannot be empty"; + return false; + } + + if (!IsWord("done")) + { + error = "Bash for-in loop is missing 'done'"; + return false; + } + + var doneToken = _tokens[_position++]; + var body = new ShellBlockSyntax + { + Statements = new[] { bodyCommand }, + SourceStart = doToken.SourceStart + doToken.SourceLength, + SourceLength = doneToken.SourceStart - + doToken.SourceStart - doToken.SourceLength, + }; + command = new ForEachSyntax + { + Binding = new LoopBindingSyntax + { + Name = bindingToken.Value, + Source = new ShellSourceFragment + { + Raw = SourceSlice(_source, bindingToken), + SourceStart = bindingToken.SourceStart, + SourceLength = bindingToken.SourceLength, + }, + }, + Iterable = new ShellSourceFragment + { + Raw = _source.Substring(iterableStart, iterableEnd - iterableStart), + SourceStart = iterableStart, + SourceLength = iterableEnd - iterableStart, + }, + IteratorCommands = iteratorCommands, + Body = body, + SourceStart = forToken.SourceStart, + SourceLength = doneToken.SourceStart + doneToken.SourceLength - + forToken.SourceStart, + }; return true; } @@ -470,7 +721,7 @@ private bool TryParseSubshell( out ShellSyntaxNode? command, out string? error) { - if (_structuralDepth + _subshellDepth >= + if (_structuralDepth + _subshellDepth + _loopDepth >= ShellAnalysisLimits.MaxStructuralNesting) { command = null; @@ -479,15 +730,18 @@ private bool TryParseSubshell( } var open = _tokens[_position++]; + var outerMutationState = _hasUnmodeledShellStateMutation; _attribution.PushForSubshell(); _subshellDepth++; var parsed = TryParseList( stopAtRightParen: true, + stopWord: null, compatibilityOperator, out var bodyCommand, out error); _subshellDepth--; _attribution.PopForSubshell(); + _hasUnmodeledShellStateMutation = outerMutationState; if (!parsed) { @@ -572,14 +826,42 @@ private bool IsOperator(string value) => _tokens[_position].Kind == BashTokenKind.Operator && string.Equals(_tokens[_position].OperatorText, value, StringComparison.Ordinal); + private bool IsWord(string value) => + _position < _tokens.Count && + _tokens[_position].Kind == BashTokenKind.Word && + HasExactLiteralValue(_tokens[_position]) && + string.Equals( + SourceSlice(_source, _tokens[_position]), + value, + StringComparison.Ordinal) && + string.Equals(_tokens[_position].Value, value, StringComparison.Ordinal); + + private bool IsListTerminator() => + IsOperator(";") || + _position < _tokens.Count && + _tokens[_position].Kind == BashTokenKind.Whitespace && + _tokens[_position].IsStatementSeparator; + + private BashParserOptions CurrentOptions() => + _attribution.HasAttribution && !_attribution.IsDynamic + ? new BashParserOptions + { + HomeDirectory = _options.HomeDirectory, + WorkingDirectory = _attribution.ResolvedCwd, + } + : _options; + private bool TryCollectCommandSubstitutions( IReadOnlyList tokens, + bool rejectCommandNameSubstitution, out IReadOnlyList substitutions, out string? error) { var discovered = new List(); - var commandNameEnd = tokens[0].SourceStart + tokens[0].SourceLength; - for (var index = 1; index < tokens.Count; index++) + var commandNameEnd = tokens.Count == 0 + ? _sourceStart + : tokens[0].SourceStart + tokens[0].SourceLength; + for (var index = 1; rejectCommandNameSubstitution && index < tokens.Count; index++) { var token = tokens[index]; if (token.Kind == BashTokenKind.Operator || @@ -637,7 +919,8 @@ private bool TryCollectCommandSubstitutions( return false; } - if (fragment.SourceStart < commandNameEnd) + if (rejectCommandNameSubstitution && + fragment.SourceStart < commandNameEnd) { substitutions = Array.Empty(); error = "Bash command-name substitution is not supported"; @@ -654,6 +937,49 @@ private bool TryCollectCommandSubstitutions( return true; } + private bool TryParseIteratorSubstitutions( + IReadOnlyList iterableWords, + BashParserOptions options, + int sourceStart, + int sourceLength, + out ShellBlockSyntax iteratorCommands, + out string? error) + { + if (!TryCollectCommandSubstitutions( + iterableWords, + rejectCommandNameSubstitution: false, + out var fragments, + out error)) + { + iteratorCommands = new ShellBlockSyntax(); + return false; + } + + if (!TryParseCommandSubstitutions( + fragments, + options, + out var substitutions, + out error)) + { + iteratorCommands = new ShellBlockSyntax(); + return false; + } + + var statements = new ShellSyntaxNode[substitutions.Count]; + for (var index = 0; index < substitutions.Count; index++) + { + statements[index] = substitutions[index]; + } + + iteratorCommands = new ShellBlockSyntax + { + Statements = statements, + SourceStart = sourceStart, + SourceLength = sourceLength, + }; + return true; + } + private bool HasAssignmentPrefix(IReadOnlyList tokens) { var spelling = _source.Substring(tokens[0].SourceStart, tokens[0].SourceLength) @@ -714,7 +1040,7 @@ private bool TryParseCommandSubstitutions( return true; } - if (_structuralDepth + _subshellDepth + 1 > + if (_structuralDepth + _subshellDepth + _loopDepth + 1 > ShellAnalysisLimits.MaxStructuralNesting) { substitutions = Array.Empty(); @@ -785,11 +1111,196 @@ private bool TryParseSubstitutionBody( shifted, options, _bashCDepth, - _structuralDepth + _subshellDepth + 1, + _structuralDepth + _subshellDepth + _loopDepth + 1, _markBashCWrapped, sourceStart, - sourceLength); - return coordinator.TryParse(out body, out error); + sourceLength, + _bindings.Clone(), + _hasUnmodeledShellStateMutation); + if (!coordinator.TryParse(out body, out error)) + { + return false; + } + + MergeFacts(coordinator); + return true; + } + + private void RegisterFacts( + SimpleCommandSyntax simple, + IReadOnlyList sourceTokens) + { + var effective = new List(); + for (var elementIndex = 0; + elementIndex < simple.Clause.Elements.Count; + elementIndex++) + { + var element = simple.Clause.Elements[elementIndex]; + if (element.Role != ClauseElementRole.Argument || + !TryGetElementValue(element, sourceTokens, out var value) || + !_bindings.TryAnalyzeEffectiveValue(value, out var domain)) + { + continue; + } + + effective.Add(new EffectiveArgument + { + ClauseElementIndex = elementIndex, + Value = domain, + }); + } + + _facts.Add(simple.Clause, new CommandOccurrenceFacts + { + EffectiveArguments = effective.ToArray(), + IsComplete = simple.Clause.Redirects.Count == 0 && + !HasUnexpandedCommandString(simple.Clause), + }); + } + + private static bool TryGetElementValue( + ClauseElement element, + IReadOnlyList sourceTokens, + out ShellValue value) + { + value = ShellValue.Literal(string.Empty); + if (element.SourceStart is null || element.SourceLength is null) + { + return false; + } + + var elementStart = element.SourceStart.Value; + var elementEnd = elementStart + element.SourceLength.Value; + var values = new List(); + var coveredStart = -1; + var coveredEnd = -1; + foreach (var token in sourceTokens) + { + var tokenEnd = token.SourceStart + token.SourceLength; + if (token.SourceStart < elementStart || tokenEnd > elementEnd) + { + continue; + } + + coveredStart = coveredStart < 0 ? token.SourceStart : coveredStart; + coveredEnd = tokenEnd; + values.Add(token.ResolverValue ?? + ShellValue.Literal(token.Value, token.SourceStart, token.SourceLength)); + } + + if (values.Count == 0 || + coveredStart != elementStart || + coveredEnd != elementEnd) + { + return false; + } + + value = values.Count == 1 ? values[0] : ShellValue.Concat(values); + return true; + } + + private void MergeFacts(StructuralCoordinator nested) + { + foreach (var pair in nested._facts) + { + _facts.Add(pair.Key, pair.Value); + } + + } + + private bool TryRegisterDecodedFacts( + ParsedCommand inner, + ShellBlockSyntax clonedBody, + out string? error) + { + if (!ShellSyntaxProjection.TryProject(clonedBody, out var clonedProjection) || + clonedProjection.Commands.Count != inner.Commands.Count) + { + error = "decoded bash -c facts could not be mapped safely"; + return false; + } + + for (var index = 0; index < inner.Commands.Count; index++) + { + var source = inner.Commands[index]; + _facts.Add(clonedProjection.Commands[index].Clause, new CommandOccurrenceFacts + { + EffectiveArguments = source.EffectiveArguments, + WorkingDirectory = source.WorkingDirectory, + Redirects = source.Redirects, + IsComplete = source.IsComplete, + }); + } + + error = null; + return true; + } + + private static bool IsPotentialBindingMutation(Clause clause) + { + if (clause.Verb.Tokens.Count == 0) + { + return true; + } + + var verb = clause.Verb.Tokens[0]; + if (verb is "unset" or "read" or "readarray" or "mapfile" or + "declare" or "typeset" or "local" or "export" or "readonly" or + "let" or "eval" or "." or "source" or "getopts" or "set" or + "cd" or "chdir" or "pushd" or "popd" or "trap") + { + return true; + } + + // These dispatch builtins can invoke every mutator above after + // option processing. Until their executable grammar is modeled, + // accepting them would let `command unset f` retain stale facts. + if (verb is "command" or "builtin") + { + return true; + } + + if (!string.Equals(verb, "printf", StringComparison.Ordinal)) + { + return false; + } + + foreach (var argument in clause.Args) + { + if (string.Equals(argument.Raw, "-v", StringComparison.Ordinal)) + { + return true; + } + } + + return false; + } + + private static bool IsBashIdentifier(string value) + { + if (value.Length == 0 || !IsBashIdentifierStart(value[0])) + { + return false; + } + + for (var index = 1; index < value.Length; index++) + { + if (!IsBashIdentifierContinuation(value[index])) + { + return false; + } + } + + return true; + } + + private sealed class ClauseReferenceComparer : IEqualityComparer + { + internal static ClauseReferenceComparer Instance { get; } = new(); + + public bool Equals(Clause? x, Clause? y) => object.ReferenceEquals(x, y); + + public int GetHashCode(Clause obj) => RuntimeHelpers.GetHashCode(obj); } } diff --git a/tests/ShellSyntaxTree.Tests/Corpus/AstAssert.cs b/tests/ShellSyntaxTree.Tests/Corpus/AstAssert.cs index 9b57622..06fc228 100644 --- a/tests/ShellSyntaxTree.Tests/Corpus/AstAssert.cs +++ b/tests/ShellSyntaxTree.Tests/Corpus/AstAssert.cs @@ -165,7 +165,14 @@ private static void AssertSyntaxEqual( wanted.SourceLength != observed.SourceLength || wanted.ClauseIndex != observed.ClauseIndex || wanted.GroupKind != observed.GroupKind || - wanted.ListOperator != observed.ListOperator) + wanted.ListOperator != observed.ListOperator || + wanted.BindingName != observed.BindingName || + wanted.BindingRaw != observed.BindingRaw || + wanted.BindingSourceStart != observed.BindingSourceStart || + wanted.BindingSourceLength != observed.BindingSourceLength || + wanted.IterableRaw != observed.IterableRaw || + wanted.IterableSourceStart != observed.IterableSourceStart || + wanted.IterableSourceLength != observed.IterableSourceLength) { throw new XunitException( prefix + $"syntax[{index}]: expected={Summarize(wanted)}, " @@ -357,6 +364,60 @@ private static void AssertCommandsEqual( prefix + $"commands[{index}].ancestry[{frameIndex}] differs"); } } + + if (wanted.EffectiveArguments is not null) + { + if (wanted.EffectiveArguments.Count != observed.EffectiveArguments.Count) + { + throw new XunitException( + prefix + $"commands[{index}].effectiveArguments.count differs"); + } + + for (var effectiveIndex = 0; + effectiveIndex < wanted.EffectiveArguments.Count; + effectiveIndex++) + { + var expectedEffective = wanted.EffectiveArguments[effectiveIndex]; + var actualEffective = observed.EffectiveArguments[effectiveIndex]; + if (expectedEffective.ClauseElementIndex != actualEffective.ClauseElementIndex) + { + throw new XunitException( + prefix + $"commands[{index}].effectiveArguments[{effectiveIndex}].clauseElementIndex differs"); + } + + AssertValueDomainEqual( + expectedEffective.Value, + actualEffective.Value, + prefix + $"commands[{index}].effectiveArguments[{effectiveIndex}].value"); + } + } + + if (wanted.WorkingDirectory is not null) + { + AssertValueDomainEqual( + wanted.WorkingDirectory, + observed.WorkingDirectory, + prefix + $"commands[{index}].workingDirectory"); + } + } + } + + private static void AssertValueDomainEqual( + ExpectedValueDomain expected, + ShellValueDomain actual, + string path) + { + var expectedValues = expected.Values ?? new List(); + if (expected.Kind != actual.Kind || + !expectedValues.SequenceEqual(actual.Values) || + expected.Pattern != actual.Pattern || + expected.CoveringDirectory != actual.CoveringDirectory) + { + throw new XunitException( + $"{path}: expected={expected.Kind}[{string.Join(",", expectedValues)}] " + + $"pattern={expected.Pattern}, covering={expected.CoveringDirectory}; " + + $"actual={actual.Kind}[{string.Join(",", actual.Values)}] " + + $"pattern={actual.Pattern}, covering={actual.CoveringDirectory}"); } } @@ -376,6 +437,7 @@ private static void AppendSyntax( } var clause = (node as SimpleCommandSyntax)?.Clause; + var forEachNode = node as ForEachSyntax; int? clauseIndex = clause is null ? null : FindClauseIndex(clauses, clause); var currentIndex = nodes.Count; nodes.Add(new ActualSyntaxNode( @@ -388,6 +450,13 @@ private static void AppendSyntax( clauseIndex, (node as GroupSyntax)?.GroupKind, listOperator, + forEachNode?.Binding.Name, + forEachNode?.Binding.Source.Raw, + forEachNode?.Binding.Source.SourceStart, + forEachNode?.Binding.Source.SourceLength, + forEachNode?.Iterable.Raw, + forEachNode?.Iterable.SourceStart, + forEachNode?.Iterable.SourceLength, clause)); switch (node) @@ -573,12 +642,14 @@ private static int FindClauseIndex(IReadOnlyList clauses, Clause clause) private static string Summarize(ExpectedSyntaxNode node) => $"{{kind={node.Kind}, parent={node.ParentIndex}, region={node.Region}, " + $"child={node.ChildIndex}, span={node.SourceStart}:{node.SourceLength}, " - + $"clause={node.ClauseIndex}, group={node.GroupKind}, listOp={node.ListOperator}}}"; + + $"clause={node.ClauseIndex}, group={node.GroupKind}, listOp={node.ListOperator}, " + + $"binding={node.BindingName}, iterable={node.IterableRaw}}}"; private static string Summarize(ActualSyntaxNode node) => $"{{kind={node.Kind}, parent={node.ParentIndex}, region={node.Region}, " + $"child={node.ChildIndex}, span={node.SourceStart}:{node.SourceLength}, " - + $"clause={node.ClauseIndex}, group={node.GroupKind}, listOp={node.ListOperator}}}"; + + $"clause={node.ClauseIndex}, group={node.GroupKind}, listOp={node.ListOperator}, " + + $"binding={node.BindingName}, iterable={node.IterableRaw}}}"; private sealed record ActualSyntaxNode( ShellSyntaxKind Kind, @@ -590,6 +661,13 @@ private sealed record ActualSyntaxNode( int? ClauseIndex, ShellGroupKind? GroupKind, CompoundOperator? ListOperator, + string? BindingName, + string? BindingRaw, + int? BindingSourceStart, + int? BindingSourceLength, + string? IterableRaw, + int? IterableSourceStart, + int? IterableSourceLength, Clause? Clause); private static void AssertClauseEqual(ExpectedClause expected, Clause actual, string path) diff --git a/tests/ShellSyntaxTree.Tests/Corpus/CorpusRunnerTests.cs b/tests/ShellSyntaxTree.Tests/Corpus/CorpusRunnerTests.cs index c645aff..1b01e1f 100644 --- a/tests/ShellSyntaxTree.Tests/Corpus/CorpusRunnerTests.cs +++ b/tests/ShellSyntaxTree.Tests/Corpus/CorpusRunnerTests.cs @@ -71,10 +71,11 @@ private static void AssertAuthoredTokenCoverage( elements, context, token.Value, - !parsed.Clauses.Any(clause => clause.IsCommandStringWrapped) - || directSegments.Contains(segment) - || isRedirectOperator - || isRedirectTarget); + (!parsed.Clauses.Any(clause => clause.IsCommandStringWrapped) + || directSegments.Contains(segment) + || isRedirectOperator + || isRedirectTarget) && + !IsBashForEachStructuralToken(parsed.Syntax, token)); redirectTargetPending = isRedirectOperator; } @@ -124,6 +125,85 @@ private static void AssertAuthoredTokenCoverage( } } + private static bool IsBashForEachStructuralToken( + ShellSyntaxNode node, + BashToken token) + { + if (node is ForEachSyntax forEach && + IsForEachStructuralToken(forEach, token)) + { + return true; + } + + return node switch + { + ShellBlockSyntax block => block.Statements.Any( + child => IsBashForEachStructuralToken(child, token)), + SimpleCommandSyntax simple => simple.Substitutions.Any( + child => IsBashForEachStructuralToken(child, token)), + PipelineSyntax pipeline => pipeline.Stages.Any( + child => IsBashForEachStructuralToken(child, token)), + CommandListSyntax list => list.Items.Any( + item => IsBashForEachStructuralToken(item.Command, token)), + GroupSyntax group => IsBashForEachStructuralToken(group.Body, token), + ForEachSyntax nested => + IsBashForEachStructuralToken(nested.IteratorCommands, token) || + IsBashForEachStructuralToken(nested.Body, token), + ConditionLoopSyntax loop => + IsBashForEachStructuralToken(loop.Condition, token) || + IsBashForEachStructuralToken(loop.Body, token), + ConditionalSyntax conditional => conditional.Branches.Any( + branch => IsBashForEachStructuralToken(branch, token)) || + conditional.Else is not null && + IsBashForEachStructuralToken(conditional.Else, token), + ConditionalBranchSyntax branch => + IsBashForEachStructuralToken(branch.Condition, token) || + IsBashForEachStructuralToken(branch.Body, token), + CommandSubstitutionSyntax substitution => + IsBashForEachStructuralToken(substitution.Body, token), + _ => false, + }; + } + + private static bool IsForEachStructuralToken( + ForEachSyntax forEach, + BashToken token) + { + if (forEach.SourceStart is null || forEach.SourceLength is null || + forEach.Binding.Source.SourceStart is null || + forEach.Binding.Source.SourceLength is null || + forEach.Iterable.SourceStart is null || + forEach.Iterable.SourceLength is null || + forEach.Body.SourceStart is null || forEach.Body.SourceLength is null) + { + return false; + } + + var tokenEnd = token.SourceStart + token.SourceLength; + var loopEnd = forEach.SourceStart.Value + forEach.SourceLength.Value; + var bindingStart = forEach.Binding.Source.SourceStart.Value; + var bindingEnd = bindingStart + forEach.Binding.Source.SourceLength.Value; + var iterableStart = forEach.Iterable.SourceStart.Value; + var iterableEnd = iterableStart + forEach.Iterable.SourceLength.Value; + var bodyStart = forEach.Body.SourceStart.Value; + var bodyEnd = bodyStart + forEach.Body.SourceLength.Value; + if (token.SourceStart >= iterableStart && tokenEnd <= iterableEnd || + token.SourceStart == bindingStart && tokenEnd == bindingEnd) + { + return true; + } + + return token.Kind == BashTokenKind.Word && + (string.Equals(token.Value, "for", StringComparison.Ordinal) && + token.SourceStart == forEach.SourceStart || + string.Equals(token.Value, "in", StringComparison.Ordinal) && + token.SourceStart >= bindingEnd && tokenEnd <= iterableStart || + string.Equals(token.Value, "do", StringComparison.Ordinal) && + token.SourceStart >= iterableEnd && tokenEnd == bodyStart || + string.Equals(token.Value, "done", StringComparison.Ordinal) && + token.SourceStart == bodyEnd && tokenEnd == loopEnd); + } + private static IReadOnlyList StandaloneSubstitutionRegions( ShellSyntaxNode syntax) { @@ -503,6 +583,20 @@ public sealed record ExpectedSyntaxNode public ShellGroupKind? GroupKind { get; init; } public CompoundOperator? ListOperator { get; init; } + + public string? BindingName { get; init; } + + public string? BindingRaw { get; init; } + + public int? BindingSourceStart { get; init; } + + public int? BindingSourceLength { get; init; } + + public string? IterableRaw { get; init; } + + public int? IterableSourceStart { get; init; } + + public int? IterableSourceLength { get; init; } } public sealed record ExpectedCommandOccurrence @@ -514,6 +608,28 @@ public sealed record ExpectedCommandOccurrence public bool IsComplete { get; init; } public List? Ancestry { get; init; } + + public List? EffectiveArguments { get; init; } + + public ExpectedValueDomain? WorkingDirectory { get; init; } +} + +public sealed record ExpectedEffectiveArgument +{ + public int ClauseElementIndex { get; init; } = -1; + + public ExpectedValueDomain Value { get; init; } = new(); +} + +public sealed record ExpectedValueDomain +{ + public ShellValueDomainKind Kind { get; init; } + + public List? Values { get; init; } + + public string? Pattern { get; init; } + + public string? CoveringDirectory { get; init; } } public sealed record ExpectedCommandAncestryFrame diff --git a/tests/ShellSyntaxTree.Tests/Corpus/bash/149_control_flow_keyword_after_newline.json b/tests/ShellSyntaxTree.Tests/Corpus/bash/149_control_flow_keyword_after_newline.json index 177b114..cf45430 100644 --- a/tests/ShellSyntaxTree.Tests/Corpus/bash/149_control_flow_keyword_after_newline.json +++ b/tests/ShellSyntaxTree.Tests/Corpus/bash/149_control_flow_keyword_after_newline.json @@ -3,7 +3,7 @@ "input": "echo hi\nfor i in 1 2 3", "expected": { "isUnparseable": true, - "unparseableReasonContains": "'for'" + "unparseableReasonContains": "missing its list terminator" }, - "notes": "SPEC §4/§11: a newline opens a new clause; a control-flow keyword at that verb slot safe-fails just as it would after ';'." + "notes": "The supported for-in grammar still fails atomically when its list terminator and do block are absent." } diff --git a/tests/ShellSyntaxTree.Tests/Corpus/bash/205_v03_for_literal_finite.json b/tests/ShellSyntaxTree.Tests/Corpus/bash/205_v03_for_literal_finite.json new file mode 100644 index 0000000..e3eb0d6 --- /dev/null +++ b/tests/ShellSyntaxTree.Tests/Corpus/bash/205_v03_for_literal_finite.json @@ -0,0 +1,42 @@ +{ + "name": "v0.3 Bash for-in literal finite binding", + "input": "for f in a.txt b.txt; do rm -- \"$f\"; done", + "expected": { + "isUnparseable": false, + "clauses": [ + { + "operator": "None", + "verb": ["rm"], + "args": [ + { "raw": "--", "kind": "Literal", "isPath": false, "resolved": "__NULL__", "isFlag": true }, + { "raw": "\"$f\"", "kind": "DynamicSkip", "isPath": false, "resolved": "__NULL__", "isFlag": false } + ], + "redirects": [] + } + ], + "syntax": [ + { "kind": "Block", "parentIndex": null, "region": "Unknown", "childIndex": null, "sourceStart": 0, "sourceLength": 41, "clauseIndex": null, "groupKind": null, "listOperator": null }, + { "kind": "ForEach", "parentIndex": 0, "region": "Root", "childIndex": 0, "sourceStart": 0, "sourceLength": 41, "clauseIndex": null, "groupKind": null, "listOperator": null, "bindingName": "f", "bindingRaw": "f", "bindingSourceStart": 4, "bindingSourceLength": 1, "iterableRaw": "a.txt b.txt", "iterableSourceStart": 9, "iterableSourceLength": 11 }, + { "kind": "Block", "parentIndex": 1, "region": "Iterator", "childIndex": null, "sourceStart": 9, "sourceLength": 11, "clauseIndex": null, "groupKind": null, "listOperator": null }, + { "kind": "Block", "parentIndex": 1, "region": "LoopBody", "childIndex": null, "sourceStart": 24, "sourceLength": 13, "clauseIndex": null, "groupKind": null, "listOperator": null }, + { "kind": "SimpleCommand", "parentIndex": 3, "region": "Statement", "childIndex": 0, "sourceStart": 25, "sourceLength": 10, "clauseIndex": 0, "groupKind": null, "listOperator": null } + ], + "commands": [ + { + "clauseIndex": 0, + "immediateRole": "LoopBody", + "isComplete": true, + "ancestry": [ + { "ancestorKind": "Block", "region": "Root", "childIndex": 0, "sourceStart": 0, "sourceLength": 41 }, + { "ancestorKind": "ForEach", "region": "LoopBody", "childIndex": null, "sourceStart": 0, "sourceLength": 41 }, + { "ancestorKind": "Block", "region": "Statement", "childIndex": 0, "sourceStart": 24, "sourceLength": 13 } + ], + "effectiveArguments": [ + { "clauseElementIndex": 2, "value": { "kind": "FiniteSet", "values": ["a.txt", "b.txt"], "pattern": null, "coveringDirectory": null } } + ], + "workingDirectory": { "kind": "Unknown", "values": [], "pattern": null, "coveringDirectory": null } + } + ] + }, + "notes": "Pins the full for-in structure, shared compatibility leaf, bounded candidates, and intentionally Unknown cwd." +} diff --git a/tests/ShellSyntaxTree.Tests/Corpus/bash/206_v03_for_iterator_substitution.json b/tests/ShellSyntaxTree.Tests/Corpus/bash/206_v03_for_iterator_substitution.json new file mode 100644 index 0000000..54a0e71 --- /dev/null +++ b/tests/ShellSyntaxTree.Tests/Corpus/bash/206_v03_for_iterator_substitution.json @@ -0,0 +1,69 @@ +{ + "name": "v0.3 Bash for-in iterator substitution", + "input": "for f in $(find /tmp -type f); do rm -- \"$f\"; done", + "expected": { + "isUnparseable": false, + "clauses": [ + { + "operator": "None", + "verb": ["find"], + "args": [ + { "raw": "/tmp", "kind": "Literal", "isPath": true, "resolved": "/tmp", "isFlag": false }, + { "raw": "-type", "kind": "Literal", "isPath": false, "resolved": "__NULL__", "isFlag": true }, + { "raw": "f", "kind": "Literal", "isPath": false, "resolved": "__NULL__", "isFlag": false } + ], + "redirects": [] + }, + { + "operator": "None", + "verb": ["rm"], + "args": [ + { "raw": "--", "kind": "Literal", "isPath": false, "resolved": "__NULL__", "isFlag": true }, + { "raw": "\"$f\"", "kind": "DynamicSkip", "isPath": false, "resolved": "__NULL__", "isFlag": false } + ], + "redirects": [] + } + ], + "syntax": [ + { "kind": "Block", "parentIndex": null, "region": "Unknown", "childIndex": null, "sourceStart": 0, "sourceLength": 50, "clauseIndex": null, "groupKind": null, "listOperator": null }, + { "kind": "ForEach", "parentIndex": 0, "region": "Root", "childIndex": 0, "sourceStart": 0, "sourceLength": 50, "clauseIndex": null, "groupKind": null, "listOperator": null, "bindingName": "f", "bindingRaw": "f", "bindingSourceStart": 4, "bindingSourceLength": 1, "iterableRaw": "$(find /tmp -type f)", "iterableSourceStart": 9, "iterableSourceLength": 20 }, + { "kind": "Block", "parentIndex": 1, "region": "Iterator", "childIndex": null, "sourceStart": 9, "sourceLength": 20, "clauseIndex": null, "groupKind": null, "listOperator": null }, + { "kind": "CommandSubstitution", "parentIndex": 2, "region": "Statement", "childIndex": 0, "sourceStart": 9, "sourceLength": 20, "clauseIndex": null, "groupKind": null, "listOperator": null }, + { "kind": "Block", "parentIndex": 3, "region": "Substitution", "childIndex": 0, "sourceStart": 11, "sourceLength": 17, "clauseIndex": null, "groupKind": null, "listOperator": null }, + { "kind": "SimpleCommand", "parentIndex": 4, "region": "Statement", "childIndex": 0, "sourceStart": 11, "sourceLength": 17, "clauseIndex": 0, "groupKind": null, "listOperator": null }, + { "kind": "Block", "parentIndex": 1, "region": "LoopBody", "childIndex": null, "sourceStart": 33, "sourceLength": 13, "clauseIndex": null, "groupKind": null, "listOperator": null }, + { "kind": "SimpleCommand", "parentIndex": 6, "region": "Statement", "childIndex": 0, "sourceStart": 34, "sourceLength": 10, "clauseIndex": 1, "groupKind": null, "listOperator": null } + ], + "commands": [ + { + "clauseIndex": 0, + "immediateRole": "Substitution", + "isComplete": true, + "ancestry": [ + { "ancestorKind": "Block", "region": "Root", "childIndex": 0, "sourceStart": 0, "sourceLength": 50 }, + { "ancestorKind": "ForEach", "region": "Iterator", "childIndex": null, "sourceStart": 0, "sourceLength": 50 }, + { "ancestorKind": "Block", "region": "Statement", "childIndex": 0, "sourceStart": 9, "sourceLength": 20 }, + { "ancestorKind": "CommandSubstitution", "region": "Substitution", "childIndex": 0, "sourceStart": 9, "sourceLength": 20 }, + { "ancestorKind": "Block", "region": "Statement", "childIndex": 0, "sourceStart": 11, "sourceLength": 17 } + ], + "effectiveArguments": [], + "workingDirectory": { "kind": "Unknown", "values": [], "pattern": null, "coveringDirectory": null } + }, + { + "clauseIndex": 1, + "immediateRole": "LoopBody", + "isComplete": true, + "ancestry": [ + { "ancestorKind": "Block", "region": "Root", "childIndex": 0, "sourceStart": 0, "sourceLength": 50 }, + { "ancestorKind": "ForEach", "region": "LoopBody", "childIndex": null, "sourceStart": 0, "sourceLength": 50 }, + { "ancestorKind": "Block", "region": "Statement", "childIndex": 0, "sourceStart": 33, "sourceLength": 13 } + ], + "effectiveArguments": [ + { "clauseElementIndex": 2, "value": { "kind": "Unknown", "values": [], "pattern": null, "coveringDirectory": null } } + ], + "workingDirectory": { "kind": "Unknown", "values": [], "pattern": null, "coveringDirectory": null } + } + ] + }, + "notes": "The iterator command is visible before the body and its produced binding remains Unknown." +} diff --git a/tests/ShellSyntaxTree.Tests/Corpus/bash/207_v03_for_static_pattern.json b/tests/ShellSyntaxTree.Tests/Corpus/bash/207_v03_for_static_pattern.json new file mode 100644 index 0000000..70a7d82 --- /dev/null +++ b/tests/ShellSyntaxTree.Tests/Corpus/bash/207_v03_for_static_pattern.json @@ -0,0 +1,144 @@ +{ + "name": "v0.3 Bash for-in static path pattern", + "input": "for f in /tmp/*.txt; do rm -- \"$f\"; done", + "expected": { + "isUnparseable": false, + "clauses": [ + { + "operator": "None", + "verb": [ + "rm" + ], + "args": [ + { + "raw": "--", + "kind": "Literal", + "isPath": false, + "resolved": "__NULL__", + "isFlag": true + }, + { + "raw": "\"$f\"", + "kind": "DynamicSkip", + "isPath": false, + "resolved": "__NULL__", + "isFlag": false + } + ], + "redirects": [] + } + ], + "syntax": [ + { + "kind": "Block", + "parentIndex": null, + "region": "Unknown", + "childIndex": null, + "sourceStart": 0, + "sourceLength": 40, + "clauseIndex": null, + "groupKind": null, + "listOperator": null + }, + { + "kind": "ForEach", + "parentIndex": 0, + "region": "Root", + "childIndex": 0, + "sourceStart": 0, + "sourceLength": 40, + "clauseIndex": null, + "groupKind": null, + "listOperator": null, + "bindingName": "f", + "bindingRaw": "f", + "bindingSourceStart": 4, + "bindingSourceLength": 1, + "iterableRaw": "/tmp/*.txt", + "iterableSourceStart": 9, + "iterableSourceLength": 10 + }, + { + "kind": "Block", + "parentIndex": 1, + "region": "Iterator", + "childIndex": null, + "sourceStart": 9, + "sourceLength": 10, + "clauseIndex": null, + "groupKind": null, + "listOperator": null + }, + { + "kind": "Block", + "parentIndex": 1, + "region": "LoopBody", + "childIndex": null, + "sourceStart": 23, + "sourceLength": 13, + "clauseIndex": null, + "groupKind": null, + "listOperator": null + }, + { + "kind": "SimpleCommand", + "parentIndex": 3, + "region": "Statement", + "childIndex": 0, + "sourceStart": 24, + "sourceLength": 10, + "clauseIndex": 0, + "groupKind": null, + "listOperator": null + } + ], + "commands": [ + { + "clauseIndex": 0, + "immediateRole": "LoopBody", + "isComplete": true, + "ancestry": [ + { + "ancestorKind": "Block", + "region": "Root", + "childIndex": 0, + "sourceStart": 0, + "sourceLength": 40 + }, + { + "ancestorKind": "ForEach", + "region": "LoopBody", + "childIndex": null, + "sourceStart": 0, + "sourceLength": 40 + }, + { + "ancestorKind": "Block", + "region": "Statement", + "childIndex": 0, + "sourceStart": 23, + "sourceLength": 13 + } + ], + "effectiveArguments": [ + { + "clauseElementIndex": 2, + "value": { + "kind": "Pattern", + "values": [], + "pattern": "/tmp/*.txt", + "coveringDirectory": "/tmp" + } + } + ], + "workingDirectory": { + "kind": "Unknown", + "values": [], + "pattern": null, + "coveringDirectory": null + } + } + ] + }, + "notes": "Pins a static iterable glob as a Pattern with its non-enumerated covering directory." +} diff --git a/tests/ShellSyntaxTree.Tests/Corpus/bash/208_v03_for_dynamic_root.json b/tests/ShellSyntaxTree.Tests/Corpus/bash/208_v03_for_dynamic_root.json new file mode 100644 index 0000000..89033b9 --- /dev/null +++ b/tests/ShellSyntaxTree.Tests/Corpus/bash/208_v03_for_dynamic_root.json @@ -0,0 +1,144 @@ +{ + "name": "v0.3 Bash for-in dynamic glob root", + "input": "for f in \"$DIR\"/*.txt; do rm -- \"$f\"; done", + "expected": { + "isUnparseable": false, + "clauses": [ + { + "operator": "None", + "verb": [ + "rm" + ], + "args": [ + { + "raw": "--", + "kind": "Literal", + "isPath": false, + "resolved": "__NULL__", + "isFlag": true + }, + { + "raw": "\"$f\"", + "kind": "DynamicSkip", + "isPath": false, + "resolved": "__NULL__", + "isFlag": false + } + ], + "redirects": [] + } + ], + "syntax": [ + { + "kind": "Block", + "parentIndex": null, + "region": "Unknown", + "childIndex": null, + "sourceStart": 0, + "sourceLength": 42, + "clauseIndex": null, + "groupKind": null, + "listOperator": null + }, + { + "kind": "ForEach", + "parentIndex": 0, + "region": "Root", + "childIndex": 0, + "sourceStart": 0, + "sourceLength": 42, + "clauseIndex": null, + "groupKind": null, + "listOperator": null, + "bindingName": "f", + "bindingRaw": "f", + "bindingSourceStart": 4, + "bindingSourceLength": 1, + "iterableRaw": "\"$DIR\"/*.txt", + "iterableSourceStart": 9, + "iterableSourceLength": 12 + }, + { + "kind": "Block", + "parentIndex": 1, + "region": "Iterator", + "childIndex": null, + "sourceStart": 9, + "sourceLength": 12, + "clauseIndex": null, + "groupKind": null, + "listOperator": null + }, + { + "kind": "Block", + "parentIndex": 1, + "region": "LoopBody", + "childIndex": null, + "sourceStart": 25, + "sourceLength": 13, + "clauseIndex": null, + "groupKind": null, + "listOperator": null + }, + { + "kind": "SimpleCommand", + "parentIndex": 3, + "region": "Statement", + "childIndex": 0, + "sourceStart": 26, + "sourceLength": 10, + "clauseIndex": 0, + "groupKind": null, + "listOperator": null + } + ], + "commands": [ + { + "clauseIndex": 0, + "immediateRole": "LoopBody", + "isComplete": true, + "ancestry": [ + { + "ancestorKind": "Block", + "region": "Root", + "childIndex": 0, + "sourceStart": 0, + "sourceLength": 42 + }, + { + "ancestorKind": "ForEach", + "region": "LoopBody", + "childIndex": null, + "sourceStart": 0, + "sourceLength": 42 + }, + { + "ancestorKind": "Block", + "region": "Statement", + "childIndex": 0, + "sourceStart": 25, + "sourceLength": 13 + } + ], + "effectiveArguments": [ + { + "clauseElementIndex": 2, + "value": { + "kind": "Unknown", + "values": [], + "pattern": null, + "coveringDirectory": null + } + } + ], + "workingDirectory": { + "kind": "Unknown", + "values": [], + "pattern": null, + "coveringDirectory": null + } + } + ] + }, + "notes": "A dynamic glob root cannot produce a safe covering directory and remains Unknown." +} diff --git a/tests/ShellSyntaxTree.Tests/Corpus/bash/209_v03_for_option_injection.json b/tests/ShellSyntaxTree.Tests/Corpus/bash/209_v03_for_option_injection.json new file mode 100644 index 0000000..5606a40 --- /dev/null +++ b/tests/ShellSyntaxTree.Tests/Corpus/bash/209_v03_for_option_injection.json @@ -0,0 +1,147 @@ +{ + "name": "v0.3 Bash for-in option-shaped candidates", + "input": "for f in -rf safe; do rm \"$f\" target; done", + "expected": { + "isUnparseable": false, + "clauses": [ + { + "operator": "None", + "verb": [ + "rm" + ], + "args": [ + { + "raw": "\"$f\"", + "kind": "DynamicSkip", + "isPath": false, + "resolved": "__NULL__", + "isFlag": false + }, + { + "raw": "target", + "kind": "Literal", + "isPath": true, + "resolved": "/work/target", + "isFlag": false + } + ], + "redirects": [] + } + ], + "syntax": [ + { + "kind": "Block", + "parentIndex": null, + "region": "Unknown", + "childIndex": null, + "sourceStart": 0, + "sourceLength": 42, + "clauseIndex": null, + "groupKind": null, + "listOperator": null + }, + { + "kind": "ForEach", + "parentIndex": 0, + "region": "Root", + "childIndex": 0, + "sourceStart": 0, + "sourceLength": 42, + "clauseIndex": null, + "groupKind": null, + "listOperator": null, + "bindingName": "f", + "bindingRaw": "f", + "bindingSourceStart": 4, + "bindingSourceLength": 1, + "iterableRaw": "-rf safe", + "iterableSourceStart": 9, + "iterableSourceLength": 8 + }, + { + "kind": "Block", + "parentIndex": 1, + "region": "Iterator", + "childIndex": null, + "sourceStart": 9, + "sourceLength": 8, + "clauseIndex": null, + "groupKind": null, + "listOperator": null + }, + { + "kind": "Block", + "parentIndex": 1, + "region": "LoopBody", + "childIndex": null, + "sourceStart": 21, + "sourceLength": 17, + "clauseIndex": null, + "groupKind": null, + "listOperator": null + }, + { + "kind": "SimpleCommand", + "parentIndex": 3, + "region": "Statement", + "childIndex": 0, + "sourceStart": 22, + "sourceLength": 14, + "clauseIndex": 0, + "groupKind": null, + "listOperator": null + } + ], + "commands": [ + { + "clauseIndex": 0, + "immediateRole": "LoopBody", + "isComplete": true, + "ancestry": [ + { + "ancestorKind": "Block", + "region": "Root", + "childIndex": 0, + "sourceStart": 0, + "sourceLength": 42 + }, + { + "ancestorKind": "ForEach", + "region": "LoopBody", + "childIndex": null, + "sourceStart": 0, + "sourceLength": 42 + }, + { + "ancestorKind": "Block", + "region": "Statement", + "childIndex": 0, + "sourceStart": 21, + "sourceLength": 17 + } + ], + "effectiveArguments": [ + { + "clauseElementIndex": 1, + "value": { + "kind": "FiniteSet", + "values": [ + "-rf", + "safe" + ], + "pattern": null, + "coveringDirectory": null + } + } + ], + "workingDirectory": { + "kind": "Unknown", + "values": [], + "pattern": null, + "coveringDirectory": null + } + } + ] + }, + "notes": "Pins option-shaped loop candidates without changing the compatibility Clause classification." +} diff --git a/tests/ShellSyntaxTree.Tests/Corpus/bash/210_v03_for_nested_correlation.json b/tests/ShellSyntaxTree.Tests/Corpus/bash/210_v03_for_nested_correlation.json new file mode 100644 index 0000000..d382fe4 --- /dev/null +++ b/tests/ShellSyntaxTree.Tests/Corpus/bash/210_v03_for_nested_correlation.json @@ -0,0 +1,196 @@ +{ + "name": "v0.3 Bash nested for-in correlation", + "input": "for d in a b; do for f in x y; do echo \"$d/$f/$d\"; done; done", + "expected": { + "isUnparseable": false, + "clauses": [ + { + "operator": "None", + "verb": [ + "echo" + ], + "args": [ + { + "raw": "\"$d/$f/$d\"", + "kind": "DynamicSkip", + "isPath": false, + "resolved": "__NULL__", + "isFlag": false + } + ], + "redirects": [] + } + ], + "syntax": [ + { + "kind": "Block", + "parentIndex": null, + "region": "Unknown", + "childIndex": null, + "sourceStart": 0, + "sourceLength": 61, + "clauseIndex": null, + "groupKind": null, + "listOperator": null + }, + { + "kind": "ForEach", + "parentIndex": 0, + "region": "Root", + "childIndex": 0, + "sourceStart": 0, + "sourceLength": 61, + "clauseIndex": null, + "groupKind": null, + "listOperator": null, + "bindingName": "d", + "bindingRaw": "d", + "bindingSourceStart": 4, + "bindingSourceLength": 1, + "iterableRaw": "a b", + "iterableSourceStart": 9, + "iterableSourceLength": 3 + }, + { + "kind": "Block", + "parentIndex": 1, + "region": "Iterator", + "childIndex": null, + "sourceStart": 9, + "sourceLength": 3, + "clauseIndex": null, + "groupKind": null, + "listOperator": null + }, + { + "kind": "Block", + "parentIndex": 1, + "region": "LoopBody", + "childIndex": null, + "sourceStart": 16, + "sourceLength": 41, + "clauseIndex": null, + "groupKind": null, + "listOperator": null + }, + { + "kind": "ForEach", + "parentIndex": 3, + "region": "Statement", + "childIndex": 0, + "sourceStart": 17, + "sourceLength": 38, + "clauseIndex": null, + "groupKind": null, + "listOperator": null, + "bindingName": "f", + "bindingRaw": "f", + "bindingSourceStart": 21, + "bindingSourceLength": 1, + "iterableRaw": "x y", + "iterableSourceStart": 26, + "iterableSourceLength": 3 + }, + { + "kind": "Block", + "parentIndex": 4, + "region": "Iterator", + "childIndex": null, + "sourceStart": 26, + "sourceLength": 3, + "clauseIndex": null, + "groupKind": null, + "listOperator": null + }, + { + "kind": "Block", + "parentIndex": 4, + "region": "LoopBody", + "childIndex": null, + "sourceStart": 33, + "sourceLength": 18, + "clauseIndex": null, + "groupKind": null, + "listOperator": null + }, + { + "kind": "SimpleCommand", + "parentIndex": 6, + "region": "Statement", + "childIndex": 0, + "sourceStart": 34, + "sourceLength": 15, + "clauseIndex": 0, + "groupKind": null, + "listOperator": null + } + ], + "commands": [ + { + "clauseIndex": 0, + "immediateRole": "LoopBody", + "isComplete": true, + "ancestry": [ + { + "ancestorKind": "Block", + "region": "Root", + "childIndex": 0, + "sourceStart": 0, + "sourceLength": 61 + }, + { + "ancestorKind": "ForEach", + "region": "LoopBody", + "childIndex": null, + "sourceStart": 0, + "sourceLength": 61 + }, + { + "ancestorKind": "Block", + "region": "Statement", + "childIndex": 0, + "sourceStart": 16, + "sourceLength": 41 + }, + { + "ancestorKind": "ForEach", + "region": "LoopBody", + "childIndex": null, + "sourceStart": 17, + "sourceLength": 38 + }, + { + "ancestorKind": "Block", + "region": "Statement", + "childIndex": 0, + "sourceStart": 33, + "sourceLength": 18 + } + ], + "effectiveArguments": [ + { + "clauseElementIndex": 1, + "value": { + "kind": "FiniteSet", + "values": [ + "a/x/a", + "a/y/a", + "b/x/b", + "b/y/b" + ], + "pattern": null, + "coveringDirectory": null + } + } + ], + "workingDirectory": { + "kind": "Unknown", + "values": [], + "pattern": null, + "coveringDirectory": null + } + } + ] + }, + "notes": "Pins the bounded nested cross-product while retaining correlation for repeated references to one binding." +} diff --git a/tests/ShellSyntaxTree.Tests/Corpus/bash/211_v03_for_pipeline_ancestry.json b/tests/ShellSyntaxTree.Tests/Corpus/bash/211_v03_for_pipeline_ancestry.json new file mode 100644 index 0000000..a964caa --- /dev/null +++ b/tests/ShellSyntaxTree.Tests/Corpus/bash/211_v03_for_pipeline_ancestry.json @@ -0,0 +1,228 @@ +{ + "name": "v0.3 Bash for-in body pipeline ancestry", + "input": "for f in a b; do find \"$f\" | xargs rm --; done", + "expected": { + "isUnparseable": false, + "clauses": [ + { + "operator": "None", + "verb": [ + "find" + ], + "args": [ + { + "raw": "\"$f\"", + "kind": "DynamicSkip", + "isPath": false, + "resolved": "__NULL__", + "isFlag": false + } + ], + "redirects": [] + }, + { + "operator": "Pipe", + "verb": [ + "xargs", + "rm" + ], + "args": [ + { + "raw": "--", + "kind": "Literal", + "isPath": false, + "resolved": "__NULL__", + "isFlag": true + } + ], + "redirects": [] + } + ], + "syntax": [ + { + "kind": "Block", + "parentIndex": null, + "region": "Unknown", + "childIndex": null, + "sourceStart": 0, + "sourceLength": 46, + "clauseIndex": null, + "groupKind": null, + "listOperator": null + }, + { + "kind": "ForEach", + "parentIndex": 0, + "region": "Root", + "childIndex": 0, + "sourceStart": 0, + "sourceLength": 46, + "clauseIndex": null, + "groupKind": null, + "listOperator": null, + "bindingName": "f", + "bindingRaw": "f", + "bindingSourceStart": 4, + "bindingSourceLength": 1, + "iterableRaw": "a b", + "iterableSourceStart": 9, + "iterableSourceLength": 3 + }, + { + "kind": "Block", + "parentIndex": 1, + "region": "Iterator", + "childIndex": null, + "sourceStart": 9, + "sourceLength": 3, + "clauseIndex": null, + "groupKind": null, + "listOperator": null + }, + { + "kind": "Block", + "parentIndex": 1, + "region": "LoopBody", + "childIndex": null, + "sourceStart": 16, + "sourceLength": 26, + "clauseIndex": null, + "groupKind": null, + "listOperator": null + }, + { + "kind": "Pipeline", + "parentIndex": 3, + "region": "Statement", + "childIndex": 0, + "sourceStart": 17, + "sourceLength": 23, + "clauseIndex": null, + "groupKind": null, + "listOperator": null + }, + { + "kind": "SimpleCommand", + "parentIndex": 4, + "region": "PipelineStage", + "childIndex": 0, + "sourceStart": 17, + "sourceLength": 9, + "clauseIndex": 0, + "groupKind": null, + "listOperator": null + }, + { + "kind": "SimpleCommand", + "parentIndex": 4, + "region": "PipelineStage", + "childIndex": 1, + "sourceStart": 29, + "sourceLength": 11, + "clauseIndex": 1, + "groupKind": null, + "listOperator": null + } + ], + "commands": [ + { + "clauseIndex": 0, + "immediateRole": "PipelineStage", + "isComplete": true, + "ancestry": [ + { + "ancestorKind": "Block", + "region": "Root", + "childIndex": 0, + "sourceStart": 0, + "sourceLength": 46 + }, + { + "ancestorKind": "ForEach", + "region": "LoopBody", + "childIndex": null, + "sourceStart": 0, + "sourceLength": 46 + }, + { + "ancestorKind": "Block", + "region": "Statement", + "childIndex": 0, + "sourceStart": 16, + "sourceLength": 26 + }, + { + "ancestorKind": "Pipeline", + "region": "PipelineStage", + "childIndex": 0, + "sourceStart": 17, + "sourceLength": 23 + } + ], + "effectiveArguments": [ + { + "clauseElementIndex": 1, + "value": { + "kind": "FiniteSet", + "values": [ + "a", + "b" + ], + "pattern": null, + "coveringDirectory": null + } + } + ], + "workingDirectory": { + "kind": "Unknown", + "values": [], + "pattern": null, + "coveringDirectory": null + } + }, + { + "clauseIndex": 1, + "immediateRole": "PipelineStage", + "isComplete": true, + "ancestry": [ + { + "ancestorKind": "Block", + "region": "Root", + "childIndex": 0, + "sourceStart": 0, + "sourceLength": 46 + }, + { + "ancestorKind": "ForEach", + "region": "LoopBody", + "childIndex": null, + "sourceStart": 0, + "sourceLength": 46 + }, + { + "ancestorKind": "Block", + "region": "Statement", + "childIndex": 0, + "sourceStart": 16, + "sourceLength": 26 + }, + { + "ancestorKind": "Pipeline", + "region": "PipelineStage", + "childIndex": 1, + "sourceStart": 17, + "sourceLength": 23 + } + ], + "effectiveArguments": [], + "workingDirectory": { + "kind": "Unknown", + "values": [], + "pattern": null, + "coveringDirectory": null + } + } + ] + }, + "notes": "Pins loop-body and pipeline ancestry plus authored Pipe compatibility ordering." +} diff --git a/tests/ShellSyntaxTree.Tests/Corpus/bash/212_v03_for_missing_done.json b/tests/ShellSyntaxTree.Tests/Corpus/bash/212_v03_for_missing_done.json new file mode 100644 index 0000000..4308883 --- /dev/null +++ b/tests/ShellSyntaxTree.Tests/Corpus/bash/212_v03_for_missing_done.json @@ -0,0 +1,9 @@ +{ + "name": "v0.3 Bash malformed for-in missing done", + "input": "for f in a b; do echo \"$f\"", + "expected": { + "isUnparseable": true, + "unparseableReasonContains": "Bash for-in loop is missing 'done'" + }, + "notes": "Malformed loop syntax fails atomically without partial compatibility or structural projections." +} diff --git a/tests/ShellSyntaxTree.Tests/Corpus/bash/213_v03_for_candidate_cap_32.json b/tests/ShellSyntaxTree.Tests/Corpus/bash/213_v03_for_candidate_cap_32.json new file mode 100644 index 0000000..a74df77 --- /dev/null +++ b/tests/ShellSyntaxTree.Tests/Corpus/bash/213_v03_for_candidate_cap_32.json @@ -0,0 +1,177 @@ +{ + "name": "v0.3 Bash for-in exact candidate cap", + "input": "for f in v0 v1 v2 v3 v4 v5 v6 v7 v8 v9 v10 v11 v12 v13 v14 v15 v16 v17 v18 v19 v20 v21 v22 v23 v24 v25 v26 v27 v28 v29 v30 v31; do printf %s \"$f\"; done", + "expected": { + "isUnparseable": false, + "clauses": [ + { + "operator": "None", + "verb": [ + "printf" + ], + "args": [ + { + "raw": "%s", + "kind": "Literal", + "isPath": false, + "resolved": "__NULL__", + "isFlag": false + }, + { + "raw": "\"$f\"", + "kind": "EnvVar", + "isPath": false, + "resolved": "__NULL__", + "isFlag": false + } + ], + "redirects": [] + } + ], + "syntax": [ + { + "kind": "Block", + "parentIndex": null, + "region": "Unknown", + "childIndex": null, + "sourceStart": 0, + "sourceLength": 151, + "clauseIndex": null, + "groupKind": null, + "listOperator": null + }, + { + "kind": "ForEach", + "parentIndex": 0, + "region": "Root", + "childIndex": 0, + "sourceStart": 0, + "sourceLength": 151, + "clauseIndex": null, + "groupKind": null, + "listOperator": null, + "bindingName": "f", + "bindingRaw": "f", + "bindingSourceStart": 4, + "bindingSourceLength": 1, + "iterableRaw": "v0 v1 v2 v3 v4 v5 v6 v7 v8 v9 v10 v11 v12 v13 v14 v15 v16 v17 v18 v19 v20 v21 v22 v23 v24 v25 v26 v27 v28 v29 v30 v31", + "iterableSourceStart": 9, + "iterableSourceLength": 117 + }, + { + "kind": "Block", + "parentIndex": 1, + "region": "Iterator", + "childIndex": null, + "sourceStart": 9, + "sourceLength": 117, + "clauseIndex": null, + "groupKind": null, + "listOperator": null + }, + { + "kind": "Block", + "parentIndex": 1, + "region": "LoopBody", + "childIndex": null, + "sourceStart": 130, + "sourceLength": 17, + "clauseIndex": null, + "groupKind": null, + "listOperator": null + }, + { + "kind": "SimpleCommand", + "parentIndex": 3, + "region": "Statement", + "childIndex": 0, + "sourceStart": 131, + "sourceLength": 14, + "clauseIndex": 0, + "groupKind": null, + "listOperator": null + } + ], + "commands": [ + { + "clauseIndex": 0, + "immediateRole": "LoopBody", + "isComplete": true, + "ancestry": [ + { + "ancestorKind": "Block", + "region": "Root", + "childIndex": 0, + "sourceStart": 0, + "sourceLength": 151 + }, + { + "ancestorKind": "ForEach", + "region": "LoopBody", + "childIndex": null, + "sourceStart": 0, + "sourceLength": 151 + }, + { + "ancestorKind": "Block", + "region": "Statement", + "childIndex": 0, + "sourceStart": 130, + "sourceLength": 17 + } + ], + "effectiveArguments": [ + { + "clauseElementIndex": 2, + "value": { + "kind": "FiniteSet", + "values": [ + "v0", + "v1", + "v2", + "v3", + "v4", + "v5", + "v6", + "v7", + "v8", + "v9", + "v10", + "v11", + "v12", + "v13", + "v14", + "v15", + "v16", + "v17", + "v18", + "v19", + "v20", + "v21", + "v22", + "v23", + "v24", + "v25", + "v26", + "v27", + "v28", + "v29", + "v30", + "v31" + ], + "pattern": null, + "coveringDirectory": null + } + } + ], + "workingDirectory": { + "kind": "Unknown", + "values": [], + "pattern": null, + "coveringDirectory": null + } + } + ] + }, + "notes": "Exactly 32 finite candidates remain representable without widening." +} diff --git a/tests/ShellSyntaxTree.Tests/Corpus/bash/214_v03_for_candidate_overflow_33.json b/tests/ShellSyntaxTree.Tests/Corpus/bash/214_v03_for_candidate_overflow_33.json new file mode 100644 index 0000000..ad7c0c3 --- /dev/null +++ b/tests/ShellSyntaxTree.Tests/Corpus/bash/214_v03_for_candidate_overflow_33.json @@ -0,0 +1,144 @@ +{ + "name": "v0.3 Bash for-in candidate overflow", + "input": "for f in v0 v1 v2 v3 v4 v5 v6 v7 v8 v9 v10 v11 v12 v13 v14 v15 v16 v17 v18 v19 v20 v21 v22 v23 v24 v25 v26 v27 v28 v29 v30 v31 v32; do printf %s \"$f\"; done", + "expected": { + "isUnparseable": false, + "clauses": [ + { + "operator": "None", + "verb": [ + "printf" + ], + "args": [ + { + "raw": "%s", + "kind": "Literal", + "isPath": false, + "resolved": "__NULL__", + "isFlag": false + }, + { + "raw": "\"$f\"", + "kind": "EnvVar", + "isPath": false, + "resolved": "__NULL__", + "isFlag": false + } + ], + "redirects": [] + } + ], + "syntax": [ + { + "kind": "Block", + "parentIndex": null, + "region": "Unknown", + "childIndex": null, + "sourceStart": 0, + "sourceLength": 155, + "clauseIndex": null, + "groupKind": null, + "listOperator": null + }, + { + "kind": "ForEach", + "parentIndex": 0, + "region": "Root", + "childIndex": 0, + "sourceStart": 0, + "sourceLength": 155, + "clauseIndex": null, + "groupKind": null, + "listOperator": null, + "bindingName": "f", + "bindingRaw": "f", + "bindingSourceStart": 4, + "bindingSourceLength": 1, + "iterableRaw": "v0 v1 v2 v3 v4 v5 v6 v7 v8 v9 v10 v11 v12 v13 v14 v15 v16 v17 v18 v19 v20 v21 v22 v23 v24 v25 v26 v27 v28 v29 v30 v31 v32", + "iterableSourceStart": 9, + "iterableSourceLength": 121 + }, + { + "kind": "Block", + "parentIndex": 1, + "region": "Iterator", + "childIndex": null, + "sourceStart": 9, + "sourceLength": 121, + "clauseIndex": null, + "groupKind": null, + "listOperator": null + }, + { + "kind": "Block", + "parentIndex": 1, + "region": "LoopBody", + "childIndex": null, + "sourceStart": 134, + "sourceLength": 17, + "clauseIndex": null, + "groupKind": null, + "listOperator": null + }, + { + "kind": "SimpleCommand", + "parentIndex": 3, + "region": "Statement", + "childIndex": 0, + "sourceStart": 135, + "sourceLength": 14, + "clauseIndex": 0, + "groupKind": null, + "listOperator": null + } + ], + "commands": [ + { + "clauseIndex": 0, + "immediateRole": "LoopBody", + "isComplete": true, + "ancestry": [ + { + "ancestorKind": "Block", + "region": "Root", + "childIndex": 0, + "sourceStart": 0, + "sourceLength": 155 + }, + { + "ancestorKind": "ForEach", + "region": "LoopBody", + "childIndex": null, + "sourceStart": 0, + "sourceLength": 155 + }, + { + "ancestorKind": "Block", + "region": "Statement", + "childIndex": 0, + "sourceStart": 134, + "sourceLength": 17 + } + ], + "effectiveArguments": [ + { + "clauseElementIndex": 2, + "value": { + "kind": "Unknown", + "values": [], + "pattern": null, + "coveringDirectory": null + } + } + ], + "workingDirectory": { + "kind": "Unknown", + "values": [], + "pattern": null, + "coveringDirectory": null + } + } + ] + }, + "notes": "The 33rd candidate exceeds the finite-set cap and widens the effective value to Unknown." +} diff --git a/tests/ShellSyntaxTree.Tests/Corpus/bash/215_v03_for_pattern_parent_traversal.json b/tests/ShellSyntaxTree.Tests/Corpus/bash/215_v03_for_pattern_parent_traversal.json new file mode 100644 index 0000000..73d3839 --- /dev/null +++ b/tests/ShellSyntaxTree.Tests/Corpus/bash/215_v03_for_pattern_parent_traversal.json @@ -0,0 +1,144 @@ +{ + "name": "v0.3 Bash for-in pattern parent traversal", + "input": "for f in /tmp/*/../../etc/passwd; do rm -- \"$f\"; done", + "expected": { + "isUnparseable": false, + "clauses": [ + { + "operator": "None", + "verb": [ + "rm" + ], + "args": [ + { + "raw": "--", + "kind": "Literal", + "isPath": false, + "resolved": "__NULL__", + "isFlag": true + }, + { + "raw": "\"$f\"", + "kind": "DynamicSkip", + "isPath": false, + "resolved": "__NULL__", + "isFlag": false + } + ], + "redirects": [] + } + ], + "syntax": [ + { + "kind": "Block", + "parentIndex": null, + "region": "Unknown", + "childIndex": null, + "sourceStart": 0, + "sourceLength": 53, + "clauseIndex": null, + "groupKind": null, + "listOperator": null + }, + { + "kind": "ForEach", + "parentIndex": 0, + "region": "Root", + "childIndex": 0, + "sourceStart": 0, + "sourceLength": 53, + "clauseIndex": null, + "groupKind": null, + "listOperator": null, + "bindingName": "f", + "bindingRaw": "f", + "bindingSourceStart": 4, + "bindingSourceLength": 1, + "iterableRaw": "/tmp/*/../../etc/passwd", + "iterableSourceStart": 9, + "iterableSourceLength": 23 + }, + { + "kind": "Block", + "parentIndex": 1, + "region": "Iterator", + "childIndex": null, + "sourceStart": 9, + "sourceLength": 23, + "clauseIndex": null, + "groupKind": null, + "listOperator": null + }, + { + "kind": "Block", + "parentIndex": 1, + "region": "LoopBody", + "childIndex": null, + "sourceStart": 36, + "sourceLength": 13, + "clauseIndex": null, + "groupKind": null, + "listOperator": null + }, + { + "kind": "SimpleCommand", + "parentIndex": 3, + "region": "Statement", + "childIndex": 0, + "sourceStart": 37, + "sourceLength": 10, + "clauseIndex": 0, + "groupKind": null, + "listOperator": null + } + ], + "commands": [ + { + "clauseIndex": 0, + "immediateRole": "LoopBody", + "isComplete": true, + "ancestry": [ + { + "ancestorKind": "Block", + "region": "Root", + "childIndex": 0, + "sourceStart": 0, + "sourceLength": 53 + }, + { + "ancestorKind": "ForEach", + "region": "LoopBody", + "childIndex": null, + "sourceStart": 0, + "sourceLength": 53 + }, + { + "ancestorKind": "Block", + "region": "Statement", + "childIndex": 0, + "sourceStart": 36, + "sourceLength": 13 + } + ], + "effectiveArguments": [ + { + "clauseElementIndex": 2, + "value": { + "kind": "Unknown", + "values": [], + "pattern": null, + "coveringDirectory": null + } + } + ], + "workingDirectory": { + "kind": "Unknown", + "values": [], + "pattern": null, + "coveringDirectory": null + } + } + ] + }, + "notes": "Parent traversal anywhere after glob expansion invalidates the covering-directory proof and widens to Unknown." +} diff --git a/tests/ShellSyntaxTree.Tests/Corpus/bash/216_v03_for_dot_glob_parent_escape.json b/tests/ShellSyntaxTree.Tests/Corpus/bash/216_v03_for_dot_glob_parent_escape.json new file mode 100644 index 0000000..a5eaec4 --- /dev/null +++ b/tests/ShellSyntaxTree.Tests/Corpus/bash/216_v03_for_dot_glob_parent_escape.json @@ -0,0 +1,144 @@ +{ + "name": "v0.3 Bash for-in dot glob parent escape", + "input": "for f in /tmp/.[.]/etc/passwd; do rm -- \"$f\"; done", + "expected": { + "isUnparseable": false, + "clauses": [ + { + "operator": "None", + "verb": [ + "rm" + ], + "args": [ + { + "raw": "--", + "kind": "Literal", + "isPath": false, + "resolved": "__NULL__", + "isFlag": true + }, + { + "raw": "\"$f\"", + "kind": "DynamicSkip", + "isPath": false, + "resolved": "__NULL__", + "isFlag": false + } + ], + "redirects": [] + } + ], + "syntax": [ + { + "kind": "Block", + "parentIndex": null, + "region": "Unknown", + "childIndex": null, + "sourceStart": 0, + "sourceLength": 50, + "clauseIndex": null, + "groupKind": null, + "listOperator": null + }, + { + "kind": "ForEach", + "parentIndex": 0, + "region": "Root", + "childIndex": 0, + "sourceStart": 0, + "sourceLength": 50, + "clauseIndex": null, + "groupKind": null, + "listOperator": null, + "bindingName": "f", + "bindingRaw": "f", + "bindingSourceStart": 4, + "bindingSourceLength": 1, + "iterableRaw": "/tmp/.[.]/etc/passwd", + "iterableSourceStart": 9, + "iterableSourceLength": 20 + }, + { + "kind": "Block", + "parentIndex": 1, + "region": "Iterator", + "childIndex": null, + "sourceStart": 9, + "sourceLength": 20, + "clauseIndex": null, + "groupKind": null, + "listOperator": null + }, + { + "kind": "Block", + "parentIndex": 1, + "region": "LoopBody", + "childIndex": null, + "sourceStart": 33, + "sourceLength": 13, + "clauseIndex": null, + "groupKind": null, + "listOperator": null + }, + { + "kind": "SimpleCommand", + "parentIndex": 3, + "region": "Statement", + "childIndex": 0, + "sourceStart": 34, + "sourceLength": 10, + "clauseIndex": 0, + "groupKind": null, + "listOperator": null + } + ], + "commands": [ + { + "clauseIndex": 0, + "immediateRole": "LoopBody", + "isComplete": true, + "ancestry": [ + { + "ancestorKind": "Block", + "region": "Root", + "childIndex": 0, + "sourceStart": 0, + "sourceLength": 50 + }, + { + "ancestorKind": "ForEach", + "region": "LoopBody", + "childIndex": null, + "sourceStart": 0, + "sourceLength": 50 + }, + { + "ancestorKind": "Block", + "region": "Statement", + "childIndex": 0, + "sourceStart": 33, + "sourceLength": 13 + } + ], + "effectiveArguments": [ + { + "clauseElementIndex": 2, + "value": { + "kind": "Unknown", + "values": [], + "pattern": null, + "coveringDirectory": null + } + } + ], + "workingDirectory": { + "kind": "Unknown", + "values": [], + "pattern": null, + "coveringDirectory": null + } + } + ] + }, + "notes": "A dot-prefixed glob can expand to the parent entry when globskipdots is disabled, so its value widens to Unknown." +} diff --git a/tests/ShellSyntaxTree.Tests/Corpus/bash/46_unparseable_for_loop.json b/tests/ShellSyntaxTree.Tests/Corpus/bash/46_unparseable_for_loop.json index 12ef66e..480d2ea 100644 --- a/tests/ShellSyntaxTree.Tests/Corpus/bash/46_unparseable_for_loop.json +++ b/tests/ShellSyntaxTree.Tests/Corpus/bash/46_unparseable_for_loop.json @@ -1,9 +1,41 @@ { - "name": "Unparseable: for-loop control flow", + "name": "Bash for-in loop with unquoted bounded binding", "input": "for i in 1 2; do echo $i; done", "expected": { - "isUnparseable": true, - "unparseableReasonContains": "'for'" + "isUnparseable": false, + "clauses": [ + { + "operator": "None", + "verb": ["echo"], + "args": [ + { "raw": "$i", "kind": "EnvVar", "isPath": false, "resolved": "__NULL__", "isFlag": false } + ], + "redirects": [] + } + ], + "syntax": [ + { "kind": "Block", "parentIndex": null, "region": "Unknown", "childIndex": null, "sourceStart": 0, "sourceLength": 30, "clauseIndex": null, "groupKind": null, "listOperator": null }, + { "kind": "ForEach", "parentIndex": 0, "region": "Root", "childIndex": 0, "sourceStart": 0, "sourceLength": 30, "clauseIndex": null, "groupKind": null, "listOperator": null, "bindingName": "i", "bindingRaw": "i", "bindingSourceStart": 4, "bindingSourceLength": 1, "iterableRaw": "1 2", "iterableSourceStart": 9, "iterableSourceLength": 3 }, + { "kind": "Block", "parentIndex": 1, "region": "Iterator", "childIndex": null, "sourceStart": 9, "sourceLength": 3, "clauseIndex": null, "groupKind": null, "listOperator": null }, + { "kind": "Block", "parentIndex": 1, "region": "LoopBody", "childIndex": null, "sourceStart": 16, "sourceLength": 10, "clauseIndex": null, "groupKind": null, "listOperator": null }, + { "kind": "SimpleCommand", "parentIndex": 3, "region": "Statement", "childIndex": 0, "sourceStart": 17, "sourceLength": 7, "clauseIndex": 0, "groupKind": null, "listOperator": null } + ], + "commands": [ + { + "clauseIndex": 0, + "immediateRole": "LoopBody", + "isComplete": true, + "ancestry": [ + { "ancestorKind": "Block", "region": "Root", "childIndex": 0, "sourceStart": 0, "sourceLength": 30 }, + { "ancestorKind": "ForEach", "region": "LoopBody", "childIndex": null, "sourceStart": 0, "sourceLength": 30 }, + { "ancestorKind": "Block", "region": "Statement", "childIndex": 0, "sourceStart": 16, "sourceLength": 10 } + ], + "effectiveArguments": [ + { "clauseElementIndex": 1, "value": { "kind": "Unknown", "values": [], "pattern": null, "coveringDirectory": null } } + ], + "workingDirectory": { "kind": "Unknown", "values": [], "pattern": null, "coveringDirectory": null } + } + ] }, - "notes": "SPEC §11: control-flow keyword as the verb sets outer IsUnparseable." + "notes": "v0.3 parses bounded for-in structure; unquoted expansion remains Unknown because Bash field splitting is possible." } diff --git a/tests/ShellSyntaxTree.Tests/DesignCorpus/v0.3/bash.json b/tests/ShellSyntaxTree.Tests/DesignCorpus/v0.3/bash.json index 17da67b..fb7a73b 100644 --- a/tests/ShellSyntaxTree.Tests/DesignCorpus/v0.3/bash.json +++ b/tests/ShellSyntaxTree.Tests/DesignCorpus/v0.3/bash.json @@ -175,6 +175,7 @@ }, { "id": "bash-for-literal-finite", + "compatibilityProjectionLanded": true, "concern": "Finite literal loop binding", "input": "for f in a.txt b.txt; do rm -- \"$f\"; done", "current": { "isUnparseable": true, "reasonContains": "'for'" }, @@ -202,6 +203,7 @@ }, { "id": "bash-for-static-glob", + "compatibilityProjectionLanded": true, "concern": "Bounded glob without filesystem enumeration", "input": "for f in /tmp/*.txt; do rm -- \"$f\"; done", "current": { "isUnparseable": true, "reasonContains": "'for'" }, @@ -229,6 +231,7 @@ }, { "id": "bash-for-dynamic-glob-root", + "compatibilityProjectionLanded": true, "concern": "Dynamic glob covering directory", "input": "for f in \"$DIR\"/*.txt; do rm -- \"$f\"; done", "current": { "isUnparseable": true, "reasonContains": "'for'" }, @@ -256,6 +259,7 @@ }, { "id": "bash-for-command-substitution-iterator", + "compatibilityProjectionLanded": true, "concern": "Executable iterator with unknown produced values", "input": "for f in $(find /tmp -type f); do rm -- \"$f\"; done", "current": { "isUnparseable": true, "reasonContains": "'for'" }, @@ -288,6 +292,7 @@ }, { "id": "bash-for-native-option-injection", + "compatibilityProjectionLanded": true, "concern": "Finite value can change native option parsing", "input": "for f in -rf /tmp/x; do rm \"$f\"; done", "current": { "isUnparseable": true, "reasonContains": "'for'" }, @@ -315,6 +320,7 @@ }, { "id": "bash-nested-for-cross-product", + "compatibilityProjectionLanded": true, "concern": "Bounded nested-loop value combination", "input": "for d in a b; do for f in x y; do echo \"$d/$f\"; done; done", "current": { "isUnparseable": true, "reasonContains": "'for'" }, @@ -344,6 +350,7 @@ }, { "id": "bash-for-body-pipeline", + "compatibilityProjectionLanded": true, "concern": "Command roles compose with loop and pipeline ancestry", "input": "for f in a b; do printf '%s\\n' \"$f\" | sort; done", "current": { "isUnparseable": true, "reasonContains": "'for'" }, @@ -375,6 +382,7 @@ }, { "id": "bash-malformed-for-missing-done", + "compatibilityProjectionLanded": true, "concern": "Incomplete delimiter safe-fail", "input": "for f in a b; do rm -- \"$f\"", "current": { "isUnparseable": true, "reasonContains": "'for'" }, @@ -532,6 +540,7 @@ }, { "id": "bash-for-candidate-cap-32", + "compatibilityProjectionLanded": true, "concern": "Finite candidate domain at the fixed cap", "input": "for f in v01 v02 v03 v04 v05 v06 v07 v08 v09 v10 v11 v12 v13 v14 v15 v16 v17 v18 v19 v20 v21 v22 v23 v24 v25 v26 v27 v28 v29 v30 v31 v32; do printf '%s\\n' \"$f\"; done", "current": { "isUnparseable": true, "reasonContains": "'for'" }, @@ -559,6 +568,7 @@ }, { "id": "bash-for-candidate-overflow-33", + "compatibilityProjectionLanded": true, "concern": "Candidate overflow collapses instead of truncating", "input": "for f in v01 v02 v03 v04 v05 v06 v07 v08 v09 v10 v11 v12 v13 v14 v15 v16 v17 v18 v19 v20 v21 v22 v23 v24 v25 v26 v27 v28 v29 v30 v31 v32 v33; do printf '%s\\n' \"$f\"; done", "current": { "isUnparseable": true, "reasonContains": "'for'" }, diff --git a/tests/ShellSyntaxTree.Tests/Lexing/BashLexerTests.cs b/tests/ShellSyntaxTree.Tests/Lexing/BashLexerTests.cs index b34f670..231962d 100644 --- a/tests/ShellSyntaxTree.Tests/Lexing/BashLexerTests.cs +++ b/tests/ShellSyntaxTree.Tests/Lexing/BashLexerTests.cs @@ -406,6 +406,18 @@ public void Complex_param_expansion_emits_unparseable_sentinel() Assert.Contains("complex parameter expansion", t.UnparseableReason); } + [Theory] + [InlineData("${f:-$(evil)}")] + [InlineData("\"${f:-$(evil)}\"")] + [InlineData("${f#prefix}")] + public void Parameter_operators_emit_unparseable_sentinel(string source) + { + var token = Assert.Single(LexNonWs(source)); + + Assert.Equal(BashTokenKind.UnparseableSentinel, token.Kind); + Assert.Contains("complex parameter expansion", token.UnparseableReason); + } + [Fact] public void Unbalanced_double_quote_emits_unparseable_sentinel() { diff --git a/tests/ShellSyntaxTree.Tests/Parsing/BashCommandParserTests.cs b/tests/ShellSyntaxTree.Tests/Parsing/BashCommandParserTests.cs index 5e873da..47776d4 100644 --- a/tests/ShellSyntaxTree.Tests/Parsing/BashCommandParserTests.cs +++ b/tests/ShellSyntaxTree.Tests/Parsing/BashCommandParserTests.cs @@ -496,11 +496,11 @@ public void Unbalanced_close_paren_marks_outer_unparseable() } [Fact] - public void Control_flow_for_keyword_marks_outer_unparseable() + public void Supported_for_in_loop_populates_body_clause() { var result = Parse("for i in 1 2 3; do echo $i; done"); - Assert.True(result.IsUnparseable); - Assert.Contains("'for'", result.UnparseableReason!); + Assert.False(result.IsUnparseable, result.UnparseableReason); + Assert.Equal(new[] { "echo" }, Assert.Single(result.Clauses).Verb.Tokens); } [Fact] @@ -910,14 +910,11 @@ public void Heredoc_terminator_newline_separates_following_clause() } [Fact] - public void Control_flow_keyword_after_newline_marks_outer_unparseable() + public void Supported_for_in_loop_after_newline_is_a_second_statement() { - // A control-flow keyword opening a newline-separated clause must - // still safe-fail per SPEC §11 — TryDetectAnomaly treats the - // newline as a verb-slot boundary. - var result = Parse("echo hi\nfor i in 1 2 3"); - Assert.True(result.IsUnparseable); - Assert.Contains("'for'", result.UnparseableReason!); + var result = Parse("echo hi\nfor i in 1 2 3; do echo $i; done"); + Assert.False(result.IsUnparseable, result.UnparseableReason); + Assert.Equal(new[] { "echo", "echo" }, result.Clauses.Select(c => c.Verb.Tokens[0])); } // ---------------- Subshell ---------------- diff --git a/tests/ShellSyntaxTree.Tests/Parsing/BashForInStructuralTests.cs b/tests/ShellSyntaxTree.Tests/Parsing/BashForInStructuralTests.cs new file mode 100644 index 0000000..3a4d671 --- /dev/null +++ b/tests/ShellSyntaxTree.Tests/Parsing/BashForInStructuralTests.cs @@ -0,0 +1,435 @@ +// ----------------------------------------------------------------------- +// +// Copyright (C) 2026 - 2026 Aaron Stannard +// +// ----------------------------------------------------------------------- +using System.Linq; +using Xunit; + +namespace ShellSyntaxTree.Tests.Parsing; + +/// Pins the bounded Bash for name in words vertical slice. +public class BashForInStructuralTests +{ + [Fact] + public void Literal_iterable_emits_one_body_occurrence_with_finite_effective_value() + { + const string source = "for f in a.txt b.txt; do rm -- \"$f\"; done"; + + var result = Parse(source); + + Assert.False(result.IsUnparseable, result.UnparseableReason); + var loop = Assert.IsType(Assert.Single(result.Syntax.Statements)); + Assert.Equal("f", loop.Binding.Name); + Assert.Equal("f", loop.Binding.Source.Raw); + Assert.Equal(4, loop.Binding.Source.SourceStart); + Assert.Equal(1, loop.Binding.Source.SourceLength); + Assert.Equal("a.txt b.txt", loop.Iterable.Raw); + Assert.Equal(9, loop.Iterable.SourceStart); + Assert.Equal(11, loop.Iterable.SourceLength); + Assert.Empty(loop.IteratorCommands.Statements); + Assert.Equal(0, loop.SourceStart); + Assert.Equal(source.Length, loop.SourceLength); + + var simple = Assert.IsType(Assert.Single(loop.Body.Statements)); + var occurrence = Assert.Single(result.Commands); + Assert.Same(simple.Clause, occurrence.Clause); + Assert.Equal(CommandOccurrenceRole.LoopBody, occurrence.ImmediateRole); + Assert.True(occurrence.IsComplete); + var effective = Assert.Single(occurrence.EffectiveArguments); + Assert.Equal("\"$f\"", simple.Clause.Elements[effective.ClauseElementIndex].Raw); + AssertDomain(effective.Value, ShellValueDomainKind.FiniteSet, "a.txt", "b.txt"); + Assert.Equal(ArgKind.DynamicSkip, simple.Clause.Elements[effective.ClauseElementIndex].Kind); + } + + [Theory] + [InlineData("for f in a\"b\"; do echo \"$f\"; done", "ab")] + [InlineData("for f in \"a\"'b'; do echo \"$f\"; done", "ab")] + [InlineData("for f in \"\"; do echo \"$f\"; done", "")] + [InlineData("for f in a a; do echo \"$f\"; done", "a")] + [InlineData("for for in a; do echo \"$for\"; done", "a")] + [InlineData("for in in a; do echo \"$in\"; done", "a")] + [InlineData("for do in a; do echo \"$do\"; done", "a")] + [InlineData("for done in a; do echo \"$done\"; done", "a")] + [InlineData("for f in a; do echo \"${f}\"; done", "a")] + [InlineData("for f in a; do echo \"${f}.txt\"; done", "a.txt")] + [InlineData("for f in a; do echo \"$f.txt\"; done", "a.txt")] + public void Bash_word_formation_and_identifier_rules_preserve_exact_values( + string source, + string expected) + { + var result = Parse(source); + + Assert.False(result.IsUnparseable, result.UnparseableReason); + var effective = Assert.Single(Assert.Single(result.Commands).EffectiveArguments); + AssertDomain(effective.Value, ShellValueDomainKind.Exact, expected); + } + + [Fact] + public void Static_path_glob_is_a_pattern_without_filesystem_enumeration() + { + var result = Parse("for f in /tmp/*.txt; do rm -- \"$f\"; done"); + + var domain = Assert.Single(Assert.Single(result.Commands).EffectiveArguments).Value; + Assert.Equal(ShellValueDomainKind.Pattern, domain.Kind); + Assert.Equal("/tmp/*.txt", domain.Pattern); + Assert.Equal("/tmp", domain.CoveringDirectory); + Assert.Empty(domain.Values); + } + + [Fact] + public void Relative_static_glob_uses_the_original_parser_working_directory() + { + var result = Parse("for f in *.txt; do rm -- \"$f\"; done"); + + var domain = Assert.Single(Assert.Single(result.Commands).EffectiveArguments).Value; + Assert.Equal(ShellValueDomainKind.Pattern, domain.Kind); + Assert.Equal("*.txt", domain.Pattern); + Assert.Equal("/work", domain.CoveringDirectory); + } + + [Theory] + [InlineData("cd /a || cd /b; for f in *.txt; do rm -- \"$f\" rel.txt; done")] + [InlineData("cd /a | cat; for f in *.txt; do rm -- \"$f\" rel.txt; done")] + [InlineData("command cd /a; for f in x; do rm -- \"$f\" rel.txt; done")] + [InlineData("builtin cd /a; for f in x; do rm -- \"$f\" rel.txt; done")] + [InlineData("eval \"cd /a\"; for f in x; do rm -- \"$f\" rel.txt; done")] + [InlineData("trap \"f=x\" DEBUG; for f in a b; do echo \"$f\"; done")] + public void Loop_after_prior_shell_state_mutation_fails_atomically(string source) + { + var result = Parse(source); + + Assert.True(result.IsUnparseable); + Assert.Empty(result.Commands); + Assert.Empty(result.Clauses); + Assert.Contains("shell-state mutation", result.UnparseableReason!); + } + + [Theory] + [InlineData("for f in \"$DIR\"/*.txt; do rm -- \"$f\"; done")] + [InlineData("for f in ../*.txt; do rm -- \"$f\"; done")] + [InlineData("for f in /tmp/*/../../etc/passwd; do rm -- \"$f\"; done")] + [InlineData("for f in */../../etc/passwd; do rm -- \"$f\"; done")] + [InlineData("for f in /tmp/.[.]/etc/passwd; do rm -- \"$f\"; done")] + [InlineData("for f in /tmp/.?/etc/passwd; do rm -- \"$f\"; done")] + [InlineData("for f in /tmp/..*/etc/passwd; do rm -- \"$f\"; done")] + [InlineData("for f in {a,b}; do rm -- \"$f\"; done")] + [InlineData("for f in $HOME; do rm -- \"$f\"; done")] + public void Unbounded_iterables_keep_the_body_value_unknown(string source) + { + var result = Parse(source); + + Assert.False(result.IsUnparseable, result.UnparseableReason); + var effective = Assert.Single(Assert.Single(result.Commands).EffectiveArguments); + Assert.Equal(ShellValueDomainKind.Unknown, effective.Value.Kind); + } + + [Fact] + public void Iterator_substitution_precedes_body_and_does_not_see_new_binding() + { + var result = Parse("for f in $(printf '%s' \"$f\"); do rm -- \"$f\"; done"); + + Assert.False(result.IsUnparseable, result.UnparseableReason); + Assert.Equal(new[] { "printf", "rm" }, result.Commands.Select(CommandVerb)); + Assert.Equal(CommandOccurrenceRole.Substitution, result.Commands[0].ImmediateRole); + Assert.Empty(result.Commands[0].EffectiveArguments); + Assert.Equal(CommandOccurrenceRole.LoopBody, result.Commands[1].ImmediateRole); + Assert.Equal( + ShellValueDomainKind.Unknown, + Assert.Single(result.Commands[1].EffectiveArguments).Value.Kind); + var loop = Assert.IsType(Assert.Single(result.Syntax.Statements)); + Assert.IsType(Assert.Single(loop.IteratorCommands.Statements)); + } + + [Fact] + public void Body_substitution_inherits_the_loop_binding() + { + var result = Parse( + "for f in a b; do printf '%s' \"$(echo \"$f\")\"; done"); + + Assert.False(result.IsUnparseable, result.UnparseableReason); + Assert.Equal(new[] { "echo", "printf" }, result.Commands.Select(CommandVerb)); + AssertDomain( + Assert.Single(result.Commands[0].EffectiveArguments).Value, + ShellValueDomainKind.FiniteSet, + "a", + "b"); + Assert.All(result.Commands, command => + Assert.Equal(ShellValueDomainKind.Unknown, command.WorkingDirectory.Kind)); + } + + [Fact] + public void Nested_bindings_cross_product_and_repeated_binding_remains_correlated() + { + var result = Parse( + "for d in a b; do for f in x y; do echo \"$d/$f/$d\"; done; done"); + + Assert.False(result.IsUnparseable, result.UnparseableReason); + var effective = Assert.Single(Assert.Single(result.Commands).EffectiveArguments); + AssertDomain( + effective.Value, + ShellValueDomainKind.FiniteSet, + "a/x/a", + "a/y/a", + "b/x/b", + "b/y/b"); + } + + [Fact] + public void Nested_reuse_of_an_active_binding_name_fails_atomically() + { + var result = Parse( + "for f in a b; do for f in x y; do printf %s \"$f\"; done; echo \"$f\"; done"); + + Assert.True(result.IsUnparseable); + Assert.Empty(result.Commands); + Assert.Empty(result.Clauses); + Assert.Contains("binding reuse", result.UnparseableReason!); + } + + [Fact] + public void Unquoted_binding_is_visible_but_unknown_due_to_field_splitting() + { + var result = Parse("for f in a b; do rm $f; done"); + + Assert.False(result.IsUnparseable, result.UnparseableReason); + var effective = Assert.Single(Assert.Single(result.Commands).EffectiveArguments); + Assert.Equal(ShellValueDomainKind.Unknown, effective.Value.Kind); + Assert.Equal(ArgKind.DynamicSkip, EffectiveValueElement(result, effective).Kind); + } + + [Fact] + public void Option_like_candidates_are_not_reclassified_by_loop_analysis() + { + var result = Parse("for f in -rf /tmp/x; do rm \"$f\"; done"); + + Assert.False(result.IsUnparseable, result.UnparseableReason); + var command = Assert.Single(result.Commands); + var effective = Assert.Single(command.EffectiveArguments); + AssertDomain(effective.Value, ShellValueDomainKind.FiniteSet, "-rf", "/tmp/x"); + Assert.Equal(ArgKind.DynamicSkip, command.Clause.Elements[effective.ClauseElementIndex].Kind); + } + + [Fact] + public void Candidate_cap_is_exact_and_overflow_becomes_unknown() + { + var exactValues = Enumerable.Range(1, ShellAnalysisLimits.MaxValueCandidates) + .Select(index => $"v{index:00}") + .ToArray(); + var exact = Parse( + $"for f in {string.Join(" ", exactValues)}; do echo \"$f\"; done"); + var overflow = Parse( + $"for f in {string.Join(" ", exactValues)} v33; do echo \"$f\"; done"); + + AssertDomain( + Assert.Single(Assert.Single(exact.Commands).EffectiveArguments).Value, + ShellValueDomainKind.FiniteSet, + exactValues); + Assert.Equal( + ShellValueDomainKind.Unknown, + Assert.Single(Assert.Single(overflow.Commands).EffectiveArguments).Value.Kind); + } + + [Fact] + public void Empty_iterable_keeps_the_authored_body_with_unknown_binding() + { + var result = Parse("for f in; do echo \"$f\"; done"); + + Assert.False(result.IsUnparseable, result.UnparseableReason); + Assert.Single(result.Commands); + Assert.Equal( + ShellValueDomainKind.Unknown, + Assert.Single(Assert.Single(result.Commands).EffectiveArguments).Value.Kind); + } + + [Fact] + public void Pipelines_compose_with_loop_ancestry_without_synthetic_operators() + { + var result = Parse("for f in a b; do printf '%s' \"$f\" | sort; done"); + + Assert.False(result.IsUnparseable, result.UnparseableReason); + Assert.Equal( + new[] { CommandOccurrenceRole.PipelineStage, CommandOccurrenceRole.PipelineStage }, + result.Commands.Select(command => command.ImmediateRole)); + Assert.Equal( + new[] { CompoundOperator.None, CompoundOperator.Pipe }, + result.Clauses.Select(clause => clause.Operator)); + Assert.Contains( + result.Commands[0].Ancestry, + frame => frame.AncestorKind == ShellSyntaxKind.ForEach && + frame.Region == CommandAncestryRegion.LoopBody); + } + + [Theory] + [InlineData("for f in a; do unset f; done")] + [InlineData("for f in a; do read f; done")] + [InlineData("for f in a; do printf -v f x; done")] + [InlineData("for f in a; do eval 'f=x'; done")] + [InlineData("for f in a; do source script.sh; done")] + [InlineData("for f in a; do cd /tmp; done")] + [InlineData("for f in a b; do trap 'f=x' DEBUG; echo \"$f\"; done")] + [InlineData("for f in a; do command unset f; echo \"$f\"; done")] + [InlineData("for f in a; do builtin unset f; echo \"$f\"; done")] + public void Binding_mutation_fails_the_whole_parse(string source) + { + var result = Parse(source); + + Assert.True(result.IsUnparseable); + Assert.Empty(result.Commands); + Assert.Empty(result.Clauses); + Assert.Contains("mutation", result.UnparseableReason!); + } + + [Theory] + [InlineData("for 1f in a; do echo ok; done")] + [InlineData("for f-x in a; do echo ok; done")] + [InlineData("for \\f in a; do echo ok; done")] + [InlineData("for f a; do echo ok; done")] + [InlineData("for f in a do echo ok; done")] + [InlineData("for f in a; echo ok; done")] + [InlineData("for f in a; do done")] + [InlineData("for f in a; do echo ok")] + [InlineData("for f; do echo ok; done")] + [InlineData("for f in a; do; done")] + [InlineData("\\for f in a; do echo ok; done")] + [InlineData("for f \\in a; do echo ok; done")] + [InlineData("for f i\\n a; do echo ok; done")] + [InlineData("for f in a; \\do echo ok; done")] + [InlineData("for f in a; d\\o echo ok; done")] + [InlineData("for f in a; do echo ok; \\done")] + public void Malformed_or_unsupported_for_forms_fail_atomically(string source) + { + var result = Parse(source); + + Assert.True(result.IsUnparseable); + Assert.Empty(result.Commands); + Assert.Empty(result.Clauses); + } + + [Theory] + [InlineData("do rm -rf /")] + [InlineData("done")] + public void Stray_loop_delimiters_fail_atomically(string source) + { + var result = Parse(source); + + Assert.True(result.IsUnparseable); + Assert.Empty(result.Commands); + Assert.Empty(result.Clauses); + } + + [Fact] + public void Hidden_execution_in_parameter_operator_fails_atomically() + { + var result = Parse("for f in a; do echo \"${f:-$(evil)}\"; done"); + + Assert.True(result.IsUnparseable); + Assert.Empty(result.Commands); + Assert.Empty(result.Clauses); + } + + [Fact] + public void Contextual_words_remain_ordinary_arguments_outside_command_position() + { + var result = Parse("echo for in do done"); + + Assert.False(result.IsUnparseable, result.UnparseableReason); + Assert.Equal(new[] { "for", "in", "do", "done" }, Assert.Single(result.Clauses).Args.Select(a => a.Raw)); + } + + [Fact] + public void Loop_structural_depth_limit_is_enforced() + { + var exact = Parse(NestedLoops(ShellAnalysisLimits.MaxStructuralNesting)); + var overflow = Parse(NestedLoops(ShellAnalysisLimits.MaxStructuralNesting + 1)); + + Assert.False(exact.IsUnparseable, exact.UnparseableReason); + Assert.True(overflow.IsUnparseable); + Assert.Empty(overflow.Commands); + Assert.Empty(overflow.Clauses); + } + + [Fact] + public void Static_bash_c_does_not_inherit_unexported_loop_binding() + { + var result = Parse("for f in a b; do bash -c 'echo \"$f\"'; done"); + + Assert.False(result.IsUnparseable, result.UnparseableReason); + Assert.Empty(Assert.Single(result.Commands).EffectiveArguments); + } + + [Fact] + public void Static_bash_c_preserves_decoded_loop_facts_without_outer_source_spans() + { + var result = Parse("bash -c 'for f in a b; do rm -- \"$f\"; done'"); + + Assert.False(result.IsUnparseable, result.UnparseableReason); + var wrapper = Assert.IsType(Assert.Single(result.Syntax.Statements)); + var loop = Assert.IsType(Assert.Single(wrapper.Body.Statements)); + Assert.Equal("f", loop.Binding.Name); + Assert.Equal("f", loop.Binding.Source.Raw); + Assert.Equal("a b", loop.Iterable.Raw); + Assert.Null(loop.SourceStart); + Assert.Null(loop.SourceLength); + Assert.Null(loop.Binding.Source.SourceStart); + Assert.Null(loop.Binding.Source.SourceLength); + Assert.Null(loop.Iterable.SourceStart); + Assert.Null(loop.Iterable.SourceLength); + Assert.Null(loop.IteratorCommands.SourceStart); + Assert.Null(loop.IteratorCommands.SourceLength); + Assert.Null(loop.Body.SourceStart); + Assert.Null(loop.Body.SourceLength); + + var simple = Assert.IsType(Assert.Single(loop.Body.Statements)); + Assert.Null(simple.SourceStart); + Assert.Null(simple.SourceLength); + Assert.All(simple.Clause.Elements, element => + { + Assert.Null(element.SourceStart); + Assert.Null(element.SourceLength); + }); + + var occurrence = Assert.Single(result.Commands); + Assert.Same(simple.Clause, occurrence.Clause); + Assert.Same(simple.Clause, Assert.Single(result.Clauses)); + AssertDomain( + Assert.Single(occurrence.EffectiveArguments).Value, + ShellValueDomainKind.FiniteSet, + "a", + "b"); + } + + private static ClauseElement EffectiveValueElement( + ParsedCommand result, + EffectiveArgument effective) => + Assert.Single(result.Commands).Clause.Elements[effective.ClauseElementIndex]; + + private static ParsedCommand Parse(string input) => + new BashParser(new BashParserOptions + { + HomeDirectory = "/home/test", + WorkingDirectory = "/work", + }).Parse(input); + + private static string CommandVerb(CommandOccurrence command) => command.Clause.Verb.Joined; + + private static void AssertDomain( + ShellValueDomain actual, + ShellValueDomainKind expectedKind, + params string[] expectedValues) + { + Assert.Equal(expectedKind, actual.Kind); + Assert.Equal(expectedValues, actual.Values); + } + + private static string NestedLoops(int depth) + { + var source = "echo ok"; + for (var index = 0; index < depth; index++) + { + source = $"for f{index} in x; do {source}; done"; + } + + return source; + } +} diff --git a/tests/ShellSyntaxTree.Tests/Parsing/ShellValueOracleTests.cs b/tests/ShellSyntaxTree.Tests/Parsing/ShellValueOracleTests.cs index 2b6999f..8f7bd15 100644 --- a/tests/ShellSyntaxTree.Tests/Parsing/ShellValueOracleTests.cs +++ b/tests/ShellSyntaxTree.Tests/Parsing/ShellValueOracleTests.cs @@ -621,6 +621,141 @@ public void Bash_continuation_and_comment_boundary_samples_are_valid() } } + [Fact] + public void Bash_for_in_runtime_oracle_matches_word_formation_and_empty_values() + { + if (!IsNativeBashAvailable()) + { + return; + } + + var output = Run( + "bash", + "-c", + "for f in a\"b\" \"\" a a; do printf '<%s>\\n' \"$f\"; done"); + + Assert.Equal(new[] { "", "<>", "", "" }, Lines(output)); + } + + [Theory] + [InlineData("for")] + [InlineData("in")] + [InlineData("do")] + [InlineData("done")] + public void Bash_for_in_runtime_oracle_accepts_reserved_word_binding(string binding) + { + if (!IsNativeBashAvailable()) + { + return; + } + + Assert.Equal( + "a", + Run( + "bash", + "-c", + $"for {binding} in a; do printf %s \"${binding}\"; done")); + } + + [Fact] + public void Bash_debug_trap_can_mutate_a_loop_binding_before_body_commands() + { + if (!IsNativeBashAvailable()) + { + return; + } + + var output = Run( + "bash", + "-c", + "for f in a b; do trap 'f=x' DEBUG; printf '<%s>\\n' \"$f\"; done"); + + Assert.Equal(new[] { "", "" }, Lines(output)); + } + + [Fact] + public void Bash_debug_trap_installed_before_a_loop_can_mutate_each_binding() + { + if (!IsNativeBashAvailable()) + { + return; + } + + var output = Run( + "bash", + "-c", + "trap 'f=x' DEBUG; for f in a b; do printf '<%s>\\n' \"$f\"; done"); + + Assert.Equal(new[] { "", "" }, Lines(output)); + } + + [Fact] + public void Bash_nested_loop_binding_reuse_does_not_restore_the_outer_value() + { + if (!IsNativeBashAvailable()) + { + return; + } + + var output = Run( + "bash", + "-c", + "for f in a b; do for f in x y; do :; done; printf 'after=<%s>\\n' \"$f\"; done"); + + Assert.Equal(new[] { "after=", "after=" }, Lines(output)); + } + + [Fact] + public void Bash_globskipdots_can_expand_dot_prefixed_globs_to_parent_traversal() + { + if (!IsNativeBashAvailable()) + { + return; + } + + var output = Run( + "bash", + "-c", + "shopt -u globskipdots; printf '<%s>\\n' /tmp/.[.] /tmp/.? /tmp/..*"); + + Assert.Equal(new[] { "", "", "" }, Lines(output)); + } + + [Theory] + [InlineData("for 1f in a; do :; done")] + [InlineData("for f-x in a; do :; done")] + [InlineData("for \\f in a; do :; done")] + public void Bash_runtime_rejects_identifiers_that_parse_only_can_misclassify(string source) + { + if (!IsNativeBashAvailable()) + { + return; + } + + var result = RunUnchecked("bash", "-c", source); + + Assert.NotEqual(0, result.ExitCode); + } + + [Theory] + [InlineData("\\for f in a; do :; done")] + [InlineData("for f \\in a; do :; done")] + [InlineData("for f i\\n a; do :; done")] + [InlineData("for f in a; \\do :; done")] + [InlineData("for f in a; d\\o :; done")] + [InlineData("for f in a; do :; \\done")] + public void Bash_runtime_rejects_escaped_contextual_loop_keywords(string source) + { + if (!IsNativeBashAvailable()) + { + return; + } + + var result = RunUnchecked("bash", "-n", "-c", source); + + Assert.NotEqual(0, result.ExitCode); + } + private static bool IsAvailable(string executable) { try diff --git a/tools/PwshCorpusTool/CorpusJson.cs b/tools/PwshCorpusTool/CorpusJson.cs index 6920229..cde55d6 100644 --- a/tools/PwshCorpusTool/CorpusJson.cs +++ b/tools/PwshCorpusTool/CorpusJson.cs @@ -33,7 +33,8 @@ internal static string BuildEntry( bool outOfScope, bool includeElements, bool includeStructure, - bool includeOptionalAssertions) + bool includeOptionalAssertions, + bool includeV03Assertions) { var obj = new JsonObject { @@ -43,7 +44,8 @@ internal static string BuildEntry( parsed, includeElements, includeStructure, - includeOptionalAssertions), + includeOptionalAssertions, + includeV03Assertions), ["notes"] = notes, }; @@ -59,7 +61,8 @@ private static JsonObject BuildExpected( ParsedCommand parsed, bool includeElements, bool includeStructure, - bool includeOptionalAssertions) + bool includeOptionalAssertions, + bool includeV03Assertions) { var expected = new JsonObject { ["isUnparseable"] = parsed.IsUnparseable }; if (parsed.IsUnparseable) @@ -77,14 +80,16 @@ private static JsonObject BuildExpected( expected["clauses"] = clauses; if (includeStructure) { - expected["syntax"] = BuildSyntax(parsed); - expected["commands"] = BuildCommands(parsed); + expected["syntax"] = BuildSyntax(parsed, includeV03Assertions); + expected["commands"] = BuildCommands(parsed, includeV03Assertions); } return expected; } - private static JsonArray BuildSyntax(ParsedCommand parsed) + private static JsonArray BuildSyntax( + ParsedCommand parsed, + bool includeV03Assertions) { var nodes = new JsonArray(); AppendSyntax( @@ -95,6 +100,7 @@ private static JsonArray BuildSyntax(ParsedCommand parsed) listOperator: null, parsed, nodes, + includeV03Assertions, isRootBlock: true); return nodes; } @@ -107,6 +113,7 @@ private static void AppendSyntax( CompoundOperator? listOperator, ParsedCommand parsed, JsonArray nodes, + bool includeV03Assertions, bool isRootBlock = false) { if (node.Kind == ShellSyntaxKind.Unknown) @@ -117,8 +124,9 @@ private static void AppendSyntax( var currentIndex = nodes.Count; var clause = (node as SimpleCommandSyntax)?.Clause; + var forEachNode = node as ForEachSyntax; var clauseIndex = clause is null ? (int?)null : FindClauseIndex(parsed, clause); - nodes.Add(new JsonObject + var syntax = new JsonObject { ["kind"] = node.Kind.ToString(), ["parentIndex"] = JsonValue.Create(parentIndex), @@ -129,7 +137,19 @@ private static void AppendSyntax( ["clauseIndex"] = JsonValue.Create(clauseIndex), ["groupKind"] = (node as GroupSyntax)?.GroupKind.ToString(), ["listOperator"] = listOperator?.ToString(), - }); + }; + if (includeV03Assertions && forEachNode is not null) + { + syntax["bindingName"] = forEachNode.Binding.Name; + syntax["bindingRaw"] = forEachNode.Binding.Source.Raw; + syntax["bindingSourceStart"] = forEachNode.Binding.Source.SourceStart; + syntax["bindingSourceLength"] = forEachNode.Binding.Source.SourceLength; + syntax["iterableRaw"] = forEachNode.Iterable.Raw; + syntax["iterableSourceStart"] = forEachNode.Iterable.SourceStart; + syntax["iterableSourceLength"] = forEachNode.Iterable.SourceLength; + } + + nodes.Add(syntax); switch (node) { @@ -146,7 +166,8 @@ private static void AppendSyntax( index, listOperator: null, parsed, - nodes); + nodes, + includeV03Assertions); } break; @@ -160,7 +181,8 @@ private static void AppendSyntax( index, listOperator: null, parsed, - nodes); + nodes, + includeV03Assertions); } break; @@ -174,7 +196,8 @@ private static void AppendSyntax( index, listOperator: null, parsed, - nodes); + nodes, + includeV03Assertions); } break; @@ -188,7 +211,8 @@ private static void AppendSyntax( index, list.Items[index].Operator, parsed, - nodes); + nodes, + includeV03Assertions); } break; @@ -200,7 +224,8 @@ private static void AppendSyntax( childIndex: null, listOperator: null, parsed, - nodes); + nodes, + includeV03Assertions); break; case ForEachSyntax forEach: AppendSyntax( @@ -210,7 +235,8 @@ private static void AppendSyntax( childIndex: null, listOperator: null, parsed, - nodes); + nodes, + includeV03Assertions); AppendSyntax( forEach.Body, currentIndex, @@ -218,7 +244,8 @@ private static void AppendSyntax( childIndex: null, listOperator: null, parsed, - nodes); + nodes, + includeV03Assertions); break; case ConditionLoopSyntax loop: AppendSyntax( @@ -228,7 +255,8 @@ private static void AppendSyntax( childIndex: null, listOperator: null, parsed, - nodes); + nodes, + includeV03Assertions); AppendSyntax( loop.Body, currentIndex, @@ -236,7 +264,8 @@ private static void AppendSyntax( childIndex: null, listOperator: null, parsed, - nodes); + nodes, + includeV03Assertions); break; case ConditionalSyntax conditional: for (var index = 0; index < conditional.Branches.Count; index++) @@ -248,7 +277,8 @@ private static void AppendSyntax( index, listOperator: null, parsed, - nodes); + nodes, + includeV03Assertions); } if (conditional.Else is not null) @@ -260,7 +290,8 @@ private static void AppendSyntax( conditional.Branches.Count, listOperator: null, parsed, - nodes); + nodes, + includeV03Assertions); } break; @@ -272,7 +303,8 @@ private static void AppendSyntax( childIndex: null, listOperator: null, parsed, - nodes); + nodes, + includeV03Assertions); AppendSyntax( branch.Body, currentIndex, @@ -280,7 +312,8 @@ private static void AppendSyntax( childIndex: null, listOperator: null, parsed, - nodes); + nodes, + includeV03Assertions); break; case CommandSubstitutionSyntax substitution: AppendSyntax( @@ -290,7 +323,8 @@ private static void AppendSyntax( childIndex, listOperator: null, parsed, - nodes); + nodes, + includeV03Assertions); break; default: throw new InvalidOperationException( @@ -298,7 +332,9 @@ private static void AppendSyntax( } } - private static JsonArray BuildCommands(ParsedCommand parsed) + private static JsonArray BuildCommands( + ParsedCommand parsed, + bool includeV03Assertions) { var commands = new JsonArray(); foreach (var command in parsed.Commands) @@ -329,18 +365,52 @@ private static JsonArray BuildCommands(ParsedCommand parsed) }); } - commands.Add(new JsonObject + var commandJson = new JsonObject { ["clauseIndex"] = FindClauseIndex(parsed, command.Clause), ["immediateRole"] = command.ImmediateRole.ToString(), ["isComplete"] = command.IsComplete, ["ancestry"] = ancestry, - }); + }; + if (includeV03Assertions) + { + var effectiveArguments = new JsonArray(); + foreach (var effective in command.EffectiveArguments) + { + effectiveArguments.Add(new JsonObject + { + ["clauseElementIndex"] = effective.ClauseElementIndex, + ["value"] = BuildValueDomain(effective.Value), + }); + } + + commandJson["effectiveArguments"] = effectiveArguments; + commandJson["workingDirectory"] = BuildValueDomain(command.WorkingDirectory); + } + + commands.Add(commandJson); } return commands; } + private static JsonObject BuildValueDomain(ShellValueDomain domain) + { + var values = new JsonArray(); + foreach (var value in domain.Values) + { + values.Add(value); + } + + return new JsonObject + { + ["kind"] = domain.Kind.ToString(), + ["values"] = values, + ["pattern"] = domain.Pattern, + ["coveringDirectory"] = domain.CoveringDirectory, + }; + } + private static int FindClauseIndex(ParsedCommand parsed, Clause clause) { for (var index = 0; index < parsed.Clauses.Count; index++) diff --git a/tools/PwshCorpusTool/Program.cs b/tools/PwshCorpusTool/Program.cs index 3a0d21e..b663541 100644 --- a/tools/PwshCorpusTool/Program.cs +++ b/tools/PwshCorpusTool/Program.cs @@ -76,7 +76,8 @@ int Generate(string outputDir) entry.OutOfScope, entry.IncludeElements, entry.IncludeStructure, - entry.IncludeOptionalAssertions); + entry.IncludeOptionalAssertions, + includeV03Assertions: false); var fileName = $"{index:D3}_{entry.Slug}.json"; File.WriteAllText(Path.Combine(outputDir, fileName), json); index++; @@ -104,7 +105,8 @@ int Check(string command) parsed.IsUnparseable, includeElements: true, includeStructure: true, - includeOptionalAssertions: true)); + includeOptionalAssertions: true, + includeV03Assertions: false)); Console.WriteLine("---- real pwsh oracle ----"); var counts = PwshOracle.CountParseErrors(new[] { command }); @@ -142,7 +144,8 @@ int CheckBash(string command) outOfScope: false, includeElements: false, includeStructure: true, - includeOptionalAssertions: true)); + includeOptionalAssertions: true, + includeV03Assertions: true)); return 0; }