From 8de7620f7e1bc8c4cca705504fdc1c2ef5fe17c3 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Tue, 4 Aug 2026 20:13:55 +0000 Subject: [PATCH 1/8] feat: preserve clause element source order --- IMPLEMENTATION_PLAN.md | 17 +- PROJECT_CONTEXT.md | 8 +- README.md | 8 +- SPEC.POWERSHELL.md | 50 ++- SPEC.md | 134 +++++++- docs/CONSUMER_GUIDE.md | 64 +++- .../.openspec.yaml | 2 + .../preserve-clause-element-order/design.md | 139 ++++++++ .../preserve-clause-element-order/proposal.md | 42 +++ .../specs/clause-element-provenance/spec.md | 102 ++++++ .../preserve-clause-element-order/tasks.md | 43 +++ src/ShellSyntaxTree/Clause.cs | 7 + src/ShellSyntaxTree/ClauseElement.cs | 61 ++++ src/ShellSyntaxTree/ClauseElementRole.cs | 21 ++ .../Bash/Parsing/BashCommandParser.cs | 171 +++++++++- .../Internal/Bash/Verbs/BashPerVerbRules.cs | 16 +- .../Internal/Bash/Verbs/BashVerbs.cs | 16 +- .../Parsing/ClauseElementProvenance.cs | 41 +++ .../Pwsh/Parsing/PwshCommandParser.cs | 176 ++++++++++- .../ShellSyntaxTree.Tests/Corpus/AstAssert.cs | 53 +++- .../Corpus/CorpusRunnerTests.cs | 79 +++++ .../152_git_global_option_provenance.json | 79 +++++ .../153_git_subcommand_option_provenance.json | 79 +++++ .../154_git_global_config_provenance.json | 65 ++++ .../155_git_subcommand_config_provenance.json | 65 ++++ .../bash/156_git_mixed_option_provenance.json | 91 ++++++ ...157_git_heuristic_boundary_provenance.json | 79 +++++ .../220_git_global_option_provenance.json | 78 +++++ .../221_git_subcommand_option_provenance.json | 78 +++++ .../222_git_global_config_provenance.json | 76 +++++ .../223_git_subcommand_config_provenance.json | 76 +++++ .../224_git_mixed_option_provenance.json | 112 +++++++ ...225_git_heuristic_boundary_provenance.json | 98 ++++++ .../Parsing/BashPerVerbRulesTests.cs | 9 +- .../Parsing/ClauseElementTests.cs | 298 ++++++++++++++++++ .../PublicApiSnapshotTests.cs | 50 +++ tools/PwshCorpusTool/CorpusJson.cs | 49 ++- tools/PwshCorpusTool/CorpusManifest.cs | 20 +- tools/PwshCorpusTool/Program.cs | 6 +- 39 files changed, 2585 insertions(+), 73 deletions(-) create mode 100644 openspec/changes/preserve-clause-element-order/.openspec.yaml create mode 100644 openspec/changes/preserve-clause-element-order/design.md create mode 100644 openspec/changes/preserve-clause-element-order/proposal.md create mode 100644 openspec/changes/preserve-clause-element-order/specs/clause-element-provenance/spec.md create mode 100644 openspec/changes/preserve-clause-element-order/tasks.md create mode 100644 src/ShellSyntaxTree/ClauseElement.cs create mode 100644 src/ShellSyntaxTree/ClauseElementRole.cs create mode 100644 src/ShellSyntaxTree/Internal/Parsing/ClauseElementProvenance.cs create mode 100644 tests/ShellSyntaxTree.Tests/Corpus/bash/152_git_global_option_provenance.json create mode 100644 tests/ShellSyntaxTree.Tests/Corpus/bash/153_git_subcommand_option_provenance.json create mode 100644 tests/ShellSyntaxTree.Tests/Corpus/bash/154_git_global_config_provenance.json create mode 100644 tests/ShellSyntaxTree.Tests/Corpus/bash/155_git_subcommand_config_provenance.json create mode 100644 tests/ShellSyntaxTree.Tests/Corpus/bash/156_git_mixed_option_provenance.json create mode 100644 tests/ShellSyntaxTree.Tests/Corpus/bash/157_git_heuristic_boundary_provenance.json create mode 100644 tests/ShellSyntaxTree.Tests/Corpus/powershell/220_git_global_option_provenance.json create mode 100644 tests/ShellSyntaxTree.Tests/Corpus/powershell/221_git_subcommand_option_provenance.json create mode 100644 tests/ShellSyntaxTree.Tests/Corpus/powershell/222_git_global_config_provenance.json create mode 100644 tests/ShellSyntaxTree.Tests/Corpus/powershell/223_git_subcommand_config_provenance.json create mode 100644 tests/ShellSyntaxTree.Tests/Corpus/powershell/224_git_mixed_option_provenance.json create mode 100644 tests/ShellSyntaxTree.Tests/Corpus/powershell/225_git_heuristic_boundary_provenance.json create mode 100644 tests/ShellSyntaxTree.Tests/Parsing/ClauseElementTests.cs diff --git a/IMPLEMENTATION_PLAN.md b/IMPLEMENTATION_PLAN.md index 05a18b2..36b5739 100644 --- a/IMPLEMENTATION_PLAN.md +++ b/IMPLEMENTATION_PLAN.md @@ -54,6 +54,17 @@ priorities. Recurse into provably static `Invoke-Expression` / `iex` payloads, safe-fail computed and pipeline-fed code, share the existing recursion limits, and preserve current-scope PowerShell location attribution. +- [x] **Issue #62 — source-ordered clause elements.** Added the additive + `Clause.Elements` provenance view for Bash and PowerShell with exact raw + spelling, decoded values, source spans when available, verb-relative + argument placement, path facts, redirects, and conservative wrapper-span + handling. Authored order is authoritative; element roles and + `PrecedingVerbElementCount` explicitly mirror the greedy parser + projection rather than executable semantics. Paired Bash/PowerShell + corpus cases cover Git `-c`/`-C`, multiple occurrences, and a valueless + option that stops the greedy walk. Existing projection shapes and + synthetic cwd attribution remain compatible; native options that differ + only by case receive corrected metadata. - [x] **Issue #64 — path-shaped operands after native verb chains.** Stop the Bash and PowerShell native greedy passes before a token that matches the shared path-shape rules. Preserve that token as a resolved @@ -70,7 +81,9 @@ priorities. corpus. Review follow-ups shipped with it: the equals-form split moved to a shared `NativeFlagSyntax` so the two parsers can't drift, a colon value under an `=`-bearing parameter name safe-fails to `DynamicSkip`, - and `-?` lexes as one parameter token. + `-?` lexes as one parameter token, and native option tables now use + ordinal spelling while PowerShell cmdlet parameters remain + case-insensitive. ### 15. Release 0.2.0 (alpha → beta → stable) — SPEC.PWSH §15 / §17 @@ -78,6 +91,8 @@ priorities. `ShellSyntaxTree.0.2.0-alpha.nupkg` and it is live on nuget.org (released 2026-05-20). - [x] `0.2.0-beta.1` so Netclaw validates the parser + the breaking rename +- [ ] Publish the next `0.2.0` prerelease with the additive issue #62 + `Clause.Elements` provenance surface and migration guidance - [ ] Promote to stable `0.2.0` after Netclaw validation ### 16. Netclaw v0.2.0 integration — SPEC.PWSH §17 #9 diff --git a/PROJECT_CONTEXT.md b/PROJECT_CONTEXT.md index f435404..0c500a4 100644 --- a/PROJECT_CONTEXT.md +++ b/PROJECT_CONTEXT.md @@ -19,6 +19,9 @@ The output is a `ParsedCommand` containing: when known, and explicit `DynamicSkip` marking for unresolved env vars / unexpanded globs - redirect operators (`>`, `>>`, `<`, `2>`, `2>>`) +- source-ordered clause elements with exact spelling, decoded values, source + spans when available, and coordinates relative to parser-classified verb + elements; executable-specific semantics remain consumer-owned - Bash `cd && cmd` and PowerShell `Set-Location ; cmd` propagation — the target is attributed to subsequent clauses - recursion into `bash -c`, `pwsh -Command`, and `pwsh -EncodedCommand` so @@ -83,8 +86,9 @@ zero-native-deps .NET parser sized to what security gates actually need. and `.ps1` file-content parsing. - Performance optimization beyond "fast enough to invoke per shell call without noticeable latency" (~1 ms typical). -- Full IDE-style source mapping. Security-motivated token provenance is under - active design in issue #62. +- Full IDE-style source mapping. `Clause.Elements` provides security-motivated + provenance for significant clause leaves, not a lossless concrete syntax + tree. ### Versioning diff --git a/README.md b/README.md index e3ce725..fc07f4c 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,9 @@ dotnet add package ShellSyntaxTree --version 0.2.0-beta.1 ``` The `0.2.0-beta.1` prerelease is required for PowerShell support. The latest -stable `0.1.x` package supports Bash only. +stable `0.1.x` package supports Bash only. The public surface documented below +tracks the `dev` branch; `Clause.Elements` is scheduled for the next `0.2.0` +prerelease and is not present in `beta.1`. ## What you get @@ -120,12 +122,14 @@ public sealed record BashParserOptions : ShellParserOptions; public sealed record PwshParserOptions : ShellParserOptions; public sealed record ParsedCommand { /* Source, Clauses, IsUnparseable, … */ } -public sealed record Clause { /* Operator, Verb, Args, Redirects, IsSubshell, IsCommandStringWrapped */ } +public sealed record Clause { /* Operator, Verb, Args, Redirects, Elements, IsSubshell, IsCommandStringWrapped */ } +public sealed record ClauseElement { /* Raw, Value, Role, source span, verb-relative position, path facts */ } public sealed record VerbChain { /* Tokens, Joined, CanonicalVerb, IsDynamic */ } public sealed record Arg { /* Raw, Resolved, Kind, IsPath, IsCwdAttribution, IsFlag */ } public sealed record Redirect { /* Direction, Target, IsDynamicSkip */ } public enum ArgKind { Literal, EnvVar, Glob, Tilde, DynamicSkip } +public enum ClauseElementRole { Verb, Argument, Redirect } public enum RedirectDirection { In, Out, Append, ErrOut, ErrAppend } public enum CompoundOperator { None, AndIf, OrIf, Sequence, Pipe } ``` diff --git a/SPEC.POWERSHELL.md b/SPEC.POWERSHELL.md index 0f9ddd5..a5fff33 100644 --- a/SPEC.POWERSHELL.md +++ b/SPEC.POWERSHELL.md @@ -75,9 +75,11 @@ syntax (§5) are all PowerShell 7 semantics. The `pwsh` validation oracle ## 2. Public API Surface -The shared interface, AST records, and enums are defined in **`SPEC.md` §2** -and are unchanged. PowerShell adds the following to namespace -`ShellSyntaxTree`; everything else is internal. +The shared interface, AST records, and enums are defined in **`SPEC.md` §2**. +The additive `Clause.Elements`, `ClauseElement`, and `ClauseElementRole` +provenance surface applies identically to both parsers. PowerShell adds the +following parser types to namespace `ShellSyntaxTree`; everything else is +internal. ```csharp namespace ShellSyntaxTree; @@ -117,14 +119,17 @@ public sealed class PwshParser : IShellParser } ``` -Two shared types gain a change (see §3): +The shared v0.2 AST gains the following changes (see §3): - `VerbChain` gains an additive `string? CanonicalVerb` field. - `VerbChain` gains an additive `bool IsDynamic` field. +- `Clause` gains the additive `Elements` provenance view shared with Bash; + `ClauseElement` and `ClauseElementRole` define its entries. - `Clause.IsBashCWrapped` is renamed `Clause.IsCommandStringWrapped`. **Versioning.** `PwshParser`, `PwshParserOptions`, `ShellParserOptions`, -`VerbChain.CanonicalVerb`, and `VerbChain.IsDynamic` are additive. The +`VerbChain.CanonicalVerb`, `VerbChain.IsDynamic`, `Clause.Elements`, +`ClauseElement`, and `ClauseElementRole` are additive. The `Clause` field rename and the `BashParserOptions` reparenting are **breaking**; `SPEC.md` Appendix A permits a breaking AST change on a `0.x` minor bump when `RELEASE_NOTES.md` carries the old→new mapping and Netclaw is @@ -136,9 +141,38 @@ never throws on a well-formed string, exactly like `BashParser`. ## 3. AST Reference -The AST records and enums are defined in **`SPEC.md` §3** and are emitted -unchanged by `PwshParser` — a consumer walks a PowerShell `ParsedCommand` -exactly as it walks a bash one. Two deltas: +The AST records and enums are defined in **`SPEC.md` §3** and are emitted by +`PwshParser` under the same shared contract — a consumer walks a PowerShell +`ParsedCommand` exactly as it walks a bash one. PowerShell has the following +deltas and provenance rules: + +### `Clause.Elements` PowerShell rules + +PowerShell parameters, native options, quoted/here-string values, and opaque +dynamic regions each occupy their authored position in `Clause.Elements`. +The leading call operator in `& command` and grouping parentheses are shell +syntax rather than verb/argument/redirect leaves and do not appear. + +Inline parameter forms remain one source element. For `-Path:C:\repo`, the +element's `Raw` and `Value` describe the full parameter token while `Kind`, +`IsPath`, and `Resolved` describe the bound `C:\repo` value. Native +`--flag=value` follows the same rule as Bash. + +Clauses recursively surfaced from `pwsh -Command` and +`pwsh -EncodedCommand` retain inner `Raw` and `Value` but have null +`SourceStart` and `SourceLength`: quote/backtick processing, script-block +stripping, and base64 decoding do not provide a generally exact map into the +outer `ParsedCommand.Source`. + +`ClauseElement.Role` and `PrecedingVerbElementCount` mirror the shared greedy +native verb projection. They are AST coordinates, not native-executable +semantic boundaries. A PowerShell consumer applies executable-specific grammar +to the complete authored element order exactly as a Bash consumer does. + +PowerShell cmdlet names, aliases, and parameter names remain +case-insensitive. Native option spelling is ordinal and reuses the shared Bash +native tables unchanged: PowerShell does not make a native executable's `-c` +and `-C` options equivalent. ### `VerbChain.CanonicalVerb` (new, additive) diff --git a/SPEC.md b/SPEC.md index 7800d7d..6b58440 100644 --- a/SPEC.md +++ b/SPEC.md @@ -135,9 +135,11 @@ public abstract record ShellParserOptions // AST records — see §3. public sealed record ParsedCommand { ... } public sealed record Clause { ... } +public sealed record ClauseElement { ... } public sealed record VerbChain { ... } public sealed record Arg { ... } public sealed record Redirect { ... } +public enum ClauseElementRole { Verb, Argument, Redirect } public enum ArgKind { Literal, EnvVar, Glob, Tilde, DynamicSkip } public enum RedirectDirection { In, Out, Append, ErrOut, ErrAppend } public enum CompoundOperator { None, AndIf, OrIf, Sequence, Pipe } @@ -211,6 +213,13 @@ public sealed record Clause /// public IReadOnlyList Redirects { get; init; } = []; + /// + /// Significant source-authored verbs, arguments, and redirects in source + /// order. This is the provenance view; Verb, Args, and Redirects remain + /// compatibility projections. Synthetic cwd attribution is excluded. + /// + public IReadOnlyList Elements { get; init; } = []; + /// /// True when this clause is wrapped in a subshell (parens). Subshells /// isolate cd state — see §9. @@ -231,6 +240,90 @@ public sealed record Clause } ``` +### `ClauseElement` + +One significant source-authored element of a clause. `Elements` preserves the +cross-projection order that `Verb`, `Args`, and `Redirects` cannot represent on +their own. + +```csharp +public sealed record ClauseElement +{ + /// Exact authored source slice, including quote delimiters. + public string Raw { get; init; } = ""; + + /// + /// Lexer-decoded logical value. For a redirect this is the decoded target; + /// for an inline binding it remains the complete decoded source token. + /// + public string Value { get; init; } = ""; + + public ClauseElementRole Role { get; init; } + + /// + /// Span in ParsedCommand.Source. Null for elements surfaced through a + /// decoded command-string wrapper when no exact outer mapping exists. + /// + public int? SourceStart { get; init; } + public int? SourceLength { get; init; } + + /// + /// Number of parser-classified verb elements authored before this element + /// in the clause. For a verb element, this is its zero-based Verb.Tokens + /// index. This is an AST coordinate, not an executable-specific semantic + /// boundary. + /// + public int PrecedingVerbElementCount { get; init; } + + /// + /// Argument classification for this token, inline bound value, or redirect + /// target. Verb elements use Literal, except dynamic command names use + /// DynamicSkip. + /// + public ArgKind Kind { get; init; } + public bool IsFlag { get; init; } + public bool IsPath { get; init; } + public string? Resolved { get; init; } +} + +public enum ClauseElementRole +{ + Verb, + Argument, + Redirect +} +``` + +The collection contains significant leaves only: whitespace, comments, +compound operators, grouping delimiters, and shell call operators are excluded. +Each verb token appears exactly once with `Role=Verb`. Each authored argument +token appears once with `Role=Argument`; inline forms such as +`--work-tree=../repo` stay one element even when `Args` exposes separate flag +and value projections. Each redirect appears once with `Role=Redirect`; its +ordinal among redirect elements matches its ordinal in `Redirects`, and `Raw` +spans the operator through its target. + +`PrecedingVerbElementCount` is clause-local and resets to zero at every clause. +For `git -C /repo commit`, `-C` and `/repo` carry `1`; for +`git commit -C HEAD~1`, `-C` and `HEAD~1` carry `2`. ShellSyntaxTree reports +that parser-relative coordinate but does not assign Git-specific meaning to +it. `Role=Verb` mirrors the greedy `Clause.Verb` heuristic. Therefore an +unrecognized option can stop verb extraction and cause a later semantic +subcommand to appear with `Role=Argument`; consumers SHALL use the complete +authored element order rather than treating this count as an executable's +semantic command boundary. + +Synthetic cwd-attribution args are deliberately absent from `Elements`: they +remain available through `Args` with `IsCwdAttribution=true`. Clauses expanded +from command-string wrappers preserve each element's inner `Raw` and `Value`, +but set `SourceStart` and `SourceLength` to null rather than guessing how a +decoded or escaped inner character maps into the outer `ParsedCommand.Source`. + +Because `Clause` is a record, `Elements` participates in its generated value +equality and hashing. Generated `ToString()` and default JSON serialization +also include the projection. The API addition is source- and binary-additive, +but these generated behaviors are observably different. + ### `VerbChain` The verb of a clause. Multi-token to handle commands like `git push`, @@ -645,10 +738,10 @@ syntactic rule disambiguates `origin` (a branch name) from `worktree` not bake per-CLI knowledge into the parser. Consumers needing security-grade verb identification should pattern-prefix -match against the raw token stream: +match against the source-ordered `Clause.Elements` view: > A command matches an approval pattern `P` if and only if the first -> `len(P.verb_prefix)` tokens of the command equal `P.verb_prefix`. +> `len(P.verb_prefix)` verb elements of the command equal `P.verb_prefix`. This punts depth choice to the consumer (via the pattern they author) and accommodates the parser's over-extraction transparently: @@ -774,11 +867,11 @@ internal static readonly IReadOnlyDictionary> FlagsWithValue = new Dictionary>( StringComparer.OrdinalIgnoreCase) { - ["git"] = new HashSet(StringComparer.OrdinalIgnoreCase) { "-C", "--git-dir", "--work-tree" }, - ["curl"] = new HashSet(StringComparer.OrdinalIgnoreCase) { "-o", "--output", "-d", "--data" }, - ["wget"] = new HashSet(StringComparer.OrdinalIgnoreCase) { "-O", "--output-document" }, - ["docker"]= new HashSet(StringComparer.OrdinalIgnoreCase) { "-v", "--volume", "-f", "--file" }, - ["tar"] = new HashSet(StringComparer.OrdinalIgnoreCase) { "-f", "--file", "-C", "--directory" }, + ["git"] = new HashSet(StringComparer.Ordinal) { "-c", "-C", "--git-dir", "--work-tree" }, + ["curl"] = new HashSet(StringComparer.Ordinal) { "-o", "--output", "-d", "--data" }, + ["wget"] = new HashSet(StringComparer.Ordinal) { "-O", "--output-document" }, + ["docker"]= new HashSet(StringComparer.Ordinal) { "-v", "--volume", "-f", "--file" }, + ["tar"] = new HashSet(StringComparer.Ordinal) { "-f", "--file", "-C", "--directory" }, // Add as corpus surfaces real cases. }; ``` @@ -787,6 +880,15 @@ internal static readonly IReadOnlyDictionary> > because `IReadOnlySet` is .NET 5+ only and the library > multi-targets `netstandard2.0`. Internal-only — no public-API impact. +> **Native option case.** The outer verb dictionary retains its existing +> case-insensitive lookup, but each native option set uses `Ordinal`. Native +> executables receive option spelling unchanged in Bash and PowerShell and may +> assign different meanings by case. Git lists both `-c` and `-C`: both consume +> a value. The generic table classifies uppercase `-C` values as paths and +> lowercase `-c` values as non-paths. Executable-aware consumers still +> reinterpret command-scoped forms such as `git commit -c/-C`, where Git uses +> the operand as a revision rather than the generic table's global meaning. + > **Note:** the verb-chain walk consumes flag-with-value pairs > transparently. For `git -C /repo log`, the walk consumes `-C /repo` > before evaluating the next token; `log` is then verb-like and extends @@ -1174,6 +1276,16 @@ Clauses = [ Args = [ Arg { Raw = "-C", IsFlag = true }, Arg { Raw = "/repo", IsPath = true, Resolved = "/repo" } + ], + Elements = [ + ClauseElement { Value = "git", Role = Verb, + PrecedingVerbElementCount = 0 }, + ClauseElement { Value = "-C", Role = Argument, + PrecedingVerbElementCount = 1 }, + ClauseElement { Value = "/repo", Role = Argument, + PrecedingVerbElementCount = 1 }, + ClauseElement { Value = "log", Role = Verb, + PrecedingVerbElementCount = 1 } ] } ] @@ -1274,7 +1386,7 @@ Each file: ], "redirects": [], "isSubshell": false, - "isBashCWrapped": false + "isCommandStringWrapped": false } ] }, @@ -1282,6 +1394,12 @@ Each file: } ``` +An entry may add an `elements` list to a clause to pin the complete +`Clause.Elements` projection (`raw`, `value`, `role`, `sourceStart`, +`sourceLength`, `precedingVerbElementCount`, `kind`, `isFlag`, `isPath`, and +`resolved`). The field is opt-in so older corpus entries remain readable; +issue-specific provenance entries SHALL include it. + ### Coverage targets for v0.1 Author at least: diff --git a/docs/CONSUMER_GUIDE.md b/docs/CONSUMER_GUIDE.md index 83ea585..97ca86f 100644 --- a/docs/CONSUMER_GUIDE.md +++ b/docs/CONSUMER_GUIDE.md @@ -44,7 +44,7 @@ flowchart TD D["PwshParser"] C --> E["Parse syntax, classify tokens, and resolve static context"] D --> E - E --> F["ParsedCommand: ordered clauses, verbs, args, redirects, cwd, and uncertainty"] + E --> F["ParsedCommand: ordered clauses and elements, semantic projections, cwd, and uncertainty"] end subgraph APP["Consumer-owned policy"] @@ -354,19 +354,65 @@ Input: ```text git -C /repo commit git commit -C HEAD~1 +git -C /repo commit -C HEAD~1 +git --no-pager commit -C HEAD~1 ``` These commands demonstrate why source provenance matters. Git assigns different meaning to `-C` based on whether it appears before or after `commit`. [Issue #62](https://github.com/Aaronontheweb/ShellSyntaxTree/issues/62) -tracks an ordered clause-element API so a Git-aware consumer can apply that -rule without re-tokenizing `ParsedCommand.Source`. Until that API ships, the -current `Verb` and `Args` projections do not preserve their interleaving. - -When ordered elements are added, this guide should be updated in the same -change with a complete command-aware-policy example. The existing projections -should remain documented as compatibility conveniences, while the ordered view -becomes the source-provenance path for consumers that need positional meaning. +introduced `Clause.Elements` so a Git-aware consumer can apply that rule +without re-tokenizing `ParsedCommand.Source`. The consumer must interpret the +complete authored stream using Git's grammar; `Role` and +`PrecedingVerbElementCount` mirror ShellSyntaxTree's greedy projection and are +not Git-semantic boundaries: + +```csharp +var authored = clause.Elements + .Where(element => element.Role != ClauseElementRole.Redirect) + .ToArray(); + +// Application-owned code: walk every authored element, apply Git's global +// option arity, locate the semantic subcommand, and bind every option operand. +if (!GitCommandGrammar.TryInterpret(authored, out var command)) +{ + return ApprovalDecision.FailClosed; +} + +foreach (var occurrence in command.Options.Where(option => option.Name is "-c" or "-C")) +{ + if (occurrence.Operand is null + || occurrence.Operand.Kind == ArgKind.DynamicSkip) + { + return ApprovalDecision.FailClosed; + } + + if (occurrence.Scope == GitOptionScope.Global) + EvaluateGitGlobalOption(occurrence.Name, occurrence.Operand); + else if (command.Subcommand == "commit") + EvaluateGitCommitOption(occurrence.Name, occurrence.Operand); +} +``` + +For `git -C /repo commit`, the `-C` and `/repo` elements report one preceding +verb element. For `git commit -C HEAD~1`, they report two. ShellSyntaxTree still +applies its generic Git flag/path tables, so a command-aware consumer may +reinterpret the latter value as a revision rather than a path. The new API +provides the missing positional evidence; it deliberately does not encode Git +semantics. `git --no-pager commit -C HEAD~1` demonstrates why the consumer +cannot use the count alone: `--no-pager` stops the generic greedy walk, so +`commit` is an argument element even though Git treats it as the subcommand. + +The grammar helper above is also responsible for attached forms and for +binding a spaced flag to the following operand. It enumerates every occurrence, +so a global `-C /repo` cannot hide a later command-scoped `-C HEAD~1`. + +`Raw` preserves exact spelling, `Value` carries the lexer-decoded value, and +`SourceStart` / `SourceLength` distinguish repeated occurrences. Existing +`Verb`, `Args`, and `Redirects` remain compatibility conveniences. Synthetic +cwd attribution remains only in `Args`; elements expanded from a command-string +wrapper have null source spans when they cannot be mapped exactly into the +outer source. ## Netclaw case study diff --git a/openspec/changes/preserve-clause-element-order/.openspec.yaml b/openspec/changes/preserve-clause-element-order/.openspec.yaml new file mode 100644 index 0000000..1b062d3 --- /dev/null +++ b/openspec/changes/preserve-clause-element-order/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-08-04 diff --git a/openspec/changes/preserve-clause-element-order/design.md b/openspec/changes/preserve-clause-element-order/design.md new file mode 100644 index 0000000..d120202 --- /dev/null +++ b/openspec/changes/preserve-clause-element-order/design.md @@ -0,0 +1,139 @@ +## Context + +Both parsers already retain exact lexer source spans and decoded token values. +Their verb passes then record verb token positions while their argument and +redirect passes build separate public projections. Once those lists are +finished, cross-list order cannot be reconstructed reliably. + +Issue #62 requires shell syntax provenance, not Git semantics. The parser must +preserve where `-c` / `-C` occurred in the authored element stream. A Git-aware +consumer decides which element is the semantic subcommand and whether an +option changes cwd, overrides configuration, or reuses a commit message. + +## Goals / Non-Goals + +**Goals:** + +- Preserve source order across verbs, arguments, and redirects. +- Identify each element's coordinate relative to parser-classified verb + elements without presenting that coordinate as executable semantics. +- Preserve exact spelling and decoded value without consumer tokenization. +- Carry the existing argument/path facts needed by security policy. +- Keep existing public projections and cwd behavior source-compatible. +- Use one shell-neutral contract for Bash and PowerShell. + +**Non-Goals:** + +- Encode Git or another executable's option grammar. +- Replace the existing projections. +- Publish whitespace, comments, compound operators, grouping delimiters, or + PowerShell's call operator as clause elements. +- Introduce recursive syntax nodes or composable public subtrees. +- Map decoded command-string payload characters back through quoting, escaping, + or base64 into an outer wrapper source. + +## Decisions + +### Add an ordered leaf projection, not a recursive tree + +`Clause.Elements` is an ordered list of significant source-authored leaves. +It solves the observed ambiguity without committing the v0.x API to a +general-purpose shell tree. A future tree can reuse this list as the leaf order +of a simple-command node. Authored order is authoritative; executable-specific +semantic interpretation remains consumer-owned. + +### Use one verb-relative coordinate + +Every element carries `PrecedingVerbElementCount`, the number of elements with +`Role=Verb` that appeared before it in the clause. For a verb element, that +value is also its zero-based index in `Clause.Verb.Tokens`. + +`Role` mirrors the parser's existing projections, including the greedy +`Clause.Verb` heuristic. Therefore the count is an AST coordinate, not an +executable-specific subcommand boundary. For example, an unrecognized global +option may stop the greedy walk even though a later element is a Git +subcommand. A Git-aware consumer uses the complete ordered list, not this +count alone, to interpret the command. + +This avoids redundant `VerbIndex` and `AfterVerbIndex` fields whose invariants +could drift. + +### Match native option spelling case-sensitively + +Native executables receive option spelling unchanged under both Bash and +PowerShell. Their option tables therefore use ordinal matching even though +PowerShell cmdlet parameter tables remain case-insensitive. Git explicitly +lists both `-c` (configuration value, not a path) and `-C` (directory path in +the generic global-option table). + +This corrects pre-existing metadata drift. It does not make the shared parser +Git-semantic: a Git-aware consumer must still reinterpret `git commit -c/-C` +operands as revisions using authored order. + +### Exclude synthetic cwd attribution + +`Clause.Elements` contains authored syntax only. A cwd attribution appended to +`Clause.Args` has no source position in that clause and is therefore excluded. +Existing consumers continue to find it through `Arg.IsCwdAttribution`. + +### Preserve source spans only when exact outer mapping exists + +For an ordinary clause, `SourceStart` and `SourceLength` index the returned +`ParsedCommand.Source`. Clauses expanded from `bash -c`, `pwsh -Command`, or +`pwsh -EncodedCommand` retain their inner `Raw` and `Value`, but their spans are +null after expansion because escaping and decoding prevent a generally exact +mapping into the outer source. + +Nullable spans are an uncertainty signal; the parser does not guess offsets. + +### Model a redirect as one ordered semantic element + +One redirect element spans its operator and target and occupies the same +ordinal among redirect elements as its corresponding `Clause.Redirects` +entry. `Raw` is the complete source slice, `Value` is the decoded target, and +the argument-like fields describe the target. A stream-merge redirect has +`Kind=DynamicSkip` and `IsPath=false`. + +This preserves placement without adding redirect-operator and redirect-target +roles that do not exist in the current semantic projection. + +### Keep inline binding forms as one source element + +Forms such as `--work-tree=../repo` and `-Path:C:\repo` are one lexer token +and therefore one clause element. Their `Raw` and `Value` describe the full +token; `IsPath`, `Kind`, and `Resolved` describe the bound value when the +parser can classify it. Existing `Args` may continue splitting such a token +into multiple semantic arguments. + +### Build elements in the classified token walk + +The argument/redirect scan emits an element when it encounters each verb +position, argument token, or redirect. It must not zip or search the completed +`Verb`, `Args`, and `Redirects` lists. Repeated values and interleaved flags +remain deterministic because lexer positions are still present at emission. + +## Risks / Trade-offs + +- The element record duplicates some `Arg` facts. This is deliberate: inline + forms are not always one-to-one with `Args`, and consumers should not need to + correlate by string value. +- Nullable wrapped-command spans provide less provenance than direct clauses, + but are safer than approximate outer offsets. +- Redirect `Raw` may contain whitespace between operator and target. That is + the exact authored region and is suitable for display. +- `Clause` is a record, so adding `Elements` changes its generated value + equality and hashing. Generated `ToString()` and default JSON serialization + also include the new projection. The surface is source- and binary-additive, + but it is behaviorally observable to consumers of those generated forms. +- Adding public types before stable `0.2.0` expands the API contract. Public API + snapshot tests and both SPECs lock the shape. + +## Migration Plan + +The change is source- and binary-additive. Existing consumers continue using +`Verb`, `Args`, and `Redirects` unchanged. Position-sensitive consumers opt +into `Elements`; consumers relying on record equality, hashing, textual +snapshots, or default JSON output must account for the new projection. + +A revert can remove the new projection before stable `0.2.0`; after release, +the normal public API compatibility rules apply. diff --git a/openspec/changes/preserve-clause-element-order/proposal.md b/openspec/changes/preserve-clause-element-order/proposal.md new file mode 100644 index 0000000..d33f6b0 --- /dev/null +++ b/openspec/changes/preserve-clause-element-order/proposal.md @@ -0,0 +1,42 @@ +## Why + +`Clause.Verb`, `Clause.Args`, and `Clause.Redirects` retain order within each +projection but lose how those projections were interleaved in the command. +That makes position-sensitive command policy impossible without re-tokenizing +`ParsedCommand.Source`. Issue #62 demonstrates this with Git global options +before a subcommand and similarly spelled options after it. + +## What Changes + +- Add a source-ordered `Clause.Elements` public projection. +- Add `ClauseElement` and `ClauseElementRole` public types. +- Preserve exact source spelling, decoded values, source spans when available, + parser-verb-relative position, and argument/path classification on each + element without claiming executable-specific semantics. +- Populate the ordered view directly while walking classified parser tokens. +- Keep `Verb`, `Args`, `Redirects`, and synthetic cwd attribution unchanged for + existing consumers. +- Apply the same contract to Bash and PowerShell. +- Match native option spelling case-sensitively in both shells so distinct + options such as Git `-c` and `-C` do not share path metadata. + +## Capabilities + +### New Capabilities + +- `clause-element-provenance`: Preserve cross-projection source order so a + command-aware consumer can interpret argument placement without duplicating + shell tokenization. + +### Modified Capabilities + +None. + +## Impact + +This is a source- and binary-additive public-API change affecting `Clause`, the +Bash and PowerShell command parsers, public API snapshot tests, parser tests, +the shared SPEC, the PowerShell delta SPEC, and the consumer guide. Generated +record equality, hashing, `ToString()`, and default JSON output observe the new +property. It adds no dependency; the native-option comparer correction may +change path classification where options differ only by case. diff --git a/openspec/changes/preserve-clause-element-order/specs/clause-element-provenance/spec.md b/openspec/changes/preserve-clause-element-order/specs/clause-element-provenance/spec.md new file mode 100644 index 0000000..f4433b2 --- /dev/null +++ b/openspec/changes/preserve-clause-element-order/specs/clause-element-provenance/spec.md @@ -0,0 +1,102 @@ +## ADDED Requirements + +### Requirement: Clauses expose significant elements in source order +Each parsed clause SHALL expose its source-authored verbs, arguments, and +redirects through `Clause.Elements` in ascending source order. + +Whitespace, comments, compound operators, grouping delimiters, shell call +operators, and synthetic cwd-attribution arguments SHALL NOT appear in the +collection. + +#### Scenario: Global Git option precedes the subcommand +- **WHEN** either parser parses `git -C /repo commit` +- **THEN** the element values are `git`, `-C`, `/repo`, and `commit` +- **THEN** `-C` and `/repo` each have `PrecedingVerbElementCount` equal to `1` +- **THEN** `commit` is a verb element with `PrecedingVerbElementCount` equal to `1` + +#### Scenario: Git option follows the subcommand +- **WHEN** either parser parses `git commit -C HEAD~1` +- **THEN** the element values are `git`, `commit`, `-C`, and `HEAD~1` +- **THEN** `-C` and `HEAD~1` each have `PrecedingVerbElementCount` equal to `2` + +#### Scenario: Multiple Git option occurrences remain distinct +- **WHEN** either parser parses `git -C /repo commit -C HEAD~1` +- **THEN** both `-C` occurrences appear at distinct source spans +- **THEN** the first has `PrecedingVerbElementCount` equal to `1` +- **THEN** the second has `PrecedingVerbElementCount` equal to `2` + +#### Scenario: Parser roles remain heuristic +- **WHEN** either parser parses `git --no-pager commit -C HEAD~1` +- **THEN** the element values preserve `git`, `--no-pager`, `commit`, `-C`, and `HEAD~1` in that order +- **THEN** `commit` may have `Role=Argument` because `--no-pager` stopped the greedy verb walk +- **THEN** `-C` has `PrecedingVerbElementCount` equal to `1` +- **THEN** consumers SHALL NOT interpret that count alone as Git's semantic option scope + +#### Scenario: Verb-relative counts reset per clause +- **WHEN** either parser parses `git -C /repo status | git commit -C HEAD~1` +- **THEN** the first clause reports `-C` after one verb token +- **THEN** the second clause reports `-C` after two verb tokens + +### Requirement: Elements preserve spelling, logical values, and occurrences +Each direct-source element SHALL expose its exact source slice as `Raw`, its +lexer-decoded logical value as `Value`, and a source span into +`ParsedCommand.Source`. + +#### Scenario: Quoted argument value +- **WHEN** Bash parses `git -c "user.name=Jane Doe" commit` +- **THEN** the quoted element has `Raw` equal to `"user.name=Jane Doe"` +- **THEN** its `Value` is `user.name=Jane Doe` +- **THEN** its span starts at the opening quote and covers the closing quote + +#### Scenario: Repeated text +- **WHEN** either parser parses `tool item --name item` +- **THEN** the two `item` elements have distinct source starts +- **THEN** the first is a verb and the second is an argument + +### Requirement: Element argument metadata matches parser classification +Argument elements SHALL carry the parser's `Kind`, `IsFlag`, `IsPath`, and +`Resolved` result for their source token or inline bound value. + +#### Scenario: Inline native path option +- **WHEN** either parser parses `git --work-tree=../repo status` +- **THEN** `--work-tree=../repo` is one argument element +- **THEN** the element is a flag and carries the bound value's path metadata + +#### Scenario: Native option spelling remains case-sensitive +- **WHEN** either parser parses `git -c key=value commit` +- **THEN** `key=value` is not classified as a path +- **WHEN** either parser parses `git -C /repo commit` +- **THEN** `/repo` is classified as a path +- **THEN** PowerShell cmdlet parameter matching remains case-insensitive + +### Requirement: Redirects occupy their authored position +Each redirect SHALL appear as one redirect element at the source position of +its operator and target. Its ordinal among redirect elements SHALL match its +ordinal in `Clause.Redirects`. + +#### Scenario: Redirect after an intervening option +- **WHEN** Bash parses `git -C /repo status > status.txt` +- **THEN** the redirect element follows the `status` verb element +- **THEN** its raw slice covers `> status.txt` +- **THEN** its value is `status.txt` + +### Requirement: Synthetic and wrapped provenance does not invent source spans +Synthetic cwd attribution SHALL remain in `Clause.Args` and SHALL NOT appear in +`Clause.Elements`. Elements from an expanded command-string wrapper SHALL have +null spans when they cannot be mapped exactly into the outer +`ParsedCommand.Source`. + +#### Scenario: Cwd attribution remains synthetic +- **WHEN** Bash parses `cd /repo && git status` +- **THEN** the second clause contains a cwd-attribution `Arg` +- **THEN** its elements are only `git` and `status` + +#### Scenario: Bash command-string expansion +- **WHEN** Bash parses `bash -c "git -C /repo status"` +- **THEN** the surfaced clause elements retain their inner raw and decoded values +- **THEN** every surfaced element has null `SourceStart` and `SourceLength` + +#### Scenario: PowerShell encoded-command expansion +- **WHEN** PowerShell parses a valid encoded command payload +- **THEN** the surfaced clause elements retain their decoded values +- **THEN** every surfaced element has null `SourceStart` and `SourceLength` diff --git a/openspec/changes/preserve-clause-element-order/tasks.md b/openspec/changes/preserve-clause-element-order/tasks.md new file mode 100644 index 0000000..8e33ac9 --- /dev/null +++ b/openspec/changes/preserve-clause-element-order/tasks.md @@ -0,0 +1,43 @@ +## 1. Contract + +- [x] 1.1 Add `ClauseElement`, `ClauseElementRole`, and `Clause.Elements` to + `SPEC.md` and the public API snapshot. +- [x] 1.2 Add the PowerShell-specific inline-binding and wrapper-span rules to + `SPEC.POWERSHELL.md`. + +## 2. Parser + +- [x] 2.1 Emit ordered elements from the Bash classified token walk. +- [x] 2.2 Emit equivalent elements from the PowerShell classified token walk. +- [x] 2.3 Clear source spans when inner command-string clauses are lifted into + an outer `ParsedCommand`. +- [x] 2.4 Keep synthetic cwd-attribution arguments out of `Elements`. + +## 3. Verification + +- [x] 3.1 Add public API and default-value assertions. +- [x] 3.2 Cover Git global/subcommand placement, quoted values, repeated text, + pipeline reset, redirects, inline bindings, wrappers, and cwd attribution + in Bash tests. +- [x] 3.3 Add equivalent PowerShell native-command and wrapper coverage. +- [x] 3.4 Verify existing `Verb`, `Args`, and `Redirects` regression coverage + remains green. +- [x] 3.5 Add paired Bash and PowerShell corpus entries for lowercase `-c`, + uppercase `-C`, mixed occurrences, and an intervening valueless option. +- [x] 3.6 Pin native-option case sensitivity and the heuristic-role boundary. + +## 4. Consumer documentation + +- [x] 4.1 Replace the issue #62 limitation in `docs/CONSUMER_GUIDE.md` with a + complete command-aware-policy example. +- [x] 4.2 Update `PROJECT_CONTEXT.md` and `IMPLEMENTATION_PLAN.md`; explicitly + schedule migration guidance with the next prerelease rather than making + this implementation change release-shaped. +- [x] 4.3 Rewrite the Git consumer example to enumerate every occurrence, + bind operands, and derive Git semantics from all authored elements. + +## 5. Completion + +- [x] 5.1 Validate the OpenSpec change. +- [x] 5.2 Run restore, Release build, full tests/corpus, header verification, + and a public API vs. SPEC diff. diff --git a/src/ShellSyntaxTree/Clause.cs b/src/ShellSyntaxTree/Clause.cs index ffd2527..d3193b9 100644 --- a/src/ShellSyntaxTree/Clause.cs +++ b/src/ShellSyntaxTree/Clause.cs @@ -39,6 +39,13 @@ public sealed record Clause /// public IReadOnlyList Redirects { get; init; } = Array.Empty(); + /// + /// Significant source-authored verbs, arguments, and redirects in source + /// order. Synthetic cwd attribution is intentionally excluded. See + /// SPEC §3 ClauseElement. + /// + public IReadOnlyList Elements { get; init; } = Array.Empty(); + /// /// True when this clause is wrapped in a subshell (parens). Subshells /// isolate cd state — see SPEC §9. diff --git a/src/ShellSyntaxTree/ClauseElement.cs b/src/ShellSyntaxTree/ClauseElement.cs new file mode 100644 index 0000000..93821cf --- /dev/null +++ b/src/ShellSyntaxTree/ClauseElement.cs @@ -0,0 +1,61 @@ +// ----------------------------------------------------------------------- +// +// Copyright (C) 2026 - 2026 Aaron Stannard +// +// ----------------------------------------------------------------------- +namespace ShellSyntaxTree; + +/// +/// One significant source-authored verb, argument, or redirect in a +/// . Elements preserve cross-projection source order; +/// executable-specific meaning remains the consumer's responsibility. +/// +public sealed record ClauseElement +{ + /// Exact authored source slice, including quote delimiters. + public string Raw { get; init; } = ""; + + /// + /// Lexer-decoded logical value. For a redirect this is its decoded target. + /// For an inline binding, this remains the complete decoded source token. + /// + public string Value { get; init; } = ""; + + /// The semantic projection represented by this element. + public ClauseElementRole Role { get; init; } + + /// + /// Zero-based start in . Null when a + /// command-string wrapper cannot be mapped exactly into the outer source. + /// + public int? SourceStart { get; init; } + + /// + /// Length of the exact source slice. Null whenever + /// is null. + /// + public int? SourceLength { get; init; } + + /// + /// Number of parser-classified verb elements authored before this element + /// in its clause. This is an AST coordinate, not an executable-specific + /// semantic boundary. + /// For a verb element, this is its zero-based + /// index. + /// + public int PrecedingVerbElementCount { get; init; } + + /// + /// Classification of this token, inline bound value, or redirect target. + /// + public ArgKind Kind { get; init; } + + /// True when the authored argument token is option-shaped. + public bool IsFlag { get; init; } + + /// True when this element carries a path-shaped operand. + public bool IsPath { get; init; } + + /// Resolved path when static resolution succeeded; otherwise null. + public string? Resolved { get; init; } +} diff --git a/src/ShellSyntaxTree/ClauseElementRole.cs b/src/ShellSyntaxTree/ClauseElementRole.cs new file mode 100644 index 0000000..a1b7d38 --- /dev/null +++ b/src/ShellSyntaxTree/ClauseElementRole.cs @@ -0,0 +1,21 @@ +// ----------------------------------------------------------------------- +// +// Copyright (C) 2026 - 2026 Aaron Stannard +// +// ----------------------------------------------------------------------- +namespace ShellSyntaxTree; + +/// +/// The semantic projection to which a source-authored clause element belongs. +/// +public enum ClauseElementRole +{ + /// A token represented in . + Verb, + + /// An authored argument token. + Argument, + + /// A redirect operator and its target. + Redirect, +} diff --git a/src/ShellSyntaxTree/Internal/Bash/Parsing/BashCommandParser.cs b/src/ShellSyntaxTree/Internal/Bash/Parsing/BashCommandParser.cs index 35fd6af..13f2faf 100644 --- a/src/ShellSyntaxTree/Internal/Bash/Parsing/BashCommandParser.cs +++ b/src/ShellSyntaxTree/Internal/Bash/Parsing/BashCommandParser.cs @@ -223,6 +223,7 @@ private static ParsedCommand ParseInternal( Operator = op, IsSubshell = isSubshell, IsCommandStringWrapped = true, + Elements = ClauseElementProvenance.WithoutOuterSourceSpans(ic.Elements), }); } @@ -893,6 +894,7 @@ private static ClauseResult ParseClauseSegment( workingDirectoryUnknown: workingDirectoryUnknown, out var emptyArgs, out var emptyRedirects, + out var emptyElements, out var redirectError); if (redirectError is not null) { @@ -905,6 +907,7 @@ private static ClauseResult ParseClauseSegment( Verb = new VerbChain(), Args = emptyArgs, Redirects = emptyRedirects, + Elements = emptyElements, IsSubshell = false, IsCommandStringWrapped = false, }); @@ -930,6 +933,7 @@ private static ClauseResult ParseClauseSegment( workingDirectoryUnknown: workingDirectoryUnknown, out var args, out var redirects, + out var elements, out var argError); if (argError is not null) { @@ -942,6 +946,7 @@ private static ClauseResult ParseClauseSegment( Verb = verbChain, Args = args, Redirects = redirects, + Elements = elements, IsSubshell = false, IsCommandStringWrapped = false, }; @@ -980,6 +985,7 @@ private static void ExtractRedirectsAndArgs( bool workingDirectoryUnknown, out IReadOnlyList args, out IReadOnlyList redirects, + out IReadOnlyList elements, out string? error) { ExtractRedirectsAndArgs( @@ -994,6 +1000,7 @@ private static void ExtractRedirectsAndArgs( workingDirectoryUnknown: workingDirectoryUnknown, out args, out redirects, + out elements, out error); } @@ -1009,12 +1016,15 @@ private static void ExtractRedirectsAndArgs( bool workingDirectoryUnknown, out IReadOnlyList args, out IReadOnlyList redirects, + out IReadOnlyList elements, out string? error) { var argList = new List(); var redirectList = new List(); + var elementList = new List(); var positionalIndex = 0; var i = start; + var precedingVerbTokenCount = 0; // Tracks "next non-flag arg is the value of this flag" — used to // attribute path-classification to the value of a flag-with-value @@ -1028,6 +1038,17 @@ private static void ExtractRedirectsAndArgs( // interleaved with consumed flag-value pairs). if (skipIndices is not null && skipIndices.Contains(i)) { + var verbToken = segmentTokens[i]; + elementList.Add(CreateElement( + source, + verbToken, + ClauseElementRole.Verb, + precedingVerbTokenCount, + ArgKind.Literal, + isFlag: false, + isPath: false, + resolved: null)); + precedingVerbTokenCount++; i++; continue; } @@ -1042,6 +1063,7 @@ private static void ExtractRedirectsAndArgs( error = $"redirect operator '{t.OperatorText}' missing target at position {t.SourceStart}"; args = argList; redirects = redirectList; + elements = elementList; return; } @@ -1051,10 +1073,21 @@ private static void ExtractRedirectsAndArgs( error = $"redirect operator '{t.OperatorText}' missing target at position {t.SourceStart}"; args = argList; redirects = redirectList; + elements = elementList; return; } - BuildRedirect(dir, target, source, options, redirectList, workingDirectoryUnknown); + BuildRedirect( + dir, + t, + target, + source, + options, + redirectList, + workingDirectoryUnknown, + precedingVerbTokenCount, + out var redirectElement); + elementList.Add(redirectElement); i += 2; continue; } @@ -1066,6 +1099,7 @@ private static void ExtractRedirectsAndArgs( error = $"heredoc operator '{t.OperatorText}' missing delimiter at position {t.SourceStart}"; args = argList; redirects = redirectList; + elements = elementList; return; } @@ -1076,6 +1110,14 @@ private static void ExtractRedirectsAndArgs( Target = "<<" + delim.Value + ">", IsDynamicSkip = false, }); + elementList.Add(CreateRedirectElement( + source, + t, + delim, + precedingVerbTokenCount, + ArgKind.Literal, + isPath: false, + resolved: null)); i += 2; continue; @@ -1084,6 +1126,7 @@ private static void ExtractRedirectsAndArgs( error = $"unexpected operator '{t.OperatorText}' at position {t.SourceStart}"; args = argList; redirects = redirectList; + elements = elementList; return; } @@ -1130,6 +1173,15 @@ private static void ExtractRedirectsAndArgs( Kind = vKind, IsPath = vIsPath, }); + elementList.Add(CreateElement( + source, + t, + ClauseElementRole.Argument, + precedingVerbTokenCount, + vKind, + isFlag: true, + isPath: vIsPath, + resolved: vResolved)); // The split form doesn't propagate to a "next-arg is // the value" pending-state — the value already @@ -1147,6 +1199,15 @@ private static void ExtractRedirectsAndArgs( Kind = ArgKind.Literal, IsPath = false, }); + elementList.Add(CreateElement( + source, + t, + ClauseElementRole.Argument, + precedingVerbTokenCount, + ArgKind.Literal, + isFlag: true, + isPath: false, + resolved: null)); // If this flag takes a value (per the verb's table), // mark the *next* non-flag arg as that value. We @@ -1192,6 +1253,15 @@ private static void ExtractRedirectsAndArgs( Kind = kind, IsPath = isPath, }); + elementList.Add(CreateElement( + source, + t, + ClauseElementRole.Argument, + precedingVerbTokenCount, + kind, + isFlag: false, + isPath: isPath, + resolved: resolved)); break; } @@ -1229,6 +1299,15 @@ private static void ExtractRedirectsAndArgs( Kind = kind, IsPath = isPath, }); + elementList.Add(CreateElement( + source, + t, + ClauseElementRole.Argument, + precedingVerbTokenCount, + kind, + isFlag: false, + isPath: isPath, + resolved: resolved)); break; } @@ -1248,6 +1327,15 @@ private static void ExtractRedirectsAndArgs( Kind = ArgKind.DynamicSkip, IsPath = false, }); + elementList.Add(CreateElement( + source, + t, + ClauseElementRole.Argument, + precedingVerbTokenCount, + ArgKind.DynamicSkip, + isFlag: false, + isPath: false, + resolved: null)); positionalIndex++; pendingFlagForValue = null; break; @@ -1262,16 +1350,20 @@ private static void ExtractRedirectsAndArgs( args = argList; redirects = redirectList; + elements = elementList; error = null; } private static void BuildRedirect( RedirectDirection direction, + BashToken redirectOperator, BashToken target, string source, BashParserOptions options, List redirectList, - bool workingDirectoryUnknown) + bool workingDirectoryUnknown, + int precedingVerbTokenCount, + out ClauseElement element) { if (target.Kind == BashTokenKind.OpaqueSubstitution) { @@ -1283,6 +1375,14 @@ private static void BuildRedirect( Target = target.Value, IsDynamicSkip = true, }); + element = CreateRedirectElement( + source, + redirectOperator, + target, + precedingVerbTokenCount, + ArgKind.DynamicSkip, + isPath: false, + resolved: null); return; } @@ -1300,6 +1400,14 @@ private static void BuildRedirect( Target = target.Value, IsDynamicSkip = true, }); + element = CreateRedirectElement( + source, + redirectOperator, + target, + precedingVerbTokenCount, + ArgKind.DynamicSkip, + isPath: false, + resolved: null); return; } @@ -1309,7 +1417,8 @@ private static void BuildRedirect( // locked interpretation #3: a glob target stays IsPath=true with // Kind=Glob; an env-var target becomes DynamicSkip; a literal // resolves against WorkingDirectory. - var (kind, resolved, _) = BashResolver.Resolve(target.Value, treatAsPath: true, options, workingDirectoryUnknown); + var (kind, resolved, isPath) = BashResolver.Resolve( + target.Value, treatAsPath: true, options, workingDirectoryUnknown); bool isDynamic; string redirectTarget; @@ -1330,6 +1439,62 @@ private static void BuildRedirect( Target = redirectTarget, IsDynamicSkip = isDynamic, }); + element = CreateRedirectElement( + source, + redirectOperator, + target, + precedingVerbTokenCount, + kind, + isPath: isPath, + resolved: kind == ArgKind.DynamicSkip ? null : resolved); + } + + private static ClauseElement CreateElement( + string source, + BashToken token, + ClauseElementRole role, + int precedingVerbTokenCount, + ArgKind kind, + bool isFlag, + bool isPath, + string? resolved) => new() + { + Raw = SourceSlice(source, token), + Value = token.Value, + Role = role, + SourceStart = token.SourceStart, + SourceLength = token.SourceLength, + PrecedingVerbElementCount = precedingVerbTokenCount, + Kind = kind, + IsFlag = isFlag, + IsPath = isPath, + Resolved = resolved, + }; + + private static ClauseElement CreateRedirectElement( + string source, + BashToken redirectOperator, + BashToken target, + int precedingVerbTokenCount, + ArgKind kind, + bool isPath, + string? resolved) + { + var sourceStart = redirectOperator.SourceStart; + var sourceEnd = target.SourceStart + target.SourceLength; + return new ClauseElement + { + Raw = source.Substring(sourceStart, sourceEnd - sourceStart), + Value = target.Value, + Role = ClauseElementRole.Redirect, + SourceStart = sourceStart, + SourceLength = sourceEnd - sourceStart, + PrecedingVerbElementCount = precedingVerbTokenCount, + Kind = kind, + IsFlag = false, + IsPath = isPath, + Resolved = resolved, + }; } private static bool IsFlag(string raw) => diff --git a/src/ShellSyntaxTree/Internal/Bash/Verbs/BashPerVerbRules.cs b/src/ShellSyntaxTree/Internal/Bash/Verbs/BashPerVerbRules.cs index e912c38..1ac542e 100644 --- a/src/ShellSyntaxTree/Internal/Bash/Verbs/BashPerVerbRules.cs +++ b/src/ShellSyntaxTree/Internal/Bash/Verbs/BashPerVerbRules.cs @@ -124,7 +124,9 @@ internal static bool IsPositionalPathArg(VerbChain verb, int positionalIndex, st private static readonly IReadOnlyDictionary<(string Verb, string Flag), bool> FlagValueIsPath = new Dictionary<(string Verb, string Flag), bool>(FlagKeyComparer.Instance) { - // git: -C / --git-dir / --work-tree all consume directory paths. + // Git native options are case-sensitive: -c consumes a config + // key/value while -C consumes a directory path. + [("git", "-c")] = false, [("git", "-C")] = true, [("git", "--git-dir")] = true, [("git", "--work-tree")] = true, @@ -171,9 +173,9 @@ internal static bool ValueOfFlagIsPath(string verb, string flag) // ---------------------------------------------------------------- key comparer /// - /// Case-insensitive equality for the (verb, flag) tuple keys. Avoids - /// allocating a wrapper record while still matching the per-verb table - /// case-insensitivity contract. + /// Verb keys retain the existing case-insensitive lookup; native flag + /// spelling is ordinal because executables may assign different meanings + /// to options that differ only by case. /// private sealed class FlagKeyComparer : IEqualityComparer<(string Verb, string Flag)> { @@ -181,11 +183,11 @@ private sealed class FlagKeyComparer : IEqualityComparer<(string Verb, string Fl public bool Equals((string Verb, string Flag) x, (string Verb, string Flag) y) => string.Equals(x.Verb, y.Verb, StringComparison.OrdinalIgnoreCase) - && string.Equals(x.Flag, y.Flag, StringComparison.OrdinalIgnoreCase); + && string.Equals(x.Flag, y.Flag, StringComparison.Ordinal); public int GetHashCode((string Verb, string Flag) obj) { - // Hash combination via ordinal-ignore-case on each component. + // Hash combination follows the mixed verb/flag comparison. // Avoid HashCode.Combine for netstandard2.0 parity. unchecked { @@ -194,7 +196,7 @@ public int GetHashCode((string Verb, string Flag) obj) : StringComparer.OrdinalIgnoreCase.GetHashCode(obj.Verb); var h2 = obj.Flag is null ? 0 - : StringComparer.OrdinalIgnoreCase.GetHashCode(obj.Flag); + : StringComparer.Ordinal.GetHashCode(obj.Flag); return (h1 * 397) ^ h2; } } diff --git a/src/ShellSyntaxTree/Internal/Bash/Verbs/BashVerbs.cs b/src/ShellSyntaxTree/Internal/Bash/Verbs/BashVerbs.cs index 393b920..3556f3e 100644 --- a/src/ShellSyntaxTree/Internal/Bash/Verbs/BashVerbs.cs +++ b/src/ShellSyntaxTree/Internal/Bash/Verbs/BashVerbs.cs @@ -91,29 +91,31 @@ internal static class BashVerbs /// Value type is rather than /// IReadOnlySet<string> for netstandard2.0 parity — /// IReadOnlySet<T> ships in net5+ only. Internally the - /// shape is identical (case-insensitive set lookup). + /// shape is identical. Native option spelling is case-sensitive even + /// when the host shell is PowerShell; the outer command lookup remains + /// case-insensitive for the existing verb-table contract. /// internal static readonly IReadOnlyDictionary> FlagsWithValue = new Dictionary>( StringComparer.OrdinalIgnoreCase) { - ["git"] = new HashSet(StringComparer.OrdinalIgnoreCase) + ["git"] = new HashSet(StringComparer.Ordinal) { - "-C", "--git-dir", "--work-tree", + "-c", "-C", "--git-dir", "--work-tree", }, - ["curl"] = new HashSet(StringComparer.OrdinalIgnoreCase) + ["curl"] = new HashSet(StringComparer.Ordinal) { "-o", "--output", "-d", "--data", }, - ["wget"] = new HashSet(StringComparer.OrdinalIgnoreCase) + ["wget"] = new HashSet(StringComparer.Ordinal) { "-O", "--output-document", }, - ["docker"] = new HashSet(StringComparer.OrdinalIgnoreCase) + ["docker"] = new HashSet(StringComparer.Ordinal) { "-v", "--volume", "-f", "--file", }, - ["tar"] = new HashSet(StringComparer.OrdinalIgnoreCase) + ["tar"] = new HashSet(StringComparer.Ordinal) { "-f", "--file", "-C", "--directory", }, diff --git a/src/ShellSyntaxTree/Internal/Parsing/ClauseElementProvenance.cs b/src/ShellSyntaxTree/Internal/Parsing/ClauseElementProvenance.cs new file mode 100644 index 0000000..afbf6ff --- /dev/null +++ b/src/ShellSyntaxTree/Internal/Parsing/ClauseElementProvenance.cs @@ -0,0 +1,41 @@ +// ----------------------------------------------------------------------- +// +// Copyright (C) 2026 - 2026 Aaron Stannard +// +// ----------------------------------------------------------------------- +using System; +using System.Collections.Generic; + +namespace ShellSyntaxTree.Internal.Parsing; + +/// +/// Shared provenance operations used when shell command-string wrappers lift +/// inner clauses into an outer . +/// +internal static class ClauseElementProvenance +{ + /// + /// Remove inner-source offsets that cannot be mapped exactly through + /// quoting, escaping, script-block stripping, or encoded-command decoding. + /// + internal static IReadOnlyList WithoutOuterSourceSpans( + IReadOnlyList elements) + { + if (elements.Count == 0) + { + return Array.Empty(); + } + + var withoutSpans = new List(elements.Count); + foreach (var element in elements) + { + withoutSpans.Add(element with + { + SourceStart = null, + SourceLength = null, + }); + } + + return withoutSpans; + } +} diff --git a/src/ShellSyntaxTree/Internal/Pwsh/Parsing/PwshCommandParser.cs b/src/ShellSyntaxTree/Internal/Pwsh/Parsing/PwshCommandParser.cs index b2a0dd8..ebb0ab7 100644 --- a/src/ShellSyntaxTree/Internal/Pwsh/Parsing/PwshCommandParser.cs +++ b/src/ShellSyntaxTree/Internal/Pwsh/Parsing/PwshCommandParser.cs @@ -688,6 +688,7 @@ private static BuildResult BuildSegment( Verb = verb, Args = argResult.Args, Redirects = argResult.Redirects, + Elements = argResult.Elements, IsSubshell = segment.Depth > 0, IsCommandStringWrapped = markWrapped, }; @@ -877,12 +878,19 @@ private readonly struct ArgResult public IReadOnlyList Redirects { get; } + public IReadOnlyList Elements { get; } + public string? Error { get; } - public ArgResult(IReadOnlyList args, IReadOnlyList redirects, string? error) + public ArgResult( + IReadOnlyList args, + IReadOnlyList redirects, + IReadOnlyList elements, + string? error) { Args = args; Redirects = redirects; + Elements = elements; Error = error; } } @@ -893,7 +901,9 @@ private static ArgResult ExtractArgsAndRedirects( { var args = new List(); var redirects = new List(); + var elements = new List(); var positionalIndex = 0; + var precedingVerbTokenCount = 0; string? pendingValueParam = null; // cmdlet/alias §6.5 value-binding string? pendingNativeFlag = null; // native flag-with-value var cmdletStyle = verb.Kind is PwshCommandKind.Cmdlet or PwshCommandKind.Alias @@ -906,6 +916,17 @@ private static ArgResult ExtractArgsAndRedirects( { if (verb.VerbPositions.Contains(i)) { + var verbToken = body[i]; + elements.Add(CreateElement( + source, + verbToken, + ClauseElementRole.Verb, + precedingVerbTokenCount, + verb.IsDynamic ? ArgKind.DynamicSkip : ArgKind.Literal, + isFlag: false, + isPath: false, + resolved: null)); + precedingVerbTokenCount++; continue; } @@ -918,19 +939,29 @@ private static ArgResult ExtractArgsAndRedirects( { // A non-leading '&' is a trailing background-job operator; // the anomaly detector already lifts this, but guard here. - return new ArgResult(args, redirects, + return new ArgResult(args, redirects, elements, "trailing '&' background-job operator is not supported in v0.2"); } pendingValueParam = null; pendingNativeFlag = null; var consumed = BuildRedirect( - body, i, source, options, workingDirectoryUnknown, redirects, out var redirectError); + body, + i, + source, + options, + workingDirectoryUnknown, + redirects, + precedingVerbTokenCount, + out var redirectElement, + out var redirectError); if (redirectError is not null) { - return new ArgResult(args, redirects, redirectError); + return new ArgResult(args, redirects, elements, redirectError); } + elements.Add(redirectElement!); + i += consumed - 1; continue; } @@ -1006,6 +1037,17 @@ private static ArgResult ExtractArgsAndRedirects( } } + var parameterMetadata = args[args.Count - 1]; + elements.Add(CreateElement( + source, + t, + ClauseElementRole.Argument, + precedingVerbTokenCount, + parameterMetadata.Kind, + isFlag: true, + isPath: parameterMetadata.IsPath, + resolved: parameterMetadata.Resolved)); + continue; } @@ -1019,6 +1061,15 @@ private static ArgResult ExtractArgsAndRedirects( Kind = ArgKind.DynamicSkip, IsPath = false, }); + elements.Add(CreateElement( + source, + t, + ClauseElementRole.Argument, + precedingVerbTokenCount, + ArgKind.DynamicSkip, + isFlag: false, + isPath: false, + resolved: null)); if (pendingValueParam is not null || pendingNativeFlag is not null) { @@ -1057,11 +1108,21 @@ private static ArgResult ExtractArgsAndRedirects( positionalIndex++; } - args.Add(ResolveValueToken( - rawValue, t.Value, treatAsPath, options, workingDirectoryUnknown, isLiteralBytes)); - } - - return new ArgResult(args, redirects, null); + var resolvedArg = ResolveValueToken( + rawValue, t.Value, treatAsPath, options, workingDirectoryUnknown, isLiteralBytes); + args.Add(resolvedArg); + elements.Add(CreateElement( + source, + t, + ClauseElementRole.Argument, + precedingVerbTokenCount, + resolvedArg.Kind, + isFlag: false, + isPath: resolvedArg.IsPath, + resolved: resolvedArg.Resolved)); + } + + return new ArgResult(args, redirects, elements, null); } private static Arg ResolveValueToken( @@ -1096,10 +1157,13 @@ private static Arg ResolveValue( private static int BuildRedirect( List body, int opIndex, string source, PwshParserOptions options, bool workingDirectoryUnknown, - List redirects, out string? error) + List redirects, int precedingVerbTokenCount, + out ClauseElement? element, out string? error) { + element = null; error = null; - var op = body[opIndex].OperatorText ?? string.Empty; + var operatorToken = body[opIndex]; + var op = operatorToken.OperatorText ?? string.Empty; var direction = MapRedirect(op, out var isMerge, out var mergeTarget); if (isMerge) @@ -1110,6 +1174,15 @@ private static int BuildRedirect( Target = mergeTarget ?? op, IsDynamicSkip = true, }); + element = CreateRedirectElement( + source, + operatorToken, + target: null, + value: mergeTarget ?? op, + precedingVerbTokenCount, + ArgKind.DynamicSkip, + isPath: false, + resolved: null); return 1; } @@ -1129,6 +1202,15 @@ private static int BuildRedirect( Target = target.Value, IsDynamicSkip = true, }); + element = CreateRedirectElement( + source, + operatorToken, + target, + target.Value, + precedingVerbTokenCount, + ArgKind.DynamicSkip, + isPath: false, + resolved: null); return 2; } @@ -1142,12 +1224,21 @@ private static int BuildRedirect( Target = "$null", IsDynamicSkip = true, }); + element = CreateRedirectElement( + source, + operatorToken, + target, + target.Value, + precedingVerbTokenCount, + ArgKind.DynamicSkip, + isPath: false, + resolved: null); return 2; } var isLiteralBytes = target.Kind == PwshTokenKind.QuotedString && target.IsSingleQuoted; var raw = SourceSlice(source, target); - var (kind, resolved, _) = PwshResolver.Resolve( + var (kind, resolved, isPath) = PwshResolver.Resolve( target.Value, treatAsPath: true, options, workingDirectoryUnknown, isLiteralBytes); redirects.Add(new Redirect @@ -1156,9 +1247,69 @@ private static int BuildRedirect( Target = kind == ArgKind.DynamicSkip ? raw : resolved ?? raw, IsDynamicSkip = kind == ArgKind.DynamicSkip, }); + element = CreateRedirectElement( + source, + operatorToken, + target, + target.Value, + precedingVerbTokenCount, + kind, + isPath, + kind == ArgKind.DynamicSkip ? null : resolved); return 2; } + private static ClauseElement CreateElement( + string source, + PwshToken token, + ClauseElementRole role, + int precedingVerbTokenCount, + ArgKind kind, + bool isFlag, + bool isPath, + string? resolved) => new() + { + Raw = SourceSlice(source, token), + Value = token.Value, + Role = role, + SourceStart = token.SourceStart, + SourceLength = token.SourceLength, + PrecedingVerbElementCount = precedingVerbTokenCount, + Kind = kind, + IsFlag = isFlag, + IsPath = isPath, + Resolved = resolved, + }; + + private static ClauseElement CreateRedirectElement( + string source, + PwshToken operatorToken, + PwshToken? target, + string value, + int precedingVerbTokenCount, + ArgKind kind, + bool isPath, + string? resolved) + { + var sourceStart = operatorToken.SourceStart; + var sourceEnd = target.HasValue + ? target.Value.SourceStart + target.Value.SourceLength + : operatorToken.SourceStart + operatorToken.SourceLength; + return new ClauseElement + { + Raw = source.Substring(sourceStart, sourceEnd - sourceStart), + Value = value, + Role = ClauseElementRole.Redirect, + SourceStart = sourceStart, + SourceLength = sourceEnd - sourceStart, + PrecedingVerbElementCount = precedingVerbTokenCount, + Kind = kind, + IsFlag = false, + IsPath = isPath, + Resolved = resolved, + }; + } + /// SPEC.POWERSHELL.md §8: map a PowerShell redirect operator /// onto the (lossy) enum. private static RedirectDirection MapRedirect(string op, out bool isMerge, out string? mergeTarget) @@ -1481,6 +1632,7 @@ private static bool TryRecurseIntoPwsh( Operator = k == 0 ? segment.PrecedingOperator : ic.Operator, IsSubshell = segment.Depth > 0 || ic.IsSubshell, IsCommandStringWrapped = true, + Elements = ClauseElementProvenance.WithoutOuterSourceSpans(ic.Elements), }); } diff --git a/tests/ShellSyntaxTree.Tests/Corpus/AstAssert.cs b/tests/ShellSyntaxTree.Tests/Corpus/AstAssert.cs index 9dce5e9..e03a80d 100644 --- a/tests/ShellSyntaxTree.Tests/Corpus/AstAssert.cs +++ b/tests/ShellSyntaxTree.Tests/Corpus/AstAssert.cs @@ -43,7 +43,8 @@ namespace ShellSyntaxTree.Tests.Corpus; /// /// Per clause: Operator, Verb.Tokens (sequence equality), /// Args.Count, per-arg fields, Redirects.Count, per- -/// redirect fields, IsSubshell, IsCommandStringWrapped. +/// redirect fields, optional Elements, IsSubshell, +/// IsCommandStringWrapped. /// /// /// Per arg: Raw, Kind, IsPath, IsCwdAttribution @@ -52,6 +53,10 @@ namespace ShellSyntaxTree.Tests.Corpus; /// opt-in via the JSON sentinel "__NULL__" for null or a literal /// string for value equality; omitting the field skips the comparison. /// +/// +/// Elements is opt-in for legacy corpus compatibility. When the +/// expectation supplies it, every provenance field is compared. +/// /// /// internal static class AstAssert @@ -163,6 +168,24 @@ private static void AssertClauseEqual(ExpectedClause expected, Clause actual, st AssertRedirectEqual(expectedRedirects[i], actual.Redirects[i], $"{path}.redirects[{i}]"); } + if (expected.Elements is not null) + { + if (expected.Elements.Count != actual.Elements.Count) + { + throw new XunitException( + $"{path}.elements.count: expected={expected.Elements.Count}, actual={actual.Elements.Count}\n" + + " actual elements: " + + string.Join(", ", actual.Elements.Select(element => + $"{{raw={element.Raw}, role={element.Role}, precedingVerbs={element.PrecedingVerbElementCount}}}"))); + } + + for (var i = 0; i < expected.Elements.Count; i++) + { + AssertClauseElementEqual( + expected.Elements[i], actual.Elements[i], $"{path}.elements[{i}]"); + } + } + if (expected.IsSubshell != actual.IsSubshell) { throw new XunitException( @@ -228,6 +251,34 @@ private static void AssertArgEqual(ExpectedArg expected, Arg actual, string path } } + private static void AssertClauseElementEqual( + ExpectedClauseElement expected, + ClauseElement actual, + string path) + { + if (expected.Raw != actual.Raw + || expected.Value != actual.Value + || expected.Role != actual.Role + || expected.SourceStart != actual.SourceStart + || expected.SourceLength != actual.SourceLength + || expected.PrecedingVerbElementCount != actual.PrecedingVerbElementCount + || expected.Kind != actual.Kind + || expected.IsFlag != actual.IsFlag + || expected.IsPath != actual.IsPath + || expected.Resolved != actual.Resolved) + { + throw new XunitException( + $"{path}: expected={{raw={Quote(expected.Raw)}, value={Quote(expected.Value)}, " + + $"role={expected.Role}, span={expected.SourceStart}:{expected.SourceLength}, " + + $"precedingVerbs={expected.PrecedingVerbElementCount}, kind={expected.Kind}, " + + $"isFlag={expected.IsFlag}, isPath={expected.IsPath}, resolved={Quote(expected.Resolved)}}}; " + + $"actual={{raw={Quote(actual.Raw)}, value={Quote(actual.Value)}, " + + $"role={actual.Role}, span={actual.SourceStart}:{actual.SourceLength}, " + + $"precedingVerbs={actual.PrecedingVerbElementCount}, kind={actual.Kind}, " + + $"isFlag={actual.IsFlag}, isPath={actual.IsPath}, resolved={Quote(actual.Resolved)}}}"); + } + } + private static void AssertRedirectEqual(ExpectedRedirect expected, Redirect actual, string path) { if (expected.Direction != actual.Direction) diff --git a/tests/ShellSyntaxTree.Tests/Corpus/CorpusRunnerTests.cs b/tests/ShellSyntaxTree.Tests/Corpus/CorpusRunnerTests.cs index 1b20ccc..d84e844 100644 --- a/tests/ShellSyntaxTree.Tests/Corpus/CorpusRunnerTests.cs +++ b/tests/ShellSyntaxTree.Tests/Corpus/CorpusRunnerTests.cs @@ -34,6 +34,56 @@ public void Corpus_entry_parses_to_expected_ast(string shell, string fileName, C var actual = CreateParser(shell).Parse(entry.Input); AstAssert.Equal(entry.Expected!, actual, $"{shell}/{fileName}"); + AssertClauseElementInvariants(actual, $"{shell}/{fileName}"); + } + + private static void AssertClauseElementInvariants(ParsedCommand parsed, string context) + { + foreach (var clause in parsed.Clauses) + { + var precedingVerbs = 0; + var redirectCount = 0; + var previousSourceStart = -1; + + foreach (var element in clause.Elements) + { + Assert.Equal(precedingVerbs, element.PrecedingVerbElementCount); + + if (element.Role == ClauseElementRole.Verb) + { + Assert.True( + precedingVerbs < clause.Verb.Tokens.Count, + $"{context}: verb element has no matching Verb.Tokens entry."); + Assert.Equal(clause.Verb.Tokens[precedingVerbs], element.Value); + precedingVerbs++; + } + else if (element.Role == ClauseElementRole.Redirect) + { + redirectCount++; + } + + Assert.Equal(element.SourceStart.HasValue, element.SourceLength.HasValue); + if (element.SourceStart.HasValue) + { + var sourceStart = element.SourceStart.Value; + var sourceLength = element.SourceLength!.Value; + Assert.True(sourceStart >= previousSourceStart, $"{context}: element order regressed."); + Assert.InRange(sourceStart, 0, parsed.Source.Length); + Assert.InRange(sourceLength, 0, parsed.Source.Length - sourceStart); + Assert.Equal(element.Raw, parsed.Source.Substring(sourceStart, sourceLength)); + previousSourceStart = sourceStart; + } + else + { + Assert.True( + clause.IsCommandStringWrapped, + $"{context}: only wrapped clauses may omit outer source spans."); + } + } + + Assert.Equal(clause.Verb.Tokens.Count, precedingVerbs); + Assert.Equal(clause.Redirects.Count, redirectCount); + } } /// @@ -158,6 +208,12 @@ public sealed record ExpectedClause public List? Redirects { get; init; } + /// + /// Optional issue #62 provenance expectation. Existing corpus entries may + /// omit it; entries that exercise ordered elements compare the full list. + /// + public List? Elements { get; init; } + public bool IsSubshell { get; init; } public bool IsCommandStringWrapped { get; init; } @@ -215,3 +271,26 @@ public sealed record ExpectedRedirect public bool? IsDynamicSkip { get; init; } } + +public sealed record ExpectedClauseElement +{ + public string Raw { get; init; } = ""; + + public string Value { get; init; } = ""; + + public ClauseElementRole Role { get; init; } + + public int? SourceStart { get; init; } + + public int? SourceLength { get; init; } + + public int PrecedingVerbElementCount { get; init; } + + public ArgKind Kind { get; init; } + + public bool IsFlag { get; init; } + + public bool IsPath { get; init; } + + public string? Resolved { get; init; } +} diff --git a/tests/ShellSyntaxTree.Tests/Corpus/bash/152_git_global_option_provenance.json b/tests/ShellSyntaxTree.Tests/Corpus/bash/152_git_global_option_provenance.json new file mode 100644 index 0000000..3e51032 --- /dev/null +++ b/tests/ShellSyntaxTree.Tests/Corpus/bash/152_git_global_option_provenance.json @@ -0,0 +1,79 @@ +{ + "name": "Issue 62: Git global option provenance", + "input": "git -C /repo commit", + "expected": { + "isUnparseable": false, + "clauses": [ + { + "operator": "None", + "verb": ["git", "commit"], + "args": [ + { + "raw": "-C", + "kind": "Literal", + "isPath": false, + "isFlag": true + }, + { + "raw": "/repo", + "kind": "Literal", + "isPath": true, + "isFlag": false, + "resolved": "/repo" + } + ], + "redirects": [], + "elements": [ + { + "raw": "git", + "value": "git", + "role": "Verb", + "sourceStart": 0, + "sourceLength": 3, + "precedingVerbElementCount": 0, + "kind": "Literal", + "isFlag": false, + "isPath": false + }, + { + "raw": "-C", + "value": "-C", + "role": "Argument", + "sourceStart": 4, + "sourceLength": 2, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": true, + "isPath": false + }, + { + "raw": "/repo", + "value": "/repo", + "role": "Argument", + "sourceStart": 7, + "sourceLength": 5, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": false, + "isPath": true, + "resolved": "/repo" + }, + { + "raw": "commit", + "value": "commit", + "role": "Verb", + "sourceStart": 13, + "sourceLength": 6, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": false, + "isPath": false + } + ], + "isSubshell": false, + "isCommandStringWrapped": false + } + ] + }, + "notes": "Issue #62: -C and /repo occur after git but before commit." +} diff --git a/tests/ShellSyntaxTree.Tests/Corpus/bash/153_git_subcommand_option_provenance.json b/tests/ShellSyntaxTree.Tests/Corpus/bash/153_git_subcommand_option_provenance.json new file mode 100644 index 0000000..b3cdbc5 --- /dev/null +++ b/tests/ShellSyntaxTree.Tests/Corpus/bash/153_git_subcommand_option_provenance.json @@ -0,0 +1,79 @@ +{ + "name": "Issue 62: Git subcommand option provenance", + "input": "git commit -C HEAD~1", + "expected": { + "isUnparseable": false, + "clauses": [ + { + "operator": "None", + "verb": ["git", "commit"], + "args": [ + { + "raw": "-C", + "kind": "Literal", + "isPath": false, + "isFlag": true + }, + { + "raw": "HEAD~1", + "kind": "Literal", + "isPath": true, + "isFlag": false, + "resolved": "/work/HEAD~1" + } + ], + "redirects": [], + "elements": [ + { + "raw": "git", + "value": "git", + "role": "Verb", + "sourceStart": 0, + "sourceLength": 3, + "precedingVerbElementCount": 0, + "kind": "Literal", + "isFlag": false, + "isPath": false + }, + { + "raw": "commit", + "value": "commit", + "role": "Verb", + "sourceStart": 4, + "sourceLength": 6, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": false, + "isPath": false + }, + { + "raw": "-C", + "value": "-C", + "role": "Argument", + "sourceStart": 11, + "sourceLength": 2, + "precedingVerbElementCount": 2, + "kind": "Literal", + "isFlag": true, + "isPath": false + }, + { + "raw": "HEAD~1", + "value": "HEAD~1", + "role": "Argument", + "sourceStart": 14, + "sourceLength": 6, + "precedingVerbElementCount": 2, + "kind": "Literal", + "isFlag": false, + "isPath": true, + "resolved": "/work/HEAD~1" + } + ], + "isSubshell": false, + "isCommandStringWrapped": false + } + ] + }, + "notes": "Issue #62: -C and HEAD~1 occur after the git commit verb boundary." +} diff --git a/tests/ShellSyntaxTree.Tests/Corpus/bash/154_git_global_config_provenance.json b/tests/ShellSyntaxTree.Tests/Corpus/bash/154_git_global_config_provenance.json new file mode 100644 index 0000000..97d77b0 --- /dev/null +++ b/tests/ShellSyntaxTree.Tests/Corpus/bash/154_git_global_config_provenance.json @@ -0,0 +1,65 @@ +{ + "name": "Issue 62: Git global config provenance", + "input": "git -c user.name=Jane commit", + "expected": { + "isUnparseable": false, + "clauses": [ + { + "operator": "None", + "verb": ["git", "commit"], + "args": [ + { "raw": "-c", "kind": "Literal", "isPath": false, "isFlag": true }, + { "raw": "user.name=Jane", "kind": "Literal", "isPath": false, "isFlag": false } + ], + "redirects": [], + "elements": [ + { + "raw": "git", + "value": "git", + "role": "Verb", + "sourceStart": 0, + "sourceLength": 3, + "precedingVerbElementCount": 0, + "kind": "Literal", + "isFlag": false, + "isPath": false + }, + { + "raw": "-c", + "value": "-c", + "role": "Argument", + "sourceStart": 4, + "sourceLength": 2, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": true, + "isPath": false + }, + { + "raw": "user.name=Jane", + "value": "user.name=Jane", + "role": "Argument", + "sourceStart": 7, + "sourceLength": 14, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": false, + "isPath": false + }, + { + "raw": "commit", + "value": "commit", + "role": "Verb", + "sourceStart": 22, + "sourceLength": 6, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": false, + "isPath": false + } + ] + } + ] + }, + "notes": "Issue #62: lowercase -c consumes a non-path configuration value before commit." +} diff --git a/tests/ShellSyntaxTree.Tests/Corpus/bash/155_git_subcommand_config_provenance.json b/tests/ShellSyntaxTree.Tests/Corpus/bash/155_git_subcommand_config_provenance.json new file mode 100644 index 0000000..5036dc0 --- /dev/null +++ b/tests/ShellSyntaxTree.Tests/Corpus/bash/155_git_subcommand_config_provenance.json @@ -0,0 +1,65 @@ +{ + "name": "Issue 62: Git subcommand config provenance", + "input": "git commit -c HEAD~1", + "expected": { + "isUnparseable": false, + "clauses": [ + { + "operator": "None", + "verb": ["git", "commit"], + "args": [ + { "raw": "-c", "kind": "Literal", "isPath": false, "isFlag": true }, + { "raw": "HEAD~1", "kind": "Literal", "isPath": false, "isFlag": false } + ], + "redirects": [], + "elements": [ + { + "raw": "git", + "value": "git", + "role": "Verb", + "sourceStart": 0, + "sourceLength": 3, + "precedingVerbElementCount": 0, + "kind": "Literal", + "isFlag": false, + "isPath": false + }, + { + "raw": "commit", + "value": "commit", + "role": "Verb", + "sourceStart": 4, + "sourceLength": 6, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": false, + "isPath": false + }, + { + "raw": "-c", + "value": "-c", + "role": "Argument", + "sourceStart": 11, + "sourceLength": 2, + "precedingVerbElementCount": 2, + "kind": "Literal", + "isFlag": true, + "isPath": false + }, + { + "raw": "HEAD~1", + "value": "HEAD~1", + "role": "Argument", + "sourceStart": 14, + "sourceLength": 6, + "precedingVerbElementCount": 2, + "kind": "Literal", + "isFlag": false, + "isPath": false + } + ] + } + ] + }, + "notes": "Issue #62: lowercase command-scoped -c preserves its post-commit position." +} diff --git a/tests/ShellSyntaxTree.Tests/Corpus/bash/156_git_mixed_option_provenance.json b/tests/ShellSyntaxTree.Tests/Corpus/bash/156_git_mixed_option_provenance.json new file mode 100644 index 0000000..39ac180 --- /dev/null +++ b/tests/ShellSyntaxTree.Tests/Corpus/bash/156_git_mixed_option_provenance.json @@ -0,0 +1,91 @@ +{ + "name": "Issue 62: Git mixed option provenance", + "input": "git -C /repo commit -C HEAD~1", + "expected": { + "isUnparseable": false, + "clauses": [ + { + "operator": "None", + "verb": ["git", "commit"], + "args": [ + { "raw": "-C", "kind": "Literal", "isPath": false, "isFlag": true }, + { "raw": "/repo", "kind": "Literal", "isPath": true, "isFlag": false, "resolved": "/repo" }, + { "raw": "-C", "kind": "Literal", "isPath": false, "isFlag": true }, + { "raw": "HEAD~1", "kind": "Literal", "isPath": true, "isFlag": false, "resolved": "/work/HEAD~1" } + ], + "redirects": [], + "elements": [ + { + "raw": "git", + "value": "git", + "role": "Verb", + "sourceStart": 0, + "sourceLength": 3, + "precedingVerbElementCount": 0, + "kind": "Literal", + "isFlag": false, + "isPath": false + }, + { + "raw": "-C", + "value": "-C", + "role": "Argument", + "sourceStart": 4, + "sourceLength": 2, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": true, + "isPath": false + }, + { + "raw": "/repo", + "value": "/repo", + "role": "Argument", + "sourceStart": 7, + "sourceLength": 5, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": false, + "isPath": true, + "resolved": "/repo" + }, + { + "raw": "commit", + "value": "commit", + "role": "Verb", + "sourceStart": 13, + "sourceLength": 6, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": false, + "isPath": false + }, + { + "raw": "-C", + "value": "-C", + "role": "Argument", + "sourceStart": 20, + "sourceLength": 2, + "precedingVerbElementCount": 2, + "kind": "Literal", + "isFlag": true, + "isPath": false + }, + { + "raw": "HEAD~1", + "value": "HEAD~1", + "role": "Argument", + "sourceStart": 23, + "sourceLength": 6, + "precedingVerbElementCount": 2, + "kind": "Literal", + "isFlag": false, + "isPath": true, + "resolved": "/work/HEAD~1" + } + ] + } + ] + }, + "notes": "Issue #62: both global and command-scoped -C occurrences remain distinct." +} diff --git a/tests/ShellSyntaxTree.Tests/Corpus/bash/157_git_heuristic_boundary_provenance.json b/tests/ShellSyntaxTree.Tests/Corpus/bash/157_git_heuristic_boundary_provenance.json new file mode 100644 index 0000000..85c60a9 --- /dev/null +++ b/tests/ShellSyntaxTree.Tests/Corpus/bash/157_git_heuristic_boundary_provenance.json @@ -0,0 +1,79 @@ +{ + "name": "Issue 62: Git heuristic boundary provenance", + "input": "git --no-pager commit -C HEAD~1", + "expected": { + "isUnparseable": false, + "clauses": [ + { + "operator": "None", + "verb": ["git"], + "args": [ + { "raw": "--no-pager", "kind": "Literal", "isPath": false, "isFlag": true }, + { "raw": "commit", "kind": "Literal", "isPath": false, "isFlag": false }, + { "raw": "-C", "kind": "Literal", "isPath": false, "isFlag": true }, + { "raw": "HEAD~1", "kind": "Literal", "isPath": true, "isFlag": false, "resolved": "/work/HEAD~1" } + ], + "redirects": [], + "elements": [ + { + "raw": "git", + "value": "git", + "role": "Verb", + "sourceStart": 0, + "sourceLength": 3, + "precedingVerbElementCount": 0, + "kind": "Literal", + "isFlag": false, + "isPath": false + }, + { + "raw": "--no-pager", + "value": "--no-pager", + "role": "Argument", + "sourceStart": 4, + "sourceLength": 10, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": true, + "isPath": false + }, + { + "raw": "commit", + "value": "commit", + "role": "Argument", + "sourceStart": 15, + "sourceLength": 6, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": false, + "isPath": false + }, + { + "raw": "-C", + "value": "-C", + "role": "Argument", + "sourceStart": 22, + "sourceLength": 2, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": true, + "isPath": false + }, + { + "raw": "HEAD~1", + "value": "HEAD~1", + "role": "Argument", + "sourceStart": 25, + "sourceLength": 6, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": false, + "isPath": true, + "resolved": "/work/HEAD~1" + } + ] + } + ] + }, + "notes": "Issue #62: authored order survives when a valueless option stops the greedy verb walk." +} diff --git a/tests/ShellSyntaxTree.Tests/Corpus/powershell/220_git_global_option_provenance.json b/tests/ShellSyntaxTree.Tests/Corpus/powershell/220_git_global_option_provenance.json new file mode 100644 index 0000000..ec376a7 --- /dev/null +++ b/tests/ShellSyntaxTree.Tests/Corpus/powershell/220_git_global_option_provenance.json @@ -0,0 +1,78 @@ +{ + "name": "Git global option provenance", + "input": "git -C C:\\repo commit", + "expected": { + "isUnparseable": false, + "clauses": [ + { + "operator": "None", + "verb": [ + "git", + "commit" + ], + "args": [ + { + "raw": "-C", + "kind": "Literal", + "isPath": false + }, + { + "raw": "C:\\repo", + "kind": "Literal", + "isPath": true, + "resolved": "C:/repo" + } + ], + "redirects": [], + "elements": [ + { + "raw": "git", + "value": "git", + "role": "Verb", + "sourceStart": 0, + "sourceLength": 3, + "precedingVerbElementCount": 0, + "kind": "Literal", + "isFlag": false, + "isPath": false + }, + { + "raw": "-C", + "value": "-C", + "role": "Argument", + "sourceStart": 4, + "sourceLength": 2, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": true, + "isPath": false + }, + { + "raw": "C:\\repo", + "value": "C:\\repo", + "role": "Argument", + "sourceStart": 7, + "sourceLength": 7, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": false, + "isPath": true, + "resolved": "C:/repo" + }, + { + "raw": "commit", + "value": "commit", + "role": "Verb", + "sourceStart": 15, + "sourceLength": 6, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": false, + "isPath": false + } + ] + } + ] + }, + "notes": "Issue #62: global -C and its path occur before the commit element." +} diff --git a/tests/ShellSyntaxTree.Tests/Corpus/powershell/221_git_subcommand_option_provenance.json b/tests/ShellSyntaxTree.Tests/Corpus/powershell/221_git_subcommand_option_provenance.json new file mode 100644 index 0000000..a18c82d --- /dev/null +++ b/tests/ShellSyntaxTree.Tests/Corpus/powershell/221_git_subcommand_option_provenance.json @@ -0,0 +1,78 @@ +{ + "name": "Git subcommand option provenance", + "input": "git commit -C HEAD~1", + "expected": { + "isUnparseable": false, + "clauses": [ + { + "operator": "None", + "verb": [ + "git", + "commit" + ], + "args": [ + { + "raw": "-C", + "kind": "Literal", + "isPath": false + }, + { + "raw": "HEAD~1", + "kind": "Literal", + "isPath": true, + "resolved": "C:/work/HEAD~1" + } + ], + "redirects": [], + "elements": [ + { + "raw": "git", + "value": "git", + "role": "Verb", + "sourceStart": 0, + "sourceLength": 3, + "precedingVerbElementCount": 0, + "kind": "Literal", + "isFlag": false, + "isPath": false + }, + { + "raw": "commit", + "value": "commit", + "role": "Verb", + "sourceStart": 4, + "sourceLength": 6, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": false, + "isPath": false + }, + { + "raw": "-C", + "value": "-C", + "role": "Argument", + "sourceStart": 11, + "sourceLength": 2, + "precedingVerbElementCount": 2, + "kind": "Literal", + "isFlag": true, + "isPath": false + }, + { + "raw": "HEAD~1", + "value": "HEAD~1", + "role": "Argument", + "sourceStart": 14, + "sourceLength": 6, + "precedingVerbElementCount": 2, + "kind": "Literal", + "isFlag": false, + "isPath": true, + "resolved": "C:/work/HEAD~1" + } + ] + } + ] + }, + "notes": "Issue #62: command-scoped -C occurs after the parser-classified commit verb." +} diff --git a/tests/ShellSyntaxTree.Tests/Corpus/powershell/222_git_global_config_provenance.json b/tests/ShellSyntaxTree.Tests/Corpus/powershell/222_git_global_config_provenance.json new file mode 100644 index 0000000..dde5d01 --- /dev/null +++ b/tests/ShellSyntaxTree.Tests/Corpus/powershell/222_git_global_config_provenance.json @@ -0,0 +1,76 @@ +{ + "name": "Git global config provenance", + "input": "git -c user.name=Jane commit", + "expected": { + "isUnparseable": false, + "clauses": [ + { + "operator": "None", + "verb": [ + "git", + "commit" + ], + "args": [ + { + "raw": "-c", + "kind": "Literal", + "isPath": false + }, + { + "raw": "user.name=Jane", + "kind": "Literal", + "isPath": false + } + ], + "redirects": [], + "elements": [ + { + "raw": "git", + "value": "git", + "role": "Verb", + "sourceStart": 0, + "sourceLength": 3, + "precedingVerbElementCount": 0, + "kind": "Literal", + "isFlag": false, + "isPath": false + }, + { + "raw": "-c", + "value": "-c", + "role": "Argument", + "sourceStart": 4, + "sourceLength": 2, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": true, + "isPath": false + }, + { + "raw": "user.name=Jane", + "value": "user.name=Jane", + "role": "Argument", + "sourceStart": 7, + "sourceLength": 14, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": false, + "isPath": false + }, + { + "raw": "commit", + "value": "commit", + "role": "Verb", + "sourceStart": 22, + "sourceLength": 6, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": false, + "isPath": false + } + ] + } + ] + }, + "notes": "Issue #62: lowercase -c consumes a non-path configuration value." +} diff --git a/tests/ShellSyntaxTree.Tests/Corpus/powershell/223_git_subcommand_config_provenance.json b/tests/ShellSyntaxTree.Tests/Corpus/powershell/223_git_subcommand_config_provenance.json new file mode 100644 index 0000000..fefc85d --- /dev/null +++ b/tests/ShellSyntaxTree.Tests/Corpus/powershell/223_git_subcommand_config_provenance.json @@ -0,0 +1,76 @@ +{ + "name": "Git subcommand config provenance", + "input": "git commit -c HEAD~1", + "expected": { + "isUnparseable": false, + "clauses": [ + { + "operator": "None", + "verb": [ + "git", + "commit" + ], + "args": [ + { + "raw": "-c", + "kind": "Literal", + "isPath": false + }, + { + "raw": "HEAD~1", + "kind": "Literal", + "isPath": false + } + ], + "redirects": [], + "elements": [ + { + "raw": "git", + "value": "git", + "role": "Verb", + "sourceStart": 0, + "sourceLength": 3, + "precedingVerbElementCount": 0, + "kind": "Literal", + "isFlag": false, + "isPath": false + }, + { + "raw": "commit", + "value": "commit", + "role": "Verb", + "sourceStart": 4, + "sourceLength": 6, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": false, + "isPath": false + }, + { + "raw": "-c", + "value": "-c", + "role": "Argument", + "sourceStart": 11, + "sourceLength": 2, + "precedingVerbElementCount": 2, + "kind": "Literal", + "isFlag": true, + "isPath": false + }, + { + "raw": "HEAD~1", + "value": "HEAD~1", + "role": "Argument", + "sourceStart": 14, + "sourceLength": 6, + "precedingVerbElementCount": 2, + "kind": "Literal", + "isFlag": false, + "isPath": false + } + ] + } + ] + }, + "notes": "Issue #62: lowercase command-scoped -c preserves its post-commit position." +} diff --git a/tests/ShellSyntaxTree.Tests/Corpus/powershell/224_git_mixed_option_provenance.json b/tests/ShellSyntaxTree.Tests/Corpus/powershell/224_git_mixed_option_provenance.json new file mode 100644 index 0000000..e5008b5 --- /dev/null +++ b/tests/ShellSyntaxTree.Tests/Corpus/powershell/224_git_mixed_option_provenance.json @@ -0,0 +1,112 @@ +{ + "name": "Git mixed option provenance", + "input": "git -C C:\\repo commit -C HEAD~1", + "expected": { + "isUnparseable": false, + "clauses": [ + { + "operator": "None", + "verb": [ + "git", + "commit" + ], + "args": [ + { + "raw": "-C", + "kind": "Literal", + "isPath": false + }, + { + "raw": "C:\\repo", + "kind": "Literal", + "isPath": true, + "resolved": "C:/repo" + }, + { + "raw": "-C", + "kind": "Literal", + "isPath": false + }, + { + "raw": "HEAD~1", + "kind": "Literal", + "isPath": true, + "resolved": "C:/work/HEAD~1" + } + ], + "redirects": [], + "elements": [ + { + "raw": "git", + "value": "git", + "role": "Verb", + "sourceStart": 0, + "sourceLength": 3, + "precedingVerbElementCount": 0, + "kind": "Literal", + "isFlag": false, + "isPath": false + }, + { + "raw": "-C", + "value": "-C", + "role": "Argument", + "sourceStart": 4, + "sourceLength": 2, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": true, + "isPath": false + }, + { + "raw": "C:\\repo", + "value": "C:\\repo", + "role": "Argument", + "sourceStart": 7, + "sourceLength": 7, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": false, + "isPath": true, + "resolved": "C:/repo" + }, + { + "raw": "commit", + "value": "commit", + "role": "Verb", + "sourceStart": 15, + "sourceLength": 6, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": false, + "isPath": false + }, + { + "raw": "-C", + "value": "-C", + "role": "Argument", + "sourceStart": 22, + "sourceLength": 2, + "precedingVerbElementCount": 2, + "kind": "Literal", + "isFlag": true, + "isPath": false + }, + { + "raw": "HEAD~1", + "value": "HEAD~1", + "role": "Argument", + "sourceStart": 25, + "sourceLength": 6, + "precedingVerbElementCount": 2, + "kind": "Literal", + "isFlag": false, + "isPath": true, + "resolved": "C:/work/HEAD~1" + } + ] + } + ] + }, + "notes": "Issue #62: both global and command-scoped -C occurrences remain distinct." +} diff --git a/tests/ShellSyntaxTree.Tests/Corpus/powershell/225_git_heuristic_boundary_provenance.json b/tests/ShellSyntaxTree.Tests/Corpus/powershell/225_git_heuristic_boundary_provenance.json new file mode 100644 index 0000000..b303eac --- /dev/null +++ b/tests/ShellSyntaxTree.Tests/Corpus/powershell/225_git_heuristic_boundary_provenance.json @@ -0,0 +1,98 @@ +{ + "name": "Git heuristic boundary provenance", + "input": "git --no-pager commit -C HEAD~1", + "expected": { + "isUnparseable": false, + "clauses": [ + { + "operator": "None", + "verb": [ + "git" + ], + "args": [ + { + "raw": "--no-pager", + "kind": "Literal", + "isPath": false + }, + { + "raw": "commit", + "kind": "Literal", + "isPath": false + }, + { + "raw": "-C", + "kind": "Literal", + "isPath": false + }, + { + "raw": "HEAD~1", + "kind": "Literal", + "isPath": true, + "resolved": "C:/work/HEAD~1" + } + ], + "redirects": [], + "elements": [ + { + "raw": "git", + "value": "git", + "role": "Verb", + "sourceStart": 0, + "sourceLength": 3, + "precedingVerbElementCount": 0, + "kind": "Literal", + "isFlag": false, + "isPath": false + }, + { + "raw": "--no-pager", + "value": "--no-pager", + "role": "Argument", + "sourceStart": 4, + "sourceLength": 10, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": true, + "isPath": false + }, + { + "raw": "commit", + "value": "commit", + "role": "Argument", + "sourceStart": 15, + "sourceLength": 6, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": false, + "isPath": false + }, + { + "raw": "-C", + "value": "-C", + "role": "Argument", + "sourceStart": 22, + "sourceLength": 2, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": true, + "isPath": false + }, + { + "raw": "HEAD~1", + "value": "HEAD~1", + "role": "Argument", + "sourceStart": 25, + "sourceLength": 6, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": false, + "isPath": true, + "resolved": "C:/work/HEAD~1" + } + ] + } + ] + }, + "notes": "Issue #62: authored order survives when a valueless option stops the greedy verb walk." +} diff --git a/tests/ShellSyntaxTree.Tests/Parsing/BashPerVerbRulesTests.cs b/tests/ShellSyntaxTree.Tests/Parsing/BashPerVerbRulesTests.cs index 394545b..9d4fa0b 100644 --- a/tests/ShellSyntaxTree.Tests/Parsing/BashPerVerbRulesTests.cs +++ b/tests/ShellSyntaxTree.Tests/Parsing/BashPerVerbRulesTests.cs @@ -284,12 +284,11 @@ public void Unknown_verb_or_flag_returns_false() } [Fact] - public void Flag_lookup_is_case_insensitive() + public void Native_flag_lookup_is_case_sensitive_while_verb_lookup_is_not() { - // The per-verb-flag table is case-insensitive; consumers may - // typo casing. - Assert.True(BashPerVerbRules.ValueOfFlagIsPath("GIT", "-c")); - Assert.True(BashPerVerbRules.ValueOfFlagIsPath("Git", "--Git-Dir")); + Assert.True(BashPerVerbRules.ValueOfFlagIsPath("GIT", "-C")); + Assert.False(BashPerVerbRules.ValueOfFlagIsPath("GIT", "-c")); + Assert.False(BashPerVerbRules.ValueOfFlagIsPath("Git", "--Git-Dir")); } // ---------------------------------------------------------------- empty verb chain diff --git a/tests/ShellSyntaxTree.Tests/Parsing/ClauseElementTests.cs b/tests/ShellSyntaxTree.Tests/Parsing/ClauseElementTests.cs new file mode 100644 index 0000000..48effe3 --- /dev/null +++ b/tests/ShellSyntaxTree.Tests/Parsing/ClauseElementTests.cs @@ -0,0 +1,298 @@ +// ----------------------------------------------------------------------- +// +// Copyright (C) 2026 - 2026 Aaron Stannard +// +// ----------------------------------------------------------------------- +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using Xunit; + +namespace ShellSyntaxTree.Tests.Parsing; + +/// +/// Cross-shell contract tests for the issue #62 source-ordered clause element +/// projection. +/// +public class ClauseElementTests +{ + private static readonly BashParser Bash = new(new BashParserOptions + { + HomeDirectory = "/home/test", + WorkingDirectory = "/work", + }); + + private static readonly PwshParser Pwsh = new(new PwshParserOptions + { + HomeDirectory = "C:/Users/user", + WorkingDirectory = "C:/work", + }); + + [Fact] + public void Git_option_position_is_preserved_in_both_parsers() + { + foreach (var (_, parser) in Parsers()) + { + var global = Assert.Single(parser.Parse("git -C /repo commit").Clauses); + Assert.Equal( + new[] { "git", "-C", "/repo", "commit" }, + global.Elements.Select(element => element.Value).ToArray()); + Assert.Equal( + new[] + { + ClauseElementRole.Verb, + ClauseElementRole.Argument, + ClauseElementRole.Argument, + ClauseElementRole.Verb, + }, + global.Elements.Select(element => element.Role).ToArray()); + Assert.Equal( + new[] { 0, 1, 1, 1 }, + global.Elements.Select(element => element.PrecedingVerbElementCount).ToArray()); + + var command = Assert.Single(parser.Parse("git commit -C HEAD~1").Clauses); + Assert.Equal( + new[] { "git", "commit", "-C", "HEAD~1" }, + command.Elements.Select(element => element.Value).ToArray()); + Assert.Equal( + new[] { 0, 1, 2, 2 }, + command.Elements.Select(element => element.PrecedingVerbElementCount).ToArray()); + + // The additive provenance view does not reshape compatibility + // projections used by existing consumers. + Assert.Equal(new[] { "git", "commit" }, command.Verb.Tokens); + Assert.Equal(new[] { "-C", "HEAD~1" }, command.Args.Select(arg => arg.Raw).ToArray()); + } + } + + [Fact] + public void Git_option_case_preserves_distinct_native_metadata() + { + foreach (var (shell, parser) in Parsers()) + { + var lower = Assert.Single(parser.Parse("git -c user.name=Jane commit").Clauses); + var lowerValue = Assert.Single( + lower.Elements, + element => element.Value == "user.name=Jane"); + Assert.False(lowerValue.IsPath, shell); + Assert.Null(lowerValue.Resolved); + + var upper = Assert.Single(parser.Parse("git -C /repo commit").Clauses); + var upperValue = Assert.Single( + upper.Elements, + element => element.Value.EndsWith("repo", StringComparison.Ordinal)); + Assert.True(upperValue.IsPath, shell); + Assert.NotNull(upperValue.Resolved); + } + } + + [Fact] + public void Multiple_git_option_occurrences_remain_distinct() + { + foreach (var (_, parser) in Parsers()) + { + var clause = Assert.Single( + parser.Parse("git -C /repo commit -C HEAD~1").Clauses); + var options = clause.Elements.Where(element => element.Value == "-C").ToArray(); + + Assert.Equal(2, options.Length); + Assert.Equal(new int?[] { 4, 20 }, options.Select(option => option.SourceStart).ToArray()); + Assert.Equal( + new[] { 1, 2 }, + options.Select(option => option.PrecedingVerbElementCount).ToArray()); + } + } + + [Fact] + public void Authored_order_survives_when_a_global_option_stops_the_greedy_walk() + { + foreach (var (_, parser) in Parsers()) + { + var clause = Assert.Single( + parser.Parse("git --no-pager commit -C HEAD~1").Clauses); + + Assert.Equal(new[] { "git" }, clause.Verb.Tokens); + Assert.Equal( + new[] { "git", "--no-pager", "commit", "-C", "HEAD~1" }, + clause.Elements.Select(element => element.Value).ToArray()); + Assert.Equal( + ClauseElementRole.Argument, + Assert.Single(clause.Elements, element => element.Value == "commit").Role); + Assert.Equal( + 1, + Assert.Single(clause.Elements, element => element.Value == "-C") + .PrecedingVerbElementCount); + } + } + + [Fact] + public void Quoted_values_preserve_exact_raw_decoded_value_and_span() + { + const string source = "git -c \"user.name=Jane Doe\" commit"; + foreach (var (_, parser) in Parsers()) + { + var clause = Assert.Single(parser.Parse(source).Clauses); + var element = Assert.Single( + clause.Elements, + candidate => candidate.Value == "user.name=Jane Doe"); + + Assert.Equal("\"user.name=Jane Doe\"", element.Raw); + Assert.Equal(source.IndexOf('"'), element.SourceStart); + Assert.Equal(element.Raw.Length, element.SourceLength); + Assert.Equal(1, element.PrecedingVerbElementCount); + } + } + + [Fact] + public void Repeated_values_are_disambiguated_by_role_and_source_span() + { + foreach (var (_, parser) in Parsers()) + { + var clause = Assert.Single(parser.Parse("tool item --name item").Clauses); + var repeated = clause.Elements.Where(element => element.Value == "item").ToArray(); + + Assert.Equal(2, repeated.Length); + Assert.Equal(ClauseElementRole.Verb, repeated[0].Role); + Assert.Equal(ClauseElementRole.Argument, repeated[1].Role); + Assert.Equal(5, repeated[0].SourceStart); + Assert.Equal(17, repeated[1].SourceStart); + } + } + + [Fact] + public void Verb_relative_position_resets_at_pipeline_boundaries() + { + const string source = "git -C /repo status | git commit -C HEAD~1"; + foreach (var (_, parser) in Parsers()) + { + var clauses = parser.Parse(source).Clauses; + Assert.Equal(2, clauses.Count); + + var firstFlag = Assert.Single( + clauses[0].Elements, + element => element.Value == "-C"); + var secondFlag = Assert.Single( + clauses[1].Elements, + element => element.Value == "-C"); + + Assert.Equal(1, firstFlag.PrecedingVerbElementCount); + Assert.Equal(2, secondFlag.PrecedingVerbElementCount); + Assert.Equal(CompoundOperator.Pipe, clauses[1].Operator); + } + } + + [Fact] + public void Redirect_is_one_element_at_its_authored_position() + { + const string source = "git -C /repo status > status.txt"; + foreach (var (shell, parser) in Parsers()) + { + var clause = Assert.Single(parser.Parse(source).Clauses); + var redirect = Assert.Single( + clause.Elements, + element => element.Role == ClauseElementRole.Redirect); + + Assert.Equal("> status.txt", redirect.Raw); + Assert.Equal("status.txt", redirect.Value); + Assert.Equal(2, redirect.PrecedingVerbElementCount); + Assert.Equal(source.IndexOf('>'), redirect.SourceStart); + Assert.Equal(ClauseElementRole.Verb, clause.Elements[3].Role); + Assert.True(redirect.IsPath, shell); + } + } + + [Fact] + public void Inline_native_path_binding_remains_one_source_element() + { + foreach (var (shell, parser) in Parsers()) + { + var clause = Assert.Single(parser.Parse("git --work-tree=../repo status").Clauses); + var option = Assert.Single( + clause.Elements, + element => element.Raw == "--work-tree=../repo"); + + Assert.Equal("--work-tree=../repo", option.Value); + Assert.Equal(ClauseElementRole.Argument, option.Role); + Assert.True(option.IsFlag, shell); + Assert.True(option.IsPath, shell); + Assert.EndsWith("/repo", option.Resolved, StringComparison.Ordinal); + Assert.DoesNotContain( + clause.Elements, + element => element.Raw is "--work-tree" or "../repo"); + } + } + + [Fact] + public void PowerShell_inline_cmdlet_binding_carries_bound_value_metadata() + { + var clause = Assert.Single(Pwsh.Parse("Remove-Item -Path:C:\\repo").Clauses); + var option = Assert.Single( + clause.Elements, + element => element.Role == ClauseElementRole.Argument); + + Assert.Equal("-Path:C:\\repo", option.Raw); + Assert.Equal("-Path:C:\\repo", option.Value); + Assert.True(option.IsFlag); + Assert.True(option.IsPath); + Assert.Equal("C:/repo", option.Resolved); + } + + [Fact] + public void Synthetic_cwd_attribution_is_not_a_source_element() + { + var bashClauses = Bash.Parse("cd /repo && git status").Clauses; + Assert.Contains(bashClauses[1].Args, arg => arg.IsCwdAttribution); + Assert.Equal( + new[] { "git", "status" }, + bashClauses[1].Elements.Select(element => element.Value).ToArray()); + + var pwshClauses = Pwsh.Parse("Set-Location C:\\repo; git status").Clauses; + Assert.Contains(pwshClauses[1].Args, arg => arg.IsCwdAttribution); + Assert.Equal( + new[] { "git", "status" }, + pwshClauses[1].Elements.Select(element => element.Value).ToArray()); + } + + [Fact] + public void Wrapped_commands_keep_values_but_clear_outer_source_spans() + { + var bashClause = Assert.Single(Bash.Parse("bash -c \"git -C /repo status\"").Clauses); + Assert.True(bashClause.IsCommandStringWrapped); + Assert.Equal( + new[] { "git", "-C", "/repo", "status" }, + bashClause.Elements.Select(element => element.Value).ToArray()); + Assert.All(bashClause.Elements, AssertSpanIsUnknown); + + var pwshClause = Assert.Single(Pwsh.Parse("pwsh -Command \"git -C C:\\repo status\"").Clauses); + Assert.True(pwshClause.IsCommandStringWrapped); + Assert.Equal( + new[] { "git", "-C", "C:\\repo", "status" }, + pwshClause.Elements.Select(element => element.Value).ToArray()); + Assert.All(pwshClause.Elements, AssertSpanIsUnknown); + } + + [Fact] + public void Encoded_command_elements_have_no_invented_outer_spans() + { + var payload = Convert.ToBase64String(Encoding.Unicode.GetBytes("git commit -C HEAD~1")); + var clause = Assert.Single(Pwsh.Parse($"pwsh -EncodedCommand {payload}").Clauses); + + Assert.Equal( + new[] { "git", "commit", "-C", "HEAD~1" }, + clause.Elements.Select(element => element.Value).ToArray()); + Assert.All(clause.Elements, AssertSpanIsUnknown); + } + + private static IEnumerable<(string Shell, IShellParser Parser)> Parsers() + { + yield return ("bash", Bash); + yield return ("pwsh", Pwsh); + } + + private static void AssertSpanIsUnknown(ClauseElement element) + { + Assert.Null(element.SourceStart); + Assert.Null(element.SourceLength); + } +} diff --git a/tests/ShellSyntaxTree.Tests/PublicApiSnapshotTests.cs b/tests/ShellSyntaxTree.Tests/PublicApiSnapshotTests.cs index d585cb1..d839f9d 100644 --- a/tests/ShellSyntaxTree.Tests/PublicApiSnapshotTests.cs +++ b/tests/ShellSyntaxTree.Tests/PublicApiSnapshotTests.cs @@ -255,6 +255,7 @@ public void Clause_has_expected_shape() AssertInitProperty(t, "Verb", typeof(VerbChain)); AssertInitProperty(t, "Args", typeof(IReadOnlyList)); AssertInitProperty(t, "Redirects", typeof(IReadOnlyList)); + AssertInitProperty(t, "Elements", typeof(IReadOnlyList)); AssertInitProperty(t, "IsSubshell", typeof(bool)); AssertInitProperty(t, "IsCommandStringWrapped", typeof(bool)); @@ -264,10 +265,45 @@ public void Clause_has_expected_shape() Assert.Empty(instance.Verb.Tokens); Assert.Empty(instance.Args); Assert.Empty(instance.Redirects); + Assert.Empty(instance.Elements); Assert.False(instance.IsSubshell); Assert.False(instance.IsCommandStringWrapped); } + // -------- ClauseElement -------- + + [Fact] + public void ClauseElement_has_expected_shape() + { + var t = typeof(ClauseElement); + Assert.True(t.IsPublic); + Assert.True(t.IsSealed); + AssertIsRecord(t); + + AssertInitProperty(t, "Raw", typeof(string)); + AssertInitProperty(t, "Value", typeof(string)); + AssertInitProperty(t, "Role", typeof(ClauseElementRole)); + AssertInitProperty(t, "SourceStart", typeof(int?), nullable: true); + AssertInitProperty(t, "SourceLength", typeof(int?), nullable: true); + AssertInitProperty(t, "PrecedingVerbElementCount", typeof(int)); + AssertInitProperty(t, "Kind", typeof(ArgKind)); + AssertInitProperty(t, "IsFlag", typeof(bool)); + AssertInitProperty(t, "IsPath", typeof(bool)); + AssertInitProperty(t, "Resolved", typeof(string), nullable: true); + + var instance = new ClauseElement(); + Assert.Equal("", instance.Raw); + Assert.Equal("", instance.Value); + Assert.Equal(ClauseElementRole.Verb, instance.Role); + Assert.Null(instance.SourceStart); + Assert.Null(instance.SourceLength); + Assert.Equal(0, instance.PrecedingVerbElementCount); + Assert.Equal(ArgKind.Literal, instance.Kind); + Assert.False(instance.IsFlag); + Assert.False(instance.IsPath); + Assert.Null(instance.Resolved); + } + // -------- VerbChain -------- [Fact] @@ -376,6 +412,18 @@ public void ArgKind_has_expected_members() Assert.Equal(4, (int)ArgKind.DynamicSkip); } + [Fact] + public void ClauseElementRole_has_expected_members() + { + Assert.Equal( + new[] { "Verb", "Argument", "Redirect" }, + Enum.GetNames(typeof(ClauseElementRole))); + + Assert.Equal(0, (int)ClauseElementRole.Verb); + Assert.Equal(1, (int)ClauseElementRole.Argument); + Assert.Equal(2, (int)ClauseElementRole.Redirect); + } + [Fact] public void RedirectDirection_has_expected_members() { @@ -423,6 +471,8 @@ public void Public_namespace_contains_only_expected_types() nameof(BashParser), nameof(BashParserOptions), nameof(Clause), + nameof(ClauseElement), + nameof(ClauseElementRole), nameof(CompoundOperator), nameof(IShellParser), nameof(ParsedCommand), diff --git a/tools/PwshCorpusTool/CorpusJson.cs b/tools/PwshCorpusTool/CorpusJson.cs index 48a1e2b..e768bbf 100644 --- a/tools/PwshCorpusTool/CorpusJson.cs +++ b/tools/PwshCorpusTool/CorpusJson.cs @@ -25,13 +25,18 @@ internal static class CorpusJson }; internal static string BuildEntry( - string name, string input, ParsedCommand parsed, string notes, bool outOfScope) + string name, + string input, + ParsedCommand parsed, + string notes, + bool outOfScope, + bool includeElements) { var obj = new JsonObject { ["name"] = name, ["input"] = input, - ["expected"] = BuildExpected(parsed), + ["expected"] = BuildExpected(parsed, includeElements), ["notes"] = notes, }; @@ -43,7 +48,7 @@ internal static string BuildEntry( return obj.ToJsonString(WriteOptions) + "\n"; } - private static JsonObject BuildExpected(ParsedCommand parsed) + private static JsonObject BuildExpected(ParsedCommand parsed, bool includeElements) { var expected = new JsonObject { ["isUnparseable"] = parsed.IsUnparseable }; if (parsed.IsUnparseable) @@ -55,14 +60,14 @@ private static JsonObject BuildExpected(ParsedCommand parsed) var clauses = new JsonArray(); foreach (var clause in parsed.Clauses) { - clauses.Add(BuildClause(clause)); + clauses.Add(BuildClause(clause, includeElements)); } expected["clauses"] = clauses; return expected; } - private static JsonObject BuildClause(Clause clause) + private static JsonObject BuildClause(Clause clause, bool includeElements) { var verb = new JsonArray(); foreach (var token in clause.Verb.Tokens) @@ -102,6 +107,17 @@ private static JsonObject BuildClause(Clause clause) obj["redirects"] = redirects; + if (includeElements) + { + var elements = new JsonArray(); + foreach (var element in clause.Elements) + { + elements.Add(BuildElement(element)); + } + + obj["elements"] = elements; + } + if (clause.IsSubshell) { obj["isSubshell"] = true; @@ -153,6 +169,29 @@ private static JsonObject BuildRedirect(Redirect redirect) return obj; } + private static JsonObject BuildElement(ClauseElement element) + { + var obj = new JsonObject + { + ["raw"] = element.Raw, + ["value"] = element.Value, + ["role"] = element.Role.ToString(), + ["sourceStart"] = element.SourceStart, + ["sourceLength"] = element.SourceLength, + ["precedingVerbElementCount"] = element.PrecedingVerbElementCount, + ["kind"] = element.Kind.ToString(), + ["isFlag"] = element.IsFlag, + ["isPath"] = element.IsPath, + }; + + if (element.Resolved is not null) + { + obj["resolved"] = element.Resolved; + } + + return obj; + } + /// /// A stable substring of an UnparseableReason for the corpus's /// unparseableReasonContains assertion — trims the variable diff --git a/tools/PwshCorpusTool/CorpusManifest.cs b/tools/PwshCorpusTool/CorpusManifest.cs index 094c534..849d9bb 100644 --- a/tools/PwshCorpusTool/CorpusManifest.cs +++ b/tools/PwshCorpusTool/CorpusManifest.cs @@ -22,7 +22,8 @@ internal sealed record ManifestEntry( string RawInput, string Notes, bool OutOfScope, - ManifestTransform Transform) + ManifestTransform Transform, + bool IncludeElements = false) { /// Human-readable corpus entry name, derived from the slug. public string Name => @@ -53,6 +54,9 @@ internal static class CorpusManifest private static ManifestEntry E(string slug, string input, string notes) => new(slug, input, notes, false, ManifestTransform.None); + private static ManifestEntry P(string slug, string input, string notes) => + new(slug, input, notes, false, ManifestTransform.None, IncludeElements: true); + private static ManifestEntry Oos(string slug, string input, string notes) => new(slug, input, notes, true, ManifestTransform.None); @@ -580,5 +584,19 @@ private static string NestIex(string inner, int depth) "A quoted module-qualified inner cmdlet safe-fails under the call operator."), Oos("iex_quoted_module_location", "Set-Location C:\\safe; iex \"& 'Microsoft.PowerShell.Management\\Set-Location' C:\\evil\"; Remove-Item child.txt", "A quoted module-qualified location mutation cannot preserve stale cwd."), + + // ---- Issue #62: authored option placement and heuristic boundaries ---- + P("git_global_option_provenance", "git -C C:\\repo commit", + "Issue #62: global -C and its path occur before the commit element."), + P("git_subcommand_option_provenance", "git commit -C HEAD~1", + "Issue #62: command-scoped -C occurs after the parser-classified commit verb."), + P("git_global_config_provenance", "git -c user.name=Jane commit", + "Issue #62: lowercase -c consumes a non-path configuration value."), + P("git_subcommand_config_provenance", "git commit -c HEAD~1", + "Issue #62: lowercase command-scoped -c preserves its post-commit position."), + P("git_mixed_option_provenance", "git -C C:\\repo commit -C HEAD~1", + "Issue #62: both global and command-scoped -C occurrences remain distinct."), + P("git_heuristic_boundary_provenance", "git --no-pager commit -C HEAD~1", + "Issue #62: authored order survives when a valueless option stops the greedy verb walk."), }; } diff --git a/tools/PwshCorpusTool/Program.cs b/tools/PwshCorpusTool/Program.cs index 72cb866..c6f19f4 100644 --- a/tools/PwshCorpusTool/Program.cs +++ b/tools/PwshCorpusTool/Program.cs @@ -59,7 +59,8 @@ int Generate(string outputDir) { var input = entry.ResolveInput(); var parsed = parser.Parse(input); - var json = CorpusJson.BuildEntry(entry.Name, input, parsed, entry.Notes, entry.OutOfScope); + var json = CorpusJson.BuildEntry( + entry.Name, input, parsed, entry.Notes, entry.OutOfScope, entry.IncludeElements); var fileName = $"{index:D3}_{entry.Slug}.json"; File.WriteAllText(Path.Combine(outputDir, fileName), json); index++; @@ -79,7 +80,8 @@ int Check(string command) var parsed = parser.Parse(command); Console.WriteLine("---- parser expected AST ----"); - Console.WriteLine(CorpusJson.BuildEntry("check", command, parsed, "ad-hoc check", parsed.IsUnparseable)); + Console.WriteLine(CorpusJson.BuildEntry( + "check", command, parsed, "ad-hoc check", parsed.IsUnparseable, includeElements: true)); Console.WriteLine("---- real pwsh oracle ----"); var counts = PwshOracle.CountParseErrors(new[] { command }); From 569b39cd8740c53e5b1c35209dc670c41686cd29 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Tue, 4 Aug 2026 20:37:47 +0000 Subject: [PATCH 2/8] fix: complete case-sensitive native option rules --- IMPLEMENTATION_PLAN.md | 8 +- SPEC.POWERSHELL.md | 3 +- SPEC.md | 10 +- .../preserve-clause-element-order/design.md | 7 ++ .../preserve-clause-element-order/proposal.md | 4 +- .../specs/clause-element-provenance/spec.md | 7 ++ .../preserve-clause-element-order/tasks.md | 2 + .../Internal/Bash/Verbs/BashPerVerbRules.cs | 9 +- .../Internal/Bash/Verbs/BashVerbs.cs | 4 +- ...158_wget_case_distinct_output_options.json | 94 ++++++++++++++ ...case_distinct_data_and_header_options.json | 93 ++++++++++++++ ...226_wget_case_distinct_output_options.json | 116 ++++++++++++++++++ ...case_distinct_data_and_header_options.json | 114 +++++++++++++++++ .../Parsing/BashPerVerbRulesTests.cs | 39 +++++- tools/PwshCorpusTool/CorpusManifest.cs | 8 ++ 15 files changed, 505 insertions(+), 13 deletions(-) create mode 100644 tests/ShellSyntaxTree.Tests/Corpus/bash/158_wget_case_distinct_output_options.json create mode 100644 tests/ShellSyntaxTree.Tests/Corpus/bash/159_curl_case_distinct_data_and_header_options.json create mode 100644 tests/ShellSyntaxTree.Tests/Corpus/powershell/226_wget_case_distinct_output_options.json create mode 100644 tests/ShellSyntaxTree.Tests/Corpus/powershell/227_curl_case_distinct_data_and_header_options.json diff --git a/IMPLEMENTATION_PLAN.md b/IMPLEMENTATION_PLAN.md index 36b5739..da521e5 100644 --- a/IMPLEMENTATION_PLAN.md +++ b/IMPLEMENTATION_PLAN.md @@ -37,7 +37,7 @@ priorities. the 64 KiB cap (§11). - [x] **11. Multi-shell corpus runner + PII audit** — directory-routed by `Corpus//`. -- [x] **12. PowerShell corpus** — 211 entries under `Corpus/powershell/`, +- [x] **12. PowerShell corpus** — 227 entries under `Corpus/powershell/`, every §13 category minimum exceeded. - [x] **13. `pwsh` validation gate + `tools/PwshCorpusTool`** — `PwshOracleTests` enforces the §13 oracle matrix + the `PwshAliases` @@ -64,7 +64,11 @@ priorities. corpus cases cover Git `-c`/`-C`, multiple occurrences, and a valueless option that stops the greedy walk. Existing projection shapes and synthetic cwd attribution remain compatible; native options that differ - only by case receive corrected metadata. + only by case receive corrected metadata. The post-implementation option + audit explicitly covers Wget `-o` / `-O`, curl `-d` / `-D` / `-o` / + `-O`, Git `-c` / `-C`, Docker `-v` / `-V`, and tar `-c` / `-C` / + `-f` / `-F`; paired corpus cases pin Wget log/document output and curl + data/header-output semantics in both shells. - [x] **Issue #64 — path-shaped operands after native verb chains.** Stop the Bash and PowerShell native greedy passes before a token that matches the shared path-shape rules. Preserve that token as a resolved diff --git a/SPEC.POWERSHELL.md b/SPEC.POWERSHELL.md index a5fff33..85b3a81 100644 --- a/SPEC.POWERSHELL.md +++ b/SPEC.POWERSHELL.md @@ -706,7 +706,8 @@ positionals are paths," exactly as `SPEC.md` §7. Native commands reuse the bash per-verb rules table verbatim — `git`, `curl`, `tar`, etc. behave identically to `SPEC.md` §7 (`curl` / `wget`: -the first positional is a URL; the `-o` / `-O` value is a path). This +the first positional is a URL; curl `-o` / `-D` values and Wget `-o` / `-O` +values are paths, while curl `-d` data is not). This includes hyphenated option names and the bash `--flag=value` split: the flag and value surface as separate args, and a curated flag's value receives the same path classification in both parsers. Native `--flag:value` has no diff --git a/SPEC.md b/SPEC.md index 6b58440..6f53488 100644 --- a/SPEC.md +++ b/SPEC.md @@ -852,7 +852,8 @@ verb chain is a path. Per-verb overrides: | `sed` | First positional is **script**; rest are paths. | | `awk` | First positional is **program**; rest are paths. | | `tar` | Action flag determines path roles; default to extracting all non-flag positionals as paths. | -| `curl`, `wget` | First positional is **URL**, not a path. `-o file` flag arg is a path. | +| `curl` | First positional is **URL**, not a path. `-o` / `--output` and `-D` / `--dump-header` values are paths; `-d` / `--data` values are request data, not paths. | +| `wget` | First positional is **URL**, not a path. `-o` / `--output-file` writes a log path; `-O` / `--output-document` writes the downloaded document path. | | `scp`, `rsync`, `sftp` | All positionals are paths (some remote). | | `cd`, `chdir`, `pushd`, `popd` | First non-flag positional is the cwd target (a path). | | Others (in FileVerbs, no override) | All non-flag positionals are paths. | @@ -868,8 +869,8 @@ internal static readonly IReadOnlyDictionary> StringComparer.OrdinalIgnoreCase) { ["git"] = new HashSet(StringComparer.Ordinal) { "-c", "-C", "--git-dir", "--work-tree" }, - ["curl"] = new HashSet(StringComparer.Ordinal) { "-o", "--output", "-d", "--data" }, - ["wget"] = new HashSet(StringComparer.Ordinal) { "-O", "--output-document" }, + ["curl"] = new HashSet(StringComparer.Ordinal) { "-o", "--output", "-d", "--data", "-D", "--dump-header" }, + ["wget"] = new HashSet(StringComparer.Ordinal) { "-o", "--output-file", "-O", "--output-document" }, ["docker"]= new HashSet(StringComparer.Ordinal) { "-v", "--volume", "-f", "--file" }, ["tar"] = new HashSet(StringComparer.Ordinal) { "-f", "--file", "-C", "--directory" }, // Add as corpus surfaces real cases. @@ -888,6 +889,9 @@ internal static readonly IReadOnlyDictionary> > lowercase `-c` values as non-paths. Executable-aware consumers still > reinterpret command-scoped forms such as `git commit -c/-C`, where Git uses > the operand as a revision rather than the generic table's global meaning. +> Every supported case-distinct spelling is listed explicitly: curl `-d` +> consumes non-path request data while `-D` consumes a header-output path; +> Wget `-o` and `-O` both consume paths but write different files. > **Note:** the verb-chain walk consumes flag-with-value pairs > transparently. For `git -C /repo log`, the walk consumes `-C /repo` diff --git a/openspec/changes/preserve-clause-element-order/design.md b/openspec/changes/preserve-clause-element-order/design.md index d120202..90fe4ad 100644 --- a/openspec/changes/preserve-clause-element-order/design.md +++ b/openspec/changes/preserve-clause-element-order/design.md @@ -66,6 +66,13 @@ PowerShell cmdlet parameter tables remain case-insensitive. Git explicitly lists both `-c` (configuration value, not a path) and `-C` (directory path in the generic global-option table). +Ordinal matching requires every supported spelling to be explicit. The shared +native table therefore distinguishes curl `-d` request data from `-D` header +output, and lists both Wget `-o` log output and `-O` document output as +path-valued bindings. An option-binding matrix pins consuming and path +semantics independently so a comparer change cannot silently alter either; +the long aliases are pinned alongside the case-colliding short forms. + This corrects pre-existing metadata drift. It does not make the shared parser Git-semantic: a Git-aware consumer must still reinterpret `git commit -c/-C` operands as revisions using authored order. diff --git a/openspec/changes/preserve-clause-element-order/proposal.md b/openspec/changes/preserve-clause-element-order/proposal.md index d33f6b0..1be9536 100644 --- a/openspec/changes/preserve-clause-element-order/proposal.md +++ b/openspec/changes/preserve-clause-element-order/proposal.md @@ -18,7 +18,9 @@ before a subcommand and similarly spelled options after it. existing consumers. - Apply the same contract to Bash and PowerShell. - Match native option spelling case-sensitively in both shells so distinct - options such as Git `-c` and `-C` do not share path metadata. + options such as Git `-c` and `-C` do not share path metadata; explicitly + model case-colliding curl and Wget bindings rather than relying on comparer + collisions. ## Capabilities diff --git a/openspec/changes/preserve-clause-element-order/specs/clause-element-provenance/spec.md b/openspec/changes/preserve-clause-element-order/specs/clause-element-provenance/spec.md index f4433b2..3f7ffe4 100644 --- a/openspec/changes/preserve-clause-element-order/specs/clause-element-provenance/spec.md +++ b/openspec/changes/preserve-clause-element-order/specs/clause-element-provenance/spec.md @@ -69,6 +69,13 @@ Argument elements SHALL carry the parser's `Kind`, `IsFlag`, `IsPath`, and - **THEN** `/repo` is classified as a path - **THEN** PowerShell cmdlet parameter matching remains case-insensitive +#### Scenario: Case-distinct native bindings are explicit +- **WHEN** either parser parses `wget -o wget.log -O download.bin URL` +- **THEN** both `wget.log` and `download.bin` are classified as paths +- **WHEN** either parser parses `curl -d payload -D headers.txt URL` +- **THEN** `payload` is not classified as a path +- **THEN** `headers.txt` is classified as a path + ### Requirement: Redirects occupy their authored position Each redirect SHALL appear as one redirect element at the source position of its operator and target. Its ordinal among redirect elements SHALL match its diff --git a/openspec/changes/preserve-clause-element-order/tasks.md b/openspec/changes/preserve-clause-element-order/tasks.md index 8e33ac9..c2bed80 100644 --- a/openspec/changes/preserve-clause-element-order/tasks.md +++ b/openspec/changes/preserve-clause-element-order/tasks.md @@ -25,6 +25,8 @@ - [x] 3.5 Add paired Bash and PowerShell corpus entries for lowercase `-c`, uppercase `-C`, mixed occurrences, and an intervening valueless option. - [x] 3.6 Pin native-option case sensitivity and the heuristic-role boundary. +- [x] 3.7 Audit case-colliding native short options; explicitly model Wget + log output and curl header output, with paired cross-shell corpus cases. ## 4. Consumer documentation diff --git a/src/ShellSyntaxTree/Internal/Bash/Verbs/BashPerVerbRules.cs b/src/ShellSyntaxTree/Internal/Bash/Verbs/BashPerVerbRules.cs index 1ac542e..b342368 100644 --- a/src/ShellSyntaxTree/Internal/Bash/Verbs/BashPerVerbRules.cs +++ b/src/ShellSyntaxTree/Internal/Bash/Verbs/BashPerVerbRules.cs @@ -131,13 +131,18 @@ internal static bool IsPositionalPathArg(VerbChain verb, int positionalIndex, st [("git", "--git-dir")] = true, [("git", "--work-tree")] = true, - // curl: -o / --output is a file path; -d / --data is body text. + // curl: output and header-dump values are file paths; data is body text. [("curl", "-o")] = true, [("curl", "--output")] = true, [("curl", "-d")] = false, [("curl", "--data")] = false, + [("curl", "-D")] = true, + [("curl", "--dump-header")] = true, - // wget: -O / --output-document is the saved file path. + // wget: -o writes logs and -O writes the downloaded document; + // both operands name files even though their meanings differ. + [("wget", "-o")] = true, + [("wget", "--output-file")] = true, [("wget", "-O")] = true, [("wget", "--output-document")] = true, diff --git a/src/ShellSyntaxTree/Internal/Bash/Verbs/BashVerbs.cs b/src/ShellSyntaxTree/Internal/Bash/Verbs/BashVerbs.cs index 3556f3e..eb7e256 100644 --- a/src/ShellSyntaxTree/Internal/Bash/Verbs/BashVerbs.cs +++ b/src/ShellSyntaxTree/Internal/Bash/Verbs/BashVerbs.cs @@ -105,11 +105,11 @@ internal static readonly IReadOnlyDictionary> }, ["curl"] = new HashSet(StringComparer.Ordinal) { - "-o", "--output", "-d", "--data", + "-o", "--output", "-d", "--data", "-D", "--dump-header", }, ["wget"] = new HashSet(StringComparer.Ordinal) { - "-O", "--output-document", + "-o", "--output-file", "-O", "--output-document", }, ["docker"] = new HashSet(StringComparer.Ordinal) { diff --git a/tests/ShellSyntaxTree.Tests/Corpus/bash/158_wget_case_distinct_output_options.json b/tests/ShellSyntaxTree.Tests/Corpus/bash/158_wget_case_distinct_output_options.json new file mode 100644 index 0000000..08d2c1b --- /dev/null +++ b/tests/ShellSyntaxTree.Tests/Corpus/bash/158_wget_case_distinct_output_options.json @@ -0,0 +1,94 @@ +{ + "name": "Wget case-distinct output options", + "input": "wget -o wget.log -O download.bin https://example.invalid/file", + "expected": { + "isUnparseable": false, + "clauses": [ + { + "operator": "None", + "verb": ["wget"], + "args": [ + { "raw": "-o", "kind": "Literal", "isPath": false, "isFlag": true }, + { "raw": "wget.log", "kind": "Literal", "isPath": true, "isFlag": false, "resolved": "/work/wget.log" }, + { "raw": "-O", "kind": "Literal", "isPath": false, "isFlag": true }, + { "raw": "download.bin", "kind": "Literal", "isPath": true, "isFlag": false, "resolved": "/work/download.bin" }, + { "raw": "https://example.invalid/file", "kind": "Literal", "isPath": false, "isFlag": false } + ], + "redirects": [], + "elements": [ + { + "raw": "wget", + "value": "wget", + "role": "Verb", + "sourceStart": 0, + "sourceLength": 4, + "precedingVerbElementCount": 0, + "kind": "Literal", + "isFlag": false, + "isPath": false + }, + { + "raw": "-o", + "value": "-o", + "role": "Argument", + "sourceStart": 5, + "sourceLength": 2, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": true, + "isPath": false + }, + { + "raw": "wget.log", + "value": "wget.log", + "role": "Argument", + "sourceStart": 8, + "sourceLength": 8, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": false, + "isPath": true, + "resolved": "/work/wget.log" + }, + { + "raw": "-O", + "value": "-O", + "role": "Argument", + "sourceStart": 17, + "sourceLength": 2, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": true, + "isPath": false + }, + { + "raw": "download.bin", + "value": "download.bin", + "role": "Argument", + "sourceStart": 20, + "sourceLength": 12, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": false, + "isPath": true, + "resolved": "/work/download.bin" + }, + { + "raw": "https://example.invalid/file", + "value": "https://example.invalid/file", + "role": "Argument", + "sourceStart": 33, + "sourceLength": 28, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": false, + "isPath": false + } + ], + "isSubshell": false, + "isCommandStringWrapped": false + } + ] + }, + "notes": "Wget -o writes a log file while -O writes the downloaded document; both operands are paths." +} diff --git a/tests/ShellSyntaxTree.Tests/Corpus/bash/159_curl_case_distinct_data_and_header_options.json b/tests/ShellSyntaxTree.Tests/Corpus/bash/159_curl_case_distinct_data_and_header_options.json new file mode 100644 index 0000000..0ba02e9 --- /dev/null +++ b/tests/ShellSyntaxTree.Tests/Corpus/bash/159_curl_case_distinct_data_and_header_options.json @@ -0,0 +1,93 @@ +{ + "name": "Curl case-distinct data and header options", + "input": "curl -d payload -D headers.txt https://example.invalid/api", + "expected": { + "isUnparseable": false, + "clauses": [ + { + "operator": "None", + "verb": ["curl"], + "args": [ + { "raw": "-d", "kind": "Literal", "isPath": false, "isFlag": true }, + { "raw": "payload", "kind": "Literal", "isPath": false, "isFlag": false }, + { "raw": "-D", "kind": "Literal", "isPath": false, "isFlag": true }, + { "raw": "headers.txt", "kind": "Literal", "isPath": true, "isFlag": false, "resolved": "/work/headers.txt" }, + { "raw": "https://example.invalid/api", "kind": "Literal", "isPath": false, "isFlag": false } + ], + "redirects": [], + "elements": [ + { + "raw": "curl", + "value": "curl", + "role": "Verb", + "sourceStart": 0, + "sourceLength": 4, + "precedingVerbElementCount": 0, + "kind": "Literal", + "isFlag": false, + "isPath": false + }, + { + "raw": "-d", + "value": "-d", + "role": "Argument", + "sourceStart": 5, + "sourceLength": 2, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": true, + "isPath": false + }, + { + "raw": "payload", + "value": "payload", + "role": "Argument", + "sourceStart": 8, + "sourceLength": 7, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": false, + "isPath": false + }, + { + "raw": "-D", + "value": "-D", + "role": "Argument", + "sourceStart": 16, + "sourceLength": 2, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": true, + "isPath": false + }, + { + "raw": "headers.txt", + "value": "headers.txt", + "role": "Argument", + "sourceStart": 19, + "sourceLength": 11, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": false, + "isPath": true, + "resolved": "/work/headers.txt" + }, + { + "raw": "https://example.invalid/api", + "value": "https://example.invalid/api", + "role": "Argument", + "sourceStart": 31, + "sourceLength": 27, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": false, + "isPath": false + } + ], + "isSubshell": false, + "isCommandStringWrapped": false + } + ] + }, + "notes": "Curl -d consumes non-path request data while -D consumes a header-output path." +} diff --git a/tests/ShellSyntaxTree.Tests/Corpus/powershell/226_wget_case_distinct_output_options.json b/tests/ShellSyntaxTree.Tests/Corpus/powershell/226_wget_case_distinct_output_options.json new file mode 100644 index 0000000..62e1c18 --- /dev/null +++ b/tests/ShellSyntaxTree.Tests/Corpus/powershell/226_wget_case_distinct_output_options.json @@ -0,0 +1,116 @@ +{ + "name": "Wget case distinct output options", + "input": "wget -o wget.log -O download.bin https://example.invalid/file", + "expected": { + "isUnparseable": false, + "clauses": [ + { + "operator": "None", + "verb": [ + "wget" + ], + "args": [ + { + "raw": "-o", + "kind": "Literal", + "isPath": false + }, + { + "raw": "wget.log", + "kind": "Literal", + "isPath": true, + "resolved": "C:/work/wget.log" + }, + { + "raw": "-O", + "kind": "Literal", + "isPath": false + }, + { + "raw": "download.bin", + "kind": "Literal", + "isPath": true, + "resolved": "C:/work/download.bin" + }, + { + "raw": "https://example.invalid/file", + "kind": "Literal", + "isPath": false + } + ], + "redirects": [], + "elements": [ + { + "raw": "wget", + "value": "wget", + "role": "Verb", + "sourceStart": 0, + "sourceLength": 4, + "precedingVerbElementCount": 0, + "kind": "Literal", + "isFlag": false, + "isPath": false + }, + { + "raw": "-o", + "value": "-o", + "role": "Argument", + "sourceStart": 5, + "sourceLength": 2, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": true, + "isPath": false + }, + { + "raw": "wget.log", + "value": "wget.log", + "role": "Argument", + "sourceStart": 8, + "sourceLength": 8, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": false, + "isPath": true, + "resolved": "C:/work/wget.log" + }, + { + "raw": "-O", + "value": "-O", + "role": "Argument", + "sourceStart": 17, + "sourceLength": 2, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": true, + "isPath": false + }, + { + "raw": "download.bin", + "value": "download.bin", + "role": "Argument", + "sourceStart": 20, + "sourceLength": 12, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": false, + "isPath": true, + "resolved": "C:/work/download.bin" + }, + { + "raw": "https://example.invalid/file", + "value": "https://example.invalid/file", + "role": "Argument", + "sourceStart": 33, + "sourceLength": 28, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": false, + "isPath": false + } + ] + } + ] + }, + "notes": "Wget -o writes a log file while -O writes the downloaded document; both operands are paths." +} diff --git a/tests/ShellSyntaxTree.Tests/Corpus/powershell/227_curl_case_distinct_data_and_header_options.json b/tests/ShellSyntaxTree.Tests/Corpus/powershell/227_curl_case_distinct_data_and_header_options.json new file mode 100644 index 0000000..1706051 --- /dev/null +++ b/tests/ShellSyntaxTree.Tests/Corpus/powershell/227_curl_case_distinct_data_and_header_options.json @@ -0,0 +1,114 @@ +{ + "name": "Curl case distinct data and header options", + "input": "curl -d payload -D headers.txt https://example.invalid/api", + "expected": { + "isUnparseable": false, + "clauses": [ + { + "operator": "None", + "verb": [ + "curl" + ], + "args": [ + { + "raw": "-d", + "kind": "Literal", + "isPath": false + }, + { + "raw": "payload", + "kind": "Literal", + "isPath": false + }, + { + "raw": "-D", + "kind": "Literal", + "isPath": false + }, + { + "raw": "headers.txt", + "kind": "Literal", + "isPath": true, + "resolved": "C:/work/headers.txt" + }, + { + "raw": "https://example.invalid/api", + "kind": "Literal", + "isPath": false + } + ], + "redirects": [], + "elements": [ + { + "raw": "curl", + "value": "curl", + "role": "Verb", + "sourceStart": 0, + "sourceLength": 4, + "precedingVerbElementCount": 0, + "kind": "Literal", + "isFlag": false, + "isPath": false + }, + { + "raw": "-d", + "value": "-d", + "role": "Argument", + "sourceStart": 5, + "sourceLength": 2, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": true, + "isPath": false + }, + { + "raw": "payload", + "value": "payload", + "role": "Argument", + "sourceStart": 8, + "sourceLength": 7, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": false, + "isPath": false + }, + { + "raw": "-D", + "value": "-D", + "role": "Argument", + "sourceStart": 16, + "sourceLength": 2, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": true, + "isPath": false + }, + { + "raw": "headers.txt", + "value": "headers.txt", + "role": "Argument", + "sourceStart": 19, + "sourceLength": 11, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": false, + "isPath": true, + "resolved": "C:/work/headers.txt" + }, + { + "raw": "https://example.invalid/api", + "value": "https://example.invalid/api", + "role": "Argument", + "sourceStart": 31, + "sourceLength": 27, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": false, + "isPath": false + } + ] + } + ] + }, + "notes": "Curl -d consumes non-path request data while -D consumes a header-output path." +} diff --git a/tests/ShellSyntaxTree.Tests/Parsing/BashPerVerbRulesTests.cs b/tests/ShellSyntaxTree.Tests/Parsing/BashPerVerbRulesTests.cs index 9d4fa0b..9ddbf76 100644 --- a/tests/ShellSyntaxTree.Tests/Parsing/BashPerVerbRulesTests.cs +++ b/tests/ShellSyntaxTree.Tests/Parsing/BashPerVerbRulesTests.cs @@ -242,17 +242,21 @@ public void Git_dash_C_value_is_a_path() } [Fact] - public void Curl_dash_o_value_is_a_path_dash_d_is_not() + public void Curl_output_and_dump_header_values_are_paths_data_is_not() { Assert.True(BashPerVerbRules.ValueOfFlagIsPath("curl", "-o")); Assert.True(BashPerVerbRules.ValueOfFlagIsPath("curl", "--output")); Assert.False(BashPerVerbRules.ValueOfFlagIsPath("curl", "-d")); Assert.False(BashPerVerbRules.ValueOfFlagIsPath("curl", "--data")); + Assert.True(BashPerVerbRules.ValueOfFlagIsPath("curl", "-D")); + Assert.True(BashPerVerbRules.ValueOfFlagIsPath("curl", "--dump-header")); } [Fact] - public void Wget_dash_O_value_is_a_path() + public void Wget_log_and_document_output_values_are_paths() { + Assert.True(BashPerVerbRules.ValueOfFlagIsPath("wget", "-o")); + Assert.True(BashPerVerbRules.ValueOfFlagIsPath("wget", "--output-file")); Assert.True(BashPerVerbRules.ValueOfFlagIsPath("wget", "-O")); Assert.True(BashPerVerbRules.ValueOfFlagIsPath("wget", "--output-document")); } @@ -291,6 +295,37 @@ public void Native_flag_lookup_is_case_sensitive_while_verb_lookup_is_not() Assert.False(BashPerVerbRules.ValueOfFlagIsPath("Git", "--Git-Dir")); } + [Theory] + [InlineData("git", "-c", true, false)] + [InlineData("git", "-C", true, true)] + [InlineData("curl", "-d", true, false)] + [InlineData("curl", "-D", true, true)] + [InlineData("curl", "-o", true, true)] + [InlineData("curl", "-O", false, false)] + [InlineData("curl", "--data", true, false)] + [InlineData("curl", "--dump-header", true, true)] + [InlineData("curl", "--output", true, true)] + [InlineData("wget", "-o", true, true)] + [InlineData("wget", "-O", true, true)] + [InlineData("wget", "--output-file", true, true)] + [InlineData("wget", "--output-document", true, true)] + [InlineData("docker", "-v", true, false)] + [InlineData("docker", "-V", false, false)] + [InlineData("tar", "-c", false, false)] + [InlineData("tar", "-C", true, true)] + [InlineData("tar", "-f", true, true)] + [InlineData("tar", "-F", false, false)] + public void Native_option_binding_matrix( + string verb, + string flag, + bool consumesValue, + bool valueIsPath) + { + Assert.True(BashVerbs.FlagsWithValue.TryGetValue(verb, out var flags)); + Assert.Equal(consumesValue, flags.Contains(flag)); + Assert.Equal(valueIsPath, BashPerVerbRules.ValueOfFlagIsPath(verb, flag)); + } + // ---------------------------------------------------------------- empty verb chain [Fact] diff --git a/tools/PwshCorpusTool/CorpusManifest.cs b/tools/PwshCorpusTool/CorpusManifest.cs index 849d9bb..62e9a80 100644 --- a/tools/PwshCorpusTool/CorpusManifest.cs +++ b/tools/PwshCorpusTool/CorpusManifest.cs @@ -598,5 +598,13 @@ private static string NestIex(string inner, int depth) "Issue #62: both global and command-scoped -C occurrences remain distinct."), P("git_heuristic_boundary_provenance", "git --no-pager commit -C HEAD~1", "Issue #62: authored order survives when a valueless option stops the greedy verb walk."), + + // ---- Issue #62 adversarial follow-up: native option case collisions ---- + P("wget_case_distinct_output_options", + "wget -o wget.log -O download.bin https://example.invalid/file", + "Wget -o writes a log file while -O writes the downloaded document; both operands are paths."), + P("curl_case_distinct_data_and_header_options", + "curl -d payload -D headers.txt https://example.invalid/api", + "Curl -d consumes non-path request data while -D consumes a header-output path."), }; } From a580fabff9d0394050671bb6ac2dbe033df11bf6 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Tue, 4 Aug 2026 21:22:59 +0000 Subject: [PATCH 3/8] fix: harden native flag value classification --- IMPLEMENTATION_PLAN.md | 11 +- SPEC.POWERSHELL.md | 10 +- SPEC.md | 23 +++- .../preserve-clause-element-order/design.md | 14 +- .../preserve-clause-element-order/proposal.md | 4 +- .../specs/clause-element-provenance/spec.md | 12 ++ .../preserve-clause-element-order/tasks.md | 2 + .../Bash/Parsing/BashCommandParser.cs | 35 +++-- .../Internal/Bash/Verbs/BashPerVerbRules.cs | 51 ++++++- .../Internal/Bash/Verbs/BashVerbs.cs | 3 +- .../Pwsh/Parsing/PwshCommandParser.cs | 26 +++- .../bash/160_tar_info_script_paths.json | 98 +++++++++++++ .../bash/161_curl_data_file_references.json | 83 +++++++++++ .../bash/162_curl_data_stdin_reference.json | 68 +++++++++ .../powershell/228_tar_info_script_paths.json | 130 ++++++++++++++++++ .../229_curl_data_file_references.json | 105 ++++++++++++++ .../230_curl_data_stdin_reference.json | 80 +++++++++++ .../Parsing/BashPerVerbRulesTests.cs | 26 +++- .../Parsing/ClauseElementTests.cs | 25 ++++ tools/PwshCorpusTool/CorpusManifest.cs | 12 ++ 20 files changed, 779 insertions(+), 39 deletions(-) create mode 100644 tests/ShellSyntaxTree.Tests/Corpus/bash/160_tar_info_script_paths.json create mode 100644 tests/ShellSyntaxTree.Tests/Corpus/bash/161_curl_data_file_references.json create mode 100644 tests/ShellSyntaxTree.Tests/Corpus/bash/162_curl_data_stdin_reference.json create mode 100644 tests/ShellSyntaxTree.Tests/Corpus/powershell/228_tar_info_script_paths.json create mode 100644 tests/ShellSyntaxTree.Tests/Corpus/powershell/229_curl_data_file_references.json create mode 100644 tests/ShellSyntaxTree.Tests/Corpus/powershell/230_curl_data_stdin_reference.json diff --git a/IMPLEMENTATION_PLAN.md b/IMPLEMENTATION_PLAN.md index da521e5..ede5557 100644 --- a/IMPLEMENTATION_PLAN.md +++ b/IMPLEMENTATION_PLAN.md @@ -37,7 +37,7 @@ priorities. the 64 KiB cap (§11). - [x] **11. Multi-shell corpus runner + PII audit** — directory-routed by `Corpus//`. -- [x] **12. PowerShell corpus** — 227 entries under `Corpus/powershell/`, +- [x] **12. PowerShell corpus** — 230 entries under `Corpus/powershell/`, every §13 category minimum exceeded. - [x] **13. `pwsh` validation gate + `tools/PwshCorpusTool`** — `PwshOracleTests` enforces the §13 oracle matrix + the `PwshAliases` @@ -66,9 +66,12 @@ priorities. synthetic cwd attribution remain compatible; native options that differ only by case receive corrected metadata. The post-implementation option audit explicitly covers Wget `-o` / `-O`, curl `-d` / `-D` / `-o` / - `-O`, Git `-c` / `-C`, Docker `-v` / `-V`, and tar `-c` / `-C` / - `-f` / `-F`; paired corpus cases pin Wget log/document output and curl - data/header-output semantics in both shells. + `-O`, Git `-c` / `-C`, and tar `-c` / `-C` / `-f` / `-F`; paired corpus + cases pin Wget log/document output, curl data/header-output and `@file` + semantics, and tar helper-script paths in both shells. Docker `-v` + remains explicitly context-sensitive: the generic table supports + `docker run`, while consumers use authored elements for global-option + interpretation. - [x] **Issue #64 — path-shaped operands after native verb chains.** Stop the Bash and PowerShell native greedy passes before a token that matches the shared path-shape rules. Preserve that token as a resolved diff --git a/SPEC.POWERSHELL.md b/SPEC.POWERSHELL.md index 85b3a81..c8e8ab7 100644 --- a/SPEC.POWERSHELL.md +++ b/SPEC.POWERSHELL.md @@ -707,11 +707,17 @@ positionals are paths," exactly as `SPEC.md` §7. Native commands reuse the bash per-verb rules table verbatim — `git`, `curl`, `tar`, etc. behave identically to `SPEC.md` §7 (`curl` / `wget`: the first positional is a URL; curl `-o` / `-D` values and Wget `-o` / `-O` -values are paths, while curl `-d` data is not). This +values are paths, while curl `-d` data is non-path unless `@file` requests a +file read; `@-` denotes stdin). Tar `-F` / `--info-script` helper values are +paths. This includes hyphenated option names and the bash `--flag=value` split: the flag and value surface as separate args, and a curated flag's value receives the same path classification in both parsers. Native `--flag:value` has no -cmdlet-binding semantics and remains verbatim. +cmdlet-binding semantics and remains verbatim. PowerShell still owns outer +tokenization: spaced curl operands beginning with `@` should be quoted because +`@name` is splatting and bare `@-` is a parse error. Use forms such as +`-d "@request.json"` / `-d "@-"`, or bind a file inline as +`--data=@request.json`, so the native command receives one value. --- diff --git a/SPEC.md b/SPEC.md index 6f53488..3703f22 100644 --- a/SPEC.md +++ b/SPEC.md @@ -851,8 +851,8 @@ verb chain is a path. Per-verb overrides: | `rg` | First positional is **pattern**; rest are paths. | | `sed` | First positional is **script**; rest are paths. | | `awk` | First positional is **program**; rest are paths. | -| `tar` | Action flag determines path roles; default to extracting all non-flag positionals as paths. | -| `curl` | First positional is **URL**, not a path. `-o` / `--output` and `-D` / `--dump-header` values are paths; `-d` / `--data` values are request data, not paths. | +| `tar` | Action flag determines path roles; default to extracting all non-flag positionals as paths. `-F` / `--info-script` / `--new-volume-script` values are helper-script paths. | +| `curl` | First positional is **URL**, not a path. `-o` / `--output` and `-D` / `--dump-header` values are paths. `-d` / `--data` values are request data unless prefixed with `@`, which reads a file; `@-` reads stdin and is not a path. | | `wget` | First positional is **URL**, not a path. `-o` / `--output-file` writes a log path; `-O` / `--output-document` writes the downloaded document path. | | `scp`, `rsync`, `sftp` | All positionals are paths (some remote). | | `cd`, `chdir`, `pushd`, `popd` | First non-flag positional is the cwd target (a path). | @@ -872,7 +872,7 @@ internal static readonly IReadOnlyDictionary> ["curl"] = new HashSet(StringComparer.Ordinal) { "-o", "--output", "-d", "--data", "-D", "--dump-header" }, ["wget"] = new HashSet(StringComparer.Ordinal) { "-o", "--output-file", "-O", "--output-document" }, ["docker"]= new HashSet(StringComparer.Ordinal) { "-v", "--volume", "-f", "--file" }, - ["tar"] = new HashSet(StringComparer.Ordinal) { "-f", "--file", "-C", "--directory" }, + ["tar"] = new HashSet(StringComparer.Ordinal) { "-f", "--file", "-C", "--directory", "-F", "--info-script", "--new-volume-script" }, // Add as corpus surfaces real cases. }; ``` @@ -890,9 +890,24 @@ internal static readonly IReadOnlyDictionary> > reinterpret command-scoped forms such as `git commit -c/-C`, where Git uses > the operand as a revision rather than the generic table's global meaning. > Every supported case-distinct spelling is listed explicitly: curl `-d` -> consumes non-path request data while `-D` consumes a header-output path; +> consumes request data while `-D` consumes a header-output path; > Wget `-o` and `-O` both consume paths but write different files. +> **Operand-sensitive values.** A fixed `(verb, flag)` boolean is insufficient +> for curl `-d` / `--data`: a value beginning with `@` names a file curl reads. +> The parser preserves the authored marker in the value `Arg.Raw` and in the +> complete `ClauseElement.Value`, strips the leading `@` only for path +> resolution, and leaves `@-` non-path because it denotes stdin. Dynamic and +> glob filenames continue through the normal §8 safe-fail rules after the +> prefix is removed. + +> **Executable context.** `FlagsWithValue` is a curated parser heuristic, not +> a complete executable grammar. In particular, Docker's global `-v` means +> `--version`, while `docker run -v` consumes a volume specification. The +> generic table preserves the established `docker run` projection; a +> Docker-aware consumer uses `Clause.Elements` to interpret placement and MUST +> NOT treat the table as universal Docker semantics. + > **Note:** the verb-chain walk consumes flag-with-value pairs > transparently. For `git -C /repo log`, the walk consumes `-C /repo` > before evaluating the next token; `log` is then verb-like and extends diff --git a/openspec/changes/preserve-clause-element-order/design.md b/openspec/changes/preserve-clause-element-order/design.md index 90fe4ad..396f9c6 100644 --- a/openspec/changes/preserve-clause-element-order/design.md +++ b/openspec/changes/preserve-clause-element-order/design.md @@ -69,9 +69,17 @@ the generic global-option table). Ordinal matching requires every supported spelling to be explicit. The shared native table therefore distinguishes curl `-d` request data from `-D` header output, and lists both Wget `-o` log output and `-O` document output as -path-valued bindings. An option-binding matrix pins consuming and path -semantics independently so a comparer change cannot silently alter either; -the long aliases are pinned alongside the case-colliding short forms. +path-valued bindings. It also lists tar `-F` independently from `-f`. An +option-binding matrix pins consuming and path semantics independently so a +comparer change cannot silently alter either; the long aliases are pinned +alongside the case-colliding short forms. + +Some native meaning is operand- or command-context-sensitive rather than +spelling-sensitive. Curl `-d` / `--data` uses a leading `@` to read a file, so +the parser preserves the authored value while stripping `@` only for path +resolution; `@-` remains stdin. Docker's global `-v` and `docker run -v` have +different meanings, so the collision matrix does not claim a universal Docker +binding. Executable-aware consumers interpret that placement from `Elements`. This corrects pre-existing metadata drift. It does not make the shared parser Git-semantic: a Git-aware consumer must still reinterpret `git commit -c/-C` diff --git a/openspec/changes/preserve-clause-element-order/proposal.md b/openspec/changes/preserve-clause-element-order/proposal.md index 1be9536..f85cd69 100644 --- a/openspec/changes/preserve-clause-element-order/proposal.md +++ b/openspec/changes/preserve-clause-element-order/proposal.md @@ -19,8 +19,8 @@ before a subcommand and similarly spelled options after it. - Apply the same contract to Bash and PowerShell. - Match native option spelling case-sensitively in both shells so distinct options such as Git `-c` and `-C` do not share path metadata; explicitly - model case-colliding curl and Wget bindings rather than relying on comparer - collisions. + model case-colliding curl, Wget, and tar bindings rather than relying on + comparer collisions. ## Capabilities diff --git a/openspec/changes/preserve-clause-element-order/specs/clause-element-provenance/spec.md b/openspec/changes/preserve-clause-element-order/specs/clause-element-provenance/spec.md index 3f7ffe4..95c0a6c 100644 --- a/openspec/changes/preserve-clause-element-order/specs/clause-element-provenance/spec.md +++ b/openspec/changes/preserve-clause-element-order/specs/clause-element-provenance/spec.md @@ -76,6 +76,18 @@ Argument elements SHALL carry the parser's `Kind`, `IsFlag`, `IsPath`, and - **THEN** `payload` is not classified as a path - **THEN** `headers.txt` is classified as a path +#### Scenario: Native values may carry path syntax +- **WHEN** either parser parses `curl -d "@request.json" --data=@payload.bin URL` +- **THEN** the spaced element value is `@request.json` +- **THEN** the inline element value remains `--data=@payload.bin` +- **THEN** both value args retain their authored `@` and resolve as paths without it +- **WHEN** either parser parses `curl -d "@-" URL` +- **THEN** `@-` is not classified as a path + +#### Scenario: Tar helper-script bindings are explicit +- **WHEN** either parser parses `tar -F=./helper.sh --info-script=./info.sh --new-volume-script ./next.sh archive.tar` +- **THEN** all three helper-script values are classified as paths + ### Requirement: Redirects occupy their authored position Each redirect SHALL appear as one redirect element at the source position of its operator and target. Its ordinal among redirect elements SHALL match its diff --git a/openspec/changes/preserve-clause-element-order/tasks.md b/openspec/changes/preserve-clause-element-order/tasks.md index c2bed80..70f64b5 100644 --- a/openspec/changes/preserve-clause-element-order/tasks.md +++ b/openspec/changes/preserve-clause-element-order/tasks.md @@ -27,6 +27,8 @@ - [x] 3.6 Pin native-option case sensitivity and the heuristic-role boundary. - [x] 3.7 Audit case-colliding native short options; explicitly model Wget log output and curl header output, with paired cross-shell corpus cases. +- [x] 3.8 Harden the audit with tar helper-script bindings, curl `@file` / + `@-` operand semantics, and scoped Docker context claims. ## 4. Consumer documentation diff --git a/src/ShellSyntaxTree/Internal/Bash/Parsing/BashCommandParser.cs b/src/ShellSyntaxTree/Internal/Bash/Parsing/BashCommandParser.cs index 13f2faf..1522fcc 100644 --- a/src/ShellSyntaxTree/Internal/Bash/Parsing/BashCommandParser.cs +++ b/src/ShellSyntaxTree/Internal/Bash/Parsing/BashCommandParser.cs @@ -1163,9 +1163,15 @@ private static void ExtractRedirectsAndArgs( // verb owns the flag, otherwise fall back to plain // literal (the equals-form is its own visible split, // so we don't apply LooksLikePath here). + var inlineValueForResolution = valuePart; var valueIsPath = verbKeyForFlagValuePaths is not null - && BashPerVerbRules.ValueOfFlagIsPath(verbKeyForFlagValuePaths, flagPart); - var (vKind, vResolved, vIsPath) = BashResolver.Resolve(valuePart, valueIsPath, options, workingDirectoryUnknown); + && BashPerVerbRules.TryGetFlagValuePath( + verbKeyForFlagValuePaths, + flagPart, + valuePart, + out inlineValueForResolution); + var (vKind, vResolved, vIsPath) = BashResolver.Resolve( + inlineValueForResolution, valueIsPath, options, workingDirectoryUnknown); argList.Add(new Arg { Raw = valuePart, @@ -1230,13 +1236,17 @@ private static void ExtractRedirectsAndArgs( } // Non-flag positional. Classify path / resolve. + var valueForResolution = t.Value; bool treatAsPath; if (pendingFlagForValue is not null && verbKeyForFlagValuePaths is not null) { // This is the value of a preceding flag — use the // flag-value rule, NOT the positional-index rule. - treatAsPath = BashPerVerbRules.ValueOfFlagIsPath( - verbKeyForFlagValuePaths, pendingFlagForValue); + treatAsPath = BashPerVerbRules.TryGetFlagValuePath( + verbKeyForFlagValuePaths, + pendingFlagForValue, + t.Value, + out valueForResolution); pendingFlagForValue = null; } else @@ -1245,7 +1255,8 @@ private static void ExtractRedirectsAndArgs( positionalIndex++; } - var (kind, resolved, isPath) = BashResolver.Resolve(t.Value, treatAsPath, options, workingDirectoryUnknown); + var (kind, resolved, isPath) = BashResolver.Resolve( + valueForResolution, treatAsPath, options, workingDirectoryUnknown); argList.Add(new Arg { Raw = sourceRaw, @@ -1274,11 +1285,15 @@ private static void ExtractRedirectsAndArgs( // a quoted string is the user's signal "literal"). They // still classify as positional path / non-path through // the per-verb rule + resolver. + var valueForResolution = t.Value; bool treatAsPath; if (pendingFlagForValue is not null && verbKeyForFlagValuePaths is not null) { - treatAsPath = BashPerVerbRules.ValueOfFlagIsPath( - verbKeyForFlagValuePaths, pendingFlagForValue); + treatAsPath = BashPerVerbRules.TryGetFlagValuePath( + verbKeyForFlagValuePaths, + pendingFlagForValue, + t.Value, + out valueForResolution); pendingFlagForValue = null; } else @@ -1291,7 +1306,11 @@ private static void ExtractRedirectsAndArgs( // — bypass tilde / $HOME / $VAR / glob handling so // `'$HOME'` doesn't expand. var (kind, resolved, isPath) = BashResolver.Resolve( - t.Value, treatAsPath, options, workingDirectoryUnknown, t.IsSingleQuoted); + valueForResolution, + treatAsPath, + options, + workingDirectoryUnknown, + t.IsSingleQuoted); argList.Add(new Arg { Raw = sourceRaw, diff --git a/src/ShellSyntaxTree/Internal/Bash/Verbs/BashPerVerbRules.cs b/src/ShellSyntaxTree/Internal/Bash/Verbs/BashPerVerbRules.cs index b342368..482d8b4 100644 --- a/src/ShellSyntaxTree/Internal/Bash/Verbs/BashPerVerbRules.cs +++ b/src/ShellSyntaxTree/Internal/Bash/Verbs/BashPerVerbRules.cs @@ -59,9 +59,9 @@ internal static class BashPerVerbRules ["sed"] = i => i >= 1, ["awk"] = i => i >= 1, - // curl / wget: first positional is a URL (not a path). The - // file-path comes from a flag-with-value (-o / -O), handled - // separately by ValueOfFlagIsPath. + // curl / wget: first positional is a URL (not a path). File + // operands come from curated flag values, including curl's + // operand-sensitive @file data form. ["curl"] = _ => false, ["wget"] = _ => false, @@ -153,17 +153,21 @@ internal static bool IsPositionalPathArg(VerbChain verb, int positionalIndex, st [("docker", "-v")] = false, [("docker", "--volume")] = false, - // tar: -f / --file is the archive path; -C / --directory is a - // directory path. + // tar: archive, directory, and multi-volume helper values are paths. [("tar", "-f")] = true, [("tar", "--file")] = true, [("tar", "-C")] = true, [("tar", "--directory")] = true, + [("tar", "-F")] = true, + [("tar", "--info-script")] = true, + [("tar", "--new-volume-script")] = true, }; /// - /// Whether the value following for - /// should be classified as a path. + /// Whether every value following for + /// should be classified as a path. Concrete + /// operand-sensitive classification goes through + /// . /// internal static bool ValueOfFlagIsPath(string verb, string flag) { @@ -175,6 +179,39 @@ internal static bool ValueOfFlagIsPath(string verb, string flag) return FlagValueIsPath.TryGetValue((verb, flag), out var isPath) && isPath; } + /// + /// Classifies a concrete native flag value and returns the logical value + /// the resolver should treat as the path. Most flags use the fixed table; + /// curl data flags additionally interpret a leading @ as a file + /// read, except @- which denotes stdin. + /// + internal static bool TryGetFlagValuePath( + string verb, + string flag, + string value, + out string pathValue) + { + pathValue = value; + + if (ValueOfFlagIsPath(verb, flag)) + { + return true; + } + + if (string.Equals(verb, "curl", StringComparison.OrdinalIgnoreCase) + && (string.Equals(flag, "-d", StringComparison.Ordinal) + || string.Equals(flag, "--data", StringComparison.Ordinal)) + && value.Length > 1 + && value[0] == '@' + && !string.Equals(value, "@-", StringComparison.Ordinal)) + { + pathValue = value.Substring(1); + return true; + } + + return false; + } + // ---------------------------------------------------------------- key comparer /// diff --git a/src/ShellSyntaxTree/Internal/Bash/Verbs/BashVerbs.cs b/src/ShellSyntaxTree/Internal/Bash/Verbs/BashVerbs.cs index eb7e256..88f1d74 100644 --- a/src/ShellSyntaxTree/Internal/Bash/Verbs/BashVerbs.cs +++ b/src/ShellSyntaxTree/Internal/Bash/Verbs/BashVerbs.cs @@ -117,7 +117,8 @@ internal static readonly IReadOnlyDictionary> }, ["tar"] = new HashSet(StringComparer.Ordinal) { - "-f", "--file", "-C", "--directory", + "-f", "--file", "-C", "--directory", "-F", + "--info-script", "--new-volume-script", }, }; diff --git a/src/ShellSyntaxTree/Internal/Pwsh/Parsing/PwshCommandParser.cs b/src/ShellSyntaxTree/Internal/Pwsh/Parsing/PwshCommandParser.cs index ebb0ab7..bd3469d 100644 --- a/src/ShellSyntaxTree/Internal/Pwsh/Parsing/PwshCommandParser.cs +++ b/src/ShellSyntaxTree/Internal/Pwsh/Parsing/PwshCommandParser.cs @@ -1017,9 +1017,15 @@ private static ArgResult ExtractArgsAndRedirects( if (NativeFlagSyntax.TrySplitEqualsFlag(raw, out var flagPart, out var valuePart)) { args.Add(new Arg { Raw = flagPart, Kind = ArgKind.Literal, IsPath = false }); - var valueIsPath = BashPerVerbRules.ValueOfFlagIsPath(verbKey, flagPart); - args.Add(ResolveValue( - valuePart, valueIsPath, options, workingDirectoryUnknown, false)); + var valueIsPath = BashPerVerbRules.TryGetFlagValuePath( + verbKey, flagPart, valuePart, out var inlineValueForResolution); + args.Add(ResolveValueToken( + valuePart, + inlineValueForResolution, + valueIsPath, + options, + workingDirectoryUnknown, + false)); } else { @@ -1088,6 +1094,7 @@ private static ArgResult ExtractArgsAndRedirects( var isLiteralBytes = t.Kind == PwshTokenKind.QuotedString && t.IsSingleQuoted; var rawValue = SourceSlice(source, t); + var valueForResolution = t.Value; bool treatAsPath; if (pendingValueParam is not null) { @@ -1097,7 +1104,11 @@ private static ArgResult ExtractArgsAndRedirects( else if (pendingNativeFlag is not null) { var verbKey = verb.VerbTokens.Count > 0 ? verb.VerbTokens[0] : string.Empty; - treatAsPath = BashPerVerbRules.ValueOfFlagIsPath(verbKey, pendingNativeFlag); + treatAsPath = BashPerVerbRules.TryGetFlagValuePath( + verbKey, + pendingNativeFlag, + t.Value, + out valueForResolution); pendingNativeFlag = null; } else @@ -1109,7 +1120,12 @@ private static ArgResult ExtractArgsAndRedirects( } var resolvedArg = ResolveValueToken( - rawValue, t.Value, treatAsPath, options, workingDirectoryUnknown, isLiteralBytes); + rawValue, + valueForResolution, + treatAsPath, + options, + workingDirectoryUnknown, + isLiteralBytes); args.Add(resolvedArg); elements.Add(CreateElement( source, diff --git a/tests/ShellSyntaxTree.Tests/Corpus/bash/160_tar_info_script_paths.json b/tests/ShellSyntaxTree.Tests/Corpus/bash/160_tar_info_script_paths.json new file mode 100644 index 0000000..b448fec --- /dev/null +++ b/tests/ShellSyntaxTree.Tests/Corpus/bash/160_tar_info_script_paths.json @@ -0,0 +1,98 @@ +{ + "name": "Tar info-script paths", + "input": "tar -F=./volume-helper.sh --info-script=./info-helper.sh --new-volume-script ./next-volume.sh archive.tar", + "expected": { + "isUnparseable": false, + "clauses": [ + { + "operator": "None", + "verb": ["tar"], + "args": [ + { "raw": "-F", "kind": "Literal", "isPath": false, "isFlag": true }, + { "raw": "./volume-helper.sh", "kind": "Literal", "isPath": true, "isFlag": false, "resolved": "/work/volume-helper.sh" }, + { "raw": "--info-script", "kind": "Literal", "isPath": false, "isFlag": true }, + { "raw": "./info-helper.sh", "kind": "Literal", "isPath": true, "isFlag": false, "resolved": "/work/info-helper.sh" }, + { "raw": "--new-volume-script", "kind": "Literal", "isPath": false, "isFlag": true }, + { "raw": "./next-volume.sh", "kind": "Literal", "isPath": true, "isFlag": false, "resolved": "/work/next-volume.sh" }, + { "raw": "archive.tar", "kind": "Literal", "isPath": true, "isFlag": false, "resolved": "/work/archive.tar" } + ], + "redirects": [], + "elements": [ + { + "raw": "tar", + "value": "tar", + "role": "Verb", + "sourceStart": 0, + "sourceLength": 3, + "precedingVerbElementCount": 0, + "kind": "Literal", + "isFlag": false, + "isPath": false + }, + { + "raw": "-F=./volume-helper.sh", + "value": "-F=./volume-helper.sh", + "role": "Argument", + "sourceStart": 4, + "sourceLength": 21, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": true, + "isPath": true, + "resolved": "/work/volume-helper.sh" + }, + { + "raw": "--info-script=./info-helper.sh", + "value": "--info-script=./info-helper.sh", + "role": "Argument", + "sourceStart": 26, + "sourceLength": 30, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": true, + "isPath": true, + "resolved": "/work/info-helper.sh" + }, + { + "raw": "--new-volume-script", + "value": "--new-volume-script", + "role": "Argument", + "sourceStart": 57, + "sourceLength": 19, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": true, + "isPath": false + }, + { + "raw": "./next-volume.sh", + "value": "./next-volume.sh", + "role": "Argument", + "sourceStart": 77, + "sourceLength": 16, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": false, + "isPath": true, + "resolved": "/work/next-volume.sh" + }, + { + "raw": "archive.tar", + "value": "archive.tar", + "role": "Argument", + "sourceStart": 94, + "sourceLength": 11, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": false, + "isPath": true, + "resolved": "/work/archive.tar" + } + ], + "isSubshell": false, + "isCommandStringWrapped": false + } + ] + }, + "notes": "Tar -F and both long aliases consume helper-script paths in equals and spaced forms." +} diff --git a/tests/ShellSyntaxTree.Tests/Corpus/bash/161_curl_data_file_references.json b/tests/ShellSyntaxTree.Tests/Corpus/bash/161_curl_data_file_references.json new file mode 100644 index 0000000..e572e90 --- /dev/null +++ b/tests/ShellSyntaxTree.Tests/Corpus/bash/161_curl_data_file_references.json @@ -0,0 +1,83 @@ +{ + "name": "Curl data-file references", + "input": "curl -d \"@request.json\" --data=@payload.bin https://example.invalid/api", + "expected": { + "isUnparseable": false, + "clauses": [ + { + "operator": "None", + "verb": ["curl"], + "args": [ + { "raw": "-d", "kind": "Literal", "isPath": false, "isFlag": true }, + { "raw": "\"@request.json\"", "kind": "Literal", "isPath": true, "isFlag": false, "resolved": "/work/request.json" }, + { "raw": "--data", "kind": "Literal", "isPath": false, "isFlag": true }, + { "raw": "@payload.bin", "kind": "Literal", "isPath": true, "isFlag": false, "resolved": "/work/payload.bin" }, + { "raw": "https://example.invalid/api", "kind": "Literal", "isPath": false, "isFlag": false } + ], + "redirects": [], + "elements": [ + { + "raw": "curl", + "value": "curl", + "role": "Verb", + "sourceStart": 0, + "sourceLength": 4, + "precedingVerbElementCount": 0, + "kind": "Literal", + "isFlag": false, + "isPath": false + }, + { + "raw": "-d", + "value": "-d", + "role": "Argument", + "sourceStart": 5, + "sourceLength": 2, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": true, + "isPath": false + }, + { + "raw": "\"@request.json\"", + "value": "@request.json", + "role": "Argument", + "sourceStart": 8, + "sourceLength": 15, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": false, + "isPath": true, + "resolved": "/work/request.json" + }, + { + "raw": "--data=@payload.bin", + "value": "--data=@payload.bin", + "role": "Argument", + "sourceStart": 24, + "sourceLength": 19, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": true, + "isPath": true, + "resolved": "/work/payload.bin" + }, + { + "raw": "https://example.invalid/api", + "value": "https://example.invalid/api", + "role": "Argument", + "sourceStart": 44, + "sourceLength": 27, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": false, + "isPath": false + } + ], + "isSubshell": false, + "isCommandStringWrapped": false + } + ] + }, + "notes": "Curl data operands prefixed with @ read files; authored values retain the @ marker." +} diff --git a/tests/ShellSyntaxTree.Tests/Corpus/bash/162_curl_data_stdin_reference.json b/tests/ShellSyntaxTree.Tests/Corpus/bash/162_curl_data_stdin_reference.json new file mode 100644 index 0000000..4abc700 --- /dev/null +++ b/tests/ShellSyntaxTree.Tests/Corpus/bash/162_curl_data_stdin_reference.json @@ -0,0 +1,68 @@ +{ + "name": "Curl data stdin reference", + "input": "curl -d \"@-\" https://example.invalid/api", + "expected": { + "isUnparseable": false, + "clauses": [ + { + "operator": "None", + "verb": ["curl"], + "args": [ + { "raw": "-d", "kind": "Literal", "isPath": false, "isFlag": true }, + { "raw": "\"@-\"", "kind": "Literal", "isPath": false, "isFlag": false }, + { "raw": "https://example.invalid/api", "kind": "Literal", "isPath": false, "isFlag": false } + ], + "redirects": [], + "elements": [ + { + "raw": "curl", + "value": "curl", + "role": "Verb", + "sourceStart": 0, + "sourceLength": 4, + "precedingVerbElementCount": 0, + "kind": "Literal", + "isFlag": false, + "isPath": false + }, + { + "raw": "-d", + "value": "-d", + "role": "Argument", + "sourceStart": 5, + "sourceLength": 2, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": true, + "isPath": false + }, + { + "raw": "\"@-\"", + "value": "@-", + "role": "Argument", + "sourceStart": 8, + "sourceLength": 4, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": false, + "isPath": false + }, + { + "raw": "https://example.invalid/api", + "value": "https://example.invalid/api", + "role": "Argument", + "sourceStart": 13, + "sourceLength": 27, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": false, + "isPath": false + } + ], + "isSubshell": false, + "isCommandStringWrapped": false + } + ] + }, + "notes": "Curl @- reads stdin and is not a filesystem path." +} diff --git a/tests/ShellSyntaxTree.Tests/Corpus/powershell/228_tar_info_script_paths.json b/tests/ShellSyntaxTree.Tests/Corpus/powershell/228_tar_info_script_paths.json new file mode 100644 index 0000000..8655ba6 --- /dev/null +++ b/tests/ShellSyntaxTree.Tests/Corpus/powershell/228_tar_info_script_paths.json @@ -0,0 +1,130 @@ +{ + "name": "Tar info script paths", + "input": "tar -F=./volume-helper.sh --info-script=./info-helper.sh --new-volume-script ./next-volume.sh archive.tar", + "expected": { + "isUnparseable": false, + "clauses": [ + { + "operator": "None", + "verb": [ + "tar" + ], + "args": [ + { + "raw": "-F", + "kind": "Literal", + "isPath": false + }, + { + "raw": "./volume-helper.sh", + "kind": "Literal", + "isPath": true, + "resolved": "C:/work/volume-helper.sh" + }, + { + "raw": "--info-script", + "kind": "Literal", + "isPath": false + }, + { + "raw": "./info-helper.sh", + "kind": "Literal", + "isPath": true, + "resolved": "C:/work/info-helper.sh" + }, + { + "raw": "--new-volume-script", + "kind": "Literal", + "isPath": false + }, + { + "raw": "./next-volume.sh", + "kind": "Literal", + "isPath": true, + "resolved": "C:/work/next-volume.sh" + }, + { + "raw": "archive.tar", + "kind": "Literal", + "isPath": true, + "resolved": "C:/work/archive.tar" + } + ], + "redirects": [], + "elements": [ + { + "raw": "tar", + "value": "tar", + "role": "Verb", + "sourceStart": 0, + "sourceLength": 3, + "precedingVerbElementCount": 0, + "kind": "Literal", + "isFlag": false, + "isPath": false + }, + { + "raw": "-F=./volume-helper.sh", + "value": "-F=./volume-helper.sh", + "role": "Argument", + "sourceStart": 4, + "sourceLength": 21, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": true, + "isPath": true, + "resolved": "C:/work/volume-helper.sh" + }, + { + "raw": "--info-script=./info-helper.sh", + "value": "--info-script=./info-helper.sh", + "role": "Argument", + "sourceStart": 26, + "sourceLength": 30, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": true, + "isPath": true, + "resolved": "C:/work/info-helper.sh" + }, + { + "raw": "--new-volume-script", + "value": "--new-volume-script", + "role": "Argument", + "sourceStart": 57, + "sourceLength": 19, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": true, + "isPath": false + }, + { + "raw": "./next-volume.sh", + "value": "./next-volume.sh", + "role": "Argument", + "sourceStart": 77, + "sourceLength": 16, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": false, + "isPath": true, + "resolved": "C:/work/next-volume.sh" + }, + { + "raw": "archive.tar", + "value": "archive.tar", + "role": "Argument", + "sourceStart": 94, + "sourceLength": 11, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": false, + "isPath": true, + "resolved": "C:/work/archive.tar" + } + ] + } + ] + }, + "notes": "Tar -F and both long aliases consume helper-script paths in equals and spaced forms." +} diff --git a/tests/ShellSyntaxTree.Tests/Corpus/powershell/229_curl_data_file_references.json b/tests/ShellSyntaxTree.Tests/Corpus/powershell/229_curl_data_file_references.json new file mode 100644 index 0000000..3725ee9 --- /dev/null +++ b/tests/ShellSyntaxTree.Tests/Corpus/powershell/229_curl_data_file_references.json @@ -0,0 +1,105 @@ +{ + "name": "Curl data file references", + "input": "curl -d \u0022@request.json\u0022 --data=@payload.bin https://example.invalid/api", + "expected": { + "isUnparseable": false, + "clauses": [ + { + "operator": "None", + "verb": [ + "curl" + ], + "args": [ + { + "raw": "-d", + "kind": "Literal", + "isPath": false + }, + { + "raw": "\u0022@request.json\u0022", + "kind": "Literal", + "isPath": true, + "resolved": "C:/work/request.json" + }, + { + "raw": "--data", + "kind": "Literal", + "isPath": false + }, + { + "raw": "@payload.bin", + "kind": "Literal", + "isPath": true, + "resolved": "C:/work/payload.bin" + }, + { + "raw": "https://example.invalid/api", + "kind": "Literal", + "isPath": false + } + ], + "redirects": [], + "elements": [ + { + "raw": "curl", + "value": "curl", + "role": "Verb", + "sourceStart": 0, + "sourceLength": 4, + "precedingVerbElementCount": 0, + "kind": "Literal", + "isFlag": false, + "isPath": false + }, + { + "raw": "-d", + "value": "-d", + "role": "Argument", + "sourceStart": 5, + "sourceLength": 2, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": true, + "isPath": false + }, + { + "raw": "\u0022@request.json\u0022", + "value": "@request.json", + "role": "Argument", + "sourceStart": 8, + "sourceLength": 15, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": false, + "isPath": true, + "resolved": "C:/work/request.json" + }, + { + "raw": "--data=@payload.bin", + "value": "--data=@payload.bin", + "role": "Argument", + "sourceStart": 24, + "sourceLength": 19, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": true, + "isPath": true, + "resolved": "C:/work/payload.bin" + }, + { + "raw": "https://example.invalid/api", + "value": "https://example.invalid/api", + "role": "Argument", + "sourceStart": 44, + "sourceLength": 27, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": false, + "isPath": false + } + ] + } + ] + }, + "notes": "Curl data operands prefixed with @ read files; authored values retain the @ marker." +} diff --git a/tests/ShellSyntaxTree.Tests/Corpus/powershell/230_curl_data_stdin_reference.json b/tests/ShellSyntaxTree.Tests/Corpus/powershell/230_curl_data_stdin_reference.json new file mode 100644 index 0000000..710e430 --- /dev/null +++ b/tests/ShellSyntaxTree.Tests/Corpus/powershell/230_curl_data_stdin_reference.json @@ -0,0 +1,80 @@ +{ + "name": "Curl data stdin reference", + "input": "curl -d \u0022@-\u0022 https://example.invalid/api", + "expected": { + "isUnparseable": false, + "clauses": [ + { + "operator": "None", + "verb": [ + "curl" + ], + "args": [ + { + "raw": "-d", + "kind": "Literal", + "isPath": false + }, + { + "raw": "\u0022@-\u0022", + "kind": "Literal", + "isPath": false + }, + { + "raw": "https://example.invalid/api", + "kind": "Literal", + "isPath": false + } + ], + "redirects": [], + "elements": [ + { + "raw": "curl", + "value": "curl", + "role": "Verb", + "sourceStart": 0, + "sourceLength": 4, + "precedingVerbElementCount": 0, + "kind": "Literal", + "isFlag": false, + "isPath": false + }, + { + "raw": "-d", + "value": "-d", + "role": "Argument", + "sourceStart": 5, + "sourceLength": 2, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": true, + "isPath": false + }, + { + "raw": "\u0022@-\u0022", + "value": "@-", + "role": "Argument", + "sourceStart": 8, + "sourceLength": 4, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": false, + "isPath": false + }, + { + "raw": "https://example.invalid/api", + "value": "https://example.invalid/api", + "role": "Argument", + "sourceStart": 13, + "sourceLength": 27, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": false, + "isPath": false + } + ] + } + ] + }, + "notes": "Curl @- reads stdin and is not a filesystem path." +} diff --git a/tests/ShellSyntaxTree.Tests/Parsing/BashPerVerbRulesTests.cs b/tests/ShellSyntaxTree.Tests/Parsing/BashPerVerbRulesTests.cs index 9ddbf76..1850aa5 100644 --- a/tests/ShellSyntaxTree.Tests/Parsing/BashPerVerbRulesTests.cs +++ b/tests/ShellSyntaxTree.Tests/Parsing/BashPerVerbRulesTests.cs @@ -309,12 +309,12 @@ public void Native_flag_lookup_is_case_sensitive_while_verb_lookup_is_not() [InlineData("wget", "-O", true, true)] [InlineData("wget", "--output-file", true, true)] [InlineData("wget", "--output-document", true, true)] - [InlineData("docker", "-v", true, false)] - [InlineData("docker", "-V", false, false)] [InlineData("tar", "-c", false, false)] [InlineData("tar", "-C", true, true)] [InlineData("tar", "-f", true, true)] - [InlineData("tar", "-F", false, false)] + [InlineData("tar", "-F", true, true)] + [InlineData("tar", "--info-script", true, true)] + [InlineData("tar", "--new-volume-script", true, true)] public void Native_option_binding_matrix( string verb, string flag, @@ -326,6 +326,26 @@ public void Native_option_binding_matrix( Assert.Equal(valueIsPath, BashPerVerbRules.ValueOfFlagIsPath(verb, flag)); } + [Theory] + [InlineData("-d", "payload", false, "payload")] + [InlineData("--data", "name=Jane", false, "name=Jane")] + [InlineData("-d", "@request.json", true, "request.json")] + [InlineData("--data", "@/etc/passwd", true, "/etc/passwd")] + [InlineData("-d", "@-", false, "@-")] + [InlineData("--data-raw", "@request.json", false, "@request.json")] + public void Curl_data_file_reference_is_operand_sensitive( + string flag, + string value, + bool isPath, + string expectedPathValue) + { + var actual = BashPerVerbRules.TryGetFlagValuePath( + "curl", flag, value, out var pathValue); + + Assert.Equal(isPath, actual); + Assert.Equal(expectedPathValue, pathValue); + } + // ---------------------------------------------------------------- empty verb chain [Fact] diff --git a/tests/ShellSyntaxTree.Tests/Parsing/ClauseElementTests.cs b/tests/ShellSyntaxTree.Tests/Parsing/ClauseElementTests.cs index 48effe3..30c5f07 100644 --- a/tests/ShellSyntaxTree.Tests/Parsing/ClauseElementTests.cs +++ b/tests/ShellSyntaxTree.Tests/Parsing/ClauseElementTests.cs @@ -87,6 +87,31 @@ public void Git_option_case_preserves_distinct_native_metadata() } } + [Fact] + public void Curl_data_file_syntax_is_classified_in_both_parsers() + { + foreach (var (shell, parser) in Parsers()) + { + var fileClause = Assert.Single( + parser.Parse("curl -d @/etc/passwd https://example.invalid/api").Clauses); + var fileValue = Assert.Single( + fileClause.Elements, + element => element.Value == "@/etc/passwd"); + Assert.True(fileValue.IsPath, shell); + Assert.NotNull(fileValue.Resolved); + Assert.EndsWith("/etc/passwd", fileValue.Resolved, StringComparison.Ordinal); + + var dynamicClause = Assert.Single( + parser.Parse("curl --data=@$PAYLOAD https://example.invalid/api").Clauses); + var dynamicValue = Assert.Single( + dynamicClause.Elements, + element => element.Value == "--data=@$PAYLOAD"); + Assert.Equal(ArgKind.DynamicSkip, dynamicValue.Kind); + Assert.False(dynamicValue.IsPath, shell); + Assert.Null(dynamicValue.Resolved); + } + } + [Fact] public void Multiple_git_option_occurrences_remain_distinct() { diff --git a/tools/PwshCorpusTool/CorpusManifest.cs b/tools/PwshCorpusTool/CorpusManifest.cs index 62e9a80..fa496a9 100644 --- a/tools/PwshCorpusTool/CorpusManifest.cs +++ b/tools/PwshCorpusTool/CorpusManifest.cs @@ -606,5 +606,17 @@ private static string NestIex(string inner, int depth) P("curl_case_distinct_data_and_header_options", "curl -d payload -D headers.txt https://example.invalid/api", "Curl -d consumes non-path request data while -D consumes a header-output path."), + + // ---- Native option hardening: value- and context-sensitive cases ---- + P("tar_info_script_paths", + "tar -F=./volume-helper.sh --info-script=./info-helper.sh " + + "--new-volume-script ./next-volume.sh archive.tar", + "Tar -F and both long aliases consume helper-script paths in equals and spaced forms."), + P("curl_data_file_references", + "curl -d \"@request.json\" --data=@payload.bin https://example.invalid/api", + "Curl data operands prefixed with @ read files; authored values retain the @ marker."), + P("curl_data_stdin_reference", + "curl -d \"@-\" https://example.invalid/api", + "Curl @- reads stdin and is not a filesystem path."), }; } From 870858d9e55c42c226342f6d86d96742259a5413 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Tue, 4 Aug 2026 21:32:53 +0000 Subject: [PATCH 4/8] fix: integrate clause provenance after dev rebase --- IMPLEMENTATION_PLAN.md | 7 +- SPEC.POWERSHELL.md | 8 +- SPEC.md | 9 +- .../preserve-clause-element-order/design.md | 14 +++- .../specs/clause-element-provenance/spec.md | 26 ++++++ .../preserve-clause-element-order/tasks.md | 2 + .../specs/invoke-expression-recursion/spec.md | 1 + .../Pwsh/Parsing/PwshCommandParser.cs | 83 +++++++++++++++++++ ... => 263_git_global_option_provenance.json} | 0 ...264_git_subcommand_option_provenance.json} | 0 ... => 265_git_global_config_provenance.json} | 0 ...266_git_subcommand_config_provenance.json} | 0 ...n => 267_git_mixed_option_provenance.json} | 0 ...68_git_heuristic_boundary_provenance.json} | 0 ...69_wget_case_distinct_output_options.json} | 0 ...ase_distinct_data_and_header_options.json} | 0 ...hs.json => 271_tar_info_script_paths.json} | 0 ...son => 272_curl_data_file_references.json} | 0 ...son => 273_curl_data_stdin_reference.json} | 0 .../Parsing/ClauseElementTests.cs | 78 +++++++++++++++++ 20 files changed, 219 insertions(+), 9 deletions(-) rename tests/ShellSyntaxTree.Tests/Corpus/powershell/{220_git_global_option_provenance.json => 263_git_global_option_provenance.json} (100%) rename tests/ShellSyntaxTree.Tests/Corpus/powershell/{221_git_subcommand_option_provenance.json => 264_git_subcommand_option_provenance.json} (100%) rename tests/ShellSyntaxTree.Tests/Corpus/powershell/{222_git_global_config_provenance.json => 265_git_global_config_provenance.json} (100%) rename tests/ShellSyntaxTree.Tests/Corpus/powershell/{223_git_subcommand_config_provenance.json => 266_git_subcommand_config_provenance.json} (100%) rename tests/ShellSyntaxTree.Tests/Corpus/powershell/{224_git_mixed_option_provenance.json => 267_git_mixed_option_provenance.json} (100%) rename tests/ShellSyntaxTree.Tests/Corpus/powershell/{225_git_heuristic_boundary_provenance.json => 268_git_heuristic_boundary_provenance.json} (100%) rename tests/ShellSyntaxTree.Tests/Corpus/powershell/{226_wget_case_distinct_output_options.json => 269_wget_case_distinct_output_options.json} (100%) rename tests/ShellSyntaxTree.Tests/Corpus/powershell/{227_curl_case_distinct_data_and_header_options.json => 270_curl_case_distinct_data_and_header_options.json} (100%) rename tests/ShellSyntaxTree.Tests/Corpus/powershell/{228_tar_info_script_paths.json => 271_tar_info_script_paths.json} (100%) rename tests/ShellSyntaxTree.Tests/Corpus/powershell/{229_curl_data_file_references.json => 272_curl_data_file_references.json} (100%) rename tests/ShellSyntaxTree.Tests/Corpus/powershell/{230_curl_data_stdin_reference.json => 273_curl_data_stdin_reference.json} (100%) diff --git a/IMPLEMENTATION_PLAN.md b/IMPLEMENTATION_PLAN.md index ede5557..c90085c 100644 --- a/IMPLEMENTATION_PLAN.md +++ b/IMPLEMENTATION_PLAN.md @@ -37,7 +37,7 @@ priorities. the 64 KiB cap (§11). - [x] **11. Multi-shell corpus runner + PII audit** — directory-routed by `Corpus//`. -- [x] **12. PowerShell corpus** — 230 entries under `Corpus/powershell/`, +- [x] **12. PowerShell corpus** — 273 entries under `Corpus/powershell/`, every §13 category minimum exceeded. - [x] **13. `pwsh` validation gate + `tools/PwshCorpusTool`** — `PwshOracleTests` enforces the §13 oracle matrix + the `PwshAliases` @@ -72,6 +72,11 @@ priorities. remains explicitly context-sensitive: the generic table supports `docker run`, while consumers use authored elements for global-option interpretation. + Command-string provenance is integrated with the later + `Invoke-Expression` recursion work: static expansion clears unmappable + outer spans, while dynamic payloads retain conservative source-aligned + elements. Nested and dynamic `bash -c` cases pin the equivalent Bash + boundary. - [x] **Issue #64 — path-shaped operands after native verb chains.** Stop the Bash and PowerShell native greedy passes before a token that matches the shared path-shape rules. Preserve that token as a resolved diff --git a/SPEC.POWERSHELL.md b/SPEC.POWERSHELL.md index c8e8ab7..72925c5 100644 --- a/SPEC.POWERSHELL.md +++ b/SPEC.POWERSHELL.md @@ -952,6 +952,9 @@ leaves the required payload missing. Dynamic inline values remain opaque. For a static payload, the parser consumes the outer expression clause and surfaces the inner clauses inline with `IsCommandStringWrapped = true`. The first inner clause takes the operator that preceded the outer expression. +Surfaced `Clause.Elements` retain their inner raw and decoded values but have +null source spans because their offsets cannot be mapped exactly into the +outer `ParsedCommand.Source`. The parse increments the same depth counter used by `pwsh -Command` and `-EncodedCommand`, and the payload passes through the same 64 KiB input cap. @@ -965,7 +968,10 @@ The parser never evaluates variables, interpolation, concatenation, subexpressions, script blocks, arrays, or other computed expressions. When a direct computed payload has a source expression, the outer expression clause remains and the entire payload source slice becomes one -`Arg { Kind=DynamicSkip, IsPath=false, Resolved=null }`. Pipeline input, +`Arg { Kind=DynamicSkip, IsPath=false, Resolved=null }`. Its authored +`Clause.Elements` retain the expression verb, an optional separate `-Command` +parameter, and one source-aligned `DynamicSkip` payload region. An inline form +such as `-Command:$code` remains one authored parameter element. Pipeline input, missing payloads, and ambiguous parameter binding set `ParsedCommand.IsUnparseable = true`; an incoming pipeline is dynamic even when an explicit literal argument also appears. These rules prevent a clean, diff --git a/SPEC.md b/SPEC.md index 3703f22..7b031f5 100644 --- a/SPEC.md +++ b/SPEC.md @@ -299,9 +299,12 @@ compound operators, grouping delimiters, and shell call operators are excluded. Each verb token appears exactly once with `Role=Verb`. Each authored argument token appears once with `Role=Argument`; inline forms such as `--work-tree=../repo` stay one element even when `Args` exposes separate flag -and value projections. Each redirect appears once with `Role=Redirect`; its -ordinal among redirect elements matches its ordinal in `Redirects`, and `Raw` -spans the operator through its target. +and value projections. A parser-defined opaque computed region that is +safe-failed as one `DynamicSkip` argument also appears as one argument element; +its `Raw` and `Value` are the complete source slice rather than a claim that +the parser understood the region's interior. Each redirect appears once with +`Role=Redirect`; its ordinal among redirect elements matches its ordinal in +`Redirects`, and `Raw` spans the operator through its target. `PrecedingVerbElementCount` is clause-local and resets to zero at every clause. For `git -C /repo commit`, `-C` and `/repo` carry `1`; for diff --git a/openspec/changes/preserve-clause-element-order/design.md b/openspec/changes/preserve-clause-element-order/design.md index 396f9c6..b09cf14 100644 --- a/openspec/changes/preserve-clause-element-order/design.md +++ b/openspec/changes/preserve-clause-element-order/design.md @@ -94,13 +94,19 @@ Existing consumers continue to find it through `Arg.IsCwdAttribution`. ### Preserve source spans only when exact outer mapping exists For an ordinary clause, `SourceStart` and `SourceLength` index the returned -`ParsedCommand.Source`. Clauses expanded from `bash -c`, `pwsh -Command`, or -`pwsh -EncodedCommand` retain their inner `Raw` and `Value`, but their spans are -null after expansion because escaping and decoding prevent a generally exact -mapping into the outer source. +`ParsedCommand.Source`. Clauses expanded from `bash -c`, `pwsh -Command`, +`pwsh -EncodedCommand`, or static `Invoke-Expression` retain their inner `Raw` +and `Value`, but their spans are null after expansion because escaping and +decoding prevent a generally exact mapping into the outer source. Nullable spans are an uncertainty signal; the parser does not guess offsets. +When another parser feature deliberately collapses a computed source +expression into one `DynamicSkip` compatibility argument, the ordered view +uses one source-aligned argument element for that same opaque region. It does +not expose interior tokens in a way that could imply partial semantic +understanding. + ### Model a redirect as one ordered semantic element One redirect element spans its operator and target and occupies the same diff --git a/openspec/changes/preserve-clause-element-order/specs/clause-element-provenance/spec.md b/openspec/changes/preserve-clause-element-order/specs/clause-element-provenance/spec.md index 95c0a6c..7528906 100644 --- a/openspec/changes/preserve-clause-element-order/specs/clause-element-provenance/spec.md +++ b/openspec/changes/preserve-clause-element-order/specs/clause-element-provenance/spec.md @@ -115,7 +115,33 @@ null spans when they cannot be mapped exactly into the outer - **THEN** the surfaced clause elements retain their inner raw and decoded values - **THEN** every surfaced element has null `SourceStart` and `SourceLength` +#### Scenario: Dynamic Bash command string remains an ordinary clause +- **WHEN** Bash parses `bash -c $code` +- **THEN** the outer `bash`, `-c`, and `$code` elements remain source-aligned +- **THEN** the clause is not marked as command-string wrapped + #### Scenario: PowerShell encoded-command expansion - **WHEN** PowerShell parses a valid encoded command payload - **THEN** the surfaced clause elements retain their decoded values - **THEN** every surfaced element has null `SourceStart` and `SourceLength` + +#### Scenario: PowerShell Invoke-Expression expansion +- **WHEN** PowerShell parses `Invoke-Expression 'git -C C:\repo status'` +- **THEN** the surfaced clause elements retain their inner raw and decoded values +- **THEN** every surfaced element has null `SourceStart` and `SourceLength` + +### Requirement: Opaque computed regions remain conservative ordered elements +When a parser feature collapses a complete computed source expression, `Clause.Elements` SHALL +represent the complete payload as one source-aligned opaque argument element +matching the one `DynamicSkip` compatibility argument, rather than exposing +partially interpreted interior tokens. + +#### Scenario: Dynamic Invoke-Expression payload +- **WHEN** PowerShell parses `Invoke-Expression $code` +- **THEN** the clause elements are the `Invoke-Expression` verb followed by one source-aligned `$code` argument +- **THEN** the payload element has `Kind=DynamicSkip` + +#### Scenario: Inline dynamic Invoke-Expression binding +- **WHEN** PowerShell parses `iex -Command:$code` +- **THEN** `-Command:$code` remains one source-aligned argument element +- **THEN** the element is both option-shaped and `DynamicSkip` diff --git a/openspec/changes/preserve-clause-element-order/tasks.md b/openspec/changes/preserve-clause-element-order/tasks.md index 70f64b5..e059587 100644 --- a/openspec/changes/preserve-clause-element-order/tasks.md +++ b/openspec/changes/preserve-clause-element-order/tasks.md @@ -29,6 +29,8 @@ log output and curl header output, with paired cross-shell corpus cases. - [x] 3.8 Harden the audit with tar helper-script bindings, curl `@file` / `@-` operand semantics, and scoped Docker context claims. +- [x] 3.9 Reconcile command-string provenance with static and dynamic + `Invoke-Expression`; audit nested and dynamic `bash -c` behavior. ## 4. Consumer documentation diff --git a/openspec/changes/recurse-static-invoke-expression/specs/invoke-expression-recursion/spec.md b/openspec/changes/recurse-static-invoke-expression/specs/invoke-expression-recursion/spec.md index 7f74be2..778aedb 100644 --- a/openspec/changes/recurse-static-invoke-expression/specs/invoke-expression-recursion/spec.md +++ b/openspec/changes/recurse-static-invoke-expression/specs/invoke-expression-recursion/spec.md @@ -57,6 +57,7 @@ unambiguously, the parser SHALL set `ParsedCommand.IsUnparseable = true`. - **WHEN** PowerShell parses `Invoke-Expression $code` - **THEN** the result retains an `Invoke-Expression` clause - **THEN** `$code` is one `DynamicSkip` argument +- **THEN** the authored verb and payload remain source-aligned clause elements #### Scenario: Interpolated payload is dynamic - **WHEN** PowerShell parses `iex "Remove-$noun C:\x"` diff --git a/src/ShellSyntaxTree/Internal/Pwsh/Parsing/PwshCommandParser.cs b/src/ShellSyntaxTree/Internal/Pwsh/Parsing/PwshCommandParser.cs index bd3469d..c660335 100644 --- a/src/ShellSyntaxTree/Internal/Pwsh/Parsing/PwshCommandParser.cs +++ b/src/ShellSyntaxTree/Internal/Pwsh/Parsing/PwshCommandParser.cs @@ -1388,6 +1388,7 @@ private static bool TryHandleInvokeExpression( } var payloadStart = start + 1; + int? commandParameterIndex = null; string? inlinePayload = null; if (payloadStart >= body.Count) { @@ -1397,6 +1398,7 @@ private static bool TryHandleInvokeExpression( if (body[payloadStart].Kind == PwshTokenKind.Parameter) { + commandParameterIndex = payloadStart; var parameter = body[payloadStart].Value; var colon = parameter.IndexOf(':'); var parameterName = colon >= 0 @@ -1500,6 +1502,16 @@ private static bool TryHandleInvokeExpression( IsPath = false, }, }, + Elements = BuildDynamicInvokeExpressionElements( + body, + start, + verb, + source, + commandParameterIndex, + payloadStart, + payloadEnd, + inlinePayload, + rawPayload!), IsSubshell = segment.Depth > 0, IsCommandStringWrapped = markWrapped, }; @@ -1538,6 +1550,8 @@ private static bool TryHandleInvokeExpression( Operator = i == 0 ? segment.PrecedingOperator : innerClause.Operator, IsSubshell = segment.Depth > 0 || innerClause.IsSubshell, IsCommandStringWrapped = true, + Elements = ClauseElementProvenance.WithoutOuterSourceSpans( + innerClause.Elements), }); } @@ -1545,6 +1559,75 @@ private static bool TryHandleInvokeExpression( return true; } + private static IReadOnlyList BuildDynamicInvokeExpressionElements( + List body, + int start, + ClassifiedVerb verb, + string source, + int? commandParameterIndex, + int payloadStart, + int payloadEnd, + string? inlinePayload, + string rawPayload) + { + var elements = new List(); + var precedingVerbTokenCount = 0; + for (var i = start; i < body.Count; i++) + { + if (!verb.VerbPositions.Contains(i)) + { + continue; + } + + elements.Add(CreateElement( + source, + body[i], + ClauseElementRole.Verb, + precedingVerbTokenCount, + verb.IsDynamic ? ArgKind.DynamicSkip : ArgKind.Literal, + isFlag: false, + isPath: false, + resolved: null)); + precedingVerbTokenCount++; + } + + if (commandParameterIndex.HasValue) + { + elements.Add(CreateElement( + source, + body[commandParameterIndex.Value], + ClauseElementRole.Argument, + precedingVerbTokenCount, + inlinePayload is null ? ArgKind.Literal : ArgKind.DynamicSkip, + isFlag: true, + isPath: false, + resolved: null)); + } + + if (inlinePayload is not null) + { + return elements; + } + + var firstPayloadToken = body[payloadStart]; + var lastPayloadToken = body[payloadEnd]; + elements.Add(new ClauseElement + { + Raw = rawPayload, + Value = payloadStart == payloadEnd ? firstPayloadToken.Value : rawPayload, + Role = ClauseElementRole.Argument, + SourceStart = firstPayloadToken.SourceStart, + SourceLength = lastPayloadToken.SourceStart + lastPayloadToken.SourceLength + - firstPayloadToken.SourceStart, + PrecedingVerbElementCount = precedingVerbTokenCount, + Kind = ArgKind.DynamicSkip, + IsFlag = false, + IsPath = false, + }); + + return elements; + } + private static bool IsInvokeExpression(ClassifiedVerb verb) { var identity = verb.CanonicalVerb diff --git a/tests/ShellSyntaxTree.Tests/Corpus/powershell/220_git_global_option_provenance.json b/tests/ShellSyntaxTree.Tests/Corpus/powershell/263_git_global_option_provenance.json similarity index 100% rename from tests/ShellSyntaxTree.Tests/Corpus/powershell/220_git_global_option_provenance.json rename to tests/ShellSyntaxTree.Tests/Corpus/powershell/263_git_global_option_provenance.json diff --git a/tests/ShellSyntaxTree.Tests/Corpus/powershell/221_git_subcommand_option_provenance.json b/tests/ShellSyntaxTree.Tests/Corpus/powershell/264_git_subcommand_option_provenance.json similarity index 100% rename from tests/ShellSyntaxTree.Tests/Corpus/powershell/221_git_subcommand_option_provenance.json rename to tests/ShellSyntaxTree.Tests/Corpus/powershell/264_git_subcommand_option_provenance.json diff --git a/tests/ShellSyntaxTree.Tests/Corpus/powershell/222_git_global_config_provenance.json b/tests/ShellSyntaxTree.Tests/Corpus/powershell/265_git_global_config_provenance.json similarity index 100% rename from tests/ShellSyntaxTree.Tests/Corpus/powershell/222_git_global_config_provenance.json rename to tests/ShellSyntaxTree.Tests/Corpus/powershell/265_git_global_config_provenance.json diff --git a/tests/ShellSyntaxTree.Tests/Corpus/powershell/223_git_subcommand_config_provenance.json b/tests/ShellSyntaxTree.Tests/Corpus/powershell/266_git_subcommand_config_provenance.json similarity index 100% rename from tests/ShellSyntaxTree.Tests/Corpus/powershell/223_git_subcommand_config_provenance.json rename to tests/ShellSyntaxTree.Tests/Corpus/powershell/266_git_subcommand_config_provenance.json diff --git a/tests/ShellSyntaxTree.Tests/Corpus/powershell/224_git_mixed_option_provenance.json b/tests/ShellSyntaxTree.Tests/Corpus/powershell/267_git_mixed_option_provenance.json similarity index 100% rename from tests/ShellSyntaxTree.Tests/Corpus/powershell/224_git_mixed_option_provenance.json rename to tests/ShellSyntaxTree.Tests/Corpus/powershell/267_git_mixed_option_provenance.json diff --git a/tests/ShellSyntaxTree.Tests/Corpus/powershell/225_git_heuristic_boundary_provenance.json b/tests/ShellSyntaxTree.Tests/Corpus/powershell/268_git_heuristic_boundary_provenance.json similarity index 100% rename from tests/ShellSyntaxTree.Tests/Corpus/powershell/225_git_heuristic_boundary_provenance.json rename to tests/ShellSyntaxTree.Tests/Corpus/powershell/268_git_heuristic_boundary_provenance.json diff --git a/tests/ShellSyntaxTree.Tests/Corpus/powershell/226_wget_case_distinct_output_options.json b/tests/ShellSyntaxTree.Tests/Corpus/powershell/269_wget_case_distinct_output_options.json similarity index 100% rename from tests/ShellSyntaxTree.Tests/Corpus/powershell/226_wget_case_distinct_output_options.json rename to tests/ShellSyntaxTree.Tests/Corpus/powershell/269_wget_case_distinct_output_options.json diff --git a/tests/ShellSyntaxTree.Tests/Corpus/powershell/227_curl_case_distinct_data_and_header_options.json b/tests/ShellSyntaxTree.Tests/Corpus/powershell/270_curl_case_distinct_data_and_header_options.json similarity index 100% rename from tests/ShellSyntaxTree.Tests/Corpus/powershell/227_curl_case_distinct_data_and_header_options.json rename to tests/ShellSyntaxTree.Tests/Corpus/powershell/270_curl_case_distinct_data_and_header_options.json diff --git a/tests/ShellSyntaxTree.Tests/Corpus/powershell/228_tar_info_script_paths.json b/tests/ShellSyntaxTree.Tests/Corpus/powershell/271_tar_info_script_paths.json similarity index 100% rename from tests/ShellSyntaxTree.Tests/Corpus/powershell/228_tar_info_script_paths.json rename to tests/ShellSyntaxTree.Tests/Corpus/powershell/271_tar_info_script_paths.json diff --git a/tests/ShellSyntaxTree.Tests/Corpus/powershell/229_curl_data_file_references.json b/tests/ShellSyntaxTree.Tests/Corpus/powershell/272_curl_data_file_references.json similarity index 100% rename from tests/ShellSyntaxTree.Tests/Corpus/powershell/229_curl_data_file_references.json rename to tests/ShellSyntaxTree.Tests/Corpus/powershell/272_curl_data_file_references.json diff --git a/tests/ShellSyntaxTree.Tests/Corpus/powershell/230_curl_data_stdin_reference.json b/tests/ShellSyntaxTree.Tests/Corpus/powershell/273_curl_data_stdin_reference.json similarity index 100% rename from tests/ShellSyntaxTree.Tests/Corpus/powershell/230_curl_data_stdin_reference.json rename to tests/ShellSyntaxTree.Tests/Corpus/powershell/273_curl_data_stdin_reference.json diff --git a/tests/ShellSyntaxTree.Tests/Parsing/ClauseElementTests.cs b/tests/ShellSyntaxTree.Tests/Parsing/ClauseElementTests.cs index 30c5f07..51c20d0 100644 --- a/tests/ShellSyntaxTree.Tests/Parsing/ClauseElementTests.cs +++ b/tests/ShellSyntaxTree.Tests/Parsing/ClauseElementTests.cs @@ -297,6 +297,77 @@ public void Wrapped_commands_keep_values_but_clear_outer_source_spans() Assert.All(pwshClause.Elements, AssertSpanIsUnknown); } + [Fact] + public void Nested_bash_command_strings_keep_values_without_outer_source_spans() + { + var clause = Assert.Single( + Bash.Parse("bash -c \"bash -c \\\"git status\\\"\"").Clauses); + + Assert.True(clause.IsCommandStringWrapped); + Assert.Equal( + new[] { "git", "status" }, + clause.Elements.Select(element => element.Value).ToArray()); + Assert.All(clause.Elements, AssertSpanIsUnknown); + } + + [Fact] + public void Dynamic_bash_command_string_stays_an_outer_source_aligned_clause() + { + const string source = "bash -c $code"; + var clause = Assert.Single(Bash.Parse(source).Clauses); + + Assert.False(clause.IsCommandStringWrapped); + Assert.Equal( + new[] { "bash", "-c", "$code" }, + clause.Elements.Select(element => element.Value).ToArray()); + Assert.All(clause.Elements, element => AssertSourceSlice(source, element)); + } + + [Fact] + public void Static_invoke_expression_keeps_values_without_outer_source_spans() + { + var clause = Assert.Single( + Pwsh.Parse("Invoke-Expression 'git -C C:\\repo status'").Clauses); + + Assert.True(clause.IsCommandStringWrapped); + Assert.Equal( + new[] { "git", "-C", "C:\\repo", "status" }, + clause.Elements.Select(element => element.Value).ToArray()); + Assert.All(clause.Elements, AssertSpanIsUnknown); + } + + [Fact] + public void Dynamic_invoke_expression_keeps_source_aligned_elements() + { + const string source = "Invoke-Expression $code"; + var clause = Assert.Single(Pwsh.Parse(source).Clauses); + + Assert.False(clause.IsCommandStringWrapped); + Assert.Equal( + new[] { "Invoke-Expression", "$code" }, + clause.Elements.Select(element => element.Value).ToArray()); + Assert.Equal( + new[] { ClauseElementRole.Verb, ClauseElementRole.Argument }, + clause.Elements.Select(element => element.Role).ToArray()); + Assert.Equal(ArgKind.DynamicSkip, clause.Elements[1].Kind); + Assert.All(clause.Elements, element => AssertSourceSlice(source, element)); + } + + [Fact] + public void Inline_dynamic_invoke_expression_binding_is_one_authored_element() + { + const string source = "iex -Command:$code"; + var clause = Assert.Single(Pwsh.Parse(source).Clauses); + + Assert.Equal(new[] { "$code" }, clause.Args.Select(arg => arg.Raw).ToArray()); + Assert.Equal( + new[] { "iex", "-Command:$code" }, + clause.Elements.Select(element => element.Value).ToArray()); + Assert.True(clause.Elements[1].IsFlag); + Assert.Equal(ArgKind.DynamicSkip, clause.Elements[1].Kind); + Assert.All(clause.Elements, element => AssertSourceSlice(source, element)); + } + [Fact] public void Encoded_command_elements_have_no_invented_outer_spans() { @@ -315,6 +386,13 @@ public void Encoded_command_elements_have_no_invented_outer_spans() yield return ("pwsh", Pwsh); } + private static void AssertSourceSlice(string source, ClauseElement element) + { + var start = Assert.IsType(element.SourceStart); + var length = Assert.IsType(element.SourceLength); + Assert.Equal(element.Raw, source.Substring(start, length)); + } + private static void AssertSpanIsUnknown(ClauseElement element) { Assert.Null(element.SourceStart); From cae0019db82b489dbe659002132bcb5144250886 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Tue, 4 Aug 2026 21:51:27 +0000 Subject: [PATCH 5/8] docs: define strict and general approval matching --- IMPLEMENTATION_PLAN.md | 5 ++- PROJECT_CONTEXT.md | 4 +- SPEC.md | 44 +++++++++++-------- docs/CONSUMER_GUIDE.md | 40 +++++++++++++++-- .../preserve-clause-element-order/design.md | 13 ++++++ .../specs/clause-element-provenance/spec.md | 6 +++ .../preserve-clause-element-order/tasks.md | 3 ++ 7 files changed, 90 insertions(+), 25 deletions(-) diff --git a/IMPLEMENTATION_PLAN.md b/IMPLEMENTATION_PLAN.md index c90085c..2b2e822 100644 --- a/IMPLEMENTATION_PLAN.md +++ b/IMPLEMENTATION_PLAN.md @@ -76,7 +76,10 @@ priorities. `Invoke-Expression` recursion work: static expansion clears unmappable outer spans, while dynamic payloads retain conservative source-aligned elements. Nested and dynamic `bash -c` cases pin the equivalent Bash - boundary. + boundary. Consumer guidance separates strict authored-stream matching + from general executable-aware normalization; Netclaw can use the latter + for reusable approvals without treating parser verb roles as semantic + command boundaries. - [x] **Issue #64 — path-shaped operands after native verb chains.** Stop the Bash and PowerShell native greedy passes before a token that matches the shared path-shape rules. Preserve that token as a resolved diff --git a/PROJECT_CONTEXT.md b/PROJECT_CONTEXT.md index 0c500a4..842081a 100644 --- a/PROJECT_CONTEXT.md +++ b/PROJECT_CONTEXT.md @@ -42,7 +42,9 @@ open-source autonomous operations agent. Netclaw's POSIX approval gate consumes ShellSyntaxTree's Bash parser to decompose approval units, identify candidate verbs and directories, propagate cwd context, inspect redirects, and fail closed when parsing is uncertain. Its PowerShell integration is the remaining -v0.2.0 downstream acceptance item. See +v0.2.0 downstream acceptance item. Netclaw is expected to use the consumer +guide's general executable-aware matching path for supported commands, with +strict authored-stream matching as the fallback for unrecognized shapes. See [`docs/CONSUMER_GUIDE.md`](./docs/CONSUMER_GUIDE.md) for the public consumer algorithm and immutable Netclaw examples. diff --git a/SPEC.md b/SPEC.md index 7b031f5..a6db16b 100644 --- a/SPEC.md +++ b/SPEC.md @@ -740,25 +740,31 @@ syntactic rule disambiguates `origin` (a branch name) from `worktree` (a subcommand verb) without per-CLI semantic knowledge — and we will not bake per-CLI knowledge into the parser. -Consumers needing security-grade verb identification should pattern-prefix -match against the source-ordered `Clause.Elements` view: - -> A command matches an approval pattern `P` if and only if the first -> `len(P.verb_prefix)` verb elements of the command equal `P.verb_prefix`. - -This punts depth choice to the consumer (via the pattern they author) -and accommodates the parser's over-extraction transparently: - -- Pattern `git push *` (verb-prefix length 2) matches `git push origin - main` because the first two command tokens are `[git, push]`. -- Pattern `kubectl get pods *` (verb-prefix length 3) matches - `kubectl get pods my-pod` because the first three tokens are - `[kubectl, get, pods]`. -- Auto-proposed patterns for unknown commands should default to - the **full** extracted verb chain (greedy match), which is the - security-correct default: a subsequent variation re-prompts rather - than silently auto-grants. Operators wanting broader grants opt in - explicitly. +Consumers needing security-grade command identification choose one of two +strategies over the source-ordered `Clause.Elements` view: + +1. **Strict authored-stream matching.** Match every modeled significant + element in source order. A strict matcher may define explicit operand slots + or wildcards, but it SHALL NOT discard an intervening argument merely + because the parser assigned it `Role=Argument`. Therefore a strict + `git commit` pattern does not match `git -C /repo commit`. +2. **General executable-aware matching.** Pass the complete authored stream to + a grammar owned by the consumer. The grammar consumes known options and + operands, identifies the executable's semantic command, and returns both a + normalized approval identity and every policy-relevant operand or scope. + Equivalent syntax may reuse an approval only after complete interpretation. + +For example, a Git-aware matcher may interpret `git -C /repo commit` as the +general identity `git commit` with effective directory `/repo`. It may then +reuse a `git commit` approval only when that approval's directory policy covers +`/repo`. Likewise, executable-aware matchers may intentionally normalize +`git push origin main` to `git push` or `kubectl get pods my-pod` to +`kubectl get pods` when their grammars establish which suffixes are operands. + +There is no shell-generic rule that selects all `Role=Verb` elements and +compares them as a contiguous semantic prefix. For unknown executables or an +unrecognized option shape, consumers should use strict matching or prompt; +they should not silently fall back to a broader general identity. False-negative (re-prompt) is recoverable. False-positive (silent destructive grant) is not. Narrow-by-default favors the recoverable diff --git a/docs/CONSUMER_GUIDE.md b/docs/CONSUMER_GUIDE.md index 97ca86f..b6cd338 100644 --- a/docs/CONSUMER_GUIDE.md +++ b/docs/CONSUMER_GUIDE.md @@ -165,9 +165,41 @@ audit UI can still show `gci`. `VerbChain` is a best-effort syntactic hint, not a complete executable grammar. The greedy native-command walk can include bare lowercase values because a generic parser cannot know whether `origin` is a Git remote or a subcommand. -Unknown commands should therefore default to the full extracted chain, which -produces narrower approvals and recoverable re-prompts. A consumer may shorten -the chain only when it owns command-specific knowledge that justifies doing so. +Unknown commands should therefore retain the complete authored shape through a +strict pattern, producing narrower approvals and recoverable re-prompts. A +consumer may normalize or shorten that shape only when it owns command-specific +knowledge that justifies doing so. + +### Choosing strict or general matching + +`Clause.Elements` supports two security-conscious consumer strategies. The +choice belongs to the approval product, not the parser. + +**Strict matching** evaluates the significant authored stream in order. A +pattern may contain explicit operand slots, but unexpected or intervening +elements prevent a match. For example, a strict `git commit` pattern does not +match `git -C /repo commit`, because `-C /repo` appears between the executable +and subcommand. This mode is easy to audit and fail-closed, but syntactic +variations can produce more prompts. + +**General matching** uses an executable-aware interpreter. The interpreter +consumes the complete element stream according to that executable's option +grammar and returns a normalized approval identity plus the policy-relevant +operands and scopes. A Git interpreter can normalize `git -C /repo commit` to +`git commit` while retaining `/repo` as its effective-directory constraint. +This preserves reusable approvals without treating the option as irrelevant. + +General matching does not mean filtering to `Role=Verb` or trusting +`PrecedingVerbElementCount` as a semantic boundary. Both fields describe the +generic parser's projection. If the executable-aware interpreter encounters an +unknown option, missing operand, dynamic value, or otherwise incomplete shape, +it should fall back to strict matching or prompt rather than broaden the +approval. + +Netclaw is expected to use general matching for supported high-frequency +commands so ordinary option placement does not create approval fatigue. Strict +matching remains the safe fallback for commands whose grammar Netclaw does not +yet understand. ## Evaluating arguments and paths @@ -347,7 +379,7 @@ The consumer can collect the attributed cwd and both path operands to propose read/write mounts. It should still apply its own cmdlet policy and access-mode rules; ShellSyntaxTree reports paths, not filesystem permissions. -### Command-aware policy +### General command-aware policy Input: diff --git a/openspec/changes/preserve-clause-element-order/design.md b/openspec/changes/preserve-clause-element-order/design.md index b09cf14..6c306ff 100644 --- a/openspec/changes/preserve-clause-element-order/design.md +++ b/openspec/changes/preserve-clause-element-order/design.md @@ -58,6 +58,19 @@ count alone, to interpret the command. This avoids redundant `VerbIndex` and `AfterVerbIndex` fields whose invariants could drift. +### Support strict and general consumer matching without defining either + +A strict consumer matches the significant authored stream in order and rejects +unexpected intervening elements. A general consumer may normalize equivalent +forms only through an executable-aware grammar that completely consumes the +stream and returns policy-relevant operands and scope alongside the normalized +identity. Selecting every `Role=Verb` element is not a general matcher: it can +silently skip options authored between an executable and subcommand. + +This division keeps approval fatigue under consumer control. High-frequency +executables can receive reusable command-aware approvals, while unknown or +partially understood shapes retain a narrow strict fallback. + ### Match native option spelling case-sensitively Native executables receive option spelling unchanged under both Bash and diff --git a/openspec/changes/preserve-clause-element-order/specs/clause-element-provenance/spec.md b/openspec/changes/preserve-clause-element-order/specs/clause-element-provenance/spec.md index 7528906..7e6bbfd 100644 --- a/openspec/changes/preserve-clause-element-order/specs/clause-element-provenance/spec.md +++ b/openspec/changes/preserve-clause-element-order/specs/clause-element-provenance/spec.md @@ -32,6 +32,12 @@ collection. - **THEN** `-C` has `PrecedingVerbElementCount` equal to `1` - **THEN** consumers SHALL NOT interpret that count alone as Git's semantic option scope +#### Scenario: Intervening arguments are not a generic verb prefix +- **WHEN** either parser parses `git -C /repo commit` +- **THEN** a strict consumer sees `git`, `-C`, `/repo`, and `commit` in that order +- **THEN** it does not discard the two argument elements and generically match `git commit` +- **THEN** a consumer may derive `git commit` only through a Git-aware interpretation that retains `/repo` as policy-relevant scope + #### Scenario: Verb-relative counts reset per clause - **WHEN** either parser parses `git -C /repo status | git commit -C HEAD~1` - **THEN** the first clause reports `-C` after one verb token diff --git a/openspec/changes/preserve-clause-element-order/tasks.md b/openspec/changes/preserve-clause-element-order/tasks.md index e059587..2f152a3 100644 --- a/openspec/changes/preserve-clause-element-order/tasks.md +++ b/openspec/changes/preserve-clause-element-order/tasks.md @@ -41,6 +41,9 @@ this implementation change release-shaped. - [x] 4.3 Rewrite the Git consumer example to enumerate every occurrence, bind operands, and derive Git semantics from all authored elements. +- [x] 4.4 Document strict authored-stream and general executable-aware + matching; forbid generic verb-role filtering and identify general + matching as Netclaw's approval-fatigue mitigation. ## 5. Completion From dc3891052b9fcad542e47f811f125a46cc52d87b Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Wed, 5 Aug 2026 14:47:21 +0000 Subject: [PATCH 6/8] fix: harden clause element provenance --- IMPLEMENTATION_PLAN.md | 9 +- SPEC.POWERSHELL.md | 30 +- SPEC.md | 22 +- docs/CONSUMER_GUIDE.md | 6 + .../preserve-clause-element-order/design.md | 12 +- .../specs/clause-element-provenance/spec.md | 28 +- .../preserve-clause-element-order/tasks.md | 10 + .../Bash/Parsing/BashCommandParser.cs | 208 ++++++++- .../Internal/Bash/Verbs/BashPerVerbRules.cs | 16 +- .../Internal/Parsing/NativeFlagSyntax.cs | 66 +++ .../Pwsh/Parsing/PwshCommandParser.cs | 425 ++++++++++++++++-- .../Corpus/CorpusRunnerTests.cs | 215 +++++++++ ...json => 160_tar_info_script_commands.json} | 52 ++- ...163_curl_data_quoted_inline_reference.json | 58 +++ ...curl_data_multiple_adjacent_fragments.json | 58 +++ ...65_curl_data_unquoted_prefix_fragment.json | 58 +++ .../166_curl_data_mixed_literal_dynamic.json | 57 +++ ...curl_data_transformed_literal_dynamic.json | 57 +++ ...json => 271_tar_info_script_commands.json} | 63 +-- ...274_curl_data_quoted_inline_reference.json | 71 +++ .../275_inline_colon_backtick_path.json | 55 +++ .../276_recursion_command_outer_redirect.json | 61 +++ ...curl_data_multiple_adjacent_fragments.json | 71 +++ ...ecursion_empty_command_outer_redirect.json | 36 ++ ...79_curl_data_unquoted_prefix_fragment.json | 71 +++ .../280_curl_data_mixed_literal_dynamic.json | 69 +++ ...curl_data_transformed_literal_dynamic.json | 69 +++ .../Parsing/BashPerVerbRulesTests.cs | 44 +- .../Parsing/ClauseElementTests.cs | 163 +++++++ tools/PwshCorpusTool/CorpusManifest.cs | 32 +- 30 files changed, 2046 insertions(+), 146 deletions(-) rename tests/ShellSyntaxTree.Tests/Corpus/bash/{160_tar_info_script_paths.json => 160_tar_info_script_commands.json} (60%) create mode 100644 tests/ShellSyntaxTree.Tests/Corpus/bash/163_curl_data_quoted_inline_reference.json create mode 100644 tests/ShellSyntaxTree.Tests/Corpus/bash/164_curl_data_multiple_adjacent_fragments.json create mode 100644 tests/ShellSyntaxTree.Tests/Corpus/bash/165_curl_data_unquoted_prefix_fragment.json create mode 100644 tests/ShellSyntaxTree.Tests/Corpus/bash/166_curl_data_mixed_literal_dynamic.json create mode 100644 tests/ShellSyntaxTree.Tests/Corpus/bash/167_curl_data_transformed_literal_dynamic.json rename tests/ShellSyntaxTree.Tests/Corpus/powershell/{271_tar_info_script_paths.json => 271_tar_info_script_commands.json} (66%) create mode 100644 tests/ShellSyntaxTree.Tests/Corpus/powershell/274_curl_data_quoted_inline_reference.json create mode 100644 tests/ShellSyntaxTree.Tests/Corpus/powershell/275_inline_colon_backtick_path.json create mode 100644 tests/ShellSyntaxTree.Tests/Corpus/powershell/276_recursion_command_outer_redirect.json create mode 100644 tests/ShellSyntaxTree.Tests/Corpus/powershell/277_curl_data_multiple_adjacent_fragments.json create mode 100644 tests/ShellSyntaxTree.Tests/Corpus/powershell/278_recursion_empty_command_outer_redirect.json create mode 100644 tests/ShellSyntaxTree.Tests/Corpus/powershell/279_curl_data_unquoted_prefix_fragment.json create mode 100644 tests/ShellSyntaxTree.Tests/Corpus/powershell/280_curl_data_mixed_literal_dynamic.json create mode 100644 tests/ShellSyntaxTree.Tests/Corpus/powershell/281_curl_data_transformed_literal_dynamic.json diff --git a/IMPLEMENTATION_PLAN.md b/IMPLEMENTATION_PLAN.md index 2b2e822..2f1f165 100644 --- a/IMPLEMENTATION_PLAN.md +++ b/IMPLEMENTATION_PLAN.md @@ -68,7 +68,14 @@ priorities. audit explicitly covers Wget `-o` / `-O`, curl `-d` / `-D` / `-o` / `-O`, Git `-c` / `-C`, and tar `-c` / `-C` / `-f` / `-F`; paired corpus cases pin Wget log/document output, curl data/header-output and `@file` - semantics, and tar helper-script paths in both shells. Docker `-v` + semantics, and tar helper-command safe-fail behavior in both shells. + Adversarial review added deterministic coverage for quoted inline native + fragment runs (including unquoted prefixes and mixed-quote safe-fail), + PowerShell backtick-decoded colon bindings, native file-verb boundaries, + and outer redirects on PowerShell command wrappers, including empty + payloads. The + corpus runner now verifies direct authored-token coverage even for legacy + entries without explicit element expectations. Docker `-v` remains explicitly context-sensitive: the generic table supports `docker run`, while consumers use authored elements for global-option interpretation. diff --git a/SPEC.POWERSHELL.md b/SPEC.POWERSHELL.md index 72925c5..296ebc1 100644 --- a/SPEC.POWERSHELL.md +++ b/SPEC.POWERSHELL.md @@ -156,13 +156,22 @@ syntax rather than verb/argument/redirect leaves and do not appear. Inline parameter forms remain one source element. For `-Path:C:\repo`, the element's `Raw` and `Value` describe the full parameter token while `Kind`, `IsPath`, and `Resolved` describe the bound `C:\repo` value. Native -`--flag=value` follows the same rule as Bash. +`--flag=value` follows the same rule as Bash. Backtick escapes in an inline +bound value are decoded before `Value` and path metadata are produced. Adjacent +native fragments such as `--data='@C:\payload file'` form one element because +PowerShell passes them to the executable as one argument. The complete +contiguous fragment run is consumed. Resolver-sensitive syntax inside a +single-quoted fragment mixed with expandable fragments safe-fails as +`DynamicSkip` rather than being expanded. Clauses recursively surfaced from `pwsh -Command` and `pwsh -EncodedCommand` retain inner `Raw` and `Value` but have null `SourceStart` and `SourceLength`: quote/backtick processing, script-block stripping, and base64 decoding do not provide a generally exact map into the outer `ParsedCommand.Source`. +An outer redirect authored after a `pwsh -Command` or `-EncodedCommand` +payload remains on the surfaced wrapped clause with its exact outer source +span; only decoded inner elements have null spans. `ClauseElement.Role` and `PrecedingVerbElementCount` mirror the shared greedy native verb projection. They are AST coordinates, not native-executable @@ -708,8 +717,9 @@ Native commands reuse the bash per-verb rules table verbatim — `git`, `curl`, `tar`, etc. behave identically to `SPEC.md` §7 (`curl` / `wget`: the first positional is a URL; curl `-o` / `-D` values and Wget `-o` / `-O` values are paths, while curl `-d` data is non-path unless `@file` requests a -file read; `@-` denotes stdin). Tar `-F` / `--info-script` helper values are -paths. This +file read; `@-` denotes stdin). Tar `-F` / `--info-script` / +`--new-volume-script` values execute commands and therefore safe-fail as +`DynamicSkip`, not paths. This includes hyphenated option names and the bash `--flag=value` split: the flag and value surface as separate args, and a curated flag's value receives the same path classification in both parsers. Native `--flag:value` has no @@ -717,7 +727,9 @@ cmdlet-binding semantics and remains verbatim. PowerShell still owns outer tokenization: spaced curl operands beginning with `@` should be quoted because `@name` is splatting and bare `@-` is a parse error. Use forms such as `-d "@request.json"` / `-d "@-"`, or bind a file inline as -`--data=@request.json`, so the native command receives one value. +`--data=@request.json`, so the native command receives one value. An equals +prefix adjacent to a quoted value, such as `--data='@C:\payload file'`, is +also one native argument and one clause element. --- @@ -894,8 +906,8 @@ not just one quoted token. The parser handles all three real forms: - **Script block** — `pwsh -Command { Remove-Item C:\tmp\x }`. Parse the script-block *interior* (braces stripped) as a fresh `ParsedCommand`. - **Bare / multi-token** — `pwsh -Command Remove-Item C:\tmp\x`. Take the - verbatim source slice from the first token after `-Command` to the end of - the statement and parse *that* as a fresh `ParsedCommand`. + verbatim source slice from the first token after `-Command` through the last + command token and parse *that* as a fresh `ParsedCommand`. In every form the inner clauses surface inline, each with `IsCommandStringWrapped = true`. **Not** recognizing the bare/multi-token @@ -906,6 +918,12 @@ yields `IsUnparseable = true`** (e.g. a `-Command` payload that decodes to a control-flow script), sets the outer `ParsedCommand.IsUnparseable = true` so the whole command routes to safe-fail (`SPEC.md` §10). +A terminal redirect belongs to the outer PowerShell invocation, not the child +command string. The parser appends each such redirect to the last surfaced +inner clause's `Redirects` and `Elements`; its outer source span remains exact. +Non-redirect arguments after a quoted, script-block, colon-bound, or encoded +payload are not modeled and set `IsUnparseable=true` rather than disappearing. + `pwsh -File script.ps1` is **not** recursion — the file content is not available to the parser. It parses as an ordinary clause with `script.ps1` as a path arg. diff --git a/SPEC.md b/SPEC.md index a6db16b..9cbcea3 100644 --- a/SPEC.md +++ b/SPEC.md @@ -299,7 +299,14 @@ compound operators, grouping delimiters, and shell call operators are excluded. Each verb token appears exactly once with `Role=Verb`. Each authored argument token appears once with `Role=Argument`; inline forms such as `--work-tree=../repo` stay one element even when `Args` exposes separate flag -and value projections. A parser-defined opaque computed region that is +and value projections. Shell-adjacent fragments that form one native argument, +such as `--data="@request file.json"`, likewise stay one element spanning the +complete authored argument. The parser consumes the full contiguous fragment +run, including an unquoted value prefix such as `--data=@request".json"`. +Mixed quoting that prevents safe reconstruction of resolver-sensitive literal +syntax (`$`, glob metacharacters, `~`, provider prefixes) safe-fails the bound +value as `DynamicSkip`, including syntax exposed only after an operand marker +such as curl's leading `@` is removed. A parser-defined opaque computed region that is safe-failed as one `DynamicSkip` argument also appears as one argument element; its `Raw` and `Value` are the complete source slice rather than a claim that the parser understood the region's interior. Each redirect appears once with @@ -860,7 +867,7 @@ verb chain is a path. Per-verb overrides: | `rg` | First positional is **pattern**; rest are paths. | | `sed` | First positional is **script**; rest are paths. | | `awk` | First positional is **program**; rest are paths. | -| `tar` | Action flag determines path roles; default to extracting all non-flag positionals as paths. `-F` / `--info-script` / `--new-volume-script` values are helper-script paths. | +| `tar` | Action flag determines path roles; default to extracting all non-flag positionals as paths. `-F` / `--info-script` / `--new-volume-script` values are executable command text and safe-fail as `DynamicSkip`, never paths. | | `curl` | First positional is **URL**, not a path. `-o` / `--output` and `-D` / `--dump-header` values are paths. `-d` / `--data` values are request data unless prefixed with `@`, which reads a file; `@-` reads stdin and is not a path. | | `wget` | First positional is **URL**, not a path. `-o` / `--output-file` writes a log path; `-O` / `--output-document` writes the downloaded document path. | | `scp`, `rsync`, `sftp` | All positionals are paths (some remote). | @@ -910,6 +917,11 @@ internal static readonly IReadOnlyDictionary> > glob filenames continue through the normal §8 safe-fail rules after the > prefix is removed. +> **Command-valued options.** GNU tar executes `-F` / `--info-script` / +> `--new-volume-script` operands. Those options still consume a value, but the +> value is `Kind=DynamicSkip`, `IsPath=false`, and `Resolved=null`; resolving +> command text as a path would give a security gate false confidence. + > **Executable context.** `FlagsWithValue` is a curated parser heuristic, not > a complete executable grammar. In particular, Docker's global `-v` means > `--version`, while `docker run -v` consumes a volume specification. The @@ -1428,6 +1440,12 @@ An entry may add an `elements` list to a clause to pin the complete `resolved`). The field is opt-in so older corpus entries remain readable; issue-specific provenance entries SHALL include it. +The corpus runner also lexes every direct, parseable input and verifies that +each authored verb, argument, opaque region, and redirect token is covered by +exactly-positioned clause-element provenance. This invariant applies even when +an older entry omits the optional field, preventing silent argument loss across +the legacy corpus. + ### Coverage targets for v0.1 Author at least: diff --git a/docs/CONSUMER_GUIDE.md b/docs/CONSUMER_GUIDE.md index b6cd338..15aba51 100644 --- a/docs/CONSUMER_GUIDE.md +++ b/docs/CONSUMER_GUIDE.md @@ -250,6 +250,9 @@ foreach (var arg in clause.Args) The policy decides whether an unknown argument matters. `echo $message` may be acceptable to one product, while `Remove-Item $target` should normally prompt. Never treat `DynamicSkip.Raw` as a statically resolved path. +Command-valued native options use the same signal. GNU tar's `-F`, +`--info-script`, and `--new-volume-script` operands execute code, so the parser +reports their values as `DynamicSkip` rather than misleading path facts. ### Working-directory attribution @@ -314,6 +317,9 @@ ShellSyntaxTree also looks through supported command-string wrappers. Clauses surfaced from `bash -c`, `pwsh -Command`, and `pwsh -EncodedCommand` carry `IsCommandStringWrapped = true`. The outer wrapper is not the action a verb-based policy should authorize; the surfaced inner clauses are. +Redirects authored on the outer PowerShell wrapper remain attached to the last +surfaced clause, so redirect policy still sees paths such as +`pwsh -Command "git status" > audit.log`. PowerShell script blocks, subexpressions, splats, and `--%` regions are opaque and surface as `DynamicSkip`. A dynamically invoked command such as `& $exe` diff --git a/openspec/changes/preserve-clause-element-order/design.md b/openspec/changes/preserve-clause-element-order/design.md index 6c306ff..8856a3a 100644 --- a/openspec/changes/preserve-clause-element-order/design.md +++ b/openspec/changes/preserve-clause-element-order/design.md @@ -83,8 +83,8 @@ Ordinal matching requires every supported spelling to be explicit. The shared native table therefore distinguishes curl `-d` request data from `-D` header output, and lists both Wget `-o` log output and `-O` document output as path-valued bindings. It also lists tar `-F` independently from `-f`. An -option-binding matrix pins consuming and path semantics independently so a -comparer change cannot silently alter either; the long aliases are pinned +option-binding matrix pins consuming, path, and executable-command semantics independently so a +comparer change cannot silently alter any of them; the long aliases are pinned alongside the case-colliding short forms. Some native meaning is operand- or command-context-sensitive rather than @@ -93,6 +93,8 @@ the parser preserves the authored value while stripping `@` only for path resolution; `@-` remains stdin. Docker's global `-v` and `docker run -v` have different meanings, so the collision matrix does not claim a universal Docker binding. Executable-aware consumers interpret that placement from `Elements`. +GNU tar executes `-F` / `--info-script` / `--new-volume-script` operands, so +those values are opaque `DynamicSkip` command text rather than resolved paths. This corrects pre-existing metadata drift. It does not make the shared parser Git-semantic: a Git-aware consumer must still reinterpret `git commit -c/-C` @@ -113,6 +115,9 @@ and `Value`, but their spans are null after expansion because escaping and decoding prevent a generally exact mapping into the outer source. Nullable spans are an uncertainty signal; the parser does not guess offsets. +An outer redirect after a PowerShell command-string payload is not decoded +inner text: it retains its exact outer span and is appended to the surfaced +wrapped clause. When another parser feature deliberately collapses a computed source expression into one `DynamicSkip` compatibility argument, the ordered view @@ -138,6 +143,9 @@ and therefore one clause element. Their `Raw` and `Value` describe the full token; `IsPath`, `Kind`, and `Resolved` describe the bound value when the parser can classify it. Existing `Args` may continue splitting such a token into multiple semantic arguments. +Adjacent fragments that the shell passes as one native argument, such as +`--data="@request file.json"`, are also one element. Its span covers every +fragment and its metadata describes the complete bound value. ### Build elements in the classified token walk diff --git a/openspec/changes/preserve-clause-element-order/specs/clause-element-provenance/spec.md b/openspec/changes/preserve-clause-element-order/specs/clause-element-provenance/spec.md index 7e6bbfd..736fda9 100644 --- a/openspec/changes/preserve-clause-element-order/specs/clause-element-provenance/spec.md +++ b/openspec/changes/preserve-clause-element-order/specs/clause-element-provenance/spec.md @@ -68,6 +68,21 @@ Argument elements SHALL carry the parser's `Kind`, `IsFlag`, `IsPath`, and - **THEN** `--work-tree=../repo` is one argument element - **THEN** the element is a flag and carries the bound value's path metadata +#### Scenario: Adjacent quoted inline native value +- **WHEN** either parser parses `curl --data="@request file.json" URL` +- **THEN** `--data="@request file.json"` is one argument element +- **THEN** its decoded value is `--data=@request file.json` +- **THEN** it carries path metadata for `request file.json` + +#### Scenario: Complete adjacent fragment run +- **WHEN** either parser parses `curl --data=@request".json" URL` +- **THEN** the unquoted prefix and quoted suffix form one argument element +- **THEN** its decoded value is `--data=@request.json` +- **WHEN** either parser parses `curl --data='@$HOME'".json" URL` +- **THEN** the mixed resolver-sensitive value is `DynamicSkip` +- **WHEN** either parser parses `curl --data='@~'"/secret.json" URL` +- **THEN** the value remains `DynamicSkip` after curl's `@` marker is removed + #### Scenario: Native option spelling remains case-sensitive - **WHEN** either parser parses `git -c key=value commit` - **THEN** `key=value` is not classified as a path @@ -90,9 +105,10 @@ Argument elements SHALL carry the parser's `Kind`, `IsFlag`, `IsPath`, and - **WHEN** either parser parses `curl -d "@-" URL` - **THEN** `@-` is not classified as a path -#### Scenario: Tar helper-script bindings are explicit -- **WHEN** either parser parses `tar -F=./helper.sh --info-script=./info.sh --new-volume-script ./next.sh archive.tar` -- **THEN** all three helper-script values are classified as paths +#### Scenario: Tar helper commands safe-fail +- **WHEN** either parser parses `tar -F ./helper.sh --info-script=./info.sh --new-volume-script ./next.sh archive` +- **THEN** all three command values are `DynamicSkip` and not paths +- **THEN** the bare `archive` operand remains a path argument rather than a verb ### Requirement: Redirects occupy their authored position Each redirect SHALL appear as one redirect element at the source position of @@ -131,6 +147,12 @@ null spans when they cannot be mapped exactly into the outer - **THEN** the surfaced clause elements retain their decoded values - **THEN** every surfaced element has null `SourceStart` and `SourceLength` +#### Scenario: PowerShell wrapper preserves an outer redirect +- **WHEN** PowerShell parses `pwsh -Command "git status" > outer.txt` +- **THEN** the surfaced clause contains the output redirect in both `Redirects` and `Elements` +- **THEN** the inner verb elements have null spans +- **THEN** the redirect element retains its exact outer source span + #### Scenario: PowerShell Invoke-Expression expansion - **WHEN** PowerShell parses `Invoke-Expression 'git -C C:\repo status'` - **THEN** the surfaced clause elements retain their inner raw and decoded values diff --git a/openspec/changes/preserve-clause-element-order/tasks.md b/openspec/changes/preserve-clause-element-order/tasks.md index 2f152a3..8fc54c4 100644 --- a/openspec/changes/preserve-clause-element-order/tasks.md +++ b/openspec/changes/preserve-clause-element-order/tasks.md @@ -31,6 +31,16 @@ `@-` operand semantics, and scoped Docker context claims. - [x] 3.9 Reconcile command-string provenance with static and dynamic `Invoke-Expression`; audit nested and dynamic `bash -c` behavior. +- [x] 3.10 Apply adversarial-review fixes: safe-fail tar command hooks, + coalesce adjacent inline native fragments, decode PowerShell colon + values, preserve PowerShell wrapper redirects, and enforce direct-token + element coverage across the corpus. +- [x] 3.11 Re-run adversarial review over the fixes; consume maximal adjacent + fragment runs, safe-fail resolver-sensitive mixed quoting, preserve + redirects for empty wrapped payloads, and require outer redirect-target + provenance. +- [x] 3.12 Safe-fail resolver-sensitive literal syntax exposed only after a + native operand marker such as curl's leading `@` is removed. ## 4. Consumer documentation diff --git a/src/ShellSyntaxTree/Internal/Bash/Parsing/BashCommandParser.cs b/src/ShellSyntaxTree/Internal/Bash/Parsing/BashCommandParser.cs index 1522fcc..2b6a347 100644 --- a/src/ShellSyntaxTree/Internal/Bash/Parsing/BashCommandParser.cs +++ b/src/ShellSyntaxTree/Internal/Bash/Parsing/BashCommandParser.cs @@ -5,6 +5,7 @@ // ----------------------------------------------------------------------- using System; using System.Collections.Generic; +using System.Text; using ShellSyntaxTree.Internal.Bash.Lexing; using ShellSyntaxTree.Internal.Bash.Verbs; using ShellSyntaxTree.Internal.Parsing; @@ -1143,6 +1144,117 @@ private static void ExtractRedirectsAndArgs( { var sourceRaw = SourceSlice(source, t); + // Bash concatenates adjacent word fragments into one + // argv entry. Preserve that behavior for an inline option + // whose value is quoted or computed: + // `--data="@request file"` / `--data=$(generate)`. + if (NativeFlagSyntax.TrySplitEqualsPrefix( + t.Value, out var adjacentFlagPart, out var adjacentValuePrefix) + && i + 1 < segmentTokens.Count + && IsAdjacent(t, segmentTokens[i + 1]) + && segmentTokens[i + 1].Kind is BashTokenKind.QuotedString + or BashTokenKind.OpaqueSubstitution) + { + var valueStart = i + 1; + var valueEnd = valueStart; + var valueBuilder = new StringBuilder(adjacentValuePrefix); + var hasOpaqueFragment = false; + var allFragmentsSingleQuoted = adjacentValuePrefix.Length == 0; + var hasSingleQuotedFragment = false; + var hasNonSingleQuotedFragment = adjacentValuePrefix.Length > 0; + var hasSensitiveLiteralFragment = false; + var previousFragment = t; + while (valueEnd < segmentTokens.Count + && IsAdjacent(previousFragment, segmentTokens[valueEnd]) + && IsNativeArgumentFragment(segmentTokens[valueEnd])) + { + var fragment = segmentTokens[valueEnd]; + valueBuilder.Append(fragment.Value); + hasOpaqueFragment |= fragment.Kind == BashTokenKind.OpaqueSubstitution; + hasSingleQuotedFragment |= fragment.Kind == BashTokenKind.QuotedString + && fragment.IsSingleQuoted; + hasNonSingleQuotedFragment |= fragment.Kind != BashTokenKind.QuotedString + || !fragment.IsSingleQuoted; + hasSensitiveLiteralFragment |= fragment.Kind == BashTokenKind.QuotedString + && fragment.IsSingleQuoted + && NativeFlagSyntax.ContainsResolverSensitiveLiteralSyntax(fragment.Value); + allFragmentsSingleQuoted &= fragment.Kind == BashTokenKind.QuotedString + && fragment.IsSingleQuoted; + previousFragment = fragment; + valueEnd++; + } + + var lastValueToken = segmentTokens[valueEnd - 1]; + var adjacentValue = valueBuilder.ToString(); + var equalsOffset = SourceSlice(source, t).IndexOf('='); + var adjacentRawStart = t.SourceStart + equalsOffset + 1; + var adjacentRaw = source.Substring( + adjacentRawStart, + lastValueToken.SourceStart + lastValueToken.SourceLength + - adjacentRawStart); + argList.Add(new Arg + { + Raw = adjacentFlagPart, + Resolved = null, + Kind = ArgKind.Literal, + IsPath = false, + }); + + Arg valueArg; + if (hasOpaqueFragment + || (hasSingleQuotedFragment + && hasNonSingleQuotedFragment + && hasSensitiveLiteralFragment) + || (verbKeyForFlagValuePaths is not null + && BashPerVerbRules.ValueOfFlagIsOpaqueCommand( + verbKeyForFlagValuePaths, adjacentFlagPart))) + { + valueArg = new Arg + { + Raw = adjacentRaw, + Kind = ArgKind.DynamicSkip, + IsPath = false, + }; + } + else + { + var adjacentValueForResolution = adjacentValue; + var adjacentValueIsPath = verbKeyForFlagValuePaths is not null + && BashPerVerbRules.TryGetFlagValuePath( + verbKeyForFlagValuePaths, + adjacentFlagPart, + adjacentValue, + out adjacentValueForResolution); + var (adjacentKind, adjacentResolved, adjacentIsPath) = BashResolver.Resolve( + adjacentValueForResolution, + adjacentValueIsPath, + options, + workingDirectoryUnknown, + allFragmentsSingleQuoted); + valueArg = new Arg + { + Raw = adjacentRaw, + Resolved = adjacentResolved, + Kind = adjacentKind, + IsPath = adjacentIsPath, + }; + } + + argList.Add(valueArg); + elementList.Add(CreateCombinedElement( + source, + t, + lastValueToken, + adjacentFlagPart + "=" + adjacentValue, + precedingVerbTokenCount, + valueArg.Kind, + isFlag: true, + valueArg.IsPath, + valueArg.Resolved)); + i = valueEnd; + continue; + } + // Equals-form flag-with-value: `--output=file.txt`. The // flag half is a Literal arg with IsFlag=true (Raw // starts with '-'); the value half is classified per @@ -1164,14 +1276,20 @@ private static void ExtractRedirectsAndArgs( // literal (the equals-form is its own visible split, // so we don't apply LooksLikePath here). var inlineValueForResolution = valuePart; - var valueIsPath = verbKeyForFlagValuePaths is not null + var inlineValueIsOpaqueCommand = verbKeyForFlagValuePaths is not null + && BashPerVerbRules.ValueOfFlagIsOpaqueCommand( + verbKeyForFlagValuePaths, flagPart); + var valueIsPath = !inlineValueIsOpaqueCommand + && verbKeyForFlagValuePaths is not null && BashPerVerbRules.TryGetFlagValuePath( verbKeyForFlagValuePaths, flagPart, valuePart, out inlineValueForResolution); - var (vKind, vResolved, vIsPath) = BashResolver.Resolve( - inlineValueForResolution, valueIsPath, options, workingDirectoryUnknown); + var (vKind, vResolved, vIsPath) = inlineValueIsOpaqueCommand + ? (ArgKind.DynamicSkip, null, false) + : BashResolver.Resolve( + inlineValueForResolution, valueIsPath, options, workingDirectoryUnknown); argList.Add(new Arg { Raw = valuePart, @@ -1237,16 +1355,20 @@ private static void ExtractRedirectsAndArgs( // Non-flag positional. Classify path / resolve. var valueForResolution = t.Value; + var valueIsOpaqueCommand = false; bool treatAsPath; if (pendingFlagForValue is not null && verbKeyForFlagValuePaths is not null) { // This is the value of a preceding flag — use the // flag-value rule, NOT the positional-index rule. - treatAsPath = BashPerVerbRules.TryGetFlagValuePath( - verbKeyForFlagValuePaths, - pendingFlagForValue, - t.Value, - out valueForResolution); + valueIsOpaqueCommand = BashPerVerbRules.ValueOfFlagIsOpaqueCommand( + verbKeyForFlagValuePaths, pendingFlagForValue); + treatAsPath = !valueIsOpaqueCommand + && BashPerVerbRules.TryGetFlagValuePath( + verbKeyForFlagValuePaths, + pendingFlagForValue, + t.Value, + out valueForResolution); pendingFlagForValue = null; } else @@ -1255,8 +1377,10 @@ private static void ExtractRedirectsAndArgs( positionalIndex++; } - var (kind, resolved, isPath) = BashResolver.Resolve( - valueForResolution, treatAsPath, options, workingDirectoryUnknown); + var (kind, resolved, isPath) = valueIsOpaqueCommand + ? (ArgKind.DynamicSkip, null, false) + : BashResolver.Resolve( + valueForResolution, treatAsPath, options, workingDirectoryUnknown); argList.Add(new Arg { Raw = sourceRaw, @@ -1286,14 +1410,18 @@ private static void ExtractRedirectsAndArgs( // still classify as positional path / non-path through // the per-verb rule + resolver. var valueForResolution = t.Value; + var valueIsOpaqueCommand = false; bool treatAsPath; if (pendingFlagForValue is not null && verbKeyForFlagValuePaths is not null) { - treatAsPath = BashPerVerbRules.TryGetFlagValuePath( - verbKeyForFlagValuePaths, - pendingFlagForValue, - t.Value, - out valueForResolution); + valueIsOpaqueCommand = BashPerVerbRules.ValueOfFlagIsOpaqueCommand( + verbKeyForFlagValuePaths, pendingFlagForValue); + treatAsPath = !valueIsOpaqueCommand + && BashPerVerbRules.TryGetFlagValuePath( + verbKeyForFlagValuePaths, + pendingFlagForValue, + t.Value, + out valueForResolution); pendingFlagForValue = null; } else @@ -1305,12 +1433,14 @@ private static void ExtractRedirectsAndArgs( // Single-quoted tokens carry literal bytes per SPEC §5 // — bypass tilde / $HOME / $VAR / glob handling so // `'$HOME'` doesn't expand. - var (kind, resolved, isPath) = BashResolver.Resolve( - valueForResolution, - treatAsPath, - options, - workingDirectoryUnknown, - t.IsSingleQuoted); + var (kind, resolved, isPath) = valueIsOpaqueCommand + ? (ArgKind.DynamicSkip, null, false) + : BashResolver.Resolve( + valueForResolution, + treatAsPath, + options, + workingDirectoryUnknown, + t.IsSingleQuoted); argList.Add(new Arg { Raw = sourceRaw, @@ -1490,6 +1620,34 @@ private static ClauseElement CreateElement( Resolved = resolved, }; + private static ClauseElement CreateCombinedElement( + string source, + BashToken first, + BashToken last, + string value, + int precedingVerbTokenCount, + ArgKind kind, + bool isFlag, + bool isPath, + string? resolved) + { + var sourceStart = first.SourceStart; + var sourceEnd = last.SourceStart + last.SourceLength; + return new ClauseElement + { + Raw = source.Substring(sourceStart, sourceEnd - sourceStart), + Value = value, + Role = ClauseElementRole.Argument, + SourceStart = sourceStart, + SourceLength = sourceEnd - sourceStart, + PrecedingVerbElementCount = precedingVerbTokenCount, + Kind = kind, + IsFlag = isFlag, + IsPath = isPath, + Resolved = resolved, + }; + } + private static ClauseElement CreateRedirectElement( string source, BashToken redirectOperator, @@ -1519,6 +1677,14 @@ private static ClauseElement CreateRedirectElement( private static bool IsFlag(string raw) => raw.Length > 0 && raw[0] == '-'; + private static bool IsAdjacent(BashToken first, BashToken second) => + first.SourceStart + first.SourceLength == second.SourceStart; + + private static bool IsNativeArgumentFragment(BashToken token) => + token.Kind is BashTokenKind.Word + or BashTokenKind.QuotedString + or BashTokenKind.OpaqueSubstitution; + private static bool IsFdDupTarget(string value) { // Recognized shapes (POSIX `[n]>&word` / `[n]<&word`): diff --git a/src/ShellSyntaxTree/Internal/Bash/Verbs/BashPerVerbRules.cs b/src/ShellSyntaxTree/Internal/Bash/Verbs/BashPerVerbRules.cs index 482d8b4..057a7a0 100644 --- a/src/ShellSyntaxTree/Internal/Bash/Verbs/BashPerVerbRules.cs +++ b/src/ShellSyntaxTree/Internal/Bash/Verbs/BashPerVerbRules.cs @@ -153,16 +153,24 @@ internal static bool IsPositionalPathArg(VerbChain verb, int positionalIndex, st [("docker", "-v")] = false, [("docker", "--volume")] = false, - // tar: archive, directory, and multi-volume helper values are paths. + // tar: archive and directory values are paths. Multi-volume + // helper values are commands and are safe-failed separately. [("tar", "-f")] = true, [("tar", "--file")] = true, [("tar", "-C")] = true, [("tar", "--directory")] = true, - [("tar", "-F")] = true, - [("tar", "--info-script")] = true, - [("tar", "--new-volume-script")] = true, }; + /// + /// True when a flag value is executable command text rather than a file + /// operand. The parser must not publish a resolved path for these values. + /// + internal static bool ValueOfFlagIsOpaqueCommand(string verb, string flag) => + string.Equals(verb, "tar", StringComparison.OrdinalIgnoreCase) + && (string.Equals(flag, "-F", StringComparison.Ordinal) + || string.Equals(flag, "--info-script", StringComparison.Ordinal) + || string.Equals(flag, "--new-volume-script", StringComparison.Ordinal)); + /// /// Whether every value following for /// should be classified as a path. Concrete diff --git a/src/ShellSyntaxTree/Internal/Parsing/NativeFlagSyntax.cs b/src/ShellSyntaxTree/Internal/Parsing/NativeFlagSyntax.cs index 1513456..c2208b3 100644 --- a/src/ShellSyntaxTree/Internal/Parsing/NativeFlagSyntax.cs +++ b/src/ShellSyntaxTree/Internal/Parsing/NativeFlagSyntax.cs @@ -46,4 +46,70 @@ internal static bool TrySplitEqualsFlag( valuePart = raw.Substring(eq + 1); return true; } + + /// + /// Recognize the first fragment of an equals-form option whose value is + /// supplied by an immediately adjacent quoted or opaque token. + /// + internal static bool TryGetTrailingEqualsFlag(string raw, out string flagPart) + { + if (raw.Length < 3 || raw[0] != '-' || raw[raw.Length - 1] != '=') + { + flagPart = ""; + return false; + } + + flagPart = raw.Substring(0, raw.Length - 1); + return true; + } + + /// + /// Split an equals-form option while allowing an empty value prefix. This + /// is used when later adjacent shell fragments complete the same argv + /// entry. + /// + internal static bool TrySplitEqualsPrefix( + string raw, out string flagPart, out string valuePrefix) + { + if (raw.Length < 2 || raw[0] != '-') + { + flagPart = ""; + valuePrefix = ""; + return false; + } + + var eq = raw.IndexOf('='); + if (eq <= 0) + { + flagPart = ""; + valuePrefix = ""; + return false; + } + + flagPart = raw.Substring(0, eq); + valuePrefix = raw.Substring(eq + 1); + return true; + } + + /// + /// Characters whose meaning changes when a shell fragment is literal + /// rather than expandable. Mixed-fragment values containing these must + /// safe-fail unless fragment-level provenance is retained. + /// + internal static bool ContainsResolverSensitiveLiteralSyntax(string value) + { + var transformedValue = value.Length > 0 && value[0] == '@' + ? value.Substring(1) + : value; + return value.IndexOf('$') >= 0 + || value.IndexOf('*') >= 0 + || value.IndexOf('?') >= 0 + || value.IndexOf('[') >= 0 + || transformedValue.StartsWith("~", System.StringComparison.Ordinal) + || transformedValue.StartsWith( + "filesystem::", System.StringComparison.OrdinalIgnoreCase) + || transformedValue.StartsWith( + "Microsoft.PowerShell.Core\\FileSystem::", + System.StringComparison.OrdinalIgnoreCase); + } } diff --git a/src/ShellSyntaxTree/Internal/Pwsh/Parsing/PwshCommandParser.cs b/src/ShellSyntaxTree/Internal/Pwsh/Parsing/PwshCommandParser.cs index c660335..0c37dc5 100644 --- a/src/ShellSyntaxTree/Internal/Pwsh/Parsing/PwshCommandParser.cs +++ b/src/ShellSyntaxTree/Internal/Pwsh/Parsing/PwshCommandParser.cs @@ -658,7 +658,8 @@ private static BuildResult BuildSegment( if (classified.Kind == PwshCommandKind.PwshInvocation) { var recursion = TryRecurseIntoPwsh( - body, start, classified, source, baseOptions, recursionDepth, + body, start, classified, source, baseOptions, effectiveOptions, + workingDirectoryUnknown, recursionDepth, segment, markWrapped, out var recursionResult); if (recursion) { @@ -814,7 +815,7 @@ private static ClassifiedVerb ClassifyVerb(List body, int start) // file utility (xcopy, robocopy, ...) gets a 1-token chain so its // arguments classify as paths. var verbTokens = new List { word }; - if (!PwshVerbs.FileVerbs.Contains(word)) + if (!PwshVerbs.FileVerbs.Contains(word) && !BashVerbs.FileVerbs.Contains(word)) { BashVerbs.FlagsWithValue.TryGetValue(word, out var flagsForVerb); var i = start + 1; @@ -973,6 +974,22 @@ private static ArgResult ExtractArgsAndRedirects( pendingNativeFlag = null; var raw = t.Value; + var bindingSeparator = FirstBindingSeparator(raw); + if (bindingSeparator >= 0 && bindingSeparator + 1 < raw.Length) + { + var encodedValue = raw.Substring(bindingSeparator + 1); + if (!PwshLexer.TryDecodeExpandableValue( + encodedValue, out var decodedValue, out _)) + { + return new ArgResult( + args, + redirects, + elements, + "invalid PowerShell Unicode escape in inline parameter value"); + } + + raw = raw.Substring(0, bindingSeparator + 1) + decodedValue; + } if (cmdletStyle) { @@ -1011,21 +1028,137 @@ private static ArgResult ExtractArgsAndRedirects( { var verbKey = verb.VerbTokens.Count > 0 ? verb.VerbTokens[0] : string.Empty; + // PowerShell passes adjacent native argument fragments as + // one argv entry. Keep an inline option and its quoted or + // opaque value together in the provenance view. + if (NativeFlagSyntax.TrySplitEqualsPrefix( + raw, out var adjacentFlagPart, out var adjacentValuePrefix) + && i + 1 < body.Count + && IsAdjacent(t, body[i + 1]) + && body[i + 1].Kind is PwshTokenKind.QuotedString + or PwshTokenKind.ScriptBlock + or PwshTokenKind.Subexpression + or PwshTokenKind.Splat) + { + var valueStart = i + 1; + var valueEnd = valueStart; + var valueBuilder = new StringBuilder(adjacentValuePrefix); + var hasOpaqueFragment = false; + var allFragmentsSingleQuoted = adjacentValuePrefix.Length == 0; + var hasSingleQuotedFragment = false; + var hasNonSingleQuotedFragment = adjacentValuePrefix.Length > 0; + var hasSensitiveLiteralFragment = false; + var previousFragment = t; + while (valueEnd < body.Count + && IsAdjacent(previousFragment, body[valueEnd]) + && IsNativeArgumentFragment(body[valueEnd])) + { + var fragment = body[valueEnd]; + valueBuilder.Append(fragment.Value); + hasOpaqueFragment |= fragment.Kind != PwshTokenKind.Word + && fragment.Kind != PwshTokenKind.QuotedString; + hasSingleQuotedFragment |= fragment.Kind == PwshTokenKind.QuotedString + && fragment.IsSingleQuoted; + hasNonSingleQuotedFragment |= fragment.Kind != PwshTokenKind.QuotedString + || !fragment.IsSingleQuoted; + hasSensitiveLiteralFragment |= fragment.Kind == PwshTokenKind.QuotedString + && fragment.IsSingleQuoted + && NativeFlagSyntax.ContainsResolverSensitiveLiteralSyntax(fragment.Value); + allFragmentsSingleQuoted &= fragment.Kind == PwshTokenKind.QuotedString + && fragment.IsSingleQuoted; + previousFragment = fragment; + valueEnd++; + } + + var lastValueToken = body[valueEnd - 1]; + var adjacentValue = valueBuilder.ToString(); + var equalsOffset = SourceSlice(source, t).IndexOf('='); + var adjacentRawStart = t.SourceStart + equalsOffset + 1; + var adjacentRaw = source.Substring( + adjacentRawStart, + lastValueToken.SourceStart + lastValueToken.SourceLength + - adjacentRawStart); + args.Add(new Arg + { + Raw = adjacentFlagPart, + Kind = ArgKind.Literal, + IsPath = false, + }); + + Arg valueArg; + if (hasOpaqueFragment + || (hasSingleQuotedFragment + && hasNonSingleQuotedFragment + && hasSensitiveLiteralFragment) + || BashPerVerbRules.ValueOfFlagIsOpaqueCommand( + verbKey, adjacentFlagPart)) + { + valueArg = new Arg + { + Raw = adjacentRaw, + Kind = ArgKind.DynamicSkip, + IsPath = false, + }; + } + else + { + var adjacentValueForResolution = adjacentValue; + var adjacentValueIsPath = BashPerVerbRules.TryGetFlagValuePath( + verbKey, + adjacentFlagPart, + adjacentValue, + out adjacentValueForResolution); + valueArg = ResolveValueToken( + adjacentRaw, + adjacentValueForResolution, + adjacentValueIsPath, + options, + workingDirectoryUnknown, + allFragmentsSingleQuoted); + } + + args.Add(valueArg); + elements.Add(CreateCombinedElement( + source, + t, + lastValueToken, + adjacentFlagPart + "=" + adjacentValue, + precedingVerbTokenCount, + valueArg.Kind, + isFlag: true, + valueArg.IsPath, + valueArg.Resolved)); + i = valueEnd - 1; + continue; + } + // Native --flag=value follows Bash exactly: surface the // flag and value separately, and classify a curated // flag's value through the shared per-verb table. if (NativeFlagSyntax.TrySplitEqualsFlag(raw, out var flagPart, out var valuePart)) { args.Add(new Arg { Raw = flagPart, Kind = ArgKind.Literal, IsPath = false }); - var valueIsPath = BashPerVerbRules.TryGetFlagValuePath( - verbKey, flagPart, valuePart, out var inlineValueForResolution); - args.Add(ResolveValueToken( - valuePart, - inlineValueForResolution, - valueIsPath, - options, - workingDirectoryUnknown, - false)); + if (BashPerVerbRules.ValueOfFlagIsOpaqueCommand(verbKey, flagPart)) + { + args.Add(new Arg + { + Raw = valuePart, + Kind = ArgKind.DynamicSkip, + IsPath = false, + }); + } + else + { + var valueIsPath = BashPerVerbRules.TryGetFlagValuePath( + verbKey, flagPart, valuePart, out var inlineValueForResolution); + args.Add(ResolveValueToken( + valuePart, + inlineValueForResolution, + valueIsPath, + options, + workingDirectoryUnknown, + false)); + } } else { @@ -1052,7 +1185,8 @@ private static ArgResult ExtractArgsAndRedirects( parameterMetadata.Kind, isFlag: true, isPath: parameterMetadata.IsPath, - resolved: parameterMetadata.Resolved)); + resolved: parameterMetadata.Resolved, + value: raw)); continue; } @@ -1095,6 +1229,7 @@ private static ArgResult ExtractArgsAndRedirects( var rawValue = SourceSlice(source, t); var valueForResolution = t.Value; + var valueIsOpaqueCommand = false; bool treatAsPath; if (pendingValueParam is not null) { @@ -1104,11 +1239,14 @@ private static ArgResult ExtractArgsAndRedirects( else if (pendingNativeFlag is not null) { var verbKey = verb.VerbTokens.Count > 0 ? verb.VerbTokens[0] : string.Empty; - treatAsPath = BashPerVerbRules.TryGetFlagValuePath( - verbKey, - pendingNativeFlag, - t.Value, - out valueForResolution); + valueIsOpaqueCommand = BashPerVerbRules.ValueOfFlagIsOpaqueCommand( + verbKey, pendingNativeFlag); + treatAsPath = !valueIsOpaqueCommand + && BashPerVerbRules.TryGetFlagValuePath( + verbKey, + pendingNativeFlag, + t.Value, + out valueForResolution); pendingNativeFlag = null; } else @@ -1119,13 +1257,15 @@ private static ArgResult ExtractArgsAndRedirects( positionalIndex++; } - var resolvedArg = ResolveValueToken( - rawValue, - valueForResolution, - treatAsPath, - options, - workingDirectoryUnknown, - isLiteralBytes); + var resolvedArg = valueIsOpaqueCommand + ? new Arg { Raw = rawValue, Kind = ArgKind.DynamicSkip, IsPath = false } + : ResolveValueToken( + rawValue, + valueForResolution, + treatAsPath, + options, + workingDirectoryUnknown, + isLiteralBytes); args.Add(resolvedArg); elements.Add(CreateElement( source, @@ -1283,10 +1423,11 @@ private static ClauseElement CreateElement( ArgKind kind, bool isFlag, bool isPath, - string? resolved) => new() + string? resolved, + string? value = null) => new() { Raw = SourceSlice(source, token), - Value = token.Value, + Value = value ?? token.Value, Role = role, SourceStart = token.SourceStart, SourceLength = token.SourceLength, @@ -1297,6 +1438,34 @@ private static ClauseElement CreateElement( Resolved = resolved, }; + private static ClauseElement CreateCombinedElement( + string source, + PwshToken first, + PwshToken last, + string value, + int precedingVerbTokenCount, + ArgKind kind, + bool isFlag, + bool isPath, + string? resolved) + { + var sourceStart = first.SourceStart; + var sourceEnd = last.SourceStart + last.SourceLength; + return new ClauseElement + { + Raw = source.Substring(sourceStart, sourceEnd - sourceStart), + Value = value, + Role = ClauseElementRole.Argument, + SourceStart = sourceStart, + SourceLength = sourceEnd - sourceStart, + PrecedingVerbElementCount = precedingVerbTokenCount, + Kind = kind, + IsFlag = isFlag, + IsPath = isPath, + Resolved = resolved, + }; + } + private static ClauseElement CreateRedirectElement( string source, PwshToken operatorToken, @@ -1326,6 +1495,33 @@ private static ClauseElement CreateRedirectElement( }; } + private static bool IsAdjacent(PwshToken first, PwshToken second) => + first.SourceStart + first.SourceLength == second.SourceStart; + + private static bool IsNativeArgumentFragment(PwshToken token) => + token.Kind is PwshTokenKind.Word + or PwshTokenKind.QuotedString + or PwshTokenKind.ScriptBlock + or PwshTokenKind.Subexpression + or PwshTokenKind.Splat; + + private static int FirstBindingSeparator(string value) + { + var colon = value.IndexOf(':'); + var equals = value.IndexOf('='); + if (colon < 0) + { + return equals; + } + + if (equals < 0) + { + return colon; + } + + return Math.Min(colon, equals); + } + /// SPEC.POWERSHELL.md §8: map a PowerShell redirect operator /// onto the (lossy) enum. private static RedirectDirection MapRedirect(string op, out bool isMerge, out string? mergeTarget) @@ -1647,7 +1843,8 @@ private static bool IsInvokeExpressionName(string identity) private static bool TryRecurseIntoPwsh( List body, int start, ClassifiedVerb verb, string source, - PwshParserOptions options, int recursionDepth, Segment segment, bool markWrapped, + PwshParserOptions options, PwshParserOptions redirectOptions, + bool workingDirectoryUnknown, int recursionDepth, Segment segment, bool markWrapped, out BuildResult result) { result = default; @@ -1665,6 +1862,8 @@ private static bool TryRecurseIntoPwsh( var colon = raw.IndexOf(':'); var name = colon > 0 ? raw.Substring(0, colon) : raw; var colonValue = colon > 0 ? raw.Substring(colon + 1) : null; + var redirectStart = FindWrapperRedirectStart(body, i + 1); + var payloadEndExclusive = redirectStart < 0 ? body.Count : redirectStart; string? inner = null; string? failure = null; @@ -1672,7 +1871,8 @@ private static bool TryRecurseIntoPwsh( if (IsCommandParameter(name)) { - inner = ResolveCommandPayload(body, i, colonValue, source); + inner = ResolveCommandPayload( + body, i, colonValue, source, payloadEndExclusive, out failure); } else if (IsEncodedCommandParameter(name)) { @@ -1684,7 +1884,15 @@ private static bool TryRecurseIntoPwsh( } else { - inner = TryDecodeEncodedCommand(payloadToken, out failure); + var expectedPayloadEnd = colonValue is null ? i + 2 : i + 1; + if (expectedPayloadEnd < payloadEndExclusive) + { + failure = "-EncodedCommand has unsupported trailing arguments"; + } + else + { + inner = TryDecodeEncodedCommand(payloadToken, out failure); + } } } else @@ -1706,6 +1914,20 @@ private static bool TryRecurseIntoPwsh( return true; } + if (!TryBuildWrapperRedirects( + body, + redirectStart, + source, + redirectOptions, + workingDirectoryUnknown, + out var wrapperRedirects, + out var wrapperRedirectElements, + out failure)) + { + result = BuildResult.Fail(failure); + return true; + } + if (recursionDepth + 1 > MaxRecursionDepth) { result = BuildResult.Fail( @@ -1735,6 +1957,46 @@ private static bool TryRecurseIntoPwsh( }); } + if (expanded.Count == 0 && wrapperRedirects.Count > 0) + { + expanded.Add(new Clause + { + Operator = segment.PrecedingOperator, + Verb = new VerbChain(), + Args = Array.Empty(), + Redirects = wrapperRedirects, + Elements = wrapperRedirectElements, + IsSubshell = segment.Depth > 0, + IsCommandStringWrapped = true, + }); + } + else if (expanded.Count > 0 && wrapperRedirects.Count > 0) + { + var lastIndex = expanded.Count - 1; + var lastClause = expanded[lastIndex]; + var redirects = new List(lastClause.Redirects.Count + wrapperRedirects.Count); + redirects.AddRange(lastClause.Redirects); + redirects.AddRange(wrapperRedirects); + + var elements = new List( + lastClause.Elements.Count + wrapperRedirectElements.Count); + elements.AddRange(lastClause.Elements); + var precedingVerbCount = lastClause.Verb.Tokens.Count; + foreach (var redirectElement in wrapperRedirectElements) + { + elements.Add(redirectElement with + { + PrecedingVerbElementCount = precedingVerbCount, + }); + } + + expanded[lastIndex] = lastClause with + { + Redirects = redirects, + Elements = elements, + }; + } + result = BuildResult.Recursion(expanded); return true; } @@ -1772,14 +2034,26 @@ private static bool IsEncodedCommandParameter(string name) /// to the end of the segment body. /// private static string? ResolveCommandPayload( - List body, int paramIndex, string? colonValue, string source) + List body, + int paramIndex, + string? colonValue, + string source, + int payloadEndExclusive, + out string? failure) { + failure = null; if (colonValue is not null) { + if (paramIndex + 1 < payloadEndExclusive) + { + failure = "-Command has unsupported trailing arguments"; + return null; + } + return colonValue; } - if (paramIndex + 1 >= body.Count) + if (paramIndex + 1 >= payloadEndExclusive) { return null; } @@ -1787,11 +2061,23 @@ private static bool IsEncodedCommandParameter(string name) var next = body[paramIndex + 1]; if (next.Kind == PwshTokenKind.QuotedString) { + if (paramIndex + 2 < payloadEndExclusive) + { + failure = "-Command has unsupported trailing arguments"; + return null; + } + return next.Value; } if (next.Kind == PwshTokenKind.ScriptBlock) { + if (paramIndex + 2 < payloadEndExclusive) + { + failure = "-Command has unsupported trailing arguments"; + return null; + } + // Strip the outer { }. var v = next.Value; if (v.Length >= 2 && v[0] == '{' && v[v.Length - 1] == '}') @@ -1803,7 +2089,7 @@ private static bool IsEncodedCommandParameter(string name) } // Bare / multi-token: verbatim slice to the end of the segment body. - var last = body[body.Count - 1]; + var last = body[payloadEndExclusive - 1]; var sliceStart = next.SourceStart; var sliceEnd = last.SourceStart + last.SourceLength; if (sliceStart < 0 || sliceStart >= source.Length) @@ -1819,6 +2105,81 @@ private static bool IsEncodedCommandParameter(string name) return source.Substring(sliceStart, sliceEnd - sliceStart); } + private static int FindWrapperRedirectStart(List body, int start) + { + for (var i = start; i < body.Count; i++) + { + if (IsRedirectOperator(body[i])) + { + return i; + } + } + + return -1; + } + + private static bool TryBuildWrapperRedirects( + List body, + int redirectStart, + string source, + PwshParserOptions options, + bool workingDirectoryUnknown, + out IReadOnlyList redirects, + out IReadOnlyList elements, + out string? failure) + { + var redirectList = new List(); + var elementList = new List(); + failure = null; + if (redirectStart < 0) + { + redirects = redirectList; + elements = elementList; + return true; + } + + var i = redirectStart; + while (i < body.Count) + { + if (!IsRedirectOperator(body[i])) + { + redirects = redirectList; + elements = elementList; + failure = "pwsh command-string wrapper has unsupported tokens after a redirect"; + return false; + } + + var consumed = BuildRedirect( + body, + i, + source, + options, + workingDirectoryUnknown, + redirectList, + precedingVerbTokenCount: 0, + out var element, + out failure); + if (failure is not null) + { + redirects = redirectList; + elements = elementList; + return false; + } + + elementList.Add(element!); + i += consumed; + } + + redirects = redirectList; + elements = elementList; + return true; + } + + private static bool IsRedirectOperator(PwshToken token) => + token.Kind == PwshTokenKind.Operator + && token.OperatorText is not null + && (token.OperatorText == "<" || token.OperatorText.IndexOf('>') >= 0); + private static string? NextTokenValue(List body, int paramIndex) => paramIndex + 1 < body.Count ? body[paramIndex + 1].Value : null; diff --git a/tests/ShellSyntaxTree.Tests/Corpus/CorpusRunnerTests.cs b/tests/ShellSyntaxTree.Tests/Corpus/CorpusRunnerTests.cs index d84e844..e4b88fe 100644 --- a/tests/ShellSyntaxTree.Tests/Corpus/CorpusRunnerTests.cs +++ b/tests/ShellSyntaxTree.Tests/Corpus/CorpusRunnerTests.cs @@ -9,6 +9,8 @@ using System.Linq; using System.Text.Json; using System.Text.Json.Serialization; +using ShellSyntaxTree.Internal.Bash.Lexing; +using ShellSyntaxTree.Internal.Pwsh.Lexing; using Xunit; namespace ShellSyntaxTree.Tests.Corpus; @@ -35,6 +37,219 @@ public void Corpus_entry_parses_to_expected_ast(string shell, string fileName, C var actual = CreateParser(shell).Parse(entry.Input); AstAssert.Equal(entry.Expected!, actual, $"{shell}/{fileName}"); AssertClauseElementInvariants(actual, $"{shell}/{fileName}"); + AssertAuthoredTokenCoverage(shell, actual, $"{shell}/{fileName}"); + } + + private static void AssertAuthoredTokenCoverage( + string shell, ParsedCommand parsed, string context) + { + if (parsed.IsUnparseable) + { + return; + } + + var elements = parsed.Clauses + .SelectMany(clause => clause.Elements) + .Where(element => element.SourceStart.HasValue) + .ToArray(); + + if (shell == "bash") + { + var tokens = BashLexer.Tokenize(parsed.Source); + var directSegments = DirectBashSegments(parsed, tokens); + var segment = 0; + var redirectTargetPending = false; + foreach (var token in tokens) + { + if (RequiresBashElement(token)) + { + var isRedirectOperator = IsBashRedirectOperator(token); + var isRedirectTarget = redirectTargetPending && !isRedirectOperator; + AssertTokenCovered( + token.SourceStart, + token.SourceLength, + elements, + context, + token.Value, + !parsed.Clauses.Any(clause => clause.IsCommandStringWrapped) + || directSegments.Contains(segment) + || isRedirectOperator + || isRedirectTarget); + redirectTargetPending = isRedirectOperator; + } + + if (IsBashSegmentSeparator(token)) + { + segment++; + } + } + + return; + } + + var pwshTokens = PwshLexer.Tokenize(parsed.Source); + var pwshDirectSegments = DirectPwshSegments(parsed, pwshTokens); + var pwshSegment = 0; + var pwshRedirectTargetPending = false; + foreach (var token in pwshTokens) + { + if (RequiresPwshElement(token)) + { + var isRedirectOperator = IsPwshRedirectOperator(token); + var isRedirectTarget = pwshRedirectTargetPending && !isRedirectOperator; + AssertTokenCovered( + token.SourceStart, + token.SourceLength, + elements, + context, + token.Value, + !parsed.Clauses.Any(clause => clause.IsCommandStringWrapped) + || pwshDirectSegments.Contains(pwshSegment) + || isRedirectOperator + || isRedirectTarget); + pwshRedirectTargetPending = isRedirectOperator + && token.OperatorText is not null + && token.OperatorText.IndexOf(">&", StringComparison.Ordinal) < 0; + } + + if (IsPwshSegmentSeparator(token)) + { + pwshSegment++; + } + } + } + + private static HashSet DirectBashSegments( + ParsedCommand parsed, IReadOnlyList tokens) + { + var segments = new HashSet(); + foreach (var element in parsed.Clauses + .Where(clause => !clause.IsCommandStringWrapped) + .SelectMany(clause => clause.Elements) + .Where(element => element.SourceStart.HasValue)) + { + segments.Add(BashSegmentAt(tokens, element.SourceStart!.Value)); + } + + return segments; + } + + private static int BashSegmentAt(IReadOnlyList tokens, int sourceStart) + { + var segment = 0; + foreach (var token in tokens) + { + if (token.SourceStart >= sourceStart) + { + break; + } + + if (IsBashSegmentSeparator(token)) + { + segment++; + } + } + + return segment; + } + + private static HashSet DirectPwshSegments( + ParsedCommand parsed, IReadOnlyList tokens) + { + var segments = new HashSet(); + foreach (var element in parsed.Clauses + .Where(clause => !clause.IsCommandStringWrapped) + .SelectMany(clause => clause.Elements) + .Where(element => element.SourceStart.HasValue)) + { + segments.Add(PwshSegmentAt(tokens, element.SourceStart!.Value)); + } + + return segments; + } + + private static int PwshSegmentAt(IReadOnlyList tokens, int sourceStart) + { + var segment = 0; + foreach (var token in tokens) + { + if (token.SourceStart >= sourceStart) + { + break; + } + + if (IsPwshSegmentSeparator(token)) + { + segment++; + } + } + + return segment; + } + + private static bool RequiresBashElement(BashToken token) + { + if (token.Kind is BashTokenKind.Whitespace + or BashTokenKind.Continuation + or BashTokenKind.Comment + or BashTokenKind.UnparseableSentinel) + { + return false; + } + + return token.Kind != BashTokenKind.Operator + || token.OperatorText is ">" or ">>" or "<" or "2>" or "2>>" or "<<" or "<<-"; + } + + private static bool IsBashRedirectOperator(BashToken token) => + token.Kind == BashTokenKind.Operator + && token.OperatorText is ">" or ">>" or "<" or "2>" or "2>>" or "<<" or "<<-"; + + private static bool IsBashSegmentSeparator(BashToken token) => + (token.Kind == BashTokenKind.Operator + && token.OperatorText is "&&" or "||" or ";" or "|") + || (token.Kind == BashTokenKind.Whitespace && token.IsStatementSeparator); + + private static bool RequiresPwshElement(PwshToken token) + { + if (token.Kind is PwshTokenKind.Whitespace + or PwshTokenKind.Continuation + or PwshTokenKind.Comment + or PwshTokenKind.UnparseableSentinel) + { + return false; + } + + return token.Kind != PwshTokenKind.Operator + || token.OperatorText == "<" + || (token.OperatorText?.IndexOf('>') ?? -1) >= 0; + } + + private static bool IsPwshRedirectOperator(PwshToken token) => + token.Kind == PwshTokenKind.Operator + && (token.OperatorText == "<" || (token.OperatorText?.IndexOf('>') ?? -1) >= 0); + + private static bool IsPwshSegmentSeparator(PwshToken token) => + (token.Kind == PwshTokenKind.Operator + && token.OperatorText is "&&" or "||" or ";" or "|") + || (token.Kind == PwshTokenKind.Whitespace && token.IsStatementSeparator); + + private static void AssertTokenCovered( + int sourceStart, + int sourceLength, + IReadOnlyList elements, + string context, + string tokenValue, + bool coverageRequired) + { + var sourceEnd = sourceStart + sourceLength; + var coveringElements = elements.Count(element => + element.SourceStart!.Value <= sourceStart + && element.SourceStart.Value + element.SourceLength!.Value >= sourceEnd); + Assert.True( + coveringElements == 1 || (!coverageRequired && coveringElements == 0), + $"{context}: authored token '{tokenValue}' at {sourceStart}:{sourceLength} " + + $"is covered by {coveringElements} clause elements; expected exactly one."); } private static void AssertClauseElementInvariants(ParsedCommand parsed, string context) diff --git a/tests/ShellSyntaxTree.Tests/Corpus/bash/160_tar_info_script_paths.json b/tests/ShellSyntaxTree.Tests/Corpus/bash/160_tar_info_script_commands.json similarity index 60% rename from tests/ShellSyntaxTree.Tests/Corpus/bash/160_tar_info_script_paths.json rename to tests/ShellSyntaxTree.Tests/Corpus/bash/160_tar_info_script_commands.json index b448fec..ba4e78e 100644 --- a/tests/ShellSyntaxTree.Tests/Corpus/bash/160_tar_info_script_paths.json +++ b/tests/ShellSyntaxTree.Tests/Corpus/bash/160_tar_info_script_commands.json @@ -1,6 +1,6 @@ { - "name": "Tar info-script paths", - "input": "tar -F=./volume-helper.sh --info-script=./info-helper.sh --new-volume-script ./next-volume.sh archive.tar", + "name": "Tar info-script commands", + "input": "tar -F ./volume-helper.sh --info-script=./info-helper.sh --new-volume-script ./next-volume.sh archive", "expected": { "isUnparseable": false, "clauses": [ @@ -9,12 +9,12 @@ "verb": ["tar"], "args": [ { "raw": "-F", "kind": "Literal", "isPath": false, "isFlag": true }, - { "raw": "./volume-helper.sh", "kind": "Literal", "isPath": true, "isFlag": false, "resolved": "/work/volume-helper.sh" }, + { "raw": "./volume-helper.sh", "kind": "DynamicSkip", "isPath": false, "isFlag": false, "resolved": "__NULL__" }, { "raw": "--info-script", "kind": "Literal", "isPath": false, "isFlag": true }, - { "raw": "./info-helper.sh", "kind": "Literal", "isPath": true, "isFlag": false, "resolved": "/work/info-helper.sh" }, + { "raw": "./info-helper.sh", "kind": "DynamicSkip", "isPath": false, "isFlag": false, "resolved": "__NULL__" }, { "raw": "--new-volume-script", "kind": "Literal", "isPath": false, "isFlag": true }, - { "raw": "./next-volume.sh", "kind": "Literal", "isPath": true, "isFlag": false, "resolved": "/work/next-volume.sh" }, - { "raw": "archive.tar", "kind": "Literal", "isPath": true, "isFlag": false, "resolved": "/work/archive.tar" } + { "raw": "./next-volume.sh", "kind": "DynamicSkip", "isPath": false, "isFlag": false, "resolved": "__NULL__" }, + { "raw": "archive", "kind": "Literal", "isPath": true, "isFlag": false, "resolved": "/work/archive" } ], "redirects": [], "elements": [ @@ -30,16 +30,26 @@ "isPath": false }, { - "raw": "-F=./volume-helper.sh", - "value": "-F=./volume-helper.sh", + "raw": "-F", + "value": "-F", "role": "Argument", "sourceStart": 4, - "sourceLength": 21, + "sourceLength": 2, "precedingVerbElementCount": 1, "kind": "Literal", "isFlag": true, - "isPath": true, - "resolved": "/work/volume-helper.sh" + "isPath": false + }, + { + "raw": "./volume-helper.sh", + "value": "./volume-helper.sh", + "role": "Argument", + "sourceStart": 7, + "sourceLength": 18, + "precedingVerbElementCount": 1, + "kind": "DynamicSkip", + "isFlag": false, + "isPath": false }, { "raw": "--info-script=./info-helper.sh", @@ -48,10 +58,9 @@ "sourceStart": 26, "sourceLength": 30, "precedingVerbElementCount": 1, - "kind": "Literal", + "kind": "DynamicSkip", "isFlag": true, - "isPath": true, - "resolved": "/work/info-helper.sh" + "isPath": false }, { "raw": "--new-volume-script", @@ -71,22 +80,21 @@ "sourceStart": 77, "sourceLength": 16, "precedingVerbElementCount": 1, - "kind": "Literal", + "kind": "DynamicSkip", "isFlag": false, - "isPath": true, - "resolved": "/work/next-volume.sh" + "isPath": false }, { - "raw": "archive.tar", - "value": "archive.tar", + "raw": "archive", + "value": "archive", "role": "Argument", "sourceStart": 94, - "sourceLength": 11, + "sourceLength": 7, "precedingVerbElementCount": 1, "kind": "Literal", "isFlag": false, "isPath": true, - "resolved": "/work/archive.tar" + "resolved": "/work/archive" } ], "isSubshell": false, @@ -94,5 +102,5 @@ } ] }, - "notes": "Tar -F and both long aliases consume helper-script paths in equals and spaced forms." + "notes": "Tar helper hooks execute commands and safe-fail; the bare archive operand remains a path argument." } diff --git a/tests/ShellSyntaxTree.Tests/Corpus/bash/163_curl_data_quoted_inline_reference.json b/tests/ShellSyntaxTree.Tests/Corpus/bash/163_curl_data_quoted_inline_reference.json new file mode 100644 index 0000000..d5fd44d --- /dev/null +++ b/tests/ShellSyntaxTree.Tests/Corpus/bash/163_curl_data_quoted_inline_reference.json @@ -0,0 +1,58 @@ +{ + "name": "Curl data quoted inline reference", + "input": "curl --data=\"@request file.json\" https://example.invalid/api", + "expected": { + "isUnparseable": false, + "clauses": [ + { + "operator": "None", + "verb": ["curl"], + "args": [ + { "raw": "--data", "kind": "Literal", "isPath": false, "isFlag": true }, + { "raw": "\"@request file.json\"", "kind": "Literal", "isPath": true, "isFlag": false, "resolved": "/work/request file.json" }, + { "raw": "https://example.invalid/api", "kind": "Literal", "isPath": false, "isFlag": false } + ], + "redirects": [], + "elements": [ + { + "raw": "curl", + "value": "curl", + "role": "Verb", + "sourceStart": 0, + "sourceLength": 4, + "precedingVerbElementCount": 0, + "kind": "Literal", + "isFlag": false, + "isPath": false + }, + { + "raw": "--data=\"@request file.json\"", + "value": "--data=@request file.json", + "role": "Argument", + "sourceStart": 5, + "sourceLength": 27, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": true, + "isPath": true, + "resolved": "/work/request file.json" + }, + { + "raw": "https://example.invalid/api", + "value": "https://example.invalid/api", + "role": "Argument", + "sourceStart": 33, + "sourceLength": 27, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": false, + "isPath": false + } + ], + "isSubshell": false, + "isCommandStringWrapped": false + } + ] + }, + "notes": "An adjacent quoted inline value is one native argument and preserves curl's file-read metadata." +} diff --git a/tests/ShellSyntaxTree.Tests/Corpus/bash/164_curl_data_multiple_adjacent_fragments.json b/tests/ShellSyntaxTree.Tests/Corpus/bash/164_curl_data_multiple_adjacent_fragments.json new file mode 100644 index 0000000..24dcb5e --- /dev/null +++ b/tests/ShellSyntaxTree.Tests/Corpus/bash/164_curl_data_multiple_adjacent_fragments.json @@ -0,0 +1,58 @@ +{ + "name": "Curl data multiple adjacent fragments", + "input": "curl --data='@request'\" file.json\" https://example.invalid/api", + "expected": { + "isUnparseable": false, + "clauses": [ + { + "operator": "None", + "verb": ["curl"], + "args": [ + { "raw": "--data", "kind": "Literal", "isPath": false, "isFlag": true }, + { "raw": "'@request'\" file.json\"", "kind": "Literal", "isPath": true, "isFlag": false, "resolved": "/work/request file.json" }, + { "raw": "https://example.invalid/api", "kind": "Literal", "isPath": false, "isFlag": false } + ], + "redirects": [], + "elements": [ + { + "raw": "curl", + "value": "curl", + "role": "Verb", + "sourceStart": 0, + "sourceLength": 4, + "precedingVerbElementCount": 0, + "kind": "Literal", + "isFlag": false, + "isPath": false + }, + { + "raw": "--data='@request'\" file.json\"", + "value": "--data=@request file.json", + "role": "Argument", + "sourceStart": 5, + "sourceLength": 29, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": true, + "isPath": true, + "resolved": "/work/request file.json" + }, + { + "raw": "https://example.invalid/api", + "value": "https://example.invalid/api", + "role": "Argument", + "sourceStart": 35, + "sourceLength": 27, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": false, + "isPath": false + } + ], + "isSubshell": false, + "isCommandStringWrapped": false + } + ] + }, + "notes": "The complete adjacent fragment run becomes one native argument and one path-aware element." +} diff --git a/tests/ShellSyntaxTree.Tests/Corpus/bash/165_curl_data_unquoted_prefix_fragment.json b/tests/ShellSyntaxTree.Tests/Corpus/bash/165_curl_data_unquoted_prefix_fragment.json new file mode 100644 index 0000000..aa24479 --- /dev/null +++ b/tests/ShellSyntaxTree.Tests/Corpus/bash/165_curl_data_unquoted_prefix_fragment.json @@ -0,0 +1,58 @@ +{ + "name": "Curl data unquoted prefix fragment", + "input": "curl --data=@request\".json\" https://example.invalid/api", + "expected": { + "isUnparseable": false, + "clauses": [ + { + "operator": "None", + "verb": ["curl"], + "args": [ + { "raw": "--data", "kind": "Literal", "isPath": false, "isFlag": true }, + { "raw": "@request\".json\"", "kind": "Literal", "isPath": true, "isFlag": false, "resolved": "/work/request.json" }, + { "raw": "https://example.invalid/api", "kind": "Literal", "isPath": false, "isFlag": false } + ], + "redirects": [], + "elements": [ + { + "raw": "curl", + "value": "curl", + "role": "Verb", + "sourceStart": 0, + "sourceLength": 4, + "precedingVerbElementCount": 0, + "kind": "Literal", + "isFlag": false, + "isPath": false + }, + { + "raw": "--data=@request\".json\"", + "value": "--data=@request.json", + "role": "Argument", + "sourceStart": 5, + "sourceLength": 22, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": true, + "isPath": true, + "resolved": "/work/request.json" + }, + { + "raw": "https://example.invalid/api", + "value": "https://example.invalid/api", + "role": "Argument", + "sourceStart": 28, + "sourceLength": 27, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": false, + "isPath": false + } + ], + "isSubshell": false, + "isCommandStringWrapped": false + } + ] + }, + "notes": "An unquoted equals-value prefix joins the complete adjacent native argument." +} diff --git a/tests/ShellSyntaxTree.Tests/Corpus/bash/166_curl_data_mixed_literal_dynamic.json b/tests/ShellSyntaxTree.Tests/Corpus/bash/166_curl_data_mixed_literal_dynamic.json new file mode 100644 index 0000000..c220181 --- /dev/null +++ b/tests/ShellSyntaxTree.Tests/Corpus/bash/166_curl_data_mixed_literal_dynamic.json @@ -0,0 +1,57 @@ +{ + "name": "Curl data mixed literal dynamic", + "input": "curl --data='@$HOME'\".json\" https://example.invalid/api", + "expected": { + "isUnparseable": false, + "clauses": [ + { + "operator": "None", + "verb": ["curl"], + "args": [ + { "raw": "--data", "kind": "Literal", "isPath": false, "isFlag": true }, + { "raw": "'@$HOME'\".json\"", "kind": "DynamicSkip", "isPath": false, "isFlag": false, "resolved": "__NULL__" }, + { "raw": "https://example.invalid/api", "kind": "Literal", "isPath": false, "isFlag": false } + ], + "redirects": [], + "elements": [ + { + "raw": "curl", + "value": "curl", + "role": "Verb", + "sourceStart": 0, + "sourceLength": 4, + "precedingVerbElementCount": 0, + "kind": "Literal", + "isFlag": false, + "isPath": false + }, + { + "raw": "--data='@$HOME'\".json\"", + "value": "--data=@$HOME.json", + "role": "Argument", + "sourceStart": 5, + "sourceLength": 22, + "precedingVerbElementCount": 1, + "kind": "DynamicSkip", + "isFlag": true, + "isPath": false + }, + { + "raw": "https://example.invalid/api", + "value": "https://example.invalid/api", + "role": "Argument", + "sourceStart": 28, + "sourceLength": 27, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": false, + "isPath": false + } + ], + "isSubshell": false, + "isCommandStringWrapped": false + } + ] + }, + "notes": "Resolver-sensitive mixed quoting safe-fails instead of expanding literal bytes." +} diff --git a/tests/ShellSyntaxTree.Tests/Corpus/bash/167_curl_data_transformed_literal_dynamic.json b/tests/ShellSyntaxTree.Tests/Corpus/bash/167_curl_data_transformed_literal_dynamic.json new file mode 100644 index 0000000..3d6eaa6 --- /dev/null +++ b/tests/ShellSyntaxTree.Tests/Corpus/bash/167_curl_data_transformed_literal_dynamic.json @@ -0,0 +1,57 @@ +{ + "name": "Curl data transformed literal dynamic", + "input": "curl --data='@~'\"/secret.json\" https://example.invalid/api", + "expected": { + "isUnparseable": false, + "clauses": [ + { + "operator": "None", + "verb": ["curl"], + "args": [ + { "raw": "--data", "kind": "Literal", "isPath": false, "isFlag": true }, + { "raw": "'@~'\"/secret.json\"", "kind": "DynamicSkip", "isPath": false, "isFlag": false, "resolved": "__NULL__" }, + { "raw": "https://example.invalid/api", "kind": "Literal", "isPath": false, "isFlag": false } + ], + "redirects": [], + "elements": [ + { + "raw": "curl", + "value": "curl", + "role": "Verb", + "sourceStart": 0, + "sourceLength": 4, + "precedingVerbElementCount": 0, + "kind": "Literal", + "isFlag": false, + "isPath": false + }, + { + "raw": "--data='@~'\"/secret.json\"", + "value": "--data=@~/secret.json", + "role": "Argument", + "sourceStart": 5, + "sourceLength": 25, + "precedingVerbElementCount": 1, + "kind": "DynamicSkip", + "isFlag": true, + "isPath": false + }, + { + "raw": "https://example.invalid/api", + "value": "https://example.invalid/api", + "role": "Argument", + "sourceStart": 31, + "sourceLength": 27, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": false, + "isPath": false + } + ], + "isSubshell": false, + "isCommandStringWrapped": false + } + ] + }, + "notes": "Resolver-sensitive syntax exposed after curl's @ marker is removed still safe-fails." +} diff --git a/tests/ShellSyntaxTree.Tests/Corpus/powershell/271_tar_info_script_paths.json b/tests/ShellSyntaxTree.Tests/Corpus/powershell/271_tar_info_script_commands.json similarity index 66% rename from tests/ShellSyntaxTree.Tests/Corpus/powershell/271_tar_info_script_paths.json rename to tests/ShellSyntaxTree.Tests/Corpus/powershell/271_tar_info_script_commands.json index 8655ba6..b26c211 100644 --- a/tests/ShellSyntaxTree.Tests/Corpus/powershell/271_tar_info_script_paths.json +++ b/tests/ShellSyntaxTree.Tests/Corpus/powershell/271_tar_info_script_commands.json @@ -1,6 +1,6 @@ { - "name": "Tar info script paths", - "input": "tar -F=./volume-helper.sh --info-script=./info-helper.sh --new-volume-script ./next-volume.sh archive.tar", + "name": "Tar info script commands", + "input": "tar -F ./volume-helper.sh --info-script=./info-helper.sh --new-volume-script ./next-volume.sh archive", "expected": { "isUnparseable": false, "clauses": [ @@ -17,9 +17,8 @@ }, { "raw": "./volume-helper.sh", - "kind": "Literal", - "isPath": true, - "resolved": "C:/work/volume-helper.sh" + "kind": "DynamicSkip", + "isPath": false }, { "raw": "--info-script", @@ -28,9 +27,8 @@ }, { "raw": "./info-helper.sh", - "kind": "Literal", - "isPath": true, - "resolved": "C:/work/info-helper.sh" + "kind": "DynamicSkip", + "isPath": false }, { "raw": "--new-volume-script", @@ -39,15 +37,14 @@ }, { "raw": "./next-volume.sh", - "kind": "Literal", - "isPath": true, - "resolved": "C:/work/next-volume.sh" + "kind": "DynamicSkip", + "isPath": false }, { - "raw": "archive.tar", + "raw": "archive", "kind": "Literal", "isPath": true, - "resolved": "C:/work/archive.tar" + "resolved": "C:/work/archive" } ], "redirects": [], @@ -64,16 +61,26 @@ "isPath": false }, { - "raw": "-F=./volume-helper.sh", - "value": "-F=./volume-helper.sh", + "raw": "-F", + "value": "-F", "role": "Argument", "sourceStart": 4, - "sourceLength": 21, + "sourceLength": 2, "precedingVerbElementCount": 1, "kind": "Literal", "isFlag": true, - "isPath": true, - "resolved": "C:/work/volume-helper.sh" + "isPath": false + }, + { + "raw": "./volume-helper.sh", + "value": "./volume-helper.sh", + "role": "Argument", + "sourceStart": 7, + "sourceLength": 18, + "precedingVerbElementCount": 1, + "kind": "DynamicSkip", + "isFlag": false, + "isPath": false }, { "raw": "--info-script=./info-helper.sh", @@ -82,10 +89,9 @@ "sourceStart": 26, "sourceLength": 30, "precedingVerbElementCount": 1, - "kind": "Literal", + "kind": "DynamicSkip", "isFlag": true, - "isPath": true, - "resolved": "C:/work/info-helper.sh" + "isPath": false }, { "raw": "--new-volume-script", @@ -105,26 +111,25 @@ "sourceStart": 77, "sourceLength": 16, "precedingVerbElementCount": 1, - "kind": "Literal", + "kind": "DynamicSkip", "isFlag": false, - "isPath": true, - "resolved": "C:/work/next-volume.sh" + "isPath": false }, { - "raw": "archive.tar", - "value": "archive.tar", + "raw": "archive", + "value": "archive", "role": "Argument", "sourceStart": 94, - "sourceLength": 11, + "sourceLength": 7, "precedingVerbElementCount": 1, "kind": "Literal", "isFlag": false, "isPath": true, - "resolved": "C:/work/archive.tar" + "resolved": "C:/work/archive" } ] } ] }, - "notes": "Tar -F and both long aliases consume helper-script paths in equals and spaced forms." + "notes": "Tar helper hooks execute commands and safe-fail; the bare archive operand remains a path argument." } diff --git a/tests/ShellSyntaxTree.Tests/Corpus/powershell/274_curl_data_quoted_inline_reference.json b/tests/ShellSyntaxTree.Tests/Corpus/powershell/274_curl_data_quoted_inline_reference.json new file mode 100644 index 0000000..c927d85 --- /dev/null +++ b/tests/ShellSyntaxTree.Tests/Corpus/powershell/274_curl_data_quoted_inline_reference.json @@ -0,0 +1,71 @@ +{ + "name": "Curl data quoted inline reference", + "input": "curl --data=\u0027@C:\\payload file\u0027 https://example.invalid/api", + "expected": { + "isUnparseable": false, + "clauses": [ + { + "operator": "None", + "verb": [ + "curl" + ], + "args": [ + { + "raw": "--data", + "kind": "Literal", + "isPath": false + }, + { + "raw": "\u0027@C:\\payload file\u0027", + "kind": "Literal", + "isPath": true, + "resolved": "C:/payload file" + }, + { + "raw": "https://example.invalid/api", + "kind": "Literal", + "isPath": false + } + ], + "redirects": [], + "elements": [ + { + "raw": "curl", + "value": "curl", + "role": "Verb", + "sourceStart": 0, + "sourceLength": 4, + "precedingVerbElementCount": 0, + "kind": "Literal", + "isFlag": false, + "isPath": false + }, + { + "raw": "--data=\u0027@C:\\payload file\u0027", + "value": "--data=@C:\\payload file", + "role": "Argument", + "sourceStart": 5, + "sourceLength": 25, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": true, + "isPath": true, + "resolved": "C:/payload file" + }, + { + "raw": "https://example.invalid/api", + "value": "https://example.invalid/api", + "role": "Argument", + "sourceStart": 31, + "sourceLength": 27, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": false, + "isPath": false + } + ] + } + ] + }, + "notes": "An adjacent quoted inline value is one native argument and preserves curl\u0027s file-read metadata." +} diff --git a/tests/ShellSyntaxTree.Tests/Corpus/powershell/275_inline_colon_backtick_path.json b/tests/ShellSyntaxTree.Tests/Corpus/powershell/275_inline_colon_backtick_path.json new file mode 100644 index 0000000..c1856e0 --- /dev/null +++ b/tests/ShellSyntaxTree.Tests/Corpus/powershell/275_inline_colon_backtick_path.json @@ -0,0 +1,55 @@ +{ + "name": "Inline colon backtick path", + "input": "Remove-Item -Path:C:\\payload\u0060 file.txt", + "expected": { + "isUnparseable": false, + "clauses": [ + { + "operator": "None", + "verb": [ + "Remove-Item" + ], + "args": [ + { + "raw": "-Path", + "kind": "Literal", + "isPath": false + }, + { + "raw": "C:\\payload file.txt", + "kind": "Literal", + "isPath": true, + "resolved": "C:/payload file.txt" + } + ], + "redirects": [], + "elements": [ + { + "raw": "Remove-Item", + "value": "Remove-Item", + "role": "Verb", + "sourceStart": 0, + "sourceLength": 11, + "precedingVerbElementCount": 0, + "kind": "Literal", + "isFlag": false, + "isPath": false + }, + { + "raw": "-Path:C:\\payload\u0060 file.txt", + "value": "-Path:C:\\payload file.txt", + "role": "Argument", + "sourceStart": 12, + "sourceLength": 26, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": true, + "isPath": true, + "resolved": "C:/payload file.txt" + } + ] + } + ] + }, + "notes": "An inline cmdlet path decodes its backtick escape before resolution." +} diff --git a/tests/ShellSyntaxTree.Tests/Corpus/powershell/276_recursion_command_outer_redirect.json b/tests/ShellSyntaxTree.Tests/Corpus/powershell/276_recursion_command_outer_redirect.json new file mode 100644 index 0000000..14c2b5b --- /dev/null +++ b/tests/ShellSyntaxTree.Tests/Corpus/powershell/276_recursion_command_outer_redirect.json @@ -0,0 +1,61 @@ +{ + "name": "Recursion command outer redirect", + "input": "pwsh -Command \u0022git status\u0022 \u003E outer.txt", + "expected": { + "isUnparseable": false, + "clauses": [ + { + "operator": "None", + "verb": [ + "git", + "status" + ], + "args": [], + "redirects": [ + { + "direction": "Out", + "target": "C:/work/outer.txt" + } + ], + "elements": [ + { + "raw": "git", + "value": "git", + "role": "Verb", + "sourceStart": null, + "sourceLength": null, + "precedingVerbElementCount": 0, + "kind": "Literal", + "isFlag": false, + "isPath": false + }, + { + "raw": "status", + "value": "status", + "role": "Verb", + "sourceStart": null, + "sourceLength": null, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": false, + "isPath": false + }, + { + "raw": "\u003E outer.txt", + "value": "outer.txt", + "role": "Redirect", + "sourceStart": 27, + "sourceLength": 11, + "precedingVerbElementCount": 2, + "kind": "Literal", + "isFlag": false, + "isPath": true, + "resolved": "C:/work/outer.txt" + } + ], + "isCommandStringWrapped": true + } + ] + }, + "notes": "A redirect authored on a pwsh wrapper remains visible after command-string recursion." +} diff --git a/tests/ShellSyntaxTree.Tests/Corpus/powershell/277_curl_data_multiple_adjacent_fragments.json b/tests/ShellSyntaxTree.Tests/Corpus/powershell/277_curl_data_multiple_adjacent_fragments.json new file mode 100644 index 0000000..10ef219 --- /dev/null +++ b/tests/ShellSyntaxTree.Tests/Corpus/powershell/277_curl_data_multiple_adjacent_fragments.json @@ -0,0 +1,71 @@ +{ + "name": "Curl data multiple adjacent fragments", + "input": "curl --data=\u0027@request\u0027\u0022 file.json\u0022 https://example.invalid/api", + "expected": { + "isUnparseable": false, + "clauses": [ + { + "operator": "None", + "verb": [ + "curl" + ], + "args": [ + { + "raw": "--data", + "kind": "Literal", + "isPath": false + }, + { + "raw": "\u0027@request\u0027\u0022 file.json\u0022", + "kind": "Literal", + "isPath": true, + "resolved": "C:/work/request file.json" + }, + { + "raw": "https://example.invalid/api", + "kind": "Literal", + "isPath": false + } + ], + "redirects": [], + "elements": [ + { + "raw": "curl", + "value": "curl", + "role": "Verb", + "sourceStart": 0, + "sourceLength": 4, + "precedingVerbElementCount": 0, + "kind": "Literal", + "isFlag": false, + "isPath": false + }, + { + "raw": "--data=\u0027@request\u0027\u0022 file.json\u0022", + "value": "--data=@request file.json", + "role": "Argument", + "sourceStart": 5, + "sourceLength": 29, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": true, + "isPath": true, + "resolved": "C:/work/request file.json" + }, + { + "raw": "https://example.invalid/api", + "value": "https://example.invalid/api", + "role": "Argument", + "sourceStart": 35, + "sourceLength": 27, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": false, + "isPath": false + } + ] + } + ] + }, + "notes": "The complete adjacent fragment run becomes one native argument and one path-aware element." +} diff --git a/tests/ShellSyntaxTree.Tests/Corpus/powershell/278_recursion_empty_command_outer_redirect.json b/tests/ShellSyntaxTree.Tests/Corpus/powershell/278_recursion_empty_command_outer_redirect.json new file mode 100644 index 0000000..a6ac08e --- /dev/null +++ b/tests/ShellSyntaxTree.Tests/Corpus/powershell/278_recursion_empty_command_outer_redirect.json @@ -0,0 +1,36 @@ +{ + "name": "Recursion empty command outer redirect", + "input": "pwsh -Command \u0022\u0022 \u003E empty.txt", + "expected": { + "isUnparseable": false, + "clauses": [ + { + "operator": "None", + "verb": [], + "args": [], + "redirects": [ + { + "direction": "Out", + "target": "C:/work/empty.txt" + } + ], + "elements": [ + { + "raw": "\u003E empty.txt", + "value": "empty.txt", + "role": "Redirect", + "sourceStart": 17, + "sourceLength": 11, + "precedingVerbElementCount": 0, + "kind": "Literal", + "isFlag": false, + "isPath": true, + "resolved": "C:/work/empty.txt" + } + ], + "isCommandStringWrapped": true + } + ] + }, + "notes": "An empty wrapped payload still surfaces its outer redirect as a redirect-only clause." +} diff --git a/tests/ShellSyntaxTree.Tests/Corpus/powershell/279_curl_data_unquoted_prefix_fragment.json b/tests/ShellSyntaxTree.Tests/Corpus/powershell/279_curl_data_unquoted_prefix_fragment.json new file mode 100644 index 0000000..478b53b --- /dev/null +++ b/tests/ShellSyntaxTree.Tests/Corpus/powershell/279_curl_data_unquoted_prefix_fragment.json @@ -0,0 +1,71 @@ +{ + "name": "Curl data unquoted prefix fragment", + "input": "curl --data=@request\u0022.json\u0022 https://example.invalid/api", + "expected": { + "isUnparseable": false, + "clauses": [ + { + "operator": "None", + "verb": [ + "curl" + ], + "args": [ + { + "raw": "--data", + "kind": "Literal", + "isPath": false + }, + { + "raw": "@request\u0022.json\u0022", + "kind": "Literal", + "isPath": true, + "resolved": "C:/work/request.json" + }, + { + "raw": "https://example.invalid/api", + "kind": "Literal", + "isPath": false + } + ], + "redirects": [], + "elements": [ + { + "raw": "curl", + "value": "curl", + "role": "Verb", + "sourceStart": 0, + "sourceLength": 4, + "precedingVerbElementCount": 0, + "kind": "Literal", + "isFlag": false, + "isPath": false + }, + { + "raw": "--data=@request\u0022.json\u0022", + "value": "--data=@request.json", + "role": "Argument", + "sourceStart": 5, + "sourceLength": 22, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": true, + "isPath": true, + "resolved": "C:/work/request.json" + }, + { + "raw": "https://example.invalid/api", + "value": "https://example.invalid/api", + "role": "Argument", + "sourceStart": 28, + "sourceLength": 27, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": false, + "isPath": false + } + ] + } + ] + }, + "notes": "An unquoted equals-value prefix joins the complete adjacent native argument." +} diff --git a/tests/ShellSyntaxTree.Tests/Corpus/powershell/280_curl_data_mixed_literal_dynamic.json b/tests/ShellSyntaxTree.Tests/Corpus/powershell/280_curl_data_mixed_literal_dynamic.json new file mode 100644 index 0000000..cf4efc9 --- /dev/null +++ b/tests/ShellSyntaxTree.Tests/Corpus/powershell/280_curl_data_mixed_literal_dynamic.json @@ -0,0 +1,69 @@ +{ + "name": "Curl data mixed literal dynamic", + "input": "curl --data=\u0027@$HOME\u0027\u0022.json\u0022 https://example.invalid/api", + "expected": { + "isUnparseable": false, + "clauses": [ + { + "operator": "None", + "verb": [ + "curl" + ], + "args": [ + { + "raw": "--data", + "kind": "Literal", + "isPath": false + }, + { + "raw": "\u0027@$HOME\u0027\u0022.json\u0022", + "kind": "DynamicSkip", + "isPath": false + }, + { + "raw": "https://example.invalid/api", + "kind": "Literal", + "isPath": false + } + ], + "redirects": [], + "elements": [ + { + "raw": "curl", + "value": "curl", + "role": "Verb", + "sourceStart": 0, + "sourceLength": 4, + "precedingVerbElementCount": 0, + "kind": "Literal", + "isFlag": false, + "isPath": false + }, + { + "raw": "--data=\u0027@$HOME\u0027\u0022.json\u0022", + "value": "--data=@$HOME.json", + "role": "Argument", + "sourceStart": 5, + "sourceLength": 22, + "precedingVerbElementCount": 1, + "kind": "DynamicSkip", + "isFlag": true, + "isPath": false + }, + { + "raw": "https://example.invalid/api", + "value": "https://example.invalid/api", + "role": "Argument", + "sourceStart": 28, + "sourceLength": 27, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": false, + "isPath": false + } + ] + } + ] + }, + "notes": "Resolver-sensitive mixed quoting safe-fails instead of expanding literal bytes." +} diff --git a/tests/ShellSyntaxTree.Tests/Corpus/powershell/281_curl_data_transformed_literal_dynamic.json b/tests/ShellSyntaxTree.Tests/Corpus/powershell/281_curl_data_transformed_literal_dynamic.json new file mode 100644 index 0000000..596f155 --- /dev/null +++ b/tests/ShellSyntaxTree.Tests/Corpus/powershell/281_curl_data_transformed_literal_dynamic.json @@ -0,0 +1,69 @@ +{ + "name": "Curl data transformed literal dynamic", + "input": "curl --data=\u0027@~\u0027\u0022/secret.json\u0022 https://example.invalid/api", + "expected": { + "isUnparseable": false, + "clauses": [ + { + "operator": "None", + "verb": [ + "curl" + ], + "args": [ + { + "raw": "--data", + "kind": "Literal", + "isPath": false + }, + { + "raw": "\u0027@~\u0027\u0022/secret.json\u0022", + "kind": "DynamicSkip", + "isPath": false + }, + { + "raw": "https://example.invalid/api", + "kind": "Literal", + "isPath": false + } + ], + "redirects": [], + "elements": [ + { + "raw": "curl", + "value": "curl", + "role": "Verb", + "sourceStart": 0, + "sourceLength": 4, + "precedingVerbElementCount": 0, + "kind": "Literal", + "isFlag": false, + "isPath": false + }, + { + "raw": "--data=\u0027@~\u0027\u0022/secret.json\u0022", + "value": "--data=@~/secret.json", + "role": "Argument", + "sourceStart": 5, + "sourceLength": 25, + "precedingVerbElementCount": 1, + "kind": "DynamicSkip", + "isFlag": true, + "isPath": false + }, + { + "raw": "https://example.invalid/api", + "value": "https://example.invalid/api", + "role": "Argument", + "sourceStart": 31, + "sourceLength": 27, + "precedingVerbElementCount": 1, + "kind": "Literal", + "isFlag": false, + "isPath": false + } + ] + } + ] + }, + "notes": "Resolver-sensitive syntax exposed after curl\u0027s @ marker is removed still safe-fails." +} diff --git a/tests/ShellSyntaxTree.Tests/Parsing/BashPerVerbRulesTests.cs b/tests/ShellSyntaxTree.Tests/Parsing/BashPerVerbRulesTests.cs index 1850aa5..d1c3403 100644 --- a/tests/ShellSyntaxTree.Tests/Parsing/BashPerVerbRulesTests.cs +++ b/tests/ShellSyntaxTree.Tests/Parsing/BashPerVerbRulesTests.cs @@ -296,34 +296,38 @@ public void Native_flag_lookup_is_case_sensitive_while_verb_lookup_is_not() } [Theory] - [InlineData("git", "-c", true, false)] - [InlineData("git", "-C", true, true)] - [InlineData("curl", "-d", true, false)] - [InlineData("curl", "-D", true, true)] - [InlineData("curl", "-o", true, true)] - [InlineData("curl", "-O", false, false)] - [InlineData("curl", "--data", true, false)] - [InlineData("curl", "--dump-header", true, true)] - [InlineData("curl", "--output", true, true)] - [InlineData("wget", "-o", true, true)] - [InlineData("wget", "-O", true, true)] - [InlineData("wget", "--output-file", true, true)] - [InlineData("wget", "--output-document", true, true)] - [InlineData("tar", "-c", false, false)] - [InlineData("tar", "-C", true, true)] - [InlineData("tar", "-f", true, true)] - [InlineData("tar", "-F", true, true)] - [InlineData("tar", "--info-script", true, true)] - [InlineData("tar", "--new-volume-script", true, true)] + [InlineData("git", "-c", true, false, false)] + [InlineData("git", "-C", true, true, false)] + [InlineData("curl", "-d", true, false, false)] + [InlineData("curl", "-D", true, true, false)] + [InlineData("curl", "-o", true, true, false)] + [InlineData("curl", "-O", false, false, false)] + [InlineData("curl", "--data", true, false, false)] + [InlineData("curl", "--dump-header", true, true, false)] + [InlineData("curl", "--output", true, true, false)] + [InlineData("wget", "-o", true, true, false)] + [InlineData("wget", "-O", true, true, false)] + [InlineData("wget", "--output-file", true, true, false)] + [InlineData("wget", "--output-document", true, true, false)] + [InlineData("tar", "-c", false, false, false)] + [InlineData("tar", "-C", true, true, false)] + [InlineData("tar", "-f", true, true, false)] + [InlineData("tar", "-F", true, false, true)] + [InlineData("tar", "--info-script", true, false, true)] + [InlineData("tar", "--new-volume-script", true, false, true)] public void Native_option_binding_matrix( string verb, string flag, bool consumesValue, - bool valueIsPath) + bool valueIsPath, + bool valueIsOpaqueCommand) { Assert.True(BashVerbs.FlagsWithValue.TryGetValue(verb, out var flags)); Assert.Equal(consumesValue, flags.Contains(flag)); Assert.Equal(valueIsPath, BashPerVerbRules.ValueOfFlagIsPath(verb, flag)); + Assert.Equal( + valueIsOpaqueCommand, + BashPerVerbRules.ValueOfFlagIsOpaqueCommand(verb, flag)); } [Theory] diff --git a/tests/ShellSyntaxTree.Tests/Parsing/ClauseElementTests.cs b/tests/ShellSyntaxTree.Tests/Parsing/ClauseElementTests.cs index 51c20d0..d87768f 100644 --- a/tests/ShellSyntaxTree.Tests/Parsing/ClauseElementTests.cs +++ b/tests/ShellSyntaxTree.Tests/Parsing/ClauseElementTests.cs @@ -248,6 +248,117 @@ public void Inline_native_path_binding_remains_one_source_element() } } + [Fact] + public void Adjacent_quoted_native_binding_is_one_path_aware_element() + { + const string source = "curl --data=\"@request file.json\" https://example.invalid/api"; + foreach (var (shell, parser) in Parsers()) + { + var clause = Assert.Single(parser.Parse(source).Clauses); + var option = Assert.Single( + clause.Elements, + element => element.Raw == "--data=\"@request file.json\""); + + Assert.Equal("--data=@request file.json", option.Value); + Assert.True(option.IsFlag, shell); + Assert.True(option.IsPath, shell); + Assert.EndsWith("/request file.json", option.Resolved, StringComparison.Ordinal); + Assert.DoesNotContain(clause.Elements, element => element.Raw == "--data="); + } + } + + [Fact] + public void Complete_adjacent_fragment_run_is_one_native_element() + { + const string source = "curl --data='@request'\" file.json\" https://example.invalid/api"; + foreach (var (shell, parser) in Parsers()) + { + var clause = Assert.Single(parser.Parse(source).Clauses); + var option = Assert.Single( + clause.Elements, + element => element.Value == "--data=@request file.json"); + + Assert.Equal("--data='@request'\" file.json\"", option.Raw); + Assert.True(option.IsPath, shell); + Assert.EndsWith("/request file.json", option.Resolved, StringComparison.Ordinal); + Assert.DoesNotContain( + clause.Elements, + element => element.Raw is "'@request'" or "\" file.json\""); + } + } + + [Fact] + public void Unquoted_value_prefix_joins_adjacent_native_fragments() + { + const string source = "curl --data=@request\".json\" https://example.invalid/api"; + foreach (var (shell, parser) in Parsers()) + { + var clause = Assert.Single(parser.Parse(source).Clauses); + var option = Assert.Single( + clause.Elements, + element => element.Value == "--data=@request.json"); + + Assert.Equal("--data=@request\".json\"", option.Raw); + Assert.True(option.IsPath, shell); + Assert.EndsWith("/request.json", option.Resolved, StringComparison.Ordinal); + } + } + + [Fact] + public void Resolver_sensitive_mixed_quoting_safe_fails() + { + const string source = "curl --data='@$HOME'\".json\" https://example.invalid/api"; + foreach (var (shell, parser) in Parsers()) + { + var clause = Assert.Single(parser.Parse(source).Clauses); + var option = Assert.Single( + clause.Elements, + element => element.Value == "--data=@$HOME.json"); + + Assert.Equal(ArgKind.DynamicSkip, option.Kind); + Assert.False(option.IsPath, shell); + Assert.Null(option.Resolved); + } + + const string transformedSource = + "curl --data='@~'\"/secret.json\" https://example.invalid/api"; + foreach (var (shell, parser) in Parsers()) + { + var clause = Assert.Single(parser.Parse(transformedSource).Clauses); + var option = Assert.Single( + clause.Elements, + element => element.Value == "--data=@~/secret.json"); + + Assert.Equal(ArgKind.DynamicSkip, option.Kind); + Assert.False(option.IsPath, shell); + Assert.Null(option.Resolved); + } + } + + [Fact] + public void Tar_helper_commands_safe_fail_and_native_paths_remain_arguments() + { + const string source = "tar -F ./helper.sh archive --file out.tar"; + foreach (var (shell, parser) in Parsers()) + { + var clause = Assert.Single(parser.Parse(source).Clauses); + Assert.Equal(new[] { "tar" }, clause.Verb.Tokens); + + var helper = Assert.Single( + clause.Elements, + element => element.Value == "./helper.sh"); + Assert.Equal(ArgKind.DynamicSkip, helper.Kind); + Assert.False(helper.IsPath, shell); + Assert.Null(helper.Resolved); + + var archive = Assert.Single( + clause.Elements, + element => element.Value == "archive"); + Assert.Equal(ClauseElementRole.Argument, archive.Role); + Assert.True(archive.IsPath, shell); + } + } + [Fact] public void PowerShell_inline_cmdlet_binding_carries_bound_value_metadata() { @@ -263,6 +374,21 @@ public void PowerShell_inline_cmdlet_binding_carries_bound_value_metadata() Assert.Equal("C:/repo", option.Resolved); } + [Fact] + public void PowerShell_inline_cmdlet_binding_decodes_backtick_escapes() + { + const string source = "Remove-Item -Path:C:\\payload` file.txt"; + var clause = Assert.Single(Pwsh.Parse(source).Clauses); + var option = Assert.Single( + clause.Elements, + element => element.Role == ClauseElementRole.Argument); + + Assert.Equal("-Path:C:\\payload` file.txt", option.Raw); + Assert.Equal("-Path:C:\\payload file.txt", option.Value); + Assert.True(option.IsPath); + Assert.Equal("C:/payload file.txt", option.Resolved); + } + [Fact] public void Synthetic_cwd_attribution_is_not_a_source_element() { @@ -380,6 +506,43 @@ public void Encoded_command_elements_have_no_invented_outer_spans() Assert.All(clause.Elements, AssertSpanIsUnknown); } + [Fact] + public void PowerShell_command_wrapper_preserves_outer_redirect() + { + const string source = "pwsh -Command \"git status\" > outer.txt"; + var clause = Assert.Single(Pwsh.Parse(source).Clauses); + var redirect = Assert.Single(clause.Redirects); + var redirectElement = Assert.Single( + clause.Elements, + element => element.Role == ClauseElementRole.Redirect); + + Assert.Equal("C:/work/outer.txt", redirect.Target); + Assert.Equal("> outer.txt", redirectElement.Raw); + Assert.Equal("outer.txt", redirectElement.Value); + Assert.Equal(source.IndexOf('>'), redirectElement.SourceStart); + Assert.True(redirectElement.IsPath); + Assert.Equal("C:/work/outer.txt", redirectElement.Resolved); + Assert.All( + clause.Elements.Where(element => element.Role != ClauseElementRole.Redirect), + AssertSpanIsUnknown); + + var payload = Convert.ToBase64String(Encoding.Unicode.GetBytes("git status")); + var encodedSource = $"pwsh -EncodedCommand {payload} > encoded.txt"; + var encodedClause = Assert.Single(Pwsh.Parse(encodedSource).Clauses); + var encodedRedirect = Assert.Single( + encodedClause.Elements, + element => element.Role == ClauseElementRole.Redirect); + Assert.Equal("> encoded.txt", encodedRedirect.Raw); + Assert.Equal("C:/work/encoded.txt", encodedRedirect.Resolved); + + var emptyClause = Assert.Single(Pwsh.Parse("pwsh -Command \"\" > empty.txt").Clauses); + Assert.Empty(emptyClause.Verb.Tokens); + Assert.Equal("C:/work/empty.txt", Assert.Single(emptyClause.Redirects).Target); + Assert.Equal( + "> empty.txt", + Assert.Single(emptyClause.Elements, element => element.Role == ClauseElementRole.Redirect).Raw); + } + private static IEnumerable<(string Shell, IShellParser Parser)> Parsers() { yield return ("bash", Bash); diff --git a/tools/PwshCorpusTool/CorpusManifest.cs b/tools/PwshCorpusTool/CorpusManifest.cs index fa496a9..600ea23 100644 --- a/tools/PwshCorpusTool/CorpusManifest.cs +++ b/tools/PwshCorpusTool/CorpusManifest.cs @@ -608,15 +608,39 @@ private static string NestIex(string inner, int depth) "Curl -d consumes non-path request data while -D consumes a header-output path."), // ---- Native option hardening: value- and context-sensitive cases ---- - P("tar_info_script_paths", - "tar -F=./volume-helper.sh --info-script=./info-helper.sh " - + "--new-volume-script ./next-volume.sh archive.tar", - "Tar -F and both long aliases consume helper-script paths in equals and spaced forms."), + P("tar_info_script_commands", + "tar -F ./volume-helper.sh --info-script=./info-helper.sh " + + "--new-volume-script ./next-volume.sh archive", + "Tar helper hooks execute commands and safe-fail; the bare archive operand remains a path argument."), P("curl_data_file_references", "curl -d \"@request.json\" --data=@payload.bin https://example.invalid/api", "Curl data operands prefixed with @ read files; authored values retain the @ marker."), P("curl_data_stdin_reference", "curl -d \"@-\" https://example.invalid/api", "Curl @- reads stdin and is not a filesystem path."), + P("curl_data_quoted_inline_reference", + "curl --data='@C:\\payload file' https://example.invalid/api", + "An adjacent quoted inline value is one native argument and preserves curl's file-read metadata."), + P("inline_colon_backtick_path", + "Remove-Item -Path:C:\\payload` file.txt", + "An inline cmdlet path decodes its backtick escape before resolution."), + P("recursion_command_outer_redirect", + "pwsh -Command \"git status\" > outer.txt", + "A redirect authored on a pwsh wrapper remains visible after command-string recursion."), + P("curl_data_multiple_adjacent_fragments", + "curl --data='@request'\" file.json\" https://example.invalid/api", + "The complete adjacent fragment run becomes one native argument and one path-aware element."), + P("recursion_empty_command_outer_redirect", + "pwsh -Command \"\" > empty.txt", + "An empty wrapped payload still surfaces its outer redirect as a redirect-only clause."), + P("curl_data_unquoted_prefix_fragment", + "curl --data=@request\".json\" https://example.invalid/api", + "An unquoted equals-value prefix joins the complete adjacent native argument."), + P("curl_data_mixed_literal_dynamic", + "curl --data='@$HOME'\".json\" https://example.invalid/api", + "Resolver-sensitive mixed quoting safe-fails instead of expanding literal bytes."), + P("curl_data_transformed_literal_dynamic", + "curl --data='@~'\"/secret.json\" https://example.invalid/api", + "Resolver-sensitive syntax exposed after curl's @ marker is removed still safe-fails."), }; } From cb93a5f7230dc53618e31f614d25080e0ab62c88 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Wed, 5 Aug 2026 14:57:15 +0000 Subject: [PATCH 7/8] refactor: remove obsolete flag helper --- .../Internal/Parsing/NativeFlagSyntax.cs | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/src/ShellSyntaxTree/Internal/Parsing/NativeFlagSyntax.cs b/src/ShellSyntaxTree/Internal/Parsing/NativeFlagSyntax.cs index c2208b3..49c9ec3 100644 --- a/src/ShellSyntaxTree/Internal/Parsing/NativeFlagSyntax.cs +++ b/src/ShellSyntaxTree/Internal/Parsing/NativeFlagSyntax.cs @@ -47,22 +47,6 @@ internal static bool TrySplitEqualsFlag( return true; } - /// - /// Recognize the first fragment of an equals-form option whose value is - /// supplied by an immediately adjacent quoted or opaque token. - /// - internal static bool TryGetTrailingEqualsFlag(string raw, out string flagPart) - { - if (raw.Length < 3 || raw[0] != '-' || raw[raw.Length - 1] != '=') - { - flagPart = ""; - return false; - } - - flagPart = raw.Substring(0, raw.Length - 1); - return true; - } - /// /// Split an equals-form option while allowing an empty value prefix. This /// is used when later adjacent shell fragments complete the same argv From 279c143c30255c51eceb31edddcf89e388bbeb04 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Wed, 5 Aug 2026 15:21:01 +0000 Subject: [PATCH 8/8] docs: track native fragment extraction --- IMPLEMENTATION_PLAN.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/IMPLEMENTATION_PLAN.md b/IMPLEMENTATION_PLAN.md index 2f1f165..812cee1 100644 --- a/IMPLEMENTATION_PLAN.md +++ b/IMPLEMENTATION_PLAN.md @@ -142,6 +142,9 @@ priorities. - PowerShell script-level constructs — control flow, `function` / `class` / `enum` definitions, `param()` / `begin` / `process` / `end` blocks, `.ps1` file parsing (`SPEC.POWERSHELL.md` §18). +- [Issue #69](https://github.com/Aaronontheweb/ShellSyntaxTree/issues/69) — + extract shared native argument-fragment classification before adding a third + shell or another fragment rule; keep shell tokenization and parsing local. - Extract a shared lexer/parser core now that two parsers exist — the seam can be designed from real duplication (`SPEC.POWERSHELL.md` §18); the path-normalization helpers duplicated between `BashResolver` and