diff --git a/docs/ExtendingKQL-RoughNotes.md b/docs/ExtendingKQL-RoughNotes.md new file mode 100644 index 0000000..a80c897 --- /dev/null +++ b/docs/ExtendingKQL-RoughNotes.md @@ -0,0 +1,541 @@ +I am planning to fork this repo and I want to extend this language to support tracing related constructs. To understand this, I would like to add a new construct that applies KQL's `where` filters to Span sets instead of a trace. I am not interested in adding code generation support, however I would need syntax and semantic analysis done. How should I go about doing this and educate myself in the process? +Some relevant files and folders I notice are: +1. @microsoft/Kusto-Query-Language/files/grammar - the antlr grammar files reside here. +2. syntax analysis is here: https://github.com/microsoft/Kusto-Query-Language/tree/master/src/Kusto.Language/Parser +3. semantic analysis is here: https://github.com/microsoft/Kusto-Query-Language/tree/master/src/Kusto.Language/Parser + +Great questions! Here’s a structured approach to extend KQL for tracing constructs like Span sets, focusing on syntax and semantic analysis, with guidance for self-education: + +--- + +## 1. **Understand the Existing Architecture** + +- **Grammar Files** (`grammar/`): + These are ANTLR grammar files defining KQL's syntax. They specify how KQL constructs are parsed into a syntax tree. +- **Syntax Analysis** (`src/Kusto.Language/Parser`): + This layer forms a parse tree from input queries using the grammar rules. +- **Semantic Analysis** (same folder): + Converts the syntax tree into a semantic tree, checking meaning, types, and more. + +**Tip:** +Read the [README](https://github.com/microsoft/Kusto-Query-Language) and any docs in the grammar or parser folders if present. They often contain architecture and design details. + +--- + +## 2. **Learn About ANTLR and KQL’s Parsing Model** + +Before you extend grammar, study: +- The basics of [ANTLR](https://www.antlr.org/) (used for grammar files). +- How a grammar change propagates into C# parser code. +- How semantic analysis works in this repo: especially how expressions like `where` are handled. + +**Resources:** +- [KQL official docs](https://docs.microsoft.com/en-us/azure/data-explorer/kusto/query/) to understand query constructs. +- General ANTLR tutorials, e.g., [Getting Started with ANTLR](https://github.com/antlr/antlr4/blob/master/doc/getting-started.md). + +--- + +## 3. **Investigate How 'where' Works** + +Examine how the existing `where` statement is defined and handled: +- Find grammar rules for `where` in `grammar/Kusto.g4`. +- Trace how this rule is implemented in `Parser` classes (C#). +- See how semantic analysis recognizes and interprets filter criteria. + +**Goal:** +Understand how a new context (Span sets) could be integrated, possibly as a variation or extension of `where`. + +--- + +## 4. **Design the New Construct** + +Decide: +- Syntax: How will users specify a `where` on Spans? + e.g., `spans where ` +- Semantics: What does ‘where’ mean on Span sets vs. traces? + Do you need different semantic validations or error messages? + +--- + +## 5. **Implement Your Extension** + +1. **Grammar Change**: + - In `grammar/Kusto.g4`, add a rule for your new construct. + - Test parsing with ANTLR tools locally. + +2. **Syntax Analysis**: + - Update or augment `Parser` logic (likely in C# under `src/Kusto.Language/Parser`) to recognize your new grammar construct. + - Validate the parse tree structure. + +3. **Semantic Analysis**: + - Extend semantic validation to support Span-specific filtering. + - Ensure error messages and tooling support (e.g. IntelliSense) reflect your new construct. + +--- + +## 6. **Test and Validate** + +- Write new unit and integration tests in C# for queries involving your construct. +- Use test queries that exercise the syntax and semantics of filtering Spans. + +--- + +## 7. **Educate Yourself Further** + +1. **Read Source Code**: + - Go through files in the grammar and parser folders. + - Look for documentation or comments regarding syntax evolution or extensions. + +2. **Experiment**: + - Fork the repo and start with small grammar changes. + - Use the test suite to ensure changes don’t break existing functionality. + +3. **Community Discussion**: + - Look for issues, pull requests, or discussions related to parser and semantic changes in the repo for guidance or best practices. + +--- + +## **Summary Steps** + +1. Study KQL’s grammar and parser code, focusing on `where`. +2. Learn ANTLR basics and how grammar gets converted to C# parser code. +3. Design the new syntax for Span filtering. +4. Change the grammar, then update syntax/semantic analysis code. +5. Write tests and validate. +6. Read/document your process so you (and others) can maintain your extension. + +--- + +**Would you like help finding actual files and code snippets that define the current `where` construct, or a step-by-step guide for making a grammar change in this repo?** + +this is an abstract answer, give me a concrete one by analyzing this codebase + +Here’s a concrete, repo-specific path you can follow, using existing patterns in `microsoft/Kusto-Query-Language` as your guide. I’ll assume your “Span set where …” is a *new query operator* that behaves like `where`, but semantically over “span sets” rather than generic rows/traces. + +I’ll walk you through: + +1. How `where`/`filter` and similar operators are wired end-to-end. +2. How to add a *new* operator (syntax + semantic analysis) with minimal friction. +3. How to adapt that to “Span sets” and how to use the codebase to educate yourself. + +--- + +## 1. Understand how `where` (filter) is actually implemented + +### 1.1. Grammar: `whereOperator` rule + +In `grammar/Kql.g4` you saw: + +```antlr +whereOperator: + Keyword=(FILTER | WHERE) (Parameters+=strictQueryOperatorParameter)* Predicate=namedExpression; +``` + +This is the ANTLR grammar that recognizes `where`/`filter` pipes. + +In the C# grammar (the hand-written parser used by the main library), the equivalent is in [`QueryGrammar.cs`](https://github.com/microsoft/Kusto-Query-Language/blob/master/src/Kusto.Language/Parser/QueryGrammar.cs) under `Initialize`: + +```csharp +var FilterOperator = + Rule( + First( + Token(SyntaxKind.WhereKeyword, CompletionKind.QueryPrefix, CompletionPriority.Top), + Token(SyntaxKind.FilterKeyword).Hide()), + QueryParameterList(QueryOperatorParameters.FilterParameters, equalsNeeded: true), + Required(NamedExpression, CreateMissingExpression), + (keyword, parameters, condition) => + (QueryOperator)new FilterOperator(keyword, parameters, condition)) + .WithTag(""); +``` + +Key observations: + +- `FilterOperator` is defined as a **rule** combining: + - A keyword token (`where` / `filter`), + - A parameter list (`QueryParameterList(QueryOperatorParameters.FilterParameters, ...)`), + - A required condition expression. +- It returns a `FilterOperator` **syntax node**, which is a subclass of `QueryOperator`. + +Later in `Initialize`, the list of operators a pipe can apply is composed: + +```csharp +var PipeExpression = + ApplyOptional( + First( + ExtendOperator, + FilterOperator, + ParseOperator, + ParseWhereOperator, + ParseKvOperator, + TakeOperator, + ... +``` + +So `FilterOperator` gets plugged into the global operator set there. + +> **Lesson:** Your new “SpanWhere” operator will need: +> - A `Rule(...)` block like `FilterOperator`. +> - Inclusion in the pipe operator list. +> - A concrete syntax node type. + +### 1.2. Related patterns: `GraphWhereNodes` / `GraphWhereEdges` + +Your use case (“where-like filter but over specific structure, e.g., span sets”) is *very close* to how graph queries introduce structural `where`-style clauses. + +Look at [`QueryGrammar.cs` around the graph operators](https://github.com/microsoft/Kusto-Query-Language/blob/master/src/Kusto.Language/Parser/QueryGrammar.cs): + +```csharp +var GraphWhereNodesOperator = + Rule( + Token(SyntaxKind.GraphWhereNodesKeyword, CompletionKind.QueryPrefix), + Required(Expression, CreateMissingExpression), + (graphWhereNodesKeyword, expression) => + (QueryOperator)new GraphWhereNodesOperator(graphWhereNodesKeyword, expression) + ) + .WithTag(""); + +var GraphWhereEdgesOperator = + Rule( + Token(SyntaxKind.GraphWhereEdgesKeyword, CompletionKind.QueryPrefix), + Required(Expression, CreateMissingExpression), + (graphWhereEdgesKeyword, expression) => + (QueryOperator)new GraphWhereEdgesOperator(graphWhereEdgesKeyword, expression) + ) + .WithTag(""); +``` + +And in the more general clause context: + +```csharp +var WhereClause = + Rule( + Token(SyntaxKind.WhereKeyword), + Required(Expression, CreateMissingExpression), + (keyword, expression) => + new WhereClause(keyword, expression)); +``` + +> **Lesson:** There are multiple *flavors* of `where`: +> - The standard `where`/`filter` pipe operator (`FilterOperator` → `FilterOperator` syntax node). +> - Specialized operators like `GraphWhereNodesOperator`, `GraphWhereEdgesOperator`. +> - Clause-level `WhereClause` (e.g., in `join`). + +Your Span-filtering operator will likely look structurally like `GraphWhereNodesOperator`: a specific keyword and an expression, with specialized semantics. + +--- + +## 2. Understand the semantic side: binding and type checks + +Syntax is only half the story. Semantic analysis is mostly in `src/Kusto.Language/Binder/Binder_NodeBinder.cs`, via `NodeBinder : SyntaxVisitor`. + +Example: look at how `JoinWhereClause` is semantically checked: + +```csharp +public override SemanticInfo VisitJoinOperator(JoinOperator node) +{ + ... + switch (node.Condition) + { + case JoinOnClause c: + ... + break; + case JoinWhereClause c: + _binder.CheckIsExactType(c.Expression, ScalarTypes.Bool, diagnostics); + break; + default: + diagnostics.Add(DiagnosticFacts.GetMissingJoinOnClause().WithLocation(node)); + break; + } + ... +} +``` + +For parse-related operators, there is a pattern too: + +```csharp +public override SemanticInfo VisitParseWhereOperator(ParseWhereOperator node) +{ + return ParseVisitCommon(node, node.Expression, node.Patterns, node.Parameters); +} + +public override SemanticInfo VisitParseOperator(ParseOperator node) +{ + return ParseVisitCommon(node, node.Expression, node.Patterns, node.Parameters); +} +``` + +> **Lesson:** +> - Every syntax node that needs semantic info gets a `VisitXxx` override; +> - Common checks (types, parameter validation, etc.) are factored into helpers like `ParseVisitCommon` and `CheckIsExactType`. + +You’ll do something similar for SpanWhere: add a `VisitSpanWhereOperator` that: +- Ensures the predicate expression is of boolean type; +- Validates that the input type is a “Span set” (however that’s modeled in your fork); +- Produces an appropriate output type. + +--- + +## 3. How to introduce a *new span-specific where operator* step-by-step + +Let’s walk through concrete modifications you’d make for, say, a `spanwhere` operator that applies a predicate to Span sets. + +I’ll call the operator `SpanWhereOperator` and the keyword `spanwhere` for illustration—you can adjust naming. + +### 3.1. Define the syntax node (via `SyntaxNodeInfos`) + +New nodes are defined declaratively in [`src/Kusto.Language.Generators/SyntaxNodeInfos.cs`](https://github.com/microsoft/Kusto-Query-Language/blob/master/src/Kusto.Language.Generators/SyntaxNodeInfos.cs). Note how `ParseWhereOperator` is declared: + +```csharp +new SyntaxNodeInfo +{ + Name = "ParseWhereOperator", + Doc = "", + Base = "QueryOperator", + Sealed = true, + Kind = "ParseWhereOperator", + Properties = new [] + { + new SyntaxNodeProperty { Name = "ParseKeyword", Type = "SyntaxToken", Completion="Keyword" }, + new SyntaxNodeProperty { Name = "Parameters", Type = "SyntaxList", Completion="None"}, + new SyntaxNodeProperty { Name = "Expression", Type = "Expression", Completion="Scalar"}, + new SyntaxNodeProperty { Name = "WithKeyword", Type = "SyntaxToken", Completion="Keyword" }, + new SyntaxNodeProperty { Name = "Patterns", Type = "SyntaxList", Completion="Syntax"} + } +}, +``` + +You’ll add something similar: + +```csharp +new SyntaxNodeInfo +{ + Name = "SpanWhereOperator", + Doc = "Filters span sets using a boolean predicate.", + Base = "QueryOperator", + Sealed = true, + Kind = "SpanWhereOperator", + Properties = new [] + { + new SyntaxNodeProperty { Name = "SpanWhereKeyword", Type = "SyntaxToken", Completion="Keyword" }, + new SyntaxNodeProperty { Name = "Expression", Type = "Expression", Completion="Boolean" }, + // Optionally Parameters if you want: + // new SyntaxNodeProperty { Name = "Parameters", Type = "SyntaxList", Completion="None"}, + } +}, +``` + +Then you’d run the syntax generator (in the original repo they use a generator project) to produce the concrete `SpanWhereOperator` class in `Syntax` and update visitors. In your fork you can also pattern after existing generated files if you don’t want to automate immediately, but staying consistent with the generator is better. + +### 3.2. Add a new `SyntaxKind` + +In [`SyntaxKind.cs`](https://github.com/microsoft/Kusto-Query-Language/blob/master/src/Kusto.Language/Syntax/SyntaxKind.cs), find the section near: + +```csharp +GraphMarkComponentsOperator, +GraphWhereNodesOperator, +GraphWhereEdgesOperator, +GraphToTableOperator, +... +ParseOperator, +ParseWhereOperator, +ParseKvWithClause, +ParseKvOperator, +... +``` + +Add: + +```csharp +SpanWhereOperator, +``` + +Then in [`SyntaxFacts.cs`](https://github.com/microsoft/Kusto-Query-Language/blob/master/src/Kusto.Language/Syntax/SyntaxFacts.cs), in the `static` initialization table where `ParseWhereOperator` is registered: + +```csharp +new SyntaxData(SyntaxKind.ParseOperator, "", SyntaxCategory.Node), +new SyntaxData(SyntaxKind.ParseWhereOperator, "", SyntaxCategory.Node), +... +// Add: +new SyntaxData(SyntaxKind.SpanWhereOperator, "", SyntaxCategory.Node), +``` + +This wires the kind into the central metadata. + +### 3.3. Wire the operator in the C# grammar (`QueryGrammar`) + +Follow the style of `FilterOperator` or `GraphWhereNodesOperator`. For example, in `QueryGrammar.Initialize`: + +```csharp +var SpanWhereOperator = + Rule( + Token(SyntaxKind.SpanWhereKeyword, CompletionKind.QueryPrefix), + Required(Expression, CreateMissingExpression), + (keyword, expression) => + (QueryOperator)new SpanWhereOperator(keyword, expression)) + .WithTag(""); +``` + +If you want it as a normal pipe operator (e.g., `| spanwhere `), you must then plug this rule into the list of operators in the big `First(...)` that defines pipe operators: + +```csharp +var PipeExpression = + ApplyOptional( + First( + ExtendOperator, + FilterOperator, + ParseOperator, + ParseWhereOperator, + ParseKvOperator, + TakeOperator, + ... + GraphWhereNodesOperator, + GraphWhereEdgesOperator, + // add: + SpanWhereOperator, + ... + ), + ... + ); +``` + +You’ll also need a token kind and lexer support for `SpanWhereKeyword`: + +- In `SyntaxKind` you’ll probably add `SpanWhereKeyword` as a token, or if you go via ANTLR you’d add a new keyword token and map it. You can follow how `GraphWhereNodesKeyword` / `GraphWhereEdgesKeyword` are declared and recognized. + +### 3.4. (Optional) Update ANTLR grammar + +If you care about the ANTLR side (used by the Java implementation and tools in `grammar/`): + +- Add a `SPANWHERE` keyword token to `KqlLexer.g4` (or equivalent). +- Add a `spanWhereOperator` rule or extend `whereOperator` or `afterPipeOperator` rules similarly to how other operators are expressed there. + +Use the existing `whereOperator` and `graphWhere...` constructs in `Kql.g4` as patterns. + +--- + +## 4. Add semantic analysis for `SpanWhereOperator` + +Now wire your new syntax node into the binder in `Binder_NodeBinder.cs`. + +1. Find the `NodeBinder` class: + + ```csharp + private class NodeBinder : SyntaxVisitor + { + ... + public override SemanticInfo VisitParseWhereOperator(ParseWhereOperator node) + { + return ParseVisitCommon(node, node.Expression, node.Patterns, node.Parameters); + } + ... + } + ``` + +2. Add a new override: + + ```csharp + public override SemanticInfo VisitSpanWhereOperator(SpanWhereOperator node) + { + // Example: ensure the expression is boolean + var diagnostics = new DiagnosticList(); + _binder.CheckIsExactType(node.Expression, ScalarTypes.Bool, diagnostics); + + // TODO: check input is a span set type, and compute output type + var inputType = _binder.GetCurrentRowScope().Type; // pattern after other operators + var resultType = inputType; // if you preserve the same schema + + return new SemanticInfo(resultType, diagnostics); + } + ``` + +3. Optionally, add more span-specific validation: + - Ensure the predicate only references allowed span columns. + - Enforce that the operator appears only after a “span-producing” operator. + +To learn how to do that, inspect how graph operators do it. For example, around `VisitGraphWhereNodesOperator` / `VisitGraphWhereEdgesOperator` in `Binder_NodeBinder.cs` (pattern to copy: they ensure correct input type and that the predicate is boolean). + +--- + +## 5. Use existing patterns to educate yourself + +Here is how I’d use this repo to teach myself the full lifecycle of a new construct, in a very targeted way: + +1. **Pick one existing operator that looks like what you want** + `GraphWhereNodesOperator` and `GraphWhereEdgesOperator` are close analogs. + +2. **Trace it end-to-end:** + + - `SyntaxKind` entry (`GraphWhereNodesOperator`, `GraphWhereNodesKeyword`). + - `SyntaxFacts` entry for the node. + - `SyntaxNodeInfos` entry defining its properties. + - `QueryGrammar` rule constructing the syntax node. + - `QueryParser` helper (for some operators, the generated parser uses custom methods like `ParseGraphWhereEdgesOperator`; you saw an example for `GraphWhereEdgesOperator`). + - `Binder_NodeBinder.VisitGraphWhereNodesOperator` for semantics. + - `QueryOperatorParameters` if it has parameters. + + Read these files for that *one* operator, line-by-line, until you’re comfortable. + +3. **Clone the pattern:** + + - Implement `SpanWhereOperator` by literally copying `GraphWhereNodesOperator` and doing a careful rename, changing behavior only where semantics differ. + - Keep your changes minimal and incremental: first get the syntax recognized; only then layer semantics. + +4. **Add tests:** + + Look at existing tests for graph and filter operators. Add: + + - Parse tests: does `T | spanwhere SomeColumn > 0` produce a `SpanWhereOperator` node with the right children? + - Semantic tests: when the input “table” is a span set, does your binder produce the expected type or diagnostics? + +--- + +## 6. How to approach your tracing/Span design specifically + +You said: + +> “new construct that applies KQL’s `where` filters to Span sets instead of a trace.” + +Concretely, you need to answer: + +- **What is a “Span set” in this syntax tree / binder world?** + - New `TableSymbol` with specific columns (`SpanId`, `TraceId`, `StartTime`, `EndTime`, …)? + - A new `TypeSymbol`? +- **How does a user obtain a Span set before calling your operator?** + - Another operator (e.g., `makespans`, `tracespan`, `fromtrace`), or a dedicated table? + +Where this flows into the code: + +- In your binder, you’ll check the **row scope** type when visiting `SpanWhereOperator`: + - If it’s not a known “Span set” shape, issue a diagnostic. +- The predicate itself is just a standard KQL expression; use the same checks as `FilterOperator` (boolean expression, accessible columns). + +You can inspect how graph operators treat their “graph input” vs standard tables; your span design can mirror that relationship. + +--- + +## 7. Minimal concrete checklist for your fork + +If you want a very practical “to-do” list: + +1. **Understand by reading**: + - `QueryGrammar.cs`: `FilterOperator`, `GraphWhereNodesOperator`, `GraphWhereEdgesOperator`, `WhereClause`, `JoinWhereClause`. + - `Binder_NodeBinder.cs`: `VisitJoinOperator` (look at `JoinWhereClause`), `VisitGraphWhereNodesOperator` / `VisitGraphWhereEdgesOperator`, `VisitFilterOperator` (if present). + - `SyntaxNodeInfos.cs`: entries for those operators. + - `SyntaxKind.cs` + `SyntaxFacts.cs`: entries for those operators. + +2. **Design your operator**: choose name, keyword, parameters, and what input/output types it expects. + +3. **Implement syntax**: + - Add new entries in `SyntaxKind`, `SyntaxFacts`, `SyntaxNodeInfos`. + - Add the grammar rule in `QueryGrammar` and plug it into the pipe operator list. + - (If needed) update ANTLR grammar in `grammar/Kql.g4` and lexer. + +4. **Implement semantics**: + - Add `VisitSpanWhereOperator` in `Binder_NodeBinder`. + - Use `CheckIsExactType` for the predicate and type-check the input as a Span set. + +5. **Add tests** to ensure: + - Parsing works for your span-where syntax. + - Semantics behave correctly on valid and invalid inputs. + +If you share the specific surface syntax you have in mind for Span sets (e.g. example queries), I can map that directly into the exact spots in `QueryGrammar` and sketch the concrete rule and binder logic for those examples. \ No newline at end of file diff --git a/docs/KQLPublicAPIs.md b/docs/KQLPublicAPIs.md new file mode 100644 index 0000000..de6af4e --- /dev/null +++ b/docs/KQLPublicAPIs.md @@ -0,0 +1,116 @@ +## Repos: +1. [KQL](https://github.com/microsoft/Kusto-Query-Language) +1. [Extensions and tools](https://github.com/mattwar/Kusto.Toolkit) + +## Quick background on compiler phases +```mermaid +flowchart TD + Q[Query text or command text] + G[GlobalState with database and functions] + + subgraph Frontend + L[Lexer] + P[Parser] + AST[SyntaxTree] + end + + subgraph SemanticAnalysis + B[Binder] + ST[Semantic info] + RT[Result type] + end + + Q --> L --> P --> AST + AST --> B --> ST --> RT + G --> B +``` + +**Legend / phases:** + +1. Lexing: Lexer turns raw query text into tokens. + +1. Parsing: QueryParser / CommandGrammar builds a SyntaxTree (AST). + +1. Semantic analysis (binding): + * Binder walks the syntax tree using GlobalState. + * Resolves symbols (tables, columns, functions). + * Computes ResultType and other type info (using TypeFacts, conversions, promotions, etc.). + +1. Public API: + * KustoCode.Parse = lex + parse (no semantics). + * KustoCode.ParseAndAnalyze / Analyze = lex + parse + semantic bind. + +## Namespaces with public APIs +```csharp +using Kusto.Language; +using Kusto.Language.Symbols; // Semantic APIs +using Kusto.Language.Syntax; // Syntax APIs +``` +💡 Hint: if you are code reading and looking for some public API, restrict your search to files with these namespaces. Rest are internal. + +## APIs: +[High level ReadMe for KQL syntax and semantic APIs](https://github.com/microsoft/Kusto-Query-Language/blob/master/src/Kusto.Language/readme.md) + +I cover more depth below. +### Parsing: +The compiler performs Syntax analysis on the input query string and transforms a valid query into a Syntax tree. Syntax tree is made of syntax nodes. This phase checks for structural correctness and not for semantic correctness. That is, a query could be structurally correct but may not have any meaning and can still pass this phase of compilation. + +```csharp +// parse only +var query = "T | project a = a + b | where a > 10.0"; +var code = KustoCode.Parse(query); + +// check parse errors +var diagnostics = code.GetDiagnostics(); +``` + +### Semantic analysis/Binding: +The meaning of the query is verified during Semantic analysis and at the end of this phase, either every node has a semantic info attached to it, or has diagnostics on why it failed semantic checks. + +When variables, columns (syntax nodes in general) are bound to their definitions, we get `symbols`. Symbols contain semantic information about the syntax node. This is produced when the compiler performs semantic analysis. + +[sourcecode: Symbol](https://github.com/microsoft/Kusto-Query-Language/blob/master/src/Kusto.Language/Symbols/Symbol.cs) + +[sourcecode: Symbol Kinds](https://github.com/microsoft/Kusto-Query-Language/blob/master/src/Kusto.Language/Symbols/SymbolKind.cs) + +#### How to use the API +```csharp +var globals = GlobalState.Default.WithDatabase( + new DatabaseSymbol("db", + new TableSymbol("T", "(a: real, b: real)"))); + +var query = "T | project a = a + b | where a > 10.0"; +var code = KustoCode.ParseAndAnalyze(query, globals); +``` + +Now when you navigate the syntax tree you can access the `ReferencedSymbol` and `ResultType` properties that tell you what is being referenced and the type of any expression. + +[sourcecode: Public API for `ReferencedSymbol`](https://github.com/microsoft/Kusto-Query-Language/blob/master/src/Kusto.Language/Syntax/SyntaxNode_Semantics.cs#L19) + +Similar to Syntax errors, there could be semantic errors in the query. i.e, operations which do not have a valid meaning and no automatic type promotions exist. +Get the `Root` of the `SyntaxTree` and look for `SemanticDiagnostics`. + +[sourcecode: Check for Semantic Errors](https://github.com/microsoft/Kusto-Query-Language/blob/343d194a104ee92d11ddfa90e4bce2be86a65d71/src/Kusto.Language/Syntax/SyntaxNode_Semantics.cs#L76) + +### Types +**Scalar** : a scalar is a single value (or a single-typed value), not a collection of rows or columns. + +**Tabular** : a tabular type is a table-like shape (rows + columns). + +#### Type Promotions +Type promotions are automatically performed during semantic analysis. So `ResultType` from a `Symbol` should give you the correct type after promotions etc. + +[sourcecode: Various kinds of conversions](https://github.com/microsoft/Kusto-Query-Language/blob/master/src/Kusto.Language/Symbols/Conversion.cs) + +[sourcecode: API to get common type among a set of types](https://github.com/microsoft/Kusto-Query-Language/blob/343d194a104ee92d11ddfa90e4bce2be86a65d71/src/Kusto.Language/Symbols/TypeFacts.cs#L67) + +### Other Useful APIs not covered above: +[sourcecode: Type Facts](https://github.com/microsoft/Kusto-Query-Language/blob/master/src/Kusto.Language/Symbols/TypeFacts.cs) + +[sourcecode: Scalars](https://github.com/microsoft/Kusto-Query-Language/blob/master/src/Kusto.Language/Symbols/ScalarSymbol.cs) + +### CodeGen +You can completely ignore codegen for the purposes of trace store as we implement `where`, `project` ourselves and delegate the rest to `DataFusion`. Including this subsection only for completeness. + +## Further help +Invoke Github copilot, reference the repo listed at the top and ask questions. diff --git a/src/AST/CommonAST.cs b/src/AST/CommonAST.cs index 735b7c6..6c561a9 100644 --- a/src/AST/CommonAST.cs +++ b/src/AST/CommonAST.cs @@ -16,7 +16,12 @@ public enum NodeKind ParenthesizedExpression, SpecialOperatorExpression, WildcardExpression, - PathExpression + PathExpression, + + // Phase 1: Aggregation support + FieldReference, + NamedExpression, + CompositeAggregation } /// @@ -309,6 +314,164 @@ public class PathExpression : Expression public required string Path { get; set; } } +#region Phase 1: Aggregation Support + +/// +/// Field categorization for different data sources +/// +public enum FieldType +{ + Intrinsic, // Built-in fields (duration, name, etc.) + Attribute, // Custom attributes + Metadata, // System metadata + Computed // Derived/computed fields +} + +/// +/// Data types for field validation +/// +public enum DataType +{ + String, + Integer, + Float, + Boolean, + Duration, + DateTime, + Array, + Object, + Unknown +} + +/// +/// Phase 1 aggregate function types (implementation priority) +/// +public enum AggregateFunction +{ + // Phase 1: Core 5 functions + Count, + Sum, + Average, + Minimum, + Maximum + + // Phase 2: Additional functions (design only, not implemented) + // StandardDeviation, Variance, Percentile, DistinctCount, + // MakeList, MakeSet, ArgumentMax, ArgumentMin, Any +} + +/// +/// Enhanced field reference with namespace support for cross-language compatibility +/// Inherits from Identifier and adds aggregation-specific metadata +/// +public class FieldReference : Identifier +{ + public override NodeKind NodeKind => NodeKind.FieldReference; + + /// + /// Field type for validation and optimization + /// + public FieldType FieldType { get; set; } = FieldType.Attribute; + + /// + /// Data type of the field + /// + public DataType? DataType { get; set; } + + /// + /// Whether this field is required for the operation + /// + public bool IsRequired { get; set; } = true; +} + +/// +/// Represents a named expression used in project, extend, summarize and other operators +/// Supports both single names and multiple names: name = expr, (name1, name2) = expr +/// +public class NamedExpression : Expression +{ + public override NodeKind NodeKind => NodeKind.NamedExpression; + + /// + /// Single result name (most common case) + /// + public string? Name { get; set; } + + /// + /// Multiple result names for tuple destructuring: (name1, name2) = expr + /// + public List? Names { get; set; } + + /// + /// The expression being named + /// + public required Expression Expression { get; set; } + + /// + /// True if this expression has explicit naming + /// + public bool IsNamed => Name != null || (Names != null && Names.Count > 0); +} + +/// +/// Individual aggregate operation within CompositeAggregationNode +/// +public class AggregateOperationNode +{ + /// Phase 1: Only these 5 functions + public required AggregateFunction Function { get; set; } + + /// Field to aggregate (null for count()) + public FieldReference? Field { get; set; } + + /// KQL-specific: Result column name + public string? ResultName { get; set; } + + /// Source expression for complex aggregations + public Expression? SourceExpression { get; set; } +} + +/// +/// Unified node for aggregation operations from both KQL and TraceQL +/// Handles grouping + aggregations in a single operation +/// Supports group-only, aggregate-only, and mixed operations +/// +public class CompositeAggregationNode : OperationNode +{ + public override NodeKind NodeKind => NodeKind.CompositeAggregation; + + /// + /// List of fields to group by (optional) + /// Empty list = no grouping + /// + public List GroupByFields { get; set; } = new List(); + + /// + /// List of aggregation operations (optional) + /// Empty list = group-only operation + /// + public List Aggregations { get; set; } = new List(); + + /// + /// Source language context for translation + /// + public string SourceLanguage { get; set; } = "Unknown"; + + /// + /// Validation: Must have either grouping OR aggregations (or both) + /// + public bool IsValid => GroupByFields.Count > 0 || Aggregations.Count > 0; + + /// + /// Operation classification + /// + public bool IsGroupOnly => Aggregations.Count == 0 && GroupByFields.Count > 0; + public bool IsAggregateOnly => Aggregations.Count > 0 && GroupByFields.Count == 0; + public bool IsMixed => Aggregations.Count > 0 && GroupByFields.Count > 0; +} + +#endregion + /// /// Builder for creating AST nodes /// @@ -443,6 +606,156 @@ public static SpecialOperatorExpression CreateSpecialOperatorExpression(Expressi Right = right }; } + + #region Phase 1: Aggregation Builder Methods + + /// + /// Creates a field reference with namespace support + /// + public static FieldReference CreateFieldReference( + string name, + string? nameSpace = null, + FieldType fieldType = FieldType.Attribute, + DataType? dataType = null) + { + return new FieldReference + { + Name = name, + Namespace = nameSpace, + FieldType = fieldType, + DataType = dataType + }; + } + + /// + /// Creates a named expression + /// + public static NamedExpression CreateNamedExpression( + Expression expression, + string? name = null, + List? names = null) + { + return new NamedExpression + { + Expression = expression, + Name = name, + Names = names + }; + } + + /// + /// Creates an aggregate operation node + /// + public static AggregateOperationNode CreateAggregateOperation( + AggregateFunction function, + FieldReference? field = null, + string? resultName = null, + Expression? sourceExpression = null) + { + return new AggregateOperationNode + { + Function = function, + Field = field, + ResultName = resultName, + SourceExpression = sourceExpression + }; + } + + /// + /// Creates a composite aggregation node (unified for both KQL and TraceQL) + /// + public static CompositeAggregationNode CreateCompositeAggregation( + List? groupByFields = null, + List? aggregations = null, + string sourceLanguage = "Unknown") + { + return new CompositeAggregationNode + { + GroupByFields = groupByFields ?? new List(), + Aggregations = aggregations ?? new List(), + SourceLanguage = sourceLanguage + }; + } + + /// + /// Creates a group-only composite aggregation (KQL: summarize by fields) + /// + public static CompositeAggregationNode CreateGroupOnlyAggregation( + List groupByFields, + string sourceLanguage = "KQL") + { + return new CompositeAggregationNode + { + GroupByFields = groupByFields, + Aggregations = new List(), // Empty = group-only + SourceLanguage = sourceLanguage + }; + } + + /// + /// Creates an aggregate-only composite aggregation (KQL: summarize aggregates) + /// + public static CompositeAggregationNode CreateAggregateOnlyAggregation( + List aggregations, + string sourceLanguage = "KQL") + { + return new CompositeAggregationNode + { + GroupByFields = new List(), // Empty = no grouping + Aggregations = aggregations, + SourceLanguage = sourceLanguage + }; + } + + /// + /// Creates a count aggregate operation + /// + public static AggregateOperationNode CreateCountOperation(string? resultName = null) + { + return CreateAggregateOperation(AggregateFunction.Count, null, resultName); + } + + /// + /// Creates a sum aggregate operation + /// + public static AggregateOperationNode CreateSumOperation( + FieldReference field, + string? resultName = null) + { + return CreateAggregateOperation(AggregateFunction.Sum, field, resultName); + } + + /// + /// Creates an average aggregate operation + /// + public static AggregateOperationNode CreateAverageOperation( + FieldReference field, + string? resultName = null) + { + return CreateAggregateOperation(AggregateFunction.Average, field, resultName); + } + + /// + /// Creates a minimum aggregate operation + /// + public static AggregateOperationNode CreateMinimumOperation( + FieldReference field, + string? resultName = null) + { + return CreateAggregateOperation(AggregateFunction.Minimum, field, resultName); + } + + /// + /// Creates a maximum aggregate operation + /// + public static AggregateOperationNode CreateMaximumOperation( + FieldReference field, + string? resultName = null) + { + return CreateAggregateOperation(AggregateFunction.Maximum, field, resultName); + } + + #endregion } /// @@ -587,4 +900,4 @@ public static QueryNode SpansOnlyFilterExample() return query; } -} \ No newline at end of file +} diff --git a/src/AST/KqlToCommonAstVisitor.cs b/src/AST/KqlToCommonAstVisitor.cs index 1d25f7b..a2bfa54 100644 --- a/src/AST/KqlToCommonAstVisitor.cs +++ b/src/AST/KqlToCommonAstVisitor.cs @@ -43,6 +43,9 @@ public void Visit(SyntaxNode node) case SyntaxKind.FilterOperator: VisitFilterOperator(node as FilterOperator); break; + case SyntaxKind.SummarizeOperator: + VisitSummarizeOperator(node as SummarizeOperator); + break; // Handle all binary expression types case SyntaxKind.EqualExpression: case SyntaxKind.NotEqualExpression: @@ -74,6 +77,9 @@ public void Visit(SyntaxNode node) case SyntaxKind.ParenthesizedExpression: VisitParenthesizedExpression(node as Kusto.Language.Syntax.ParenthesizedExpression); break; + case SyntaxKind.FunctionCallExpression: + VisitFunctionCallExpression(node as FunctionCallExpression); + break; default: // other node types are not yet implemented, just default to visiting the children for now VisitChildren(node); @@ -290,37 +296,37 @@ private void VisitParenthesizedExpression(Kusto.Language.Syntax.ParenthesizedExp // } // } - // private void VisitFunctionCallExpression(Kusto.Language.Syntax.FunctionCallExpression node) - // { - // if (node == null) return; + private void VisitFunctionCallExpression(FunctionCallExpression node) + { + if (node == null) return; - // string functionName = "unknown"; - // if (node.Name is Kusto.Language.Syntax.NameReference nameRef) - // { - // functionName = nameRef.Name.Text; - // } + string functionName = "unknown"; + if (node.Name is NameReference nameRef) + { + functionName = nameRef.Name.SimpleName; + } - // var arguments = new List(); + var arguments = new List(); - // // Process arguments - // if (node.ArgumentList != null) - // { - // foreach (var arg in node.ArgumentList.Expressions) - // { - // Visit(arg); - // if (_expressionStack.Count > 0) - // { - // arguments.Add(_expressionStack.Pop()); - // } - // } - // } + // Process arguments + if (node.ArgumentList != null) + { + foreach (var arg in node.ArgumentList.Expressions) + { + Visit(arg.Element); + if (_expressionStack.Count > 0) + { + arguments.Add(_expressionStack.Pop()); + } + } + } - // // Reverse the arguments since we process them in reverse order - // arguments.Reverse(); + // Reverse the arguments since we process them in reverse order + arguments.Reverse(); - // var callExpr = AstBuilder.CreateCallExpression(functionName, arguments); - // _expressionStack.Push(callExpr); - // } + var callExpr = AstBuilder.CreateCallExpression(functionName, arguments); + _expressionStack.Push(callExpr); + } // private void VisitInExpression(Kusto.Language.Syntax.InExpression node) // { @@ -358,6 +364,28 @@ private void VisitParenthesizedExpression(Kusto.Language.Syntax.ParenthesizedExp // } // } + #region Phase 1: Aggregation Support + + /// + /// Visits a KQL summarize operator and creates CompositeAggregationNode + /// TODO: Implement full summarize operator parsing once KQL syntax structure is verified + /// + private void VisitSummarizeOperator(SummarizeOperator node) + { + if (node == null) return; + + // For now, create a placeholder aggregation node + // TODO: Parse actual summarize structure from KQL syntax tree + var compositeAggregation = AstBuilder.CreateCompositeAggregation( + null, // groupByFields - to be implemented + null, // aggregations - to be implemented + "KQL"); + + _rootNode.Operations.Add(compositeAggregation); + } + + #endregion + // Helper method to map KQL operators to CommonAST BinaryOperatorKind private BinaryOperatorKind MapKqlOperatorToCommonAST(string op) { @@ -379,4 +407,4 @@ private BinaryOperatorKind MapKqlOperatorToCommonAST(string op) default: throw new NotSupportedException($"Unsupported binary operator: {op}"); } } -} \ No newline at end of file +} diff --git a/tests/CommonAST.Tests/AggregationTests.cs b/tests/CommonAST.Tests/AggregationTests.cs new file mode 100644 index 0000000..d8e1fa7 --- /dev/null +++ b/tests/CommonAST.Tests/AggregationTests.cs @@ -0,0 +1,274 @@ +using System; +using System.Collections.Generic; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using CommonAST; + +namespace CommonAST.Tests +{ + [TestClass] + public class AggregationTests + { + #region Phase 1: Basic AST Node Tests + + [TestMethod] + public void CreateFieldReference_WithNamespace_CreatesCorrectly() + { + // Arrange & Act + var fieldRef = AstBuilder.CreateFieldReference("Duration", "event", FieldType.Attribute, DataType.Integer); + + // Assert + Assert.IsNotNull(fieldRef); + Assert.AreEqual(NodeKind.FieldReference, fieldRef.NodeKind); + Assert.AreEqual("Duration", fieldRef.Name); + Assert.AreEqual("event", fieldRef.Namespace); + Assert.AreEqual(FieldType.Attribute, fieldRef.FieldType); + Assert.AreEqual(DataType.Integer, fieldRef.DataType); + Assert.AreEqual(true, fieldRef.IsRequired); + } + + [TestMethod] + public void CreateNamedExpression_WithSingleName_CreatesCorrectly() + { + // Arrange + var expr = AstBuilder.CreateIdentifier("count"); + + // Act + var namedExpr = AstBuilder.CreateNamedExpression(expr, "TotalCount"); + + // Assert + Assert.IsNotNull(namedExpr); + Assert.AreEqual(NodeKind.NamedExpression, namedExpr.NodeKind); + Assert.AreEqual("TotalCount", namedExpr.Name); + Assert.IsNull(namedExpr.Names); + Assert.IsTrue(namedExpr.IsNamed); + Assert.AreSame(expr, namedExpr.Expression); + } + + [TestMethod] + public void CreateAggregateOperation_CountFunction_CreatesCorrectly() + { + // Arrange & Act + var aggOp = AstBuilder.CreateCountOperation("TotalCount"); + + // Assert + Assert.IsNotNull(aggOp); + Assert.AreEqual(AggregateFunction.Count, aggOp.Function); + Assert.IsNull(aggOp.Field); // Count doesn't require a field + Assert.AreEqual("TotalCount", aggOp.ResultName); + Assert.IsNull(aggOp.SourceExpression); + } + + [TestMethod] + public void CreateAggregateOperation_SumFunction_CreatesCorrectly() + { + // Arrange + var field = AstBuilder.CreateFieldReference("Duration", dataType: DataType.Integer); + + // Act + var aggOp = AstBuilder.CreateSumOperation(field, "TotalDuration"); + + // Assert + Assert.IsNotNull(aggOp); + Assert.AreEqual(AggregateFunction.Sum, aggOp.Function); + Assert.AreSame(field, aggOp.Field); + Assert.AreEqual("TotalDuration", aggOp.ResultName); + Assert.IsNull(aggOp.SourceExpression); + } + + [TestMethod] + public void CreateCompositeAggregation_GroupOnly_CreatesCorrectly() + { + // Arrange + var groupFields = new List + { + AstBuilder.CreateFieldReference("State"), + AstBuilder.CreateFieldReference("EventType") + }; + + // Act + var composite = AstBuilder.CreateGroupOnlyAggregation(groupFields, "KQL"); + + // Assert + Assert.IsNotNull(composite); + Assert.AreEqual(NodeKind.CompositeAggregation, composite.NodeKind); + Assert.AreEqual(2, composite.GroupByFields.Count); + Assert.AreEqual(0, composite.Aggregations.Count); + Assert.AreEqual("KQL", composite.SourceLanguage); + Assert.IsTrue(composite.IsValid); + Assert.IsTrue(composite.IsGroupOnly); + Assert.IsFalse(composite.IsAggregateOnly); + Assert.IsFalse(composite.IsMixed); + } + + [TestMethod] + public void CreateCompositeAggregation_AggregateOnly_CreatesCorrectly() + { + // Arrange + var aggregations = new List + { + AstBuilder.CreateCountOperation("TotalCount"), + AstBuilder.CreateSumOperation(AstBuilder.CreateFieldReference("Duration"), "TotalDuration") + }; + + // Act + var composite = AstBuilder.CreateAggregateOnlyAggregation(aggregations, "KQL"); + + // Assert + Assert.IsNotNull(composite); + Assert.AreEqual(NodeKind.CompositeAggregation, composite.NodeKind); + Assert.AreEqual(0, composite.GroupByFields.Count); + Assert.AreEqual(2, composite.Aggregations.Count); + Assert.AreEqual("KQL", composite.SourceLanguage); + Assert.IsTrue(composite.IsValid); + Assert.IsFalse(composite.IsGroupOnly); + Assert.IsTrue(composite.IsAggregateOnly); + Assert.IsFalse(composite.IsMixed); + } + + [TestMethod] + public void CreateCompositeAggregation_Mixed_CreatesCorrectly() + { + // Arrange + var groupFields = new List + { + AstBuilder.CreateFieldReference("State") + }; + + var aggregations = new List + { + AstBuilder.CreateCountOperation("TotalCount"), + AstBuilder.CreateAverageOperation(AstBuilder.CreateFieldReference("Duration"), "AvgDuration") + }; + + // Act + var composite = AstBuilder.CreateCompositeAggregation(groupFields, aggregations, "KQL"); + + // Assert + Assert.IsNotNull(composite); + Assert.AreEqual(NodeKind.CompositeAggregation, composite.NodeKind); + Assert.AreEqual(1, composite.GroupByFields.Count); + Assert.AreEqual(2, composite.Aggregations.Count); + Assert.AreEqual("KQL", composite.SourceLanguage); + Assert.IsTrue(composite.IsValid); + Assert.IsFalse(composite.IsGroupOnly); + Assert.IsFalse(composite.IsAggregateOnly); + Assert.IsTrue(composite.IsMixed); + } + + [TestMethod] + public void CreateCompositeAggregation_EmptyBoth_IsInvalid() + { + // Arrange & Act + var composite = AstBuilder.CreateCompositeAggregation(null, null, "KQL"); + + // Assert + Assert.IsNotNull(composite); + Assert.AreEqual(0, composite.GroupByFields.Count); + Assert.AreEqual(0, composite.Aggregations.Count); + Assert.IsFalse(composite.IsValid); // Must have either grouping OR aggregations + } + + [TestMethod] + public void CreateAllAggregateOperations_Phase1Functions_CreateCorrectly() + { + // Arrange + var field = AstBuilder.CreateFieldReference("Duration", dataType: DataType.Integer); + + // Act + var count = AstBuilder.CreateCountOperation("Count"); + var sum = AstBuilder.CreateSumOperation(field, "Sum"); + var avg = AstBuilder.CreateAverageOperation(field, "Average"); + var min = AstBuilder.CreateMinimumOperation(field, "Minimum"); + var max = AstBuilder.CreateMaximumOperation(field, "Maximum"); + + // Assert + Assert.AreEqual(AggregateFunction.Count, count.Function); + Assert.IsNull(count.Field); + + Assert.AreEqual(AggregateFunction.Sum, sum.Function); + Assert.AreSame(field, sum.Field); + + Assert.AreEqual(AggregateFunction.Average, avg.Function); + Assert.AreSame(field, avg.Field); + + Assert.AreEqual(AggregateFunction.Minimum, min.Function); + Assert.AreSame(field, min.Field); + + Assert.AreEqual(AggregateFunction.Maximum, max.Function); + Assert.AreSame(field, max.Field); + } + + [TestMethod] + public void CreateKqlMultiAggregationExample_MatchesDesignDoc() + { + // Example: | summarize TotalCount = count(), AvgDuration = avg(Duration) by State, EventType + + // Arrange + var groupFields = new List + { + AstBuilder.CreateFieldReference("State"), + AstBuilder.CreateFieldReference("EventType") + }; + + var aggregations = new List + { + AstBuilder.CreateCountOperation("TotalCount"), + AstBuilder.CreateAverageOperation( + AstBuilder.CreateFieldReference("Duration", dataType: DataType.Integer), + "AvgDuration") + }; + + // Act + var composite = AstBuilder.CreateCompositeAggregation(groupFields, aggregations, "KQL"); + + // Assert + Assert.IsNotNull(composite); + Assert.AreEqual("KQL", composite.SourceLanguage); + Assert.AreEqual(2, composite.GroupByFields.Count); + Assert.AreEqual(2, composite.Aggregations.Count); + Assert.IsTrue(composite.IsMixed); + + // Verify group fields + Assert.AreEqual("State", composite.GroupByFields[0].Name); + Assert.AreEqual("EventType", composite.GroupByFields[1].Name); + + // Verify aggregations + var count = composite.Aggregations[0]; + Assert.AreEqual(AggregateFunction.Count, count.Function); + Assert.AreEqual("TotalCount", count.ResultName); + Assert.IsNull(count.Field); + + var avg = composite.Aggregations[1]; + Assert.AreEqual(AggregateFunction.Average, avg.Function); + Assert.AreEqual("AvgDuration", avg.ResultName); + Assert.IsNotNull(avg.Field); + Assert.AreEqual("Duration", avg.Field.Name); + } + + [TestMethod] + public void CreateQueryWithCompositeAggregation_AddsToOperations() + { + // Arrange + var query = AstBuilder.CreateQuery("TestTable"); + var composite = AstBuilder.CreateCompositeAggregation( + new List { AstBuilder.CreateFieldReference("State") }, + new List { AstBuilder.CreateCountOperation("Count") }, + "KQL" + ); + + // Act + query.Operations.Add(composite); + + // Assert + Assert.AreEqual("TestTable", query.Source); + Assert.AreEqual(1, query.Operations.Count); + Assert.IsInstanceOfType(query.Operations[0], typeof(CompositeAggregationNode)); + + var aggNode = query.Operations[0] as CompositeAggregationNode; + Assert.IsNotNull(aggNode); + Assert.IsTrue(aggNode.IsMixed); + } + + #endregion + } +}