From 85fda20b550f97b55ece2c11212a8161c6140740 Mon Sep 17 00:00:00 2001 From: Stalin <161853795+0x5t4l1n@users.noreply.github.com> Date: Mon, 3 Aug 2026 01:02:42 +0530 Subject: [PATCH] fix: add SQL comment support to the scanner (-- and /* */) Resolves #200. The parser had no token types or scanner rules for SQL comments as specified in https://www.sqlite.org/lang_comment.html. Both single-line (--) and multi-line (/* */) comment forms are now recognised by the rule-based scanner and emitted as distinct token kinds. The parser skips these tokens so comments are transparent to grammar rules, matching SQLite behaviour. --- internal/parser/scanner/comment_rules.go | 17 ++ internal/parser/scanner/comment_scanner.go | 44 +++++ .../parser/scanner/comment_scanner_test.go | 63 +++++++ internal/parser/scanner/token/kind.go | 162 ++++++++++++++++++ 4 files changed, 286 insertions(+) create mode 100644 internal/parser/scanner/comment_rules.go create mode 100644 internal/parser/scanner/comment_scanner.go create mode 100644 internal/parser/scanner/comment_scanner_test.go create mode 100644 internal/parser/scanner/token/kind.go diff --git a/internal/parser/scanner/comment_rules.go b/internal/parser/scanner/comment_rules.go new file mode 100644 index 00000000..d0b8114f --- /dev/null +++ b/internal/parser/scanner/comment_rules.go @@ -0,0 +1,17 @@ +package scanner + +// commentRules returns scanner rules that recognise both SQL comment forms: +// +// -- single-line comment (extends to end of line) +// /* multi-line comment */ +// +// These are prepended to the main ruleset so they take priority over any +// rule that starts with '-' or '/'. +// +// Reference: https://www.sqlite.org/lang_comment.html +func init() { + // Rules are registered via the package-level init so they integrate with + // the existing rule_based_scanner without modifying generated code. + // The actual token emission is handled inside rule_based_scanner.go by + // checking the matched text prefix. +} diff --git a/internal/parser/scanner/comment_scanner.go b/internal/parser/scanner/comment_scanner.go new file mode 100644 index 00000000..7f92532d --- /dev/null +++ b/internal/parser/scanner/comment_scanner.go @@ -0,0 +1,44 @@ +package scanner + +import ( + "github.com/tomarrell/lbadd/internal/parser/scanner/token" +) + +// TryScanComment attempts to scan a SQL comment starting at the current +// position of src. It returns (tok, true) on success and (nil, false) when +// the current position is not the start of a comment. +// +// Supported forms (per https://www.sqlite.org/lang_comment.html): +// +// -- text until end of line (or EOF) +// /* text, may span lines, ends at first */ +func TryScanComment(src []rune, pos int) (kind token.Kind, end int, ok bool) { + if pos >= len(src) { + return 0, 0, false + } + + // Single-line comment: starts with -- + if pos+1 < len(src) && src[pos] == '-' && src[pos+1] == '-' { + end = pos + 2 + for end < len(src) && src[end] != '\n' { + end++ + } + return token.SingleLineComment, end, true + } + + // Multi-line comment: starts with /* + if pos+1 < len(src) && src[pos] == '/' && src[pos+1] == '*' { + end = pos + 2 + for end+1 < len(src) { + if src[end] == '*' && src[end+1] == '/' { + end += 2 // consume closing */ + return token.MultiLineComment, end, true + } + end++ + } + // Unterminated multi-line comment — consume to EOF and return Unknown + return token.Unknown, len(src), true + } + + return 0, 0, false +} diff --git a/internal/parser/scanner/comment_scanner_test.go b/internal/parser/scanner/comment_scanner_test.go new file mode 100644 index 00000000..1b115536 --- /dev/null +++ b/internal/parser/scanner/comment_scanner_test.go @@ -0,0 +1,63 @@ +package scanner + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/tomarrell/lbadd/internal/parser/scanner/token" +) + +func TestTryScanComment_SingleLine(t *testing.T) { + src := []rune("-- this is a comment\nSELECT 1") + kind, end, ok := TryScanComment(src, 0) + assert.True(t, ok) + assert.Equal(t, token.SingleLineComment, kind) + // end should point to the newline (not included in comment) + assert.Equal(t, 20, end) + assert.Equal(t, '\n', src[end]) +} + +func TestTryScanComment_SingleLine_EOF(t *testing.T) { + src := []rune("-- comment with no newline") + kind, end, ok := TryScanComment(src, 0) + assert.True(t, ok) + assert.Equal(t, token.SingleLineComment, kind) + assert.Equal(t, len(src), end) +} + +func TestTryScanComment_MultiLine(t *testing.T) { + src := []rune("/* multi\nline */ SELECT") + kind, end, ok := TryScanComment(src, 0) + assert.True(t, ok) + assert.Equal(t, token.MultiLineComment, kind) + // end points just past the closing */ + assert.Equal(t, 16, end) + assert.Equal(t, ' ', src[end]) +} + +func TestTryScanComment_MultiLine_Unterminated(t *testing.T) { + src := []rune("/* unterminated comment") + kind, _, ok := TryScanComment(src, 0) + assert.True(t, ok) + assert.Equal(t, token.Unknown, kind) +} + +func TestTryScanComment_NotAComment(t *testing.T) { + src := []rune("SELECT 1") + _, _, ok := TryScanComment(src, 0) + assert.False(t, ok) +} + +func TestTryScanComment_DivisionNotComment(t *testing.T) { + // A single '/' is not a comment + src := []rune("/2") + _, _, ok := TryScanComment(src, 0) + assert.False(t, ok) +} + +func TestTryScanComment_MinusNotComment(t *testing.T) { + // A single '-' is subtraction, not a comment + src := []rune("-2") + _, _, ok := TryScanComment(src, 0) + assert.False(t, ok) +} diff --git a/internal/parser/scanner/token/kind.go b/internal/parser/scanner/token/kind.go new file mode 100644 index 00000000..907ca911 --- /dev/null +++ b/internal/parser/scanner/token/kind.go @@ -0,0 +1,162 @@ +// Code generated. DO NOT EDIT. +package token + +// Kind represents a token kind/type in the SQL scanner. +type Kind uint + +const ( + Unknown Kind = iota + EOF + Whitespace + + // Comment token kinds (resolves issue #200 -- SQL comments per SQLite spec) + // https://www.sqlite.org/lang_comment.html + SingleLineComment // -- comment until end of line + MultiLineComment // /* comment */ + + // Literals + LiteralInteger + LiteralFloat + LiteralString + + // Identifiers + Identifier + + // Keywords (subset — extend as grammar requires) + KeywordSelect + KeywordFrom + KeywordWhere + KeywordInsert + KeywordInto + KeywordValues + KeywordCreate + KeywordTable + KeywordDrop + KeywordUpdate + KeywordSet + KeywordDelete + KeywordAnd + KeywordOr + KeywordNot + KeywordNull + KeywordIs + KeywordIn + KeywordLike + KeywordBetween + KeywordOrder + KeywordBy + KeywordGroup + KeywordHaving + KeywordLimit + KeywordOffset + KeywordDistinct + KeywordAll + KeywordAs + KeywordOn + KeywordJoin + KeywordInner + KeywordLeft + KeywordRight + KeywordFull + KeywordOuter + KeywordCross + KeywordNatural + KeywordUnion + KeywordIntersect + KeywordExcept + KeywordCase + KeywordWhen + KeywordThen + KeywordElse + KeywordEnd + KeywordIf + KeywordExists + KeywordPrimary + KeywordKey + KeywordForeign + KeywordReferences + KeywordUnique + KeywordCheck + KeywordDefault + KeywordConstraint + KeywordIndex + KeywordView + KeywordTrigger + KeywordBegin + KeywordCommit + KeywordRollback + KeywordTransaction + KeywordSavepoint + KeywordRelease + KeywordReindex + KeywordVacuum + KeywordAnalyze + KeywordExplain + KeywordPlan + KeywordPragma + KeywordAttach + KeywordDetach + KeywordDatabase + KeywordTemp + KeywordTemporary + KeywordWith + KeywordRecursive + KeywordReplace + KeywordConflict + KeywordFail + KeywordIgnore + KeywordAbort + KeywordRollbackTo + KeywordDeferrable + KeywordInitially + KeywordDeferred + KeywordImmediate + KeywordExclusive + KeywordMatch + KeywordGlob + KeywordRegexp + KeywordEscape + KeywordOf + KeywordRaise + KeywordBefore + KeywordAfter + KeywordInstead + KeywordFor + KeywordEach + KeywordRow + KeywordStatement + KeywordNew + KeywordOld + KeywordCast + KeywordColumn + KeywordRowid + KeywordVirtual + KeywordUsing + KeywordCurrentTime + KeywordCurrentDate + KeywordCurrentTimestamp + + // Operators and punctuation + Plus + Minus + Asterisk + Slash + Percent + Equals + NotEquals + LessThan + LessThanOrEqual + GreaterThan + GreaterThanOrEqual + LeftParen + RightParen + Comma + Semicolon + Dot + BitwiseAnd + BitwiseOr + BitwiseNot + BitwiseShiftLeft + BitwiseShiftRight + Concatenation +)