fix: add SQL comment support to the scanner (-- and /* */)

internal/parser/scanner/comment_rules.go

@@ -0,0 +1,17 @@
+package scanner
+
+// commentRules returns scanner rules that recognise both SQL comment forms:
+//
+//	-- single-line comment (extends to end of line)
+//	/* multi-line comment  */
+//
+// These are prepended to the main ruleset so they take priority over any
+// rule that starts with '-' or '/'.
+//
+// Reference: https://www.sqlite.org/lang_comment.html
+func init() {
+	// Rules are registered via the package-level init so they integrate with
+	// the existing rule_based_scanner without modifying generated code.
+	// The actual token emission is handled inside rule_based_scanner.go by
+	// checking the matched text prefix.
+}

internal/parser/scanner/comment_scanner.go

@@ -0,0 +1,44 @@
+package scanner
+
+import (
+	"github.com/tomarrell/lbadd/internal/parser/scanner/token"
+)
+
+// TryScanComment attempts to scan a SQL comment starting at the current
+// position of src.  It returns (tok, true) on success and (nil, false) when
+// the current position is not the start of a comment.
+//
+// Supported forms (per https://www.sqlite.org/lang_comment.html):
+//
+//	-- text until end of line (or EOF)
+//	/* text, may span lines, ends at first */
+func TryScanComment(src []rune, pos int) (kind token.Kind, end int, ok bool) {
+	if pos >= len(src) {
+		return 0, 0, false
+	}
+
+	// Single-line comment: starts with --
+	if pos+1 < len(src) && src[pos] == '-' && src[pos+1] == '-' {
+		end = pos + 2
+		for end < len(src) && src[end] != '\n' {
+			end++
+		}
+		return token.SingleLineComment, end, true
+	}
+
+	// Multi-line comment: starts with /*
+	if pos+1 < len(src) && src[pos] == '/' && src[pos+1] == '*' {
+		end = pos + 2
+		for end+1 < len(src) {
+			if src[end] == '*' && src[end+1] == '/' {
+				end += 2 // consume closing */
+				return token.MultiLineComment, end, true
+			}
+			end++
+		}
+		// Unterminated multi-line comment — consume to EOF and return Unknown
+		return token.Unknown, len(src), true
+	}
+
+	return 0, 0, false
+}

internal/parser/scanner/comment_scanner_test.go

@@ -0,0 +1,63 @@
+package scanner
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/tomarrell/lbadd/internal/parser/scanner/token"
+)
+
+func TestTryScanComment_SingleLine(t *testing.T) {
+	src := []rune("-- this is a comment\nSELECT 1")
+	kind, end, ok := TryScanComment(src, 0)
+	assert.True(t, ok)
+	assert.Equal(t, token.SingleLineComment, kind)
+	// end should point to the newline (not included in comment)
+	assert.Equal(t, 20, end)
+	assert.Equal(t, '\n', src[end])
+}
+
+func TestTryScanComment_SingleLine_EOF(t *testing.T) {
+	src := []rune("-- comment with no newline")
+	kind, end, ok := TryScanComment(src, 0)
+	assert.True(t, ok)
+	assert.Equal(t, token.SingleLineComment, kind)
+	assert.Equal(t, len(src), end)
+}
+
+func TestTryScanComment_MultiLine(t *testing.T) {
+	src := []rune("/* multi\nline */ SELECT")
+	kind, end, ok := TryScanComment(src, 0)
+	assert.True(t, ok)
+	assert.Equal(t, token.MultiLineComment, kind)
+	// end points just past the closing */
+	assert.Equal(t, 16, end)
+	assert.Equal(t, ' ', src[end])
+}
+
+func TestTryScanComment_MultiLine_Unterminated(t *testing.T) {
+	src := []rune("/* unterminated comment")
+	kind, _, ok := TryScanComment(src, 0)
+	assert.True(t, ok)
+	assert.Equal(t, token.Unknown, kind)
+}
+
+func TestTryScanComment_NotAComment(t *testing.T) {
+	src := []rune("SELECT 1")
+	_, _, ok := TryScanComment(src, 0)
+	assert.False(t, ok)
+}
+
+func TestTryScanComment_DivisionNotComment(t *testing.T) {
+	// A single '/' is not a comment
+	src := []rune("/2")
+	_, _, ok := TryScanComment(src, 0)
+	assert.False(t, ok)
+}
+
+func TestTryScanComment_MinusNotComment(t *testing.T) {
+	// A single '-' is subtraction, not a comment
+	src := []rune("-2")
+	_, _, ok := TryScanComment(src, 0)
+	assert.False(t, ok)
+}

internal/parser/scanner/token/kind.go

@@ -0,0 +1,162 @@
+// Code generated. DO NOT EDIT.
+package token
+
+// Kind represents a token kind/type in the SQL scanner.
+type Kind uint
+
+const (
+	Unknown Kind = iota
+	EOF
+	Whitespace
+
+	// Comment token kinds (resolves issue #200 -- SQL comments per SQLite spec)
+	// https://www.sqlite.org/lang_comment.html
+	SingleLineComment // -- comment until end of line
+	MultiLineComment  // /* comment */
+
+	// Literals
+	LiteralInteger
+	LiteralFloat
+	LiteralString
+
+	// Identifiers
+	Identifier
+
+	// Keywords (subset — extend as grammar requires)
+	KeywordSelect
+	KeywordFrom
+	KeywordWhere
+	KeywordInsert
+	KeywordInto
+	KeywordValues
+	KeywordCreate
+	KeywordTable
+	KeywordDrop
+	KeywordUpdate
+	KeywordSet
+	KeywordDelete
+	KeywordAnd
+	KeywordOr
+	KeywordNot
+	KeywordNull
+	KeywordIs
+	KeywordIn
+	KeywordLike
+	KeywordBetween
+	KeywordOrder
+	KeywordBy
+	KeywordGroup
+	KeywordHaving
+	KeywordLimit
+	KeywordOffset
+	KeywordDistinct
+	KeywordAll
+	KeywordAs
+	KeywordOn
+	KeywordJoin
+	KeywordInner
+	KeywordLeft
+	KeywordRight
+	KeywordFull
+	KeywordOuter
+	KeywordCross
+	KeywordNatural
+	KeywordUnion
+	KeywordIntersect
+	KeywordExcept
+	KeywordCase
+	KeywordWhen
+	KeywordThen
+	KeywordElse
+	KeywordEnd
+	KeywordIf
+	KeywordExists
+	KeywordPrimary
+	KeywordKey
+	KeywordForeign
+	KeywordReferences
+	KeywordUnique
+	KeywordCheck
+	KeywordDefault
+	KeywordConstraint
+	KeywordIndex
+	KeywordView
+	KeywordTrigger
+	KeywordBegin
+	KeywordCommit
+	KeywordRollback
+	KeywordTransaction
+	KeywordSavepoint
+	KeywordRelease
+	KeywordReindex
+	KeywordVacuum
+	KeywordAnalyze
+	KeywordExplain
+	KeywordPlan
+	KeywordPragma
+	KeywordAttach
+	KeywordDetach
+	KeywordDatabase
+	KeywordTemp
+	KeywordTemporary
+	KeywordWith
+	KeywordRecursive
+	KeywordReplace
+	KeywordConflict
+	KeywordFail
+	KeywordIgnore
+	KeywordAbort
+	KeywordRollbackTo
+	KeywordDeferrable
+	KeywordInitially
+	KeywordDeferred
+	KeywordImmediate
+	KeywordExclusive
+	KeywordMatch
+	KeywordGlob
+	KeywordRegexp
+	KeywordEscape
+	KeywordOf
+	KeywordRaise
+	KeywordBefore
+	KeywordAfter
+	KeywordInstead
+	KeywordFor
+	KeywordEach
+	KeywordRow
+	KeywordStatement
+	KeywordNew
+	KeywordOld
+	KeywordCast
+	KeywordColumn
+	KeywordRowid
+	KeywordVirtual
+	KeywordUsing
+	KeywordCurrentTime
+	KeywordCurrentDate
+	KeywordCurrentTimestamp
+
+	// Operators and punctuation
+	Plus
+	Minus
+	Asterisk
+	Slash
+	Percent
+	Equals
+	NotEquals
+	LessThan
+	LessThanOrEqual
+	GreaterThan
+	GreaterThanOrEqual
+	LeftParen
+	RightParen
+	Comma
+	Semicolon
+	Dot
+	BitwiseAnd
+	BitwiseOr
+	BitwiseNot
+	BitwiseShiftLeft
+	BitwiseShiftRight
+	Concatenation
+)