UBNFとLSP補完語彙の乖離を自動検出する

tools/tinyexpression-p4-lsp-vscode/README.md

@@ -127,6 +127,11 @@ hand-written adapter only supplies language-specific runtime binding through the
 `runtimeVariables(...)` hook. Debug Console evaluation delegates to TinyExpression itself; it
 does not contain a second arithmetic parser.
 
+The module's vocabulary conformance test extracts word literals from the UBNF and compares them
+with the static LSP completion vocabulary. An intentional omission must carry a reason in the
+test allow-list. The extractor is kept independent of LSP internals so the same check can later
+cover TextMate grammar scopes, DAP value vocabularies, and files bundled in the VSIX.
+
 AST stepping is currently structural: the formula result and parity snapshot are evaluated with
 the launch variables, while F10 changes the selected AST node. Per-node mutable runtime state and
 reverse/time-travel debugging are not provided.

tools/tinyexpression-p4-lsp-vscode/pom.xml

@@ -210,7 +210,7 @@
                         <groupId>org.apache.maven.plugins</groupId>
                         <artifactId>maven-surefire-plugin</artifactId>
                         <configuration>
-                            <test>org.unlaxer.tinyexpression.lsp.p4.TinyExpressionP4LanguageServerExtTest</test>
+                            <test>org.unlaxer.tinyexpression.lsp.p4.TinyExpressionP4LanguageServerExtTest,org.unlaxer.tinyexpression.lsp.p4.TinyExpressionP4VocabularyConformanceTest</test>
                             <failIfNoTests>true</failIfNoTests>
                         </configuration>
                     </plugin>

tools/tinyexpression-p4-lsp-vscode/src/main/java/org/unlaxer/tinyexpression/lsp/p4/TinyExpressionP4LanguageServerExt.java

@@ -141,7 +141,7 @@ public class TinyExpressionP4LanguageServerExt extends TinyExpressionP4LanguageS
       "var", "variable", "as",
       "number", "string", "boolean", "object", "float",
       "set", "not", "exists", "description", "call",
-      "import", "external", "returning",
+      "import", "external", "returning", "internal",
       "true", "false");
 
   // ── Operator set ──
@@ -157,10 +157,15 @@ public class TinyExpressionP4LanguageServerExt extends TinyExpressionP4LanguageS
       "if", "else", "match", "default",
       "var", "variable", "as",
       "number", "string", "boolean", "object", "float",
-      "set", "not", "exists", "call",
-      "import", "external", "returning",
+      "set", "not", "exists", "description", "call",
+      "import", "external", "returning", "internal",
       "true", "false");
 
+  /** Closed grammar values that are useful as expression completions. */
+  private static final List<String> COMPLETION_VALUES = List.of(
+      "MONDAY", "TUESDAY", "WEDNESDAY", "THURSDAY",
+      "FRIDAY", "SATURDAY", "SUNDAY");
+
   /**
    * Snippet completions for callable functions. Accepting one of these inserts
    * the function name with balanced parens and a tab-stop on the first argument
@@ -180,7 +185,7 @@ public class TinyExpressionP4LanguageServerExt extends TinyExpressionP4LanguageS
       FUNCTION_SNIPPETS.put(f, f + "($1)$0");
     }
     // Math / string — two arguments
-    for (String f : List.of("pow", "indexOf", "startsWith", "endsWith", "contains")) {
+    for (String f : List.of("pow", "startsWith", "endsWith", "contains")) {
       FUNCTION_SNIPPETS.put(f, f + "($1, $2)$0");
     }
     // Variadic — surface a 2-arg starter
@@ -218,6 +223,19 @@ public class TinyExpressionP4LanguageServerExt extends TinyExpressionP4LanguageS
         "external returning as ${1:number} ${2:name}($3)$0");
   }
 
+  /**
+   * Package-private conformance seam. Tests compare this vocabulary with the
+   * UBNF source so a grammar addition cannot silently disappear from LSP
+   * completion.
+   */
+  static Set<String> staticCompletionVocabulary() {
+    Set<String> vocabulary = new LinkedHashSet<>(COMPLETION_KEYWORDS);
+    vocabulary.addAll(FUNCTION_SNIPPETS.keySet());
+    vocabulary.addAll(BLOCK_SNIPPETS.keySet());
+    vocabulary.addAll(COMPLETION_VALUES);
+    return Collections.unmodifiableSet(vocabulary);
+  }
+
   /** Pattern for extracting $variable references from document text. */
   private static final Pattern VARIABLE_PATTERN =
       Pattern.compile("\\$([a-zA-Z_][a-zA-Z0-9_]*)");
@@ -1514,6 +1532,15 @@ public CompletableFuture<Either<List<CompletionItem>, CompletionList>> completio
         }
       }
 
+      // 1a. Closed grammar values (currently DayOfWeek).
+      for (String value : COMPLETION_VALUES) {
+        if (value.startsWith(prefix)) {
+          CompletionItem item = new CompletionItem(value);
+          item.setKind(CompletionItemKind.EnumMember);
+          items.add(item);
+        }
+      }
+
       // 1b. Function snippets — paren-balanced completions (issue #11 §3)
       for (Map.Entry<String, String> e : FUNCTION_SNIPPETS.entrySet()) {
         String fn = e.getKey();

tools/tinyexpression-p4-lsp-vscode/src/test/java/org/unlaxer/tinyexpression/lsp/p4/TinyExpressionP4LanguageServerExtTest.java

@@ -241,6 +241,31 @@ public void testCompletionFunctionSnippet() throws Exception {
         assertEquals("sin($1)$0", sin.getInsertText());
     }
 
+    @Test
+    public void testCompletionIncludesGrammarOnlyValuesAndInternalKeyword() throws Exception {
+        String content = "";
+        server.parseAndEnrich(TEST_URI, content, 0, content);
+
+        CompletionParams params = new CompletionParams();
+        params.setTextDocument(new TextDocumentIdentifier(TEST_URI));
+        params.setPosition(new Position(0, 0));
+
+        List<CompletionItem> items = service.completion(params).get().getLeft();
+        CompletionItem monday = items.stream()
+            .filter(i -> "MONDAY".equals(i.getLabel()))
+            .findFirst()
+            .orElse(null);
+
+        assertNotNull("DayOfWeek grammar values should be suggested", monday);
+        assertEquals(CompletionItemKind.EnumMember, monday.getKind());
+        assertTrue("internal invocation keyword should be suggested",
+            items.stream().anyMatch(i -> "internal".equals(i.getLabel())));
+        assertTrue("description declaration keyword should be suggested",
+            items.stream().anyMatch(i -> "description".equals(i.getLabel())));
+        assertFalse("unsupported indexOf must not be suggested",
+            items.stream().anyMatch(i -> "indexOf".equals(i.getLabel())));
+    }
+
     /**
      * Block-keyword snippet completion — semicolon completion 由来。issue #11
      * §3 セミコロン補完を declaration テンプレート経由で復元する。"var" を

tools/tinyexpression-p4-lsp-vscode/src/test/java/org/unlaxer/tinyexpression/lsp/p4/TinyExpressionP4VocabularyConformanceTest.java

@@ -0,0 +1,72 @@
+package org.unlaxer.tinyexpression.lsp.p4;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Set;
+import org.junit.Test;
+
+public class TinyExpressionP4VocabularyConformanceTest {
+    /**
+     * Compatibility spellings are accepted by the grammar, but completion
+     * deliberately teaches the canonical lower-case type spelling.
+     */
+    private static final Map<String, String> INTENTIONALLY_OMITTED = Map.of(
+        "Number", "compatibility alias; suggest canonical number",
+        "Float", "compatibility alias; suggest canonical float",
+        "String", "compatibility alias; suggest canonical string",
+        "Boolean", "compatibility alias; suggest canonical boolean",
+        "Object", "compatibility alias; suggest canonical object");
+
+    @Test
+    public void everyGrammarWordIsCompletedOrExplicitlyOmitted() throws IOException {
+        String grammar = Files.readString(grammarPath());
+        Set<String> completions = TinyExpressionP4LanguageServerExt.staticCompletionVocabulary();
+        Set<String> missing = UbnfVocabulary.missingCompletions(
+            grammar, completions, INTENTIONALLY_OMITTED.keySet());
+
+        assertTrue("UBNF words missing from LSP completion classification: " + missing,
+            missing.isEmpty());
+
+        Set<String> grammarWords = UbnfVocabulary.wordLiterals(grammar);
+        Set<String> stale = new java.util.LinkedHashSet<>(completions);
+        stale.removeAll(grammarWords);
+        assertTrue("LSP static completions absent from UBNF: " + stale, stale.isEmpty());
+    }
+
+    @Test
+    public void syntheticGrammarAdditionIsReportedAsMissing() {
+        String fixture = "grammar Fixture { Root ::= 'existing' | 'futureKeyword' ; }";
+        Set<String> missing = UbnfVocabulary.missingCompletions(
+            fixture, Set.of("existing"), Set.of());
+
+        assertEquals(Set.of("futureKeyword"), missing);
+    }
+
+    @Test
+    public void intentionalDifferencesAlwaysCarryAReason() {
+        Map<String, String> missingReasons = new LinkedHashMap<>();
+        INTENTIONALLY_OMITTED.forEach((word, reason) -> {
+            if (reason == null || reason.isBlank()) {
+                missingReasons.put(word, reason);
+            }
+        });
+        assertTrue("Every allow-list entry needs a reason: " + missingReasons,
+            missingReasons.isEmpty());
+    }
+
+    private static Path grammarPath() {
+        Path modulePath = Path.of(System.getProperty("basedir", "."),
+            "grammar", "tinyexpression-p4.ubnf");
+        if (Files.isRegularFile(modulePath)) {
+            return modulePath;
+        }
+        return Path.of("tools", "tinyexpression-p4-lsp-vscode", "grammar",
+            "tinyexpression-p4.ubnf");
+    }
+}

tools/tinyexpression-p4-lsp-vscode/src/test/java/org/unlaxer/tinyexpression/lsp/p4/UbnfVocabulary.java

@@ -0,0 +1,97 @@
+package org.unlaxer.tinyexpression.lsp.p4;
+
+import java.util.LinkedHashSet;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/** Lightweight UBNF literal extraction used by tooling conformance tests. */
+final class UbnfVocabulary {
+    private static final Pattern QUOTED_LITERAL =
+        Pattern.compile("'((?:\\\\.|[^'\\\\])*)'");
+    private static final Pattern WORD = Pattern.compile("[A-Za-z][A-Za-z0-9]*");
+
+    private UbnfVocabulary() {}
+
+    static Set<String> wordLiterals(String ubnf) {
+        Set<String> words = new LinkedHashSet<>();
+        boolean inRule = false;
+
+        for (String rawLine : ubnf.split("\\R", -1)) {
+            String line = stripLineComment(rawLine);
+            if (!inRule && line.contains("::=")) {
+                inRule = true;
+            }
+            if (!inRule) {
+                continue;
+            }
+
+            Matcher matcher = QUOTED_LITERAL.matcher(line);
+            while (matcher.find()) {
+                String literal = matcher.group(1).replace("\\'", "'");
+                if (WORD.matcher(literal).matches()) {
+                    words.add(literal);
+                }
+            }
+
+            if (hasUnquotedSemicolon(line)) {
+                inRule = false;
+            }
+        }
+        return words;
+    }
+
+    static Set<String> missingCompletions(
+            String ubnf, Set<String> completions, Set<String> intentionallyOmitted) {
+        Set<String> missing = wordLiterals(ubnf);
+        missing.removeAll(completions);
+        missing.removeAll(intentionallyOmitted);
+        return missing;
+    }
+
+    private static String stripLineComment(String line) {
+        boolean quoted = false;
+        boolean escaped = false;
+        for (int i = 0; i + 1 < line.length(); i++) {
+            char current = line.charAt(i);
+            if (escaped) {
+                escaped = false;
+                continue;
+            }
+            if (current == '\\' && quoted) {
+                escaped = true;
+                continue;
+            }
+            if (current == '\'') {
+                quoted = !quoted;
+                continue;
+            }
+            if (!quoted && current == '/' && line.charAt(i + 1) == '/') {
+                return line.substring(0, i);
+            }
+        }
+        return line;
+    }
+
+    private static boolean hasUnquotedSemicolon(String line) {
+        boolean quoted = false;
+        boolean escaped = false;
+        for (int i = 0; i < line.length(); i++) {
+            char current = line.charAt(i);
+            if (escaped) {
+                escaped = false;
+                continue;
+            }
+            if (current == '\\' && quoted) {
+                escaped = true;
+                continue;
+            }
+            if (current == '\'') {
+                quoted = !quoted;
+            } else if (!quoted && current == ';') {
+                return true;
+            }
+        }
+        return false;
+    }
+}