From a98bd11af4f62bb326cafcb763d36a1430d5345c Mon Sep 17 00:00:00 2001 From: Sai Date: Wed, 29 Jul 2026 23:14:31 -0700 Subject: [PATCH] Add `begin_keywords / `end_keywords with "VAMS-2023" (VAMS-2023 10.6) VAMS-2023 10.6 (Mantis 7921) extends the IEEE 1364 `begin_keywords / `end_keywords directives with a "VAMS-2023" version specifier. The directives select which identifiers are reserved as keywords, apply to everything that follows (across `include boundaries) until the matching `end_keywords, and may only appear outside a design element. OpenVAF had no handling for them at all - they fell through to the "unknown macro" path. Keyword sets ------------ `tokens::KeywordSet` models the five specifiers the standard requires ("1364-1995", "1364-2001", "1364-2005", "VAMS-2.3", "VAMS-2023") plus OpenVAF's default set, ordered by inclusion. Every standard set is a subset of what OpenVAF reserves today, so the only table needed is the IEEE 1364 keyword lists: under a Verilog specifier a word stays reserved iff that revision reserves it, and under a Verilog-AMS specifier everything OpenVAF reserves stays reserved. That keeps the default behaviour bit-for-bit unchanged and makes the Verilog-AMS keywords (`analog`, `discipline`, `from`, `string`, `ground`, `inf`, ...), `localparam`/`genvar` (1364-2001) and `uwire` (1364-2005) available as ordinary identifiers when an older specifier is selected. Two places decide whether a word is a keyword, and both are now version-aware: - lexing: `TokenKind::to_syntax` takes the active set, so released words lex as IDENT. - the reserved-identifier check in `syntax::validation`, which uses the much larger Annex B list and would otherwise still reject a declaration named `sin` or `discipline`. The active set travels on each `preprocessor::Token`; the tree builder turns that into sorted regions of tree text and `validate_name` looks the position up. This is what makes the example from 10.6 (`sin` as a port name under "1364-2005") actually compile. Preprocessor ------------ The keyword stack lives on the `Processor` so it survives `include` boundaries, while the active set is shared with every `Parser` through an `Rc`. Reported: unknown version specifiers, `end_keywords without a matching `begin_keywords, a `begin_keywords that is never closed, and either directive inside a module (tracked with a module depth counter that saturates at zero, so it can only ever miss a violation, never invent one). Also fixes a pre-existing hang: an unexpected compiler directive inside a `define body was diagnosed without consuming the token, so the macro-body loop spun forever. `begin_keywords in a `define body would have hit exactly that path. Tests: - tokens: unit tests for the set model (Verilog-AMS keywords released by the 1364 sets, revision boundaries for localparam/genvar/uwire, core keywords reserved everywhere, shadowable builtins reserved nowhere). - preprocessor: token-stream snapshots for a "1364-2005" region, nested directives, a directive applying across `include, and a directive in a disabled `ifdef branch; diagnostics for an unknown specifier, both unbalanced forms, a directive inside a module and one inside a `define. - syn_ui/begin_keywords.va: the 10.6 example plus Verilog-AMS keywords used as identifiers, compiling without a single diagnostic, and ordinary Verilog-AMS code outside the region. - syn_ui/begin_keywords_err.va: rendered reports for all four diagnostics. Co-Authored-By: Claude Opus 5 --- .../src/diagnostics/preprocessor_error.rs | 53 +++++ openvaf/preprocessor/src/diagnostics.rs | 9 + openvaf/preprocessor/src/grammar.rs | 41 +++- openvaf/preprocessor/src/lib.rs | 6 + openvaf/preprocessor/src/parser.rs | 77 ++++++- openvaf/preprocessor/src/processor.rs | 88 ++++++- .../test_data/begin_keywords_1364_2005.tokens | 44 ++++ .../begin_keywords_1364_2005_expanded.va | 7 + .../begin_keywords_across_include.tokens | 12 + .../begin_keywords_across_include_expanded.va | 3 + .../begin_keywords_in_disabled_branch.tokens | 5 + ...in_keywords_in_disabled_branch_expanded.va | 2 + .../test_data/begin_keywords_nested.tokens | 13 ++ .../begin_keywords_nested_expanded.va | 4 + openvaf/preprocessor/src/tests.rs | 132 ++++++++++- openvaf/syntax/src/lib.rs | 7 +- openvaf/syntax/src/parsing.rs | 12 +- openvaf/syntax/src/parsing/tree_builder.rs | 66 +++++- openvaf/syntax/src/validation.rs | 17 +- openvaf/test_data/syn_ui/begin_keywords.log | 0 openvaf/test_data/syn_ui/begin_keywords.va | 34 +++ .../test_data/syn_ui/begin_keywords_err.log | 38 ++++ .../test_data/syn_ui/begin_keywords_err.va | 16 ++ openvaf/tokens/src/keywords.rs | 215 ++++++++++++++++++ openvaf/tokens/src/lib.rs | 15 +- 25 files changed, 877 insertions(+), 39 deletions(-) create mode 100644 openvaf/preprocessor/src/test_data/begin_keywords_1364_2005.tokens create mode 100644 openvaf/preprocessor/src/test_data/begin_keywords_1364_2005_expanded.va create mode 100644 openvaf/preprocessor/src/test_data/begin_keywords_across_include.tokens create mode 100644 openvaf/preprocessor/src/test_data/begin_keywords_across_include_expanded.va create mode 100644 openvaf/preprocessor/src/test_data/begin_keywords_in_disabled_branch.tokens create mode 100644 openvaf/preprocessor/src/test_data/begin_keywords_in_disabled_branch_expanded.va create mode 100644 openvaf/preprocessor/src/test_data/begin_keywords_nested.tokens create mode 100644 openvaf/preprocessor/src/test_data/begin_keywords_nested_expanded.va create mode 100644 openvaf/test_data/syn_ui/begin_keywords.log create mode 100644 openvaf/test_data/syn_ui/begin_keywords.va create mode 100644 openvaf/test_data/syn_ui/begin_keywords_err.log create mode 100644 openvaf/test_data/syn_ui/begin_keywords_err.va create mode 100644 openvaf/tokens/src/keywords.rs diff --git a/openvaf/basedb/src/diagnostics/preprocessor_error.rs b/openvaf/basedb/src/diagnostics/preprocessor_error.rs index 61a91042..dd1897f4 100755 --- a/openvaf/basedb/src/diagnostics/preprocessor_error.rs +++ b/openvaf/basedb/src/diagnostics/preprocessor_error.rs @@ -270,6 +270,59 @@ impl Diagnostic for PreprocessorDiagnostic { }, ]) } + PreprocessorDiagnostic::UnknownKeywordVersion { span, .. } => { + let span = span.to_file_span(&sm); + Report::error() + .with_labels(vec![Label { + style: LabelStyle::Primary, + file_id: span.file, + range: span.range.into(), + message: "unknown version specifier".to_owned(), + }]) + .with_notes(vec![format!( + "expected one of {}", + syntax::KeywordSet::VERSION_SPECIFIERS + .iter() + .map(|specifier| format!("\"{specifier}\"")) + .collect::>() + .join(", ") + )]) + } + PreprocessorDiagnostic::UnmatchedEndKeywords { span } => { + let span = span.to_file_span(&sm); + Report::error().with_labels(vec![Label { + style: LabelStyle::Primary, + file_id: span.file, + range: span.range.into(), + message: "no keyword set is active here".to_owned(), + }]) + } + PreprocessorDiagnostic::UnterminatedKeywords { span } => { + let span = span.to_file_span(&sm); + Report::error() + .with_labels(vec![Label { + style: LabelStyle::Primary, + file_id: span.file, + range: span.range.into(), + message: "keyword set is opened here".to_owned(), + }]) + .with_notes(vec![ + "add '`end_keywords' to restore the default keywords".to_owned() + ]) + } + PreprocessorDiagnostic::KeywordsInDesignElement { span, .. } => { + let span = span.to_file_span(&sm); + Report::error() + .with_labels(vec![Label { + style: LabelStyle::Primary, + file_id: span.file, + range: span.range.into(), + message: "directive is used inside a module".to_owned(), + }]) + .with_notes(vec![ + "keyword directives may only appear outside of design elements".to_owned(), + ]) + } }; report.with_message(self.to_string()) diff --git a/openvaf/preprocessor/src/diagnostics.rs b/openvaf/preprocessor/src/diagnostics.rs index 5a840df4..b9fa1696 100644 --- a/openvaf/preprocessor/src/diagnostics.rs +++ b/openvaf/preprocessor/src/diagnostics.rs @@ -18,6 +18,11 @@ pub enum PreprocessorDiagnostic { MissingOrUnexpectedToken { expected: &'static str, expected_at: CtxSpan, span: CtxSpan }, UnexpectedToken(CtxSpan), MacroOverwritten { old: CtxSpan, new: CtxSpan, name: String }, + // `begin_keywords / `end_keywords (VAMS-2023 10.6) + UnknownKeywordVersion { version: String, span: CtxSpan }, + UnmatchedEndKeywords { span: CtxSpan }, + UnterminatedKeywords { span: CtxSpan }, + KeywordsInDesignElement { name: &'static str, span: CtxSpan }, } use PreprocessorDiagnostic::*; @@ -34,5 +39,9 @@ impl_display! { MissingOrUnexpectedToken { expected, ..} => "unexpected token, expected '{}'", expected; UnexpectedToken(_) => "encountered unexpected token!"; MacroOverwritten { name, .. } => "macro '`{}' was overwritten", name; + UnknownKeywordVersion { version, .. } => "unknown keyword version specifier \"{}\"", version; + UnmatchedEndKeywords { .. } => "'`end_keywords' without a matching '`begin_keywords'"; + UnterminatedKeywords { .. } => "'`begin_keywords' without a matching '`end_keywords'"; + KeywordsInDesignElement { name, .. } => "'`{}' is not allowed inside a design element", name; } } diff --git a/openvaf/preprocessor/src/grammar.rs b/openvaf/preprocessor/src/grammar.rs index 97f22a6f..b764abd0 100644 --- a/openvaf/preprocessor/src/grammar.rs +++ b/openvaf/preprocessor/src/grammar.rs @@ -9,6 +9,7 @@ */ use text_size::TextRange; +use tokens::KeywordSet; // use tracing::{debug, trace, trace_span}; use typed_index_collections::TiVec; @@ -111,6 +112,41 @@ pub(crate) fn parse_include<'a>( } } +/// Parses `` `begin_keywords "" `` (VAMS-2023 10.6). +/// +/// Returns the selected keyword set together with the span of the whole +/// directive. `None` is returned (and a diagnostic emitted) if the specifier is +/// missing or is not one of the specifiers the standard defines; the caller +/// keeps the currently active set in that case. +pub(crate) fn parse_begin_keywords( + p: &mut Parser<'_, '_>, + err: &mut Diagnostics, +) -> Option<(KeywordSet, CtxSpan)> { + let start = p.current_range().start(); + p.bump(); + + let specifier = p.current_text(); + if !p.expect(PreprocessorToken::StrLit, "a version specifier", err) { + return None; + } + + let range = TextRange::new(start, p.previous_range().end()); + let span = CtxSpan { ctx: p.ctx(), range }; + // strip the surrounding quotes + let specifier = &specifier[1..specifier.len() - 1]; + + match KeywordSet::from_version_specifier(specifier) { + Some(set) => Some((set, span)), + None => { + err.push(PreprocessorDiagnostic::UnknownKeywordVersion { + version: specifier.to_owned(), + span, + }); + None + } + } +} + // const MACRO_ARG_DEF_TERMINATOR_SET: TokenSet = // TokenSet::new(&[RawToken::ParenClose]).union(MACRO_TERMINATOR_SET); @@ -220,7 +256,10 @@ fn parse_macro_token<'a>( err.push(PreprocessorDiagnostic::UnexpectedToken(CtxSpan { ctx: p.ctx, range: p.current_range(), - })) + })); + // the directive still has to be consumed: without this the caller's + // `while p.before(end)` loop never makes progress and hangs. + p.bump(); } return; } diff --git a/openvaf/preprocessor/src/lib.rs b/openvaf/preprocessor/src/lib.rs index f9bd4fe2..fa48189c 100644 --- a/openvaf/preprocessor/src/lib.rs +++ b/openvaf/preprocessor/src/lib.rs @@ -77,4 +77,10 @@ pub trait SourceProvider { pub struct Token { pub span: CtxSpan, pub kind: tokens::parser::SyntaxKind, + /// The keyword set that was active where this token was produced. + /// + /// Reserved-identifier checking happens on the syntax tree, long after the + /// `` `begin_keywords `` regions have been consumed, so the active set + /// travels with the tokens (VAMS-2023 10.6). + pub keywords: tokens::KeywordSet, } diff --git a/openvaf/preprocessor/src/parser.rs b/openvaf/preprocessor/src/parser.rs index 707fe630..138d2abc 100755 --- a/openvaf/preprocessor/src/parser.rs +++ b/openvaf/preprocessor/src/parser.rs @@ -1,11 +1,13 @@ +use std::cell::Cell; use std::cmp::min; use std::ops::Range; +use std::rc::Rc; use stdx::impl_idx_math_from; use text_size::{TextRange, TextSize}; use tokens::lexer::{LiteralKind, Token, TokenKind}; use tokens::parser::SyntaxKind; -use tokens::LexerErrorKind; +use tokens::{KeywordSet, LexerErrorKind}; // use tracing::debug; use typed_index_collections::{TiSlice, TiVec}; use vfs::VfsPath; @@ -23,6 +25,45 @@ impl_idx_math_from!(FullTokenIdx(u32)); pub struct RelevantTokenIdx(u32); impl_idx_math_from!(RelevantTokenIdx(u32)); +/// Lexer state that outlives an individual source file. +/// +/// `` `begin_keywords `` affects "all source code that follows the directive, +/// even across source code file boundaries" (VAMS-2023 10.6), so the active +/// keyword set cannot live in the per-file [`Parser`]. It is owned by the +/// [`Processor`](crate::processor::Processor) and shared with every parser it +/// creates. +#[derive(Debug, Default)] +pub(crate) struct LexerState { + keywords: Cell, + /// Number of `module` tokens without a matching `endmodule` seen so far. + /// Used to reject keyword directives inside a design element. + module_depth: Cell, +} + +impl LexerState { + pub(crate) fn keywords(&self) -> KeywordSet { + self.keywords.get() + } + + pub(crate) fn set_keywords(&self, keywords: KeywordSet) { + self.keywords.set(keywords) + } + + pub(crate) fn in_design_element(&self) -> bool { + self.module_depth.get() != 0 + } + + fn track_design_element(&self, kind: SyntaxKind) { + match kind { + SyntaxKind::MODULE_KW => self.module_depth.set(self.module_depth.get() + 1), + SyntaxKind::ENDMODULE_KW => { + self.module_depth.set(self.module_depth.get().saturating_sub(1)) + } + _ => (), + } + } +} + pub(crate) struct Parser<'a, 'd> { full_tokens: TiVec, relevant_tokens: TiVec, @@ -35,6 +76,7 @@ pub(crate) struct Parser<'a, 'd> { pub(crate) ctx: SourceContext, pub(crate) dst: &'d mut Vec, pub(crate) working_dir: VfsPath, + pub(crate) state: Rc, } fn mk_token( @@ -53,6 +95,7 @@ impl<'a, 'd> Parser<'a, 'd> { ctx: SourceContext, working_dir: VfsPath, dst: &'d mut Vec, + state: Rc, err: &mut Vec, ) -> Self { let full_tokens = TiVec::from(lexer::tokenize(src)); @@ -93,6 +136,7 @@ impl<'a, 'd> Parser<'a, 'd> { ctx, dst, working_dir, + state, previous_offset: 0.into(), offset: 0.into(), token, @@ -170,11 +214,21 @@ impl<'a, 'd> Parser<'a, 'd> { fn advance(&mut self, save: bool, start: FullTokenIdx, err: &mut Vec) { let range = start..self.full_token_pos; if save { + let state = &*self.state; self.dst.extend(self.full_tokens[range].iter().filter_map(|token| { - let res = Self::convert_lexer_token(*token, self.offset, self.src, err, self.ctx); + let keywords = state.keywords(); + let res = Self::convert_lexer_token( + *token, + self.offset, + self.src, + err, + self.ctx, + keywords, + ); self.offset += token.len; let (kind, range) = res?; - Some(crate::Token { span: CtxSpan { range, ctx: self.ctx }, kind }) + state.track_design_element(kind); + Some(crate::Token { span: CtxSpan { range, ctx: self.ctx }, kind, keywords }) })) } else { let len: TextSize = self.full_tokens[range].iter().map(|token| token.len).sum(); @@ -188,9 +242,10 @@ impl<'a, 'd> Parser<'a, 'd> { src: &str, err: &mut Vec, ctx: SourceContext, + keywords: KeywordSet, ) -> Option<(SyntaxKind, TextRange)> { let range = TextRange::at(offset, token.len); - let (syntax, error) = token.kind.to_syntax(&src[range]); + let (syntax, error) = token.kind.to_syntax(&src[range], keywords); if let Some(error) = error { let span = CtxSpan { range, ctx }; match error { @@ -214,8 +269,13 @@ impl<'a, 'd> Parser<'a, 'd> { dst: &mut Vec>, err: &mut Vec, ) { + // NOTE: macro bodies are resolved to syntax tokens at definition time, so + // they capture the keyword set in effect where the `define appears rather + // than the one at the expansion site. + let keywords = self.state.keywords(); dst.extend(self.full_tokens[range].iter().filter_map(|token| { - let res = Self::convert_lexer_token(*token, self.offset, self.src, err, self.ctx); + let res = + Self::convert_lexer_token(*token, self.offset, self.src, err, self.ctx, keywords); self.offset += token.len; let (kind, range) = res?; Some(ParsedToken { kind: kind.into(), range }) @@ -306,6 +366,9 @@ impl<'a, 'd> Parser<'a, 'd> { "`endif" => CompilerDirective::EndIf, "`undef" => CompilerDirective::Undef, "`resetall" => CompilerDirective::ResetAll, + // VAMS-2023 10.6: select the set of reserved keywords. + "`begin_keywords" => CompilerDirective::BeginKeywords, + "`end_keywords" => CompilerDirective::EndKeywords, _ => CompilerDirective::Macro, } } @@ -334,5 +397,9 @@ pub enum CompilerDirective { EndIf, Undef, ResetAll, + /// `` `begin_keywords "" `` — push a keyword set. + BeginKeywords, + /// `` `end_keywords `` — pop back to the previous keyword set. + EndKeywords, Macro, } diff --git a/openvaf/preprocessor/src/processor.rs b/openvaf/preprocessor/src/processor.rs index a9cb641d..7b882a94 100755 --- a/openvaf/preprocessor/src/processor.rs +++ b/openvaf/preprocessor/src/processor.rs @@ -1,11 +1,13 @@ use std::io; use std::iter::once; +use std::rc::Rc; use std::sync::Arc; use ahash::AHashMap; use stdx::{impl_debug_display, impl_idx_from}; use text_size::{TextRange, TextSize}; use tokens::parser::SyntaxKind; +use tokens::KeywordSet; use tokens::SyntaxKind::{L_PAREN, R_PAREN}; // use tracing::{debug, debug_span, trace}; use typed_index_collections::{TiSlice, TiVec}; @@ -14,8 +16,10 @@ use vfs::{FileId, VfsPath}; use crate::diagnostics::PreprocessorDiagnostic::{ self, MacroArgumentCountMismatch, MacroNotFound, UnexpectedToken, }; -use crate::grammar::{parse_condition, parse_define, parse_include, parse_macro_call}; -use crate::parser::{CompilerDirective, Parser, PreprocessorToken}; +use crate::grammar::{ + parse_begin_keywords, parse_condition, parse_define, parse_include, parse_macro_call, +}; +use crate::parser::{CompilerDirective, LexerState, Parser, PreprocessorToken}; use crate::sourcemap::{CtxSpan, FileSpan, SourceContext, SourceMap}; use crate::{Diagnostics, FileReadError, ScopedTextArea, SourceProvider, Token}; @@ -25,6 +29,13 @@ pub(crate) struct Processor<'a> { arena: &'a ScopedTextArea, macros: AHashMap<&'a str, Macro<'a>>, include_dirs: Arc<[VfsPath]>, + /// Lexer state shared with every [`Parser`] this processor creates. + lexer_state: Rc, + /// The `` `begin_keywords `` directives that are still open, innermost last. + /// + /// The directives nest and span file boundaries (VAMS-2023 10.6), so the + /// stack belongs to the processor rather than to a single parser. + keyword_stack: Vec<(KeywordSet, CtxSpan)>, } impl<'a> Processor<'a> { @@ -51,6 +62,8 @@ impl<'a> Processor<'a> { arena: storage, sources, include_dirs: sources.include_dirs(root_file), + lexer_state: Rc::default(), + keyword_stack: Vec::new(), }; Ok(res) } @@ -60,13 +73,48 @@ impl<'a> Processor<'a> { let mut err = Diagnostics::new(); let mut dst = Vec::new(); - let parser = - Parser::new(self.arena.get(0), SourceContext::ROOT, working_dir, &mut dst, &mut err); + let parser = Parser::new( + self.arena.get(0), + SourceContext::ROOT, + working_dir, + &mut dst, + self.lexer_state.clone(), + &mut err, + ); self.process_file(parser, &mut err); + // every `begin_keywords must be closed by the end of the compilation unit + for (_, span) in self.keyword_stack.drain(..) { + err.push(PreprocessorDiagnostic::UnterminatedKeywords { span }) + } + (dst, err) } + /// Applies the innermost open `` `begin_keywords `` directive, or OpenVAF's + /// default keyword set if there is none. + fn sync_keywords(&self) { + let keywords = + self.keyword_stack.last().map_or(KeywordSet::default(), |&(keywords, _)| keywords); + self.lexer_state.set_keywords(keywords) + } + + /// VAMS-2023 10.6: the keyword directives may only appear outside a design + /// element. Returns `true` (and reports) when that is violated. + fn reject_keywords_in_design_element( + &self, + name: &'static str, + span: CtxSpan, + err: &mut Diagnostics, + ) -> bool { + if self.lexer_state.in_design_element() { + err.push(PreprocessorDiagnostic::KeywordsInDesignElement { name, span }); + true + } else { + false + } + } + pub(crate) fn is_macro_defined(&mut self, name: &'a str) -> bool { self.macros.contains_key(name) } @@ -102,7 +150,7 @@ impl<'a> Processor<'a> { .source_map .add_ctx(FileSpan { file, range: TextRange::up_to(TextSize::of(src)) }, span); - let parser = Parser::new(src, ctx, workdir, dst, errors); + let parser = Parser::new(src, ctx, workdir, dst, self.lexer_state.clone(), errors); self.process_file(parser, errors); Ok(()) @@ -133,7 +181,12 @@ impl<'a> Processor<'a> { errors: &mut Diagnostics, ) { match *token { - ParsedTokenKind::ResolvedToken(kind) => dst.push(Token { kind, span }), + // the token kinds of a macro body are resolved where the `define is + // parsed, but the expansion site decides which identifiers are + // reserved names there + ParsedTokenKind::ResolvedToken(kind) => { + dst.push(Token { kind, span, keywords: self.lexer_state.keywords() }) + } ParsedTokenKind::ArgumentReference(arg) => { dst.extend(&args[arg]); } @@ -176,13 +229,14 @@ impl<'a> Processor<'a> { if new_args.len() > def.arg_cnt { // macro definition has no arguments, but some were parsed as part of the call // so put the arguments back - dst.push(Token { kind: L_PAREN, span }); + let keywords = self.lexer_state.keywords(); + dst.push(Token { kind: L_PAREN, span, keywords }); for arg in new_args { for tok in arg { dst.push(tok) } } - dst.push(Token { kind: R_PAREN, span }); + dst.push(Token { kind: R_PAREN, span, keywords }); } } else { errors.push(MacroArgumentCountMismatch { @@ -268,6 +322,24 @@ impl<'a> Processor<'a> { }); p.bump(); } + CompilerDirective::BeginKeywords => { + if let Some((keywords, span)) = parse_begin_keywords(p, err) { + if !self.reject_keywords_in_design_element("begin_keywords", span, err) { + self.keyword_stack.push((keywords, span)); + self.sync_keywords(); + } + } + } + CompilerDirective::EndKeywords => { + let span = p.current_span(); + p.bump(); + if !self.reject_keywords_in_design_element("end_keywords", span, err) { + if self.keyword_stack.pop().is_none() { + err.push(PreprocessorDiagnostic::UnmatchedEndKeywords { span }) + } + self.sync_keywords(); + } + } CompilerDirective::Macro => { let (call, range) = parse_macro_call(p, err, &[], &mut self.source_map, p.end()); diff --git a/openvaf/preprocessor/src/test_data/begin_keywords_1364_2005.tokens b/openvaf/preprocessor/src/test_data/begin_keywords_1364_2005.tokens new file mode 100644 index 00000000..fdfec66f --- /dev/null +++ b/openvaf/preprocessor/src/test_data/begin_keywords_1364_2005.tokens @@ -0,0 +1,44 @@ +WHITESPACE +MODULE_KW +WHITESPACE +IDENT +L_PAREN +IDENT +COMMA +WHITESPACE +IDENT +COMMA +WHITESPACE +IDENT +R_PAREN +SEMICOLON +WHITESPACE +INPUT_KW +WHITESPACE +IDENT +COMMA +WHITESPACE +IDENT +COMMA +WHITESPACE +IDENT +SEMICOLON +WHITESPACE +ENDMODULE_KW +WHITESPACE +MODULE_KW +WHITESPACE +IDENT +L_PAREN +IDENT +R_PAREN +SEMICOLON +WHITESPACE +ANALOG_KW +WHITESPACE +BEGIN_KW +WHITESPACE +END_KW +WHITESPACE +ENDMODULE_KW +WHITESPACE diff --git a/openvaf/preprocessor/src/test_data/begin_keywords_1364_2005_expanded.va b/openvaf/preprocessor/src/test_data/begin_keywords_1364_2005_expanded.va new file mode 100644 index 00000000..0bdac42f --- /dev/null +++ b/openvaf/preprocessor/src/test_data/begin_keywords_1364_2005_expanded.va @@ -0,0 +1,7 @@ + +module legacy(from, string, ground); + input from, string, ground; +endmodule +module ams(a); + analog begin end +endmodule diff --git a/openvaf/preprocessor/src/test_data/begin_keywords_across_include.tokens b/openvaf/preprocessor/src/test_data/begin_keywords_across_include.tokens new file mode 100644 index 00000000..4b5b625f --- /dev/null +++ b/openvaf/preprocessor/src/test_data/begin_keywords_across_include.tokens @@ -0,0 +1,12 @@ +IDENT +WHITESPACE +IDENT +WHITESPACE +IDENT +WHITESPACE +IDENT +WHITESPACE +ANALOG_KW +WHITESPACE +STRING_KW +WHITESPACE diff --git a/openvaf/preprocessor/src/test_data/begin_keywords_across_include_expanded.va b/openvaf/preprocessor/src/test_data/begin_keywords_across_include_expanded.va new file mode 100644 index 00000000..13623be7 --- /dev/null +++ b/openvaf/preprocessor/src/test_data/begin_keywords_across_include_expanded.va @@ -0,0 +1,3 @@ +analog string +analog string +analog string diff --git a/openvaf/preprocessor/src/test_data/begin_keywords_in_disabled_branch.tokens b/openvaf/preprocessor/src/test_data/begin_keywords_in_disabled_branch.tokens new file mode 100644 index 00000000..1e708cbb --- /dev/null +++ b/openvaf/preprocessor/src/test_data/begin_keywords_in_disabled_branch.tokens @@ -0,0 +1,5 @@ +WHITESPACE +ANALOG_KW +WHITESPACE +STRING_KW +WHITESPACE diff --git a/openvaf/preprocessor/src/test_data/begin_keywords_in_disabled_branch_expanded.va b/openvaf/preprocessor/src/test_data/begin_keywords_in_disabled_branch_expanded.va new file mode 100644 index 00000000..943c5731 --- /dev/null +++ b/openvaf/preprocessor/src/test_data/begin_keywords_in_disabled_branch_expanded.va @@ -0,0 +1,2 @@ + +analog string diff --git a/openvaf/preprocessor/src/test_data/begin_keywords_nested.tokens b/openvaf/preprocessor/src/test_data/begin_keywords_nested.tokens new file mode 100644 index 00000000..bb442274 --- /dev/null +++ b/openvaf/preprocessor/src/test_data/begin_keywords_nested.tokens @@ -0,0 +1,13 @@ +WHITESPACE +IDENT +WHITESPACE +IDENT +WHITESPACE +LOCALPARAM_KW +WHITESPACE +GENVAR_KW +WHITESPACE +LOCALPARAM_KW +WHITESPACE +GENVAR_KW +WHITESPACE diff --git a/openvaf/preprocessor/src/test_data/begin_keywords_nested_expanded.va b/openvaf/preprocessor/src/test_data/begin_keywords_nested_expanded.va new file mode 100644 index 00000000..4ca42474 --- /dev/null +++ b/openvaf/preprocessor/src/test_data/begin_keywords_nested_expanded.va @@ -0,0 +1,4 @@ + +localparam genvar +localparam genvar +localparam genvar diff --git a/openvaf/preprocessor/src/tests.rs b/openvaf/preprocessor/src/tests.rs index b4a6efe3..51a48781 100644 --- a/openvaf/preprocessor/src/tests.rs +++ b/openvaf/preprocessor/src/tests.rs @@ -1,7 +1,7 @@ use std::sync::Arc; use std::{cell::RefCell, path::PathBuf}; -use expect_test::expect_file; +use expect_test::{expect, expect_file}; use vfs::{FileId, Vfs, VfsPath}; use crate::{preprocess, Preprocess, SourceProvider}; @@ -181,3 +181,133 @@ fn source_map_triple_replacement() { "source_map_triple_replacement", ) } + +fn preprocessor_diagnostics(src: &str) -> String { + let sources = TestSourceProvider::new(vec![]); + let file = sources.vfs.borrow_mut().add_virt_file("/keywords_test.va", src.to_owned().into()); + let Preprocess { diagnostics, .. } = preprocess(&sources, file); + diagnostics.iter().map(|diagnostic| format!("{diagnostic}\n")).collect() +} + +/// VAMS-2023 10.6: `` `begin_keywords "1364-2005" `` releases the Verilog-AMS +/// keywords, so `from`, `string` and `ground` lex as plain identifiers until the +/// matching `` `end_keywords ``. +#[test] +fn begin_keywords_1364_2005() { + check_prepocessor_single_file( + r#" +`begin_keywords "1364-2005" +module legacy(from, string, ground); + input from, string, ground; +endmodule +`end_keywords +module ams(a); + analog begin end +endmodule +"#, + "begin_keywords_1364_2005", + ) +} + +/// Nested directives form a stack: `` `end_keywords `` restores the enclosing +/// keyword set rather than the default one. +#[test] +fn begin_keywords_nested() { + check_prepocessor_single_file( + r#" +`begin_keywords "VAMS-2023" +`begin_keywords "1364-1995" +localparam genvar +`end_keywords +localparam genvar +`end_keywords +localparam genvar +"#, + "begin_keywords_nested", + ) +} + +/// The directive "affects all source code that follows the directive, even +/// across source code file boundaries". +#[test] +fn begin_keywords_across_include() { + let sources = TestSourceProvider::new(vec![]); + let root = { + let mut vfs = sources.vfs.borrow_mut(); + vfs.add_virt_file("/inc.va", "analog string\n".to_owned().into()); + vfs.add_virt_file( + "/parent.va", + concat!( + "`begin_keywords \"1364-2005\"\n", + "`include \"inc.va\"\n", + "analog string\n", + "`end_keywords\n", + "analog string\n" + ) + .to_owned() + .into(), + ) + }; + check_prepocessor(sources, root, "begin_keywords_across_include"); +} + +/// Only the specifiers listed in VAMS-2023 10.6 are accepted; an unknown one is +/// reported and leaves the active keyword set alone. +#[test] +fn begin_keywords_unknown_version() { + expect![[r#" + unknown keyword version specifier "VAMS-2.5" + "#]] + .assert_eq(&preprocessor_diagnostics("`begin_keywords \"VAMS-2.5\"\nanalog\n")); +} + +/// An unmatched `` `end_keywords `` and a `` `begin_keywords `` that is never +/// closed are both reported. +#[test] +fn begin_keywords_unbalanced() { + expect![[r#" + '`end_keywords' without a matching '`begin_keywords' + "#]] + .assert_eq(&preprocessor_diagnostics("`end_keywords\n")); + + expect![[r#" + '`begin_keywords' without a matching '`end_keywords' + "#]] + .assert_eq(&preprocessor_diagnostics("`begin_keywords \"1364-2005\"\nanalog\n")); +} + +/// A directive inside a disabled `` `ifdef `` branch is never taken. +#[test] +fn begin_keywords_in_disabled_branch() { + check_prepocessor_single_file( + r#" +`ifdef NOT_DEFINED +`begin_keywords "1364-2005" +`endif +analog string +"#, + "begin_keywords_in_disabled_branch", + ) +} + +/// A keyword directive inside a `` `define `` body is rejected. The parser has +/// to consume it: leaving it in place used to spin the macro-body loop forever. +#[test] +fn begin_keywords_inside_define() { + expect![[r#" + encountered unexpected token! + "#]] + .assert_eq(&preprocessor_diagnostics("`define BAD `begin_keywords \"1364-2005\"\n`BAD\n")); +} + +/// VAMS-2023 10.6: the directives may only be specified outside of a design +/// element. +#[test] +fn begin_keywords_inside_module() { + expect![[r#" + '`begin_keywords' is not allowed inside a design element + "#]] + .assert_eq(&preprocessor_diagnostics( + "module m;\n`begin_keywords \"1364-2005\"\nendmodule\n", + )); +} diff --git a/openvaf/syntax/src/lib.rs b/openvaf/syntax/src/lib.rs index a64d9fff..26bcace0 100644 --- a/openvaf/syntax/src/lib.rs +++ b/openvaf/syntax/src/lib.rs @@ -22,7 +22,7 @@ pub use rowan::{ }; pub use syntax_node::{SyntaxNode, SyntaxToken}; pub use token_text::TokenText; -pub use tokens::{SyntaxKind, T}; +pub use tokens::{KeywordSet, SyntaxKind, T}; use vfs::FileId; pub use ast::ConstExprValue; @@ -192,10 +192,11 @@ impl SourceFile { root_file: FileId, preprocess: &Preprocess, ) -> Parse { - let (green, mut errors, ctx_map) = parsing::parse_text(db, root_file, preprocess); + let parsing::Built { tree: green, mut errors, ranges: ctx_map, keyword_regions } = + parsing::parse_text(db, root_file, preprocess); let root = SyntaxNode::new_root(green.clone()); - validation::validate(&root, &mut errors); + validation::validate(&root, &keyword_regions, &mut errors); assert_eq!(root.kind(), SyntaxKind::SOURCE_FILE); diff --git a/openvaf/syntax/src/parsing.rs b/openvaf/syntax/src/parsing.rs index d86fa6fd..0d15d2d7 100644 --- a/openvaf/syntax/src/parsing.rs +++ b/openvaf/syntax/src/parsing.rs @@ -1,19 +1,17 @@ mod tree_builder; -use ::preprocessor::sourcemap::SourceContext; use ::preprocessor::{Preprocess, SourceProvider}; -use rowan::{TextRange, TextSize}; use vfs::FileId; +pub(crate) use tree_builder::{Built, KeywordRegions}; + use crate::parsing::tree_builder::SyntaxTreeBuilder; -use crate::syntax_node::GreenNode; -use crate::SyntaxError; pub(crate) fn parse_text( sources: &dyn SourceProvider, root_file: FileId, Preprocess { ts, sm, .. }: &Preprocess, -) -> (GreenNode, Vec, Vec<(TextRange, SourceContext, TextSize)>) { +) -> Built { // tokens without whitespaces/comments let parser_tokens: Vec<_> = ts .iter() @@ -34,7 +32,5 @@ pub(crate) fn parse_text( } } - let (tree, parser_errors, ctx_map) = builder.finish(); - - (tree, parser_errors, ctx_map) + builder.finish() } diff --git a/openvaf/syntax/src/parsing/tree_builder.rs b/openvaf/syntax/src/parsing/tree_builder.rs index ebee41ba..14a1ce88 100644 --- a/openvaf/syntax/src/parsing/tree_builder.rs +++ b/openvaf/syntax/src/parsing/tree_builder.rs @@ -4,6 +4,7 @@ use std::sync::Arc; use preprocessor::sourcemap::{CtxSpan, SourceContext, SourceMap}; use preprocessor::{SourceProvider, Token}; use rowan::{GreenNodeBuilder, Language}; +use tokens::KeywordSet; use vfs::FileId; use crate::syntax_node::{GreenNode, VerilogALanguage}; @@ -29,6 +30,10 @@ pub(crate) struct SyntaxTreeBuilder<'a> { sm: &'a SourceMap, ranges: Vec<(TextRange, SourceContext, TextSize)>, current_range: CtxSpan, + /// Runs of tree text that a `` `begin_keywords `` directive applies to, + /// sorted and non-overlapping. Regions using the default keyword set are not + /// recorded. + keyword_regions: Vec<(TextRange, KeywordSet)>, } enum State { @@ -37,6 +42,29 @@ enum State { PendingFinish, } +pub(crate) struct Built { + pub tree: GreenNode, + pub errors: Vec, + pub ranges: Vec<(TextRange, SourceContext, TextSize)>, + pub keyword_regions: KeywordRegions, +} + +/// The `` `begin_keywords `` regions of a parsed file, in tree coordinates. +#[derive(Debug, Default)] +pub(crate) struct KeywordRegions(Vec<(TextRange, KeywordSet)>); + +impl KeywordRegions { + /// The keyword set in effect at `pos`, which is OpenVAF's default set unless + /// a `` `begin_keywords `` directive covers it. + pub fn get(&self, pos: TextSize) -> KeywordSet { + let idx = self.0.partition_point(|(range, _)| range.end() <= pos); + match self.0.get(idx) { + Some(&(range, set)) if range.contains(pos) => set, + _ => KeywordSet::default(), + } + } +} + impl<'a> SyntaxTreeBuilder<'a> { pub(super) fn token(&mut self, kind: SyntaxKind) { match mem::replace(&mut self.state, State::Normal) { @@ -45,12 +73,12 @@ impl<'a> SyntaxTreeBuilder<'a> { State::Normal => (), } self.eat_trivia(); - let span = self.tokens[self.token_pos].span; + let token = self.tokens[self.token_pos]; self.panic &= !matches!( kind, T![;] | T![end] | T![endnature] | T![endmodule] | T![enddiscipline] | T![endfunction] ) || self.err_depth != u32::MAX; - self.do_token(kind, span); + self.do_token(kind, token.span, token.keywords); } pub(super) fn start_node(&mut self, kind: SyntaxKind) { @@ -164,6 +192,7 @@ impl<'a> SyntaxTreeBuilder<'a> { db, sm, current_src, + keyword_regions: Vec::new(), ranges: Vec::with_capacity(128), current_range: CtxSpan { ctx: SourceContext::ROOT, @@ -176,9 +205,7 @@ impl<'a> SyntaxTreeBuilder<'a> { } } - pub(super) fn finish( - mut self, - ) -> (GreenNode, Vec, Vec<(TextRange, SourceContext, TextSize)>) { + pub(super) fn finish(mut self) -> Built { match mem::replace(&mut self.state, State::Normal) { State::PendingFinish => { self.eat_trivia(); @@ -189,7 +216,12 @@ impl<'a> SyntaxTreeBuilder<'a> { let start = self.ranges.last().map_or(0.into(), |(range, _, _)| range.end()); let range = TextRange::new(start, self.text_pos); self.ranges.push((range, self.current_range.ctx, self.current_range.range.start())); - (self.inner.finish(), self.errors, self.ranges) + Built { + tree: self.inner.finish(), + errors: self.errors, + ranges: self.ranges, + keyword_regions: KeywordRegions(self.keyword_regions), + } } fn eat_trivia(&mut self) { @@ -197,11 +229,28 @@ impl<'a> SyntaxTreeBuilder<'a> { if !token.kind.is_trivia() { break; } - self.do_token(token.kind, token.span); + self.do_token(token.kind, token.span, token.keywords); + } + } + + /// Extends the current `` `begin_keywords `` region, or starts a new one. + /// + /// Called for every token in tree order, so `self.text_pos` is the start of + /// the token that is about to be appended. + fn record_keywords(&mut self, keywords: KeywordSet, len: TextSize) { + if keywords == KeywordSet::default() { + return; + } + let end = self.text_pos + len; + match self.keyword_regions.last_mut() { + Some((range, set)) if *set == keywords && range.end() == self.text_pos => { + *range = TextRange::new(range.start(), end) + } + _ => self.keyword_regions.push((TextRange::new(self.text_pos, end), keywords)), } } - fn do_token(&mut self, kind: SyntaxKind, span: CtxSpan) { + fn do_token(&mut self, kind: SyntaxKind, span: CtxSpan, keywords: KeywordSet) { let same_ctx = span.ctx == self.current_range.ctx; let is_continuous = same_ctx && span.range.start() == self.current_range.range.end(); if is_continuous { @@ -223,6 +272,7 @@ impl<'a> SyntaxTreeBuilder<'a> { } let range = span.to_file_span(self.sm).range; + self.record_keywords(keywords, range.len()); let text = &self.current_src[range]; self.text_pos += range.len(); self.token_pos += 1; diff --git a/openvaf/syntax/src/validation.rs b/openvaf/syntax/src/validation.rs index 31aad1dc..41237c25 100644 --- a/openvaf/syntax/src/validation.rs +++ b/openvaf/syntax/src/validation.rs @@ -7,9 +7,14 @@ use crate::ast::{ LiteralKind, ModulePortKind, ModulePorts, Name, PathSegmentKind, }; use crate::name::{kw, kw_comp}; +use crate::parsing::KeywordRegions; use crate::{match_ast, AstNode, AstPtr, SyntaxError, SyntaxNode, SyntaxNodePtr, T}; -pub(crate) fn validate(root: &SyntaxNode, errors: &mut Vec) { +pub(crate) fn validate( + root: &SyntaxNode, + keywords: &KeywordRegions, + errors: &mut Vec, +) { for node in root.descendants() { match_ast! { match node { @@ -21,7 +26,7 @@ pub(crate) fn validate(root: &SyntaxNode, errors: &mut Vec) { ast::NatureDecl(decl) => validate_nature_decl(decl,errors), ast::NatureAttr(attr) => validate_nature_attr(attr,errors), ast::Literal(decl) => validate_literal(decl, errors), - ast::Name(name) => validate_name(name,errors), + ast::Name(name) => validate_name(name,keywords,errors), ast::ModuleDecl(module) => validate_module(module,errors), ast::ParamDecl(param) => validate_param(param, errors), _ => validate_net_type_token(node,errors) @@ -274,8 +279,14 @@ fn validate_function(fun: ast::Function, errors: &mut Vec) { } } -fn validate_name(name: Name, errors: &mut Vec) { +fn validate_name(name: Name, keywords: &KeywordRegions, errors: &mut Vec) { if let Some(ident) = name.ident_token() { + // VAMS-2023 10.6: a `begin_keywords region may release words that are + // reserved by default, which then become ordinary identifiers. + if !keywords.get(ident.text_range().start()).reserves(ident.text()) { + return; + } + let parent = name.syntax().parent(); let p = parent.as_ref(); diff --git a/openvaf/test_data/syn_ui/begin_keywords.log b/openvaf/test_data/syn_ui/begin_keywords.log new file mode 100644 index 00000000..e69de29b diff --git a/openvaf/test_data/syn_ui/begin_keywords.va b/openvaf/test_data/syn_ui/begin_keywords.va new file mode 100644 index 00000000..8f9e85c6 --- /dev/null +++ b/openvaf/test_data/syn_ui/begin_keywords.va @@ -0,0 +1,34 @@ +// VAMS-2023 10.6 (Mantis 7921): `begin_keywords / `end_keywords select the set +// of reserved keywords. Under a plain Verilog specifier the Verilog-AMS +// keywords are ordinary identifiers again. + +// The example from 10.6: `sin` is a Verilog-AMS keyword but not a 1364-2005 one. +`begin_keywords "1364-2005" +module m2(sin); + input sin; +endmodule + +module legacy(clk); + input clk; + // `discipline`, `from`, `string` and `ground` are Verilog-AMS keywords but + // ordinary identifiers in IEEE Std 1364-2005 + integer discipline; + integer from; + integer string; + integer ground; + // ... while `localparam` and `uwire` are reserved by 1364-2005 + localparam integer two = 2; +endmodule +`end_keywords + +// ... and the Verilog-AMS keywords are keywords again once the region is closed +module ams(a); + inout a; + electrical a; + parameter real r = 1.0 from [0.0:inf); + string label; + analog begin + label = "ok"; + I(a) <+ V(a) / r; + end +endmodule diff --git a/openvaf/test_data/syn_ui/begin_keywords_err.log b/openvaf/test_data/syn_ui/begin_keywords_err.log new file mode 100644 index 00000000..a2de21dc --- /dev/null +++ b/openvaf/test_data/syn_ui/begin_keywords_err.log @@ -0,0 +1,38 @@ +error: unknown keyword version specifier "VAMS-2.5" + --> /begin_keywords_err.va:4:1 + | +4 | `begin_keywords "VAMS-2.5" + | ^^^^^^^^^^^^^^^^^^^^^^^^^^ unknown version specifier + | + = expected one of "1364-1995", "1364-2001", "1364-2005", "VAMS-2.3", "VAMS-2023" + +error: '`end_keywords' without a matching '`begin_keywords' + --> /begin_keywords_err.va:7:1 + | +7 | `end_keywords + | ^^^^^^^^^^^^^ no keyword set is active here + +error: '`begin_keywords' is not allowed inside a design element + --> /begin_keywords_err.va:11:5 + | +11 | `begin_keywords "1364-2005" + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ directive is used inside a module + | + = keyword directives may only appear outside of design elements + +error: '`end_keywords' is not allowed inside a design element + --> /begin_keywords_err.va:12:5 + | +12 | `end_keywords + | ^^^^^^^^^^^^^ directive is used inside a module + | + = keyword directives may only appear outside of design elements + +error: '`begin_keywords' without a matching '`end_keywords' + --> /begin_keywords_err.va:16:1 + | +16 | `begin_keywords "1364-1995" + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ keyword set is opened here + | + = add '`end_keywords' to restore the default keywords + diff --git a/openvaf/test_data/syn_ui/begin_keywords_err.va b/openvaf/test_data/syn_ui/begin_keywords_err.va new file mode 100644 index 00000000..5977e406 --- /dev/null +++ b/openvaf/test_data/syn_ui/begin_keywords_err.va @@ -0,0 +1,16 @@ +// Error cases for the VAMS-2023 10.6 keyword directives. + +// only the version specifiers named by the standard are accepted +`begin_keywords "VAMS-2.5" + +// `end_keywords without a matching `begin_keywords +`end_keywords + +module m; + // the directives may only appear outside of a design element + `begin_keywords "1364-2005" + `end_keywords +endmodule + +// ... and this one is never closed +`begin_keywords "1364-1995" diff --git a/openvaf/tokens/src/keywords.rs b/openvaf/tokens/src/keywords.rs new file mode 100644 index 00000000..b3c532b4 --- /dev/null +++ b/openvaf/tokens/src/keywords.rs @@ -0,0 +1,215 @@ +//! Selectable sets of reserved keywords. +//! +//! Verilog-AMS `` `begin_keywords ``/`` `end_keywords `` (VAMS-2023 10.6, Mantis +//! 7921) let a source file pick which identifiers are reserved as keywords. The +//! directives change *only* the set of reserved words; they do not change the +//! grammar, the token kinds or any other aspect of the language. +//! +//! The interesting direction is *releasing* words: every set the standard +//! defines is a subset of what OpenVAF reserves by default, so +//! [`KeywordSet::reserves`] answers "does the selected set still reserve this +//! word that OpenVAF reserves by default". For the Verilog-AMS specifiers that +//! is always the case, which leaves the IEEE Std 1364 keyword lists as the only +//! table this module has to carry. + +use crate::parser::SyntaxKind; + +/// A set of reserved keywords, selected by a `` `begin_keywords `` version +/// specifier. +/// +/// Ordered by inclusion: every set reserves everything the sets before it +/// reserve. +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug, Default, Hash)] +pub enum KeywordSet { + /// `"1364-1995"` — IEEE Std 1364-1995 Verilog. + Verilog1995, + /// `"1364-2001"` — IEEE Std 1364-2001 Verilog. + Verilog2001, + /// `"1364-2005"` — IEEE Std 1364-2005 Verilog. + Verilog2005, + /// `"VAMS-2.3"` — Verilog-AMS 2.3, Annex B. + Vams23, + /// `"VAMS-2023"` — Verilog-AMS 2023, Annex B. + Vams2023, + /// The set used when no `` `begin_keywords `` directive is in effect: the + /// Verilog-AMS keywords plus the extensions OpenVAF recognises. + #[default] + OpenVaf, +} + +impl KeywordSet { + /// Maps a `` `begin_keywords `` version specifier (without the surrounding + /// quotes) onto the keyword set it selects. + /// + /// VAMS-2023 10.6 requires `"1364-1995"`, `"1364-2001"`, `"1364-2005"`, + /// `"VAMS-2.3"` and `"VAMS-2023"` to be supported; no other specifier is + /// defined by the standard. + pub fn from_version_specifier(specifier: &str) -> Option { + let set = match specifier { + "1364-1995" => KeywordSet::Verilog1995, + "1364-2001" => KeywordSet::Verilog2001, + "1364-2005" => KeywordSet::Verilog2005, + "VAMS-2.3" => KeywordSet::Vams23, + "VAMS-2023" => KeywordSet::Vams2023, + _ => return None, + }; + Some(set) + } + + /// The specifiers accepted by [`KeywordSet::from_version_specifier`], for + /// use in diagnostics. + pub const VERSION_SPECIFIERS: &'static [&'static str] = + &["1364-1995", "1364-2001", "1364-2005", "VAMS-2.3", "VAMS-2023"]; + + /// Whether this set is one of the plain Verilog specifiers, which do not + /// reserve any of the Verilog-AMS keywords. + pub fn is_verilog(self) -> bool { + self <= KeywordSet::Verilog2005 + } + + /// Whether `ident` is reserved in this set. + /// + /// This is only meaningful for words that OpenVAF reserves in the first + /// place: it answers "does the selected set still reserve this", never + /// "does the selected set additionally reserve this". Words that OpenVAF + /// does not reserve by default (`sin`, `abs`, ... in expression position, + /// which are shadowable builtins) stay unreserved in every set. + pub fn reserves(self, ident: &str) -> bool { + if !self.is_verilog() { + // Verilog-AMS 2.3 and 2023 reserve everything OpenVAF reserves: + // none of the words added by VAMS-2023 over 2.3 (`break`, + // `continue`, `return`, `expm1`, `ln1p`, ...) are reserved words in + // OpenVAF, and OpenVAF's own extensions stay reserved so that + // opting into a Verilog-AMS specifier never disables a language + // feature. + return true; + } + verilog_keyword_since(ident).is_some_and(|since| since <= self) + } +} + +/// The earliest IEEE Std 1364 revision that reserves `ident`, or `None` if +/// `ident` is not a Verilog keyword at all (i.e. it is a Verilog-AMS keyword or +/// an OpenVAF extension). +fn verilog_keyword_since(ident: &str) -> Option { + use KeywordSet::{Verilog1995, Verilog2001, Verilog2005}; + let since = match ident { + // IEEE Std 1364-1995, Table 3-1 + "always" | "and" | "assign" | "begin" | "buf" | "bufif0" | "bufif1" | "case" | "casex" + | "casez" | "cmos" | "deassign" | "default" | "defparam" | "disable" | "edge" | "else" + | "end" | "endcase" | "endfunction" | "endmodule" | "endprimitive" | "endspecify" + | "endtable" | "endtask" | "event" | "for" | "force" | "forever" | "fork" | "function" + | "highz0" | "highz1" | "if" | "ifnone" | "initial" | "inout" | "input" | "integer" + | "join" | "large" | "macromodule" | "medium" | "module" | "nand" | "negedge" | "nmos" + | "nor" | "not" | "notif0" | "notif1" | "or" | "output" | "parameter" | "pmos" + | "posedge" | "primitive" | "pull0" | "pull1" | "pulldown" | "pullup" | "rcmos" + | "real" | "realtime" | "reg" | "release" | "repeat" | "rnmos" | "rpmos" | "rtran" + | "rtranif0" | "rtranif1" | "scalared" | "small" | "specify" | "specparam" | "strong0" + | "strong1" | "supply0" | "supply1" | "table" | "task" | "time" | "tran" | "tranif0" + | "tranif1" | "tri" | "tri0" | "tri1" | "triand" | "trior" | "trireg" | "vectored" + | "wait" | "wand" | "weak0" | "weak1" | "while" | "wire" | "wor" | "xnor" | "xor" => { + Verilog1995 + } + + // added by IEEE Std 1364-2001 + "automatic" + | "cell" + | "config" + | "design" + | "endconfig" + | "endgenerate" + | "generate" + | "genvar" + | "incdir" + | "include" + | "instance" + | "liblist" + | "library" + | "localparam" + | "noshowcancelled" + | "pulsestyle_ondetect" + | "pulsestyle_onevent" + | "showcancelled" + | "signed" + | "unsigned" + | "use" => Verilog2001, + + // added by IEEE Std 1364-2005 + "uwire" => Verilog2005, + + _ => return None, + }; + Some(since) +} + +/// [`SyntaxKind::from_keyword`], restricted to the keywords `set` reserves. +/// +/// Identifiers that are keywords in OpenVAF's default set but not in `set` lex +/// as plain [`SyntaxKind::IDENT`]. +pub fn from_keyword_in(ident: &str, set: KeywordSet) -> Option { + let kind = SyntaxKind::from_keyword(ident)?; + set.reserves(ident).then_some(kind) +} + +#[cfg(test)] +mod tests { + use super::{from_keyword_in, KeywordSet}; + use crate::parser::SyntaxKind; + + #[test] + fn verilog_sets_release_verilog_ams_keywords() { + for kw in ["analog", "string", "ground", "discipline", "from", "inf", "aliasparam"] { + assert_eq!(from_keyword_in(kw, KeywordSet::Verilog2005), None, "{kw}"); + assert_eq!( + from_keyword_in(kw, KeywordSet::Vams23), + SyntaxKind::from_keyword(kw), + "{kw}" + ); + } + } + + #[test] + fn verilog_revisions_release_later_keywords() { + assert_eq!(from_keyword_in("localparam", KeywordSet::Verilog1995), None); + assert_eq!( + from_keyword_in("localparam", KeywordSet::Verilog2001), + Some(SyntaxKind::LOCALPARAM_KW) + ); + assert_eq!(from_keyword_in("genvar", KeywordSet::Verilog1995), None); + assert_eq!(from_keyword_in("genvar", KeywordSet::Verilog2001), Some(SyntaxKind::GENVAR_KW)); + assert_eq!(from_keyword_in("uwire", KeywordSet::Verilog2001), None); + assert_eq!(from_keyword_in("uwire", KeywordSet::Verilog2005), Some(SyntaxKind::NET_TYPE)); + } + + #[test] + fn core_keywords_are_reserved_everywhere() { + for kw in ["module", "endmodule", "begin", "end", "input", "real", "parameter", "wire"] { + for set in [ + KeywordSet::Verilog1995, + KeywordSet::Verilog2005, + KeywordSet::Vams2023, + KeywordSet::OpenVaf, + ] { + assert_eq!(from_keyword_in(kw, set), SyntaxKind::from_keyword(kw), "{kw} {set:?}"); + } + } + } + + #[test] + fn unreserved_words_stay_unreserved() { + // shadowable builtins are not keywords in OpenVAF, in any set + for set in [KeywordSet::Verilog1995, KeywordSet::Vams2023, KeywordSet::OpenVaf] { + assert_eq!(from_keyword_in("sin", set), None); + assert_eq!(from_keyword_in("expm1", set), None); + } + } + + #[test] + fn version_specifiers_round_trip() { + for specifier in KeywordSet::VERSION_SPECIFIERS { + assert!(KeywordSet::from_version_specifier(specifier).is_some(), "{specifier}"); + } + assert_eq!(KeywordSet::from_version_specifier("VAMS-2.4"), None); + assert_eq!(KeywordSet::from_version_specifier(""), None); + } +} diff --git a/openvaf/tokens/src/lib.rs b/openvaf/tokens/src/lib.rs index e682e596..05a84ead 100644 --- a/openvaf/tokens/src/lib.rs +++ b/openvaf/tokens/src/lib.rs @@ -1,6 +1,8 @@ +pub mod keywords; pub mod lexer; pub mod parser; +pub use keywords::KeywordSet; use lexer::LiteralKind; use lexer::TokenKind::*; pub use parser::SyntaxKind; @@ -12,7 +14,16 @@ pub enum LexerErrorKind { } impl lexer::TokenKind { - pub fn to_syntax(self, src: &str) -> (Option, Option) { + /// Converts a lexer token into a syntax token. + /// + /// `keywords` selects which identifiers are reserved words; it is changed by + /// the `` `begin_keywords ``/`` `end_keywords `` directives and is + /// [`KeywordSet::OpenVaf`] unless one is in effect. + pub fn to_syntax( + self, + src: &str, + keywords: KeywordSet, + ) -> (Option, Option) { let token = match self { // Combined operators LineComment | BlockComment { terminated: true } => SyntaxKind::COMMENT, @@ -20,7 +31,7 @@ impl lexer::TokenKind { return (Some(SyntaxKind::COMMENT), Some(LexerErrorKind::UnterminatedBlockComment)) } Whitespace => SyntaxKind::WHITESPACE, - SimpleIdent => SyntaxKind::from_keyword(src).unwrap_or(SyntaxKind::IDENT), + SimpleIdent => keywords::from_keyword_in(src, keywords).unwrap_or(SyntaxKind::IDENT), EscapedIdent => SyntaxKind::IDENT, SystemCallIdent if src == "$root" => SyntaxKind::ROOT_KW, SystemCallIdent => SyntaxKind::SYSFUN,