cove_syntax/
token.rs

1//! Tokens produced by the Cove lexer.
2
3use cove_diag::Span;
4
5/// A lexed token.
6#[derive(Clone, Debug, PartialEq)]
7pub struct Token {
8    pub kind: TokenKind,
9    pub span: Span,
10    /// True when a line break separates this token from the previous one.
11    ///
12    /// Newlines are not tokens; the parser reads this flag only where a
13    /// statement could end, so most parsing routines never see line breaks.
14    /// Line breaks hidden inside a `//` or `/* */` comment count as well, so
15    /// commenting out the tail of a line cannot join two statements.
16    pub preceded_by_newline: bool,
17}
18
19/// Keywords recognised by the MVP grammar.
20#[derive(Clone, Copy, Debug, PartialEq, Eq)]
21pub enum Keyword {
22    Async,
23    Await,
24    Break,
25    Continue,
26    Dyn,
27    Else,
28    Enum,
29    Export,
30    Fn,
31    For,
32    If,
33    Impl,
34    In,
35    Is,
36    Let,
37    Match,
38    Opaque,
39    Return,
40    Scope,
41    SelfValue,
42    Struct,
43    Test,
44    Trait,
45    Type,
46    Use,
47    Var,
48    While,
49}
50
51impl Keyword {
52    pub fn from_text(text: &str) -> Option<Keyword> {
53        Some(match text {
54            "async" => Keyword::Async,
55            "await" => Keyword::Await,
56            "break" => Keyword::Break,
57            "continue" => Keyword::Continue,
58            "dyn" => Keyword::Dyn,
59            "else" => Keyword::Else,
60            "enum" => Keyword::Enum,
61            "export" => Keyword::Export,
62            "fn" => Keyword::Fn,
63            "for" => Keyword::For,
64            "if" => Keyword::If,
65            "impl" => Keyword::Impl,
66            "in" => Keyword::In,
67            "is" => Keyword::Is,
68            "let" => Keyword::Let,
69            "match" => Keyword::Match,
70            "opaque" => Keyword::Opaque,
71            "return" => Keyword::Return,
72            "scope" => Keyword::Scope,
73            "self" => Keyword::SelfValue,
74            "struct" => Keyword::Struct,
75            "test" => Keyword::Test,
76            "trait" => Keyword::Trait,
77            "type" => Keyword::Type,
78            "use" => Keyword::Use,
79            "var" => Keyword::Var,
80            "while" => Keyword::While,
81            _ => return None,
82        })
83    }
84
85    pub fn as_str(self) -> &'static str {
86        match self {
87            Keyword::Async => "async",
88            Keyword::Await => "await",
89            Keyword::Break => "break",
90            Keyword::Continue => "continue",
91            Keyword::Dyn => "dyn",
92            Keyword::Else => "else",
93            Keyword::Enum => "enum",
94            Keyword::Export => "export",
95            Keyword::Fn => "fn",
96            Keyword::For => "for",
97            Keyword::If => "if",
98            Keyword::Impl => "impl",
99            Keyword::In => "in",
100            Keyword::Is => "is",
101            Keyword::Let => "let",
102            Keyword::Match => "match",
103            Keyword::Opaque => "opaque",
104            Keyword::Return => "return",
105            Keyword::Scope => "scope",
106            Keyword::SelfValue => "self",
107            Keyword::Struct => "struct",
108            Keyword::Test => "test",
109            Keyword::Trait => "trait",
110            Keyword::Type => "type",
111            Keyword::Use => "use",
112            Keyword::Var => "var",
113            Keyword::While => "while",
114        }
115    }
116}
117
118/// One piece of a string literal.
119///
120/// `"Hello, {name}!"` lexes to `[Text("Hello, "), Interpolation("name"), Text("!")]`.
121#[derive(Clone, Debug, PartialEq)]
122pub enum StringPart {
123    /// Literal text with escape sequences already resolved.
124    Text(String),
125    /// Source text between `{` and `}`, re-parsed as an expression by the parser.
126    Interpolation { source: String, span: Span },
127}
128
129#[derive(Clone, Debug, PartialEq)]
130pub enum TokenKind {
131    Ident(String),
132    Keyword(Keyword),
133    /// `true` / `false`.
134    Bool(bool),
135    Int(i64),
136    Float(f64),
137    /// A duration literal such as `500ms` or `5s`, normalised to nanoseconds.
138    Duration(i64),
139    Str(Vec<StringPart>),
140    /// `/// text` attached to the following declaration.
141    DocComment(String),
142
143    // Delimiters
144    LParen,
145    RParen,
146    LBrace,
147    RBrace,
148    LBracket,
149    RBracket,
150
151    // Punctuation
152    Comma,
153    Colon,
154    Dot,
155    DotDot,
156    DotDotLt,
157    Ellipsis,
158    Arrow,
159    FatArrow,
160    Question,
161    Underscore,
162
163    // Operators
164    Eq,
165    EqEq,
166    Bang,
167    BangEq,
168    Lt,
169    LtEq,
170    Gt,
171    GtEq,
172    Plus,
173    Minus,
174    Star,
175    Slash,
176    Percent,
177    AmpAmp,
178    PipePipe,
179    PlusEq,
180    MinusEq,
181    StarEq,
182    SlashEq,
183    PercentEq,
184
185    Eof,
186}
187
188impl TokenKind {
189    /// A short human-readable name used in diagnostics.
190    pub fn describe(&self) -> String {
191        match self {
192            TokenKind::Ident(name) => format!("identifier `{name}`"),
193            TokenKind::Keyword(k) => format!("keyword `{}`", k.as_str()),
194            TokenKind::Bool(b) => format!("`{b}`"),
195            TokenKind::Int(_) => "integer literal".into(),
196            TokenKind::Float(_) => "float literal".into(),
197            TokenKind::Duration(_) => "duration literal".into(),
198            TokenKind::Str(_) => "string literal".into(),
199            TokenKind::DocComment(_) => "doc comment".into(),
200            TokenKind::Eof => "end of file".into(),
201            other => format!("`{}`", other.symbol().unwrap_or("?")),
202        }
203    }
204
205    /// The literal spelling of a punctuation or operator token.
206    pub fn symbol(&self) -> Option<&'static str> {
207        Some(match self {
208            TokenKind::LParen => "(",
209            TokenKind::RParen => ")",
210            TokenKind::LBrace => "{",
211            TokenKind::RBrace => "}",
212            TokenKind::LBracket => "[",
213            TokenKind::RBracket => "]",
214            TokenKind::Comma => ",",
215            TokenKind::Colon => ":",
216            TokenKind::Dot => ".",
217            TokenKind::DotDot => "..",
218            TokenKind::DotDotLt => "..<",
219            TokenKind::Ellipsis => "...",
220            TokenKind::Arrow => "->",
221            TokenKind::FatArrow => "=>",
222            TokenKind::Question => "?",
223            TokenKind::Underscore => "_",
224            TokenKind::Eq => "=",
225            TokenKind::EqEq => "==",
226            TokenKind::Bang => "!",
227            TokenKind::BangEq => "!=",
228            TokenKind::Lt => "<",
229            TokenKind::LtEq => "<=",
230            TokenKind::Gt => ">",
231            TokenKind::GtEq => ">=",
232            TokenKind::Plus => "+",
233            TokenKind::Minus => "-",
234            TokenKind::Star => "*",
235            TokenKind::Slash => "/",
236            TokenKind::Percent => "%",
237            TokenKind::AmpAmp => "&&",
238            TokenKind::PipePipe => "||",
239            TokenKind::PlusEq => "+=",
240            TokenKind::MinusEq => "-=",
241            TokenKind::StarEq => "*=",
242            TokenKind::SlashEq => "/=",
243            TokenKind::PercentEq => "%=",
244            _ => return None,
245        })
246    }
247}
cove_syntax/token.rs

cove_syntax/
token.rs
