cove_syntax/

lexer.rs

//! The Cove lexer.
//!
//! Converts one source file into a flat token stream. Lexing never panics on
//! malformed input: every lexical error is collected as a [`Diagnostic`] and
//! the lexer recovers by skipping the offending text, so a single call to
//! [`lex`] can report every problem in a file at once.

use cove_diag::{Diagnostic, FileId, SourceMap, Span};

use crate::token::{Keyword, StringPart, Token, TokenKind};

/// One level of the nesting [`Lexer::skip_interpolation_body`] steps over: a
/// `{` that opened an interpolation, or a `"` that opened a string literal.
enum Unclosed {
    Brace,
    Quote,
    Apostrophe,
}

/// Lexes `file` out of `sources` into a token stream.
///
/// The returned stream always ends with exactly one [`TokenKind::Eof`] whose
/// span is the empty range at end of file. On any lexical error, every error
/// found in the file is collected and returned; no tokens are produced in
/// that case.
pub fn lex(sources: &SourceMap, file: FileId) -> Result<Vec<Token>, Vec<Diagnostic>> {
    let (tokens, diagnostics) = lex_recovered(sources, file);
    if diagnostics.is_empty() {
        Ok(tokens)
    } else {
        Err(diagnostics)
    }
}

/// Lexes `file` and answers both halves of what the lexer found: the tokens it
/// recovered *and* everything it complained about.
///
/// [`lex`] is this function with the tokens thrown away whenever there is a
/// diagnostic, which is the right contract for a compiler — a parser handed a
/// token stream with a hole in it reports a second, invented error at the
/// hole. It is the wrong contract for a reader looking at the text. The
/// recovery this exposes is not new: the lexer has always skipped past a
/// lexical error and carried on so that one call reports every problem in a
/// file, and these are the tokens that pass produced.
///
/// The caller is what makes the difference. `crates/cove-wasm`'s highlighter
/// colours source that is being typed, which is a state that does not lex for
/// most of the time a string literal is being written, and a highlighter that
/// had nothing to say about a file with one open quote in it would have
/// nothing to say most of the time.
///
/// The tokens are the same tokens [`lex`] would have answered had there been
/// no diagnostic; nothing is invented to fill a gap. Text the lexer skipped is
/// simply absent from the stream, so a caller that cares what is between two
/// tokens must look at the source.
pub fn lex_recovered(sources: &SourceMap, file: FileId) -> (Vec<Token>, Vec<Diagnostic>) {
    let text = sources.get(file).text.as_str();
    let mut lexer = Lexer {
        text,
        file,
        pos: 0,
        tokens: Vec::new(),
        diagnostics: Vec::new(),
        pending_newline: false,
    };
    lexer.run();
    (lexer.tokens, lexer.diagnostics)
}

struct Lexer<'a> {
    text: &'a str,
    file: FileId,
    pos: usize,
    tokens: Vec<Token>,
    diagnostics: Vec<Diagnostic>,
    /// Set once a line break is seen in the trivia before the next token, and
    /// cleared when that token is produced.
    pending_newline: bool,
}

/// The character an escape sequence spells, or `None` when it spells none.
///
/// One table for both literal forms, because the rule is one: **a code-point
/// literal takes the escapes a string takes, and `\'` besides.** `\'` is
/// therefore legal in a string too, where the apostrophe needs no escaping —
/// which is a widening, and a deliberate one: a second table would be a
/// second rule to remember and a second place for the two to drift apart.
///
/// There is no `\u{...}`. Nothing in the corpus has needed one — every
/// code point a program in this repository names is ASCII — and adding it to
/// one literal form and not the other would be exactly the asymmetry the
/// paragraph above avoids. When something needs it, it goes in here and
/// both forms gain it at once.
fn escaped_char(escaped: char) -> Option<char> {
    Some(match escaped {
        '\\' => '\\',
        '"' => '"',
        '\'' => '\'',
        'n' => '\n',
        't' => '\t',
        'r' => '\r',
        '0' => '\0',
        '{' => '{',
        '}' => '}',
        _ => return None,
    })
}

fn is_ident_start(c: char) -> bool {
    c.is_ascii_alphabetic() || c == '_'
}

fn is_ident_continue(c: char) -> bool {
    c.is_ascii_alphanumeric() || c == '_'
}

/// Nanoseconds per unit for a duration suffix, or `None` if `unit` is not one
/// of `ns`, `us`, `ms`, `s`, `m`, `h`.
fn duration_factor(unit: &str) -> Option<i64> {
    Some(match unit {
        "ns" => 1,
        "us" => 1_000,
        "ms" => 1_000_000,
        "s" => 1_000_000_000,
        "m" => 60_000_000_000,
        "h" => 3_600_000_000_000,
        _ => return None,
    })
}

impl<'a> Lexer<'a> {
    fn peek_char(&self) -> Option<char> {
        self.text[self.pos..].chars().next()
    }

    fn peek_char_at(&self, n: usize) -> Option<char> {
        self.text[self.pos..].chars().nth(n)
    }

    fn bump(&mut self) -> Option<char> {
        let c = self.peek_char()?;
        self.pos += c.len_utf8();
        Some(c)
    }

    /// Produces one token, transferring any line break seen in the trivia
    /// before it onto [`Token::preceded_by_newline`].
    fn push_token(&mut self, kind: TokenKind, start: usize) {
        let span = Span::new(self.file, start as u32, self.pos as u32);
        let preceded_by_newline = std::mem::take(&mut self.pending_newline);
        self.tokens.push(Token {
            kind,
            span,
            preceded_by_newline,
        });
    }

    fn run(&mut self) {
        loop {
            self.skip_whitespace();
            let start = self.pos;
            let Some(c) = self.peek_char() else { break };

            if c == '/' {
                if let Some(kind) = self.handle_slash() {
                    self.push_token(kind, start);
                }
                continue;
            }

            if c == '"' {
                self.bump();
                if let Some(kind) = self.lex_string(start) {
                    self.push_token(kind, start);
                }
                continue;
            }

            if c == '\'' {
                self.bump();
                if let Some(kind) = self.lex_code_point(start) {
                    self.push_token(kind, start);
                }
                continue;
            }

            if c.is_ascii_digit() {
                let kind = self.lex_number(start);
                self.push_token(kind, start);
                continue;
            }

            if is_ident_start(c) {
                let kind = self.lex_ident(start);
                self.push_token(kind, start);
                continue;
            }

            if let Some(kind) = self.lex_operator() {
                self.push_token(kind, start);
                continue;
            }

            if c == '@' {
                self.bump();
                self.reserved_annotation(start);
                continue;
            }

            self.bump();
            self.unexpected_character(c, start);
        }

        let eof = self.pos as u32;
        self.push_token(TokenKind::Eof, eof as usize);
    }

    fn skip_whitespace(&mut self) {
        while let Some(c) = self.peek_char() {
            match c {
                '\n' => self.pending_newline = true,
                ' ' | '\t' | '\r' => {}
                _ => return,
            }
            self.bump();
        }
    }

    fn unexpected_character(&mut self, c: char, start: usize) {
        let span = Span::new(self.file, start as u32, self.pos as u32);
        let mut diag = Diagnostic::error(
            "cove::lex::unexpected_character",
            format!("unexpected character `{c}`"),
        )
        .at(span);
        if c == ';' {
            diag = diag
                .rule("Cove statements are not terminated by `;`.")
                .help("Remove the `;`; the next token or a newline ends the statement.");
        }
        self.diagnostics.push(diag);
    }

    /// `@` is reserved surface, not merely an unknown character: the
    /// Language Card reserves decorator syntax for behavior with specified
    /// compiler or runtime semantics, and the MVP defines none. This is
    /// reported distinctly from [`Lexer::unexpected_character`] so the
    /// message states that rule instead of reading as a stray-character typo.
    fn reserved_annotation(&mut self, start: usize) {
        let span = Span::new(self.file, start as u32, self.pos as u32);
        self.diagnostics.push(
            Diagnostic::error(
                "cove::parse::reserved_annotation",
                "`@` is reserved decorator syntax",
            )
            .at(span)
            .rule(
                "Decorator syntax is reserved for behavior with specified compiler or runtime \
                 semantics; the MVP defines no annotations, so an unknown annotation is an error.",
            )
            .help("remove the `@...`; there is no annotation the MVP recognizes yet"),
        );
    }

    fn find_line_end(&self) -> usize {
        match self.text[self.pos..].find('\n') {
            Some(i) => self.pos + i,
            None => self.text.len(),
        }
    }

    /// Handles everything that can start with `/`: line comments, doc
    /// comments, block comments, `/=`, and plain `/`.
    ///
    /// Returns `Some(kind)` when a token should be produced, or `None` when a
    /// comment was discarded.
    fn handle_slash(&mut self) -> Option<TokenKind> {
        let rest = &self.text[self.pos..];

        if rest.starts_with("///") && !rest.starts_with("////") {
            self.pos += 3;
            let content_start = self.pos;
            let line_end = self.find_line_end();
            let content = &self.text[content_start..line_end];
            let content = content.strip_prefix(' ').unwrap_or(content);
            let text = content.trim_end().to_string();
            self.pos = line_end;
            return Some(TokenKind::DocComment(text));
        }

        if rest.starts_with("//") {
            self.pos = self.find_line_end();
            return None;
        }

        if rest.starts_with("/*") {
            let start = self.pos;
            self.pos += 2;
            self.skip_block_comment(start);
            if self.text[start..self.pos].contains('\n') {
                self.pending_newline = true;
            }
            return None;
        }

        if rest.starts_with("/=") {
            self.pos += 2;
            return Some(TokenKind::SlashEq);
        }

        self.pos += 1;
        Some(TokenKind::Slash)
    }

    /// Skips a `/* ... */` comment, `start` being the offset of its opening
    /// `/`. Block comments nest. `self.pos` must already be past the opening
    /// `/*`.
    fn skip_block_comment(&mut self, start: usize) {
        let mut depth = 1u32;
        loop {
            if self.text[self.pos..].starts_with("*/") {
                self.pos += 2;
                depth -= 1;
                if depth == 0 {
                    return;
                }
                continue;
            }
            if self.text[self.pos..].starts_with("/*") {
                self.pos += 2;
                depth += 1;
                continue;
            }
            if self.bump().is_none() {
                let span = Span::new(self.file, start as u32, self.pos as u32);
                self.diagnostics.push(
                    Diagnostic::error(
                        "cove::lex::unterminated_block_comment",
                        "block comment is never closed",
                    )
                    .at(span)
                    .help("Add a matching `*/`."),
                );
                return;
            }
        }
    }

    fn lex_ident(&mut self, start: usize) -> TokenKind {
        while let Some(c) = self.peek_char() {
            if is_ident_continue(c) {
                self.bump();
            } else {
                break;
            }
        }
        let text = &self.text[start..self.pos];
        match text {
            "_" => TokenKind::Underscore,
            "true" => TokenKind::Bool(true),
            "false" => TokenKind::Bool(false),
            _ => match Keyword::from_text(text) {
                Some(keyword) => TokenKind::Keyword(keyword),
                None => TokenKind::Ident(text.to_string()),
            },
        }
    }

    fn lex_number(&mut self, start: usize) -> TokenKind {
        if self.text[start..].starts_with("0x") {
            let digits_start = start + 2;
            if matches!(self.text[digits_start..].chars().next(), Some(c) if c.is_ascii_hexdigit())
            {
                self.pos = digits_start;
                return self.lex_radix_integer(start, 16, |c| c.is_ascii_hexdigit());
            }
        } else if self.text[start..].starts_with("0b") {
            let digits_start = start + 2;
            if matches!(self.text[digits_start..].chars().next(), Some('0' | '1')) {
                self.pos = digits_start;
                return self.lex_radix_integer(start, 2, |c| c == '0' || c == '1');
            }
        }
        self.lex_decimal_number(start)
    }

    fn lex_radix_integer(
        &mut self,
        start: usize,
        radix: u32,
        pred: impl Fn(char) -> bool,
    ) -> TokenKind {
        let digits_start = self.pos;
        while let Some(c) = self.peek_char() {
            if pred(c) || c == '_' {
                self.bump();
            } else {
                break;
            }
        }
        let raw = self.text[digits_start..self.pos].replace('_', "");
        self.finish_integer_or_duration(start, &raw, radix)
    }

    fn lex_decimal_number(&mut self, start: usize) -> TokenKind {
        while let Some(c) = self.peek_char() {
            if c.is_ascii_digit() || c == '_' {
                self.bump();
            } else {
                break;
            }
        }

        let mut is_float = false;

        if self.peek_char() == Some('.')
            && matches!(self.peek_char_at(1), Some(d) if d.is_ascii_digit())
        {
            is_float = true;
            self.bump();
            while let Some(c) = self.peek_char() {
                if c.is_ascii_digit() || c == '_' {
                    self.bump();
                } else {
                    break;
                }
            }
        }

        if matches!(self.peek_char(), Some('e' | 'E')) {
            let mark = self.pos;
            self.bump();
            if matches!(self.peek_char(), Some('+' | '-')) {
                self.bump();
            }
            if matches!(self.peek_char(), Some(d) if d.is_ascii_digit()) {
                is_float = true;
                while let Some(c) = self.peek_char() {
                    if c.is_ascii_digit() || c == '_' {
                        self.bump();
                    } else {
                        break;
                    }
                }
            } else {
                self.pos = mark;
            }
        }

        if is_float {
            let text = self.text[start..self.pos].replace('_', "");
            let value: f64 = text
                .parse()
                .expect("lexer produced a malformed float literal");
            TokenKind::Float(value)
        } else {
            let raw = self.text[start..self.pos].replace('_', "");
            self.finish_integer_or_duration(start, &raw, 10)
        }
    }

    /// Given the digits of an integer literal already scanned (`raw_digits`,
    /// in `radix`), scans an optional duration-unit suffix and produces
    /// either an `Int` or a `Duration` token.
    fn finish_integer_or_duration(
        &mut self,
        start: usize,
        raw_digits: &str,
        radix: u32,
    ) -> TokenKind {
        let suffix_start = self.pos;
        while let Some(c) = self.peek_char() {
            if is_ident_continue(c) {
                self.bump();
            } else {
                break;
            }
        }
        let suffix = self.text[suffix_start..self.pos].to_string();
        let span = Span::new(self.file, start as u32, self.pos as u32);

        if suffix.is_empty() {
            return match i64::from_str_radix(raw_digits, radix) {
                Ok(value) => TokenKind::Int(value),
                Err(_) => {
                    let message = format!(
                        "integer literal `{}` does not fit in a 64-bit integer",
                        &self.text[start..self.pos]
                    );
                    self.diagnostics.push(
                        Diagnostic::error("cove::lex::integer_out_of_range", message)
                            .at(span)
                            .help("Use a smaller value, or split the computation across multiple steps."),
                    );
                    TokenKind::Int(0)
                }
            };
        }

        if let Some(factor) = duration_factor(&suffix) {
            let ns = i64::from_str_radix(raw_digits, radix)
                .ok()
                .and_then(|value| value.checked_mul(factor));
            match ns {
                Some(ns) => TokenKind::Duration(ns),
                None => {
                    let message = format!(
                        "duration literal `{}` overflows a 64-bit nanosecond count",
                        &self.text[start..self.pos]
                    );
                    self.diagnostics.push(
                        Diagnostic::error("cove::lex::duration_out_of_range", message)
                            .at(span)
                            .help("Use a smaller value or a coarser unit."),
                    );
                    TokenKind::Duration(0)
                }
            }
        } else {
            let message = format!("`{suffix}` is not a valid literal suffix");
            self.diagnostics.push(
                Diagnostic::error("cove::lex::invalid_number_suffix", message)
                    .at(span)
                    .help("Valid duration suffixes are `ns`, `us`, `ms`, `s`, `m`, and `h`."),
            );
            TokenKind::Int(0)
        }
    }

    /// Lexes a string literal, having already consumed its opening `"` at
    /// `quote_start`. Returns `None` (after recording a diagnostic) if the
    /// string or one of its interpolations is never closed.
    fn lex_string(&mut self, quote_start: usize) -> Option<TokenKind> {
        let mut parts = Vec::new();
        let mut current = String::new();

        loop {
            match self.peek_char() {
                None => {
                    self.unterminated_string(quote_start);
                    return None;
                }
                Some('"') => {
                    self.bump();
                    if !current.is_empty() {
                        parts.push(StringPart::Text(current));
                    }
                    return Some(TokenKind::Str(parts));
                }
                Some('\\') => {
                    let esc_start = self.pos;
                    self.bump();
                    match self.peek_char() {
                        None => {
                            self.unterminated_string(quote_start);
                            return None;
                        }
                        Some(escaped) => {
                            self.bump();
                            match escaped_char(escaped) {
                                Some(character) => current.push(character),
                                None => self.unknown_escape(escaped, esc_start),
                            }
                        }
                    }
                }
                Some('{') => {
                    let brace_start = self.pos;
                    self.bump();
                    if !current.is_empty() {
                        parts.push(StringPart::Text(std::mem::take(&mut current)));
                    }
                    let interp_start = self.pos;
                    match self.skip_interpolation_body() {
                        Ok(()) => {
                            let interp_end = self.pos - 1;
                            let source = self.text[interp_start..interp_end].to_string();
                            let span = Span::new(self.file, interp_start as u32, interp_end as u32);
                            parts.push(StringPart::Interpolation { source, span });
                        }
                        Err(()) => {
                            let span = Span::new(self.file, brace_start as u32, self.pos as u32);
                            self.diagnostics.push(
                                Diagnostic::error(
                                    "cove::lex::unterminated_interpolation",
                                    "string interpolation is never closed",
                                )
                                .at(span)
                                .help("Add a matching `}`."),
                            );
                            return None;
                        }
                    }
                }
                Some(c) => {
                    self.bump();
                    current.push(c);
                }
            }
        }
    }

    /// A code-point literal: `'a'`, whose value is a Unicode scalar value and
    /// whose type is `Int`.
    ///
    /// There is no `Char` type and this does not add one —
    /// [ADR 0046](../../../docs/adr/0046-a-byte-offset-is-a-value-a-string-hands-out.md)
    /// decided a code point is an `Int`, and this is a literal for one rather
    /// than a type for one. It answers [`TokenKind::Int`], so nothing past
    /// this function knows the form exists, which is the whole of what it
    /// costs the rest of the compiler.
    ///
    /// A literal holding no scalar or more than one is reported and then
    /// answers something anyway, so that one bad literal does not cascade
    /// into every expression that reads it.
    fn lex_code_point(&mut self, quote_start: usize) -> Option<TokenKind> {
        let mut scalars = String::new();
        loop {
            match self.peek_char() {
                None => {
                    let span = Span::new(self.file, quote_start as u32, self.pos as u32);
                    self.diagnostics.push(
                        Diagnostic::error(
                            "cove::lex::unterminated_code_point",
                            "code-point literal is never closed",
                        )
                        .at(span)
                        .help("Add a closing `'`."),
                    );
                    return None;
                }
                Some('\'') => {
                    self.bump();
                    let span = Span::new(self.file, quote_start as u32, self.pos as u32);
                    let mut characters = scalars.chars();
                    let Some(first) = characters.next() else {
                        self.diagnostics.push(
                            Diagnostic::error(
                                "cove::lex::empty_code_point",
                                "code-point literal names no character",
                            )
                            .at(span)
                            .rule("A code-point literal holds exactly one Unicode scalar value.")
                            .help("Write the character between the quotes, as `'a'`. The empty string is `\"\"`."),
                        );
                        return Some(TokenKind::Int(0));
                    };
                    if characters.next().is_some() {
                        let count = scalars.chars().count();
                        self.diagnostics.push(
                            Diagnostic::error(
                                "cove::lex::code_point_is_not_one_scalar",
                                format!("code-point literal holds {count} scalar values, not one"),
                            )
                            .at(span)
                            .rule(
                                "A code-point literal holds exactly one Unicode scalar value, so a \
                                 combining pair or an emoji sequence is more than one and is not a \
                                 code point.",
                            )
                            .help("Write one scalar, or use a `\"...\"` string for text of any length."),
                        );
                    }
                    return Some(TokenKind::Int(first as i64));
                }
                Some('\\') => {
                    let esc_start = self.pos;
                    self.bump();
                    match self.peek_char() {
                        None => {
                            let span = Span::new(self.file, quote_start as u32, self.pos as u32);
                            self.diagnostics.push(
                                Diagnostic::error(
                                    "cove::lex::unterminated_code_point",
                                    "code-point literal is never closed",
                                )
                                .at(span)
                                .help("Add a closing `'`."),
                            );
                            return None;
                        }
                        Some(escaped) => {
                            self.bump();
                            match escaped_char(escaped) {
                                Some(character) => scalars.push(character),
                                None => self.unknown_escape(escaped, esc_start),
                            }
                        }
                    }
                }
                Some(c) => {
                    self.bump();
                    scalars.push(c);
                }
            }
        }
    }

    /// Reports an escape neither literal form spells anything with.
    fn unknown_escape(&mut self, escaped: char, esc_start: usize) {
        let span = Span::new(self.file, esc_start as u32, self.pos as u32);
        let message = format!("unknown escape sequence `\\{escaped}`");
        self.diagnostics.push(
            Diagnostic::error("cove::lex::unknown_escape", message)
                .at(span)
                .help(
                    "Use one of the supported escapes: \\\\, \\\", \\', \\n, \\t, \\r, \\0, \\{, \\}.",
                ),
        );
    }

    fn unterminated_string(&mut self, quote_start: usize) {
        let span = Span::new(self.file, quote_start as u32, self.pos as u32);
        self.diagnostics.push(
            Diagnostic::error(
                "cove::lex::unterminated_string",
                "string literal is never closed",
            )
            .at(span)
            .help("Add a closing `\"`."),
        );
    }

    /// Consumes source text up to and including the `}` matching the `{`
    /// that was just consumed by the caller, stepping over any nested `{ }`
    /// blocks and any nested string literals, so that a `}` inside a nested
    /// string does not end the interpolation early.
    ///
    /// A string may hold an interpolation, an interpolation may hold a
    /// string, and either may hold more of itself, so what is stepped over is
    /// a nesting. It is held in a `Vec` rather than in the call stack on
    /// purpose. This runs before the parser, and therefore before
    /// [`crate::parser`]'s nesting limit can refuse anything, so a recursive
    /// scan here would let a string literal of nothing but `{` end the
    /// process — which is the failure that limit exists to prevent. What this
    /// spends instead is one byte of heap per unclosed level, bounded by the
    /// length of the file.
    fn skip_interpolation_body(&mut self) -> Result<(), ()> {
        let mut unclosed = vec![Unclosed::Brace];
        while let Some(innermost) = unclosed.last() {
            // A `'...'` is stepped over whole, the way a `"..."` is, and for
            // one reason more: inside it a brace is a *character*. Without
            // that, `"{ head == '\{' }"` ends the interpolation at a brace
            // the program meant as text. A `{` inside a `"..."` is not the
            // same case — a nested string may itself interpolate — which is
            // why only the apostrophe suppresses it.
            let inside_scalar = matches!(innermost, Unclosed::Apostrophe);
            let escaping = matches!(innermost, Unclosed::Quote | Unclosed::Apostrophe);
            match self.peek_char() {
                None => return Err(()),
                Some('\\') if escaping => {
                    self.bump();
                    if self.peek_char().is_none() {
                        return Err(());
                    }
                    self.bump();
                }
                Some('"') if !inside_scalar => {
                    self.bump();
                    if matches!(innermost, Unclosed::Quote) {
                        unclosed.pop();
                    } else {
                        unclosed.push(Unclosed::Quote);
                    }
                }
                // An apostrophe inside a string is ordinary text, so only a
                // brace's body opens one and only its own closes it.
                Some('\'') if !matches!(innermost, Unclosed::Quote) => {
                    self.bump();
                    if inside_scalar {
                        unclosed.pop();
                    } else {
                        unclosed.push(Unclosed::Apostrophe);
                    }
                }
                Some('{') if !inside_scalar => {
                    self.bump();
                    unclosed.push(Unclosed::Brace);
                }
                Some('}') if matches!(innermost, Unclosed::Brace) => {
                    self.bump();
                    unclosed.pop();
                }
                Some(_) => {
                    self.bump();
                }
            }
        }
        Ok(())
    }

    /// Matches punctuation and operators, longest match first. `/` and
    /// everything that can start with it (comments, `/=`) is handled
    /// separately by [`Lexer::handle_slash`].
    fn lex_operator(&mut self) -> Option<TokenKind> {
        let rest = &self.text[self.pos..];

        macro_rules! op {
            ($lit:literal, $kind:expr) => {
                if rest.starts_with($lit) {
                    self.pos += $lit.len();
                    return Some($kind);
                }
            };
        }

        op!("...", TokenKind::Ellipsis);
        op!("..<", TokenKind::DotDotLt);
        op!("..", TokenKind::DotDot);
        op!(".", TokenKind::Dot);
        op!("->", TokenKind::Arrow);
        op!("=>", TokenKind::FatArrow);
        op!("==", TokenKind::EqEq);
        op!("!=", TokenKind::BangEq);
        op!("<=", TokenKind::LtEq);
        op!(">=", TokenKind::GtEq);
        op!("+=", TokenKind::PlusEq);
        op!("-=", TokenKind::MinusEq);
        op!("*=", TokenKind::StarEq);
        op!("%=", TokenKind::PercentEq);
        op!("&&", TokenKind::AmpAmp);
        op!("||", TokenKind::PipePipe);
        op!("=", TokenKind::Eq);
        op!("!", TokenKind::Bang);
        op!("<", TokenKind::Lt);
        op!(">", TokenKind::Gt);
        op!("+", TokenKind::Plus);
        op!("-", TokenKind::Minus);
        op!("*", TokenKind::Star);
        op!("%", TokenKind::Percent);
        op!("?", TokenKind::Question);
        op!(",", TokenKind::Comma);
        op!(":", TokenKind::Colon);
        op!("(", TokenKind::LParen);
        op!(")", TokenKind::RParen);
        op!("{", TokenKind::LBrace);
        op!("}", TokenKind::RBrace);
        op!("[", TokenKind::LBracket);
        op!("]", TokenKind::RBracket);

        None
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    fn lex_ok(src: &str) -> Vec<Token> {
        let mut sources = SourceMap::new();
        let file = sources.add("test.cove", src);
        lex(&sources, file).unwrap_or_else(|diags| {
            panic!("expected `{src}` to lex successfully, got errors: {diags:?}")
        })
    }

    fn lex_err(src: &str) -> Vec<Diagnostic> {
        let mut sources = SourceMap::new();
        let file = sources.add("test.cove", src);
        match lex(&sources, file) {
            Ok(tokens) => panic!("expected `{src}` to fail to lex, got tokens: {tokens:?}"),
            Err(diags) => diags,
        }
    }

    fn kinds(src: &str) -> Vec<TokenKind> {
        let tokens = lex_ok(src);
        assert!(
            matches!(tokens.last().unwrap().kind, TokenKind::Eof),
            "token stream must end with Eof, got {tokens:?}"
        );
        tokens[..tokens.len() - 1]
            .iter()
            .map(|t| t.kind.clone())
            .collect()
    }

    #[test]
    fn empty_file_is_just_eof() {
        let tokens = lex_ok("");
        assert_eq!(tokens.len(), 1);
        assert_eq!(tokens[0].kind, TokenKind::Eof);
        assert_eq!(tokens[0].span.start, 0);
        assert_eq!(tokens[0].span.end, 0);
    }

    #[test]
    fn whitespace_produces_no_tokens() {
        assert_eq!(kinds(" \t\r\n foo"), vec![TokenKind::Ident("foo".into())]);
    }

    /// A line break is recorded on the token that follows it rather than
    /// becoming a token of its own, and comments do not hide it.
    #[test]
    fn tokens_record_a_preceding_line_break() {
        let tokens = lex_ok("a b\nc /* x\ny */ d // e\nf");
        let flags: Vec<bool> = tokens.iter().map(|t| t.preceded_by_newline).collect();
        // a, b, c, d, f, Eof
        assert_eq!(flags, vec![false, false, true, true, true, false]);
    }

    #[test]
    fn a_block_comment_on_one_line_is_not_a_line_break() {
        let tokens = lex_ok("a /* x */ b");
        assert!(!tokens[1].preceded_by_newline);
    }

    #[test]
    fn keywords_vs_identifiers() {
        assert_eq!(
            kinds("fn foo let bar self"),
            vec![
                TokenKind::Keyword(Keyword::Fn),
                TokenKind::Ident("foo".into()),
                TokenKind::Keyword(Keyword::Let),
                TokenKind::Ident("bar".into()),
                TokenKind::Keyword(Keyword::SelfValue),
            ]
        );
        // Maximal munch: `forever` is one identifier, not `for` + `ever`.
        assert_eq!(kinds("forever"), vec![TokenKind::Ident("forever".into())]);
        assert_eq!(
            kinds("true false"),
            vec![TokenKind::Bool(true), TokenKind::Bool(false)]
        );
    }

    #[test]
    fn is_is_a_keyword_and_maximal_munch_still_applies() {
        assert_eq!(
            kinds("a is b"),
            vec![
                TokenKind::Ident("a".into()),
                TokenKind::Keyword(Keyword::Is),
                TokenKind::Ident("b".into()),
            ]
        );
        // `island` is one identifier, not `is` + `land`.
        assert_eq!(kinds("island"), vec![TokenKind::Ident("island".into())]);
    }

    #[test]
    fn underscore_is_its_own_token() {
        assert_eq!(kinds("_"), vec![TokenKind::Underscore]);
        assert_eq!(kinds("_foo"), vec![TokenKind::Ident("_foo".into())]);
        assert_eq!(kinds("foo_"), vec![TokenKind::Ident("foo_".into())]);
    }

    #[test]
    fn doc_comments_strip_one_leading_space_and_trailing_whitespace() {
        assert_eq!(
            kinds("/// hello\n/// world"),
            vec![
                TokenKind::DocComment("hello".into()),
                TokenKind::DocComment("world".into()),
            ]
        );
        assert_eq!(
            kinds("///no-space"),
            vec![TokenKind::DocComment("no-space".into())]
        );
        assert_eq!(
            kinds("///  two spaces"),
            vec![TokenKind::DocComment(" two spaces".into())]
        );
        assert_eq!(
            kinds("///trailing   \nfn"),
            vec![
                TokenKind::DocComment("trailing".into()),
                TokenKind::Keyword(Keyword::Fn)
            ]
        );
    }

    #[test]
    fn four_slashes_is_a_plain_comment_not_a_doc_comment() {
        assert_eq!(kinds("//// not a doc\n42"), vec![TokenKind::Int(42)]);
    }

    #[test]
    fn line_comments_are_discarded() {
        assert_eq!(kinds("// hello\n42"), vec![TokenKind::Int(42)]);
    }

    #[test]
    fn nested_block_comments_are_discarded() {
        assert_eq!(
            kinds("/* outer /* inner */ still outer */ 42"),
            vec![TokenKind::Int(42)]
        );
    }

    #[test]
    fn unterminated_block_comment_is_an_error() {
        let diags = lex_err("/* never closed");
        assert_eq!(diags.len(), 1);
        assert_eq!(diags[0].code, "cove::lex::unterminated_block_comment");
    }

    #[test]
    fn decimal_integers_and_underscores() {
        assert_eq!(kinds("123"), vec![TokenKind::Int(123)]);
        assert_eq!(kinds("1_000"), vec![TokenKind::Int(1000)]);
        assert_eq!(kinds("0"), vec![TokenKind::Int(0)]);
    }

    #[test]
    fn hex_and_binary_integers() {
        assert_eq!(kinds("0xFF"), vec![TokenKind::Int(255)]);
        assert_eq!(kinds("0b1010"), vec![TokenKind::Int(10)]);
    }

    #[test]
    fn floats() {
        assert_eq!(kinds("1.5"), vec![TokenKind::Float(1.5)]);
        assert_eq!(kinds("1.5e10"), vec![TokenKind::Float(1.5e10)]);
        assert_eq!(kinds("1e-3"), vec![TokenKind::Float(1e-3)]);
    }

    #[test]
    fn range_dot_is_only_part_of_a_float_before_a_digit() {
        assert_eq!(
            kinds("0..<10"),
            vec![TokenKind::Int(0), TokenKind::DotDotLt, TokenKind::Int(10)]
        );
        assert_eq!(
            kinds("0..n"),
            vec![
                TokenKind::Int(0),
                TokenKind::DotDot,
                TokenKind::Ident("n".into())
            ]
        );
    }

    #[test]
    fn every_duration_unit() {
        assert_eq!(kinds("1ns"), vec![TokenKind::Duration(1)]);
        assert_eq!(kinds("1us"), vec![TokenKind::Duration(1_000)]);
        assert_eq!(kinds("1ms"), vec![TokenKind::Duration(1_000_000)]);
        assert_eq!(kinds("1s"), vec![TokenKind::Duration(1_000_000_000)]);
        assert_eq!(kinds("1m"), vec![TokenKind::Duration(60_000_000_000)]);
        assert_eq!(kinds("1h"), vec![TokenKind::Duration(3_600_000_000_000)]);
        assert_eq!(kinds("500ms"), vec![TokenKind::Duration(500_000_000)]);
        assert_eq!(kinds("60s"), vec![TokenKind::Duration(60_000_000_000)]);
        assert_eq!(kinds("5s"), vec![TokenKind::Duration(5_000_000_000)]);
    }

    #[test]
    fn integer_out_of_range_is_an_error() {
        let diags = lex_err("99999999999999999999");
        assert_eq!(diags.len(), 1);
        assert_eq!(diags[0].code, "cove::lex::integer_out_of_range");
    }

    #[test]
    fn duration_out_of_range_is_an_error() {
        let diags = lex_err("9999999999h");
        assert_eq!(diags.len(), 1);
        assert_eq!(diags[0].code, "cove::lex::duration_out_of_range");
    }

    #[test]
    fn invalid_number_suffix_is_an_error() {
        let diags = lex_err("5sx");
        assert_eq!(diags.len(), 1);
        assert_eq!(diags[0].code, "cove::lex::invalid_number_suffix");
    }

    #[test]
    fn string_escapes() {
        let tokens = kinds(r#""\\ \" \n \t \r \0 \{ \}""#);
        assert_eq!(
            tokens,
            vec![TokenKind::Str(vec![StringPart::Text(
                "\\ \" \n \t \r \0 { }".into()
            )])]
        );
    }

    /// A code-point literal is an `Int` token and nothing else.
    ///
    /// There is no `Char`, so what the lexer answers here is the same token
    /// `97` answers, and every test below is written against that fact
    /// rather than against a form the rest of the compiler would have to
    /// know about.
    #[test]
    fn code_point_literals_are_int_tokens() {
        assert_eq!(kinds("'a'"), vec![TokenKind::Int(97)]);
        assert_eq!(kinds("'0'"), vec![TokenKind::Int(48)]);
        assert_eq!(kinds("' '"), vec![TokenKind::Int(32)]);
        assert_eq!(kinds("'{'"), vec![TokenKind::Int(123)]);
        assert_eq!(kinds("'\u{e9}'"), vec![TokenKind::Int(233)]);
        assert_eq!(kinds("'\u{3042}'"), vec![TokenKind::Int(12354)]);
        assert_eq!(kinds("'\u{1F600}'"), vec![TokenKind::Int(128512)]);
    }

    /// The escapes are the string's, and `\'` besides.
    #[test]
    fn code_point_literals_take_the_escapes_a_string_takes() {
        assert_eq!(kinds(r"'\n'"), vec![TokenKind::Int(10)]);
        assert_eq!(kinds(r"'\t'"), vec![TokenKind::Int(9)]);
        assert_eq!(kinds(r"'\r'"), vec![TokenKind::Int(13)]);
        assert_eq!(kinds(r"'\0'"), vec![TokenKind::Int(0)]);
        assert_eq!(kinds(r"'\\'"), vec![TokenKind::Int(92)]);
        assert_eq!(kinds(r"'\''"), vec![TokenKind::Int(39)]);
        assert_eq!(kinds(r#"'\"'"#), vec![TokenKind::Int(34)]);
        assert_eq!(kinds(r"'\{'"), vec![TokenKind::Int(123)]);
        assert_eq!(kinds(r"'\}'"), vec![TokenKind::Int(125)]);
    }

    /// `\'` is legal in a string too, because there is one escape table and
    /// the apostrophe is in it. It spells the apostrophe it always did.
    #[test]
    fn a_string_may_escape_an_apostrophe() {
        assert_eq!(
            kinds(r#""\'""#),
            vec![TokenKind::Str(vec![StringPart::Text("'".to_string())])]
        );
    }

    #[test]
    fn an_empty_code_point_literal_is_an_error() {
        let diags = lex_err("''");
        assert_eq!(diags.len(), 1);
        assert_eq!(diags[0].code, "cove::lex::empty_code_point");
    }

    /// Two scalars is an error, and so is a grapheme cluster that is written
    /// as one character and is not one scalar — which is the case the rule
    /// exists for.
    #[test]
    fn a_code_point_literal_holding_more_than_one_scalar_is_an_error() {
        let diags = lex_err("'ab'");
        assert_eq!(diags.len(), 1);
        assert_eq!(diags[0].code, "cove::lex::code_point_is_not_one_scalar");
        assert_eq!(
            diags[0].message,
            "code-point literal holds 2 scalar values, not one"
        );

        let diags = lex_err("'\u{1F468}\u{200D}\u{1F469}\u{200D}\u{1F466}'");
        assert_eq!(diags.len(), 1);
        assert_eq!(
            diags[0].message,
            "code-point literal holds 5 scalar values, not one"
        );
    }

    #[test]
    fn an_unterminated_code_point_literal_is_an_error() {
        let diags = lex_err("'a");
        assert_eq!(diags.len(), 1);
        assert_eq!(diags[0].code, "cove::lex::unterminated_code_point");
    }

    /// An interpolation is scanned for its closing brace before it is parsed,
    /// and a brace inside a code-point literal is a character rather than a
    /// nesting. Without that, this string ends at the wrong place.
    #[test]
    fn a_code_point_literal_inside_an_interpolation_may_hold_a_brace() {
        let tokens = kinds(r#""{ head == '{' }""#);
        assert_eq!(tokens.len(), 1);
        let TokenKind::Str(parts) = &tokens[0] else {
            panic!("expected a string token, got {tokens:?}");
        };
        assert_eq!(parts.len(), 1);
        let StringPart::Interpolation { source, .. } = &parts[0] else {
            panic!("expected one interpolation, got {parts:?}");
        };
        assert_eq!(source.trim(), "head == '{'");
    }

    #[test]
    fn unknown_escape_is_an_error() {
        let diags = lex_err(r#""\q""#);
        assert_eq!(diags.len(), 1);
        assert_eq!(diags[0].code, "cove::lex::unknown_escape");
    }

    #[test]
    fn unterminated_string_is_an_error() {
        let diags = lex_err("\"never closed");
        assert_eq!(diags.len(), 1);
        assert_eq!(diags[0].code, "cove::lex::unterminated_string");
    }

    #[test]
    fn simple_interpolation() {
        let tokens = kinds(r#""a{b}c""#);
        assert_eq!(
            tokens,
            vec![TokenKind::Str(vec![
                StringPart::Text("a".into()),
                StringPart::Interpolation {
                    source: "b".into(),
                    span: Span::new(FileId(0), 3, 4),
                },
                StringPart::Text("c".into()),
            ])]
        );
    }

    #[test]
    fn interpolation_with_nested_braces_and_nested_strings() {
        // `"{f("}")}"` must lex `f("}")` as the interpolation source: the
        // brace inside the nested string must not end the interpolation
        // early, and the nested string's own quotes must not be confused
        // with the outer string's.
        let tokens = kinds("\"{f(\"}\")}\"");
        assert_eq!(
            tokens,
            vec![TokenKind::Str(vec![StringPart::Interpolation {
                source: "f(\"}\")".into(),
                span: Span::new(FileId(0), 2, 8),
            }])]
        );
    }

    #[test]
    fn multiple_interpolations_in_one_string() {
        let tokens = kinds(r#""{a}-{b}""#);
        assert_eq!(
            tokens,
            vec![TokenKind::Str(vec![
                StringPart::Interpolation {
                    source: "a".into(),
                    span: Span::new(FileId(0), 2, 3),
                },
                StringPart::Text("-".into()),
                StringPart::Interpolation {
                    source: "b".into(),
                    span: Span::new(FileId(0), 6, 7),
                },
            ])]
        );
    }

    #[test]
    fn unterminated_interpolation_is_an_error() {
        let diags = lex_err(r#""{a"#);
        assert_eq!(diags.len(), 1);
        assert_eq!(diags[0].code, "cove::lex::unterminated_interpolation");
    }

    #[test]
    fn longest_match_operators() {
        assert_eq!(
            kinds("... ..< .. ."),
            vec![
                TokenKind::Ellipsis,
                TokenKind::DotDotLt,
                TokenKind::DotDot,
                TokenKind::Dot,
            ]
        );
        assert_eq!(
            kinds("-> => == != <= >= += -= *= /= %= && ||"),
            vec![
                TokenKind::Arrow,
                TokenKind::FatArrow,
                TokenKind::EqEq,
                TokenKind::BangEq,
                TokenKind::LtEq,
                TokenKind::GtEq,
                TokenKind::PlusEq,
                TokenKind::MinusEq,
                TokenKind::StarEq,
                TokenKind::SlashEq,
                TokenKind::PercentEq,
                TokenKind::AmpAmp,
                TokenKind::PipePipe,
            ]
        );
        assert_eq!(
            kinds("= ! < > + - * / % ? , : ( ) { } [ ]"),
            vec![
                TokenKind::Eq,
                TokenKind::Bang,
                TokenKind::Lt,
                TokenKind::Gt,
                TokenKind::Plus,
                TokenKind::Minus,
                TokenKind::Star,
                TokenKind::Slash,
                TokenKind::Percent,
                TokenKind::Question,
                TokenKind::Comma,
                TokenKind::Colon,
                TokenKind::LParen,
                TokenKind::RParen,
                TokenKind::LBrace,
                TokenKind::RBrace,
                TokenKind::LBracket,
                TokenKind::RBracket,
            ]
        );
        // No space: greedy longest match still applies.
        assert_eq!(kinds("<=="), vec![TokenKind::LtEq, TokenKind::Eq]);
    }

    #[test]
    fn semicolon_is_an_error_with_the_cove_rule() {
        let diags = lex_err(";");
        assert_eq!(diags.len(), 1);
        assert_eq!(diags[0].code, "cove::lex::unexpected_character");
        assert!(diags[0].rule.as_deref().unwrap().contains(';'));
    }

    #[test]
    fn unexpected_character_is_an_error() {
        let diags = lex_err("`");
        assert_eq!(diags.len(), 1);
        assert_eq!(diags[0].code, "cove::lex::unexpected_character");
    }

    #[test]
    fn at_sign_is_reserved_decorator_syntax() {
        let diags = lex_err("@decorate");
        assert_eq!(diags.len(), 1);
        assert_eq!(diags[0].code, "cove::parse::reserved_annotation");
        assert!(diags[0].rule.as_deref().unwrap().contains("Decorator"));
    }

    #[test]
    fn all_errors_in_a_file_are_collected() {
        let diags = lex_err("` ~ #");
        assert_eq!(diags.len(), 3);
        for diag in &diags {
            assert_eq!(diag.code, "cove::lex::unexpected_character");
        }
    }

    /// The half of the lexer's work that [`lex`] discards. A highlighter
    /// reads it, and the case it reads it for is this one: a string that is
    /// open because the reader has not typed the closing quote yet.
    #[test]
    fn recovery_answers_the_tokens_before_an_error_as_well_as_the_error() {
        let mut sources = SourceMap::new();
        let file = sources.add("test.cove", "let n = 1\nlet greeting = \"open");
        let (tokens, diagnostics) = lex_recovered(&sources, file);

        assert_eq!(diagnostics.len(), 1);
        assert_eq!(diagnostics[0].code, "cove::lex::unterminated_string");
        assert!(
            lex(&sources, file).is_err(),
            "`lex` still refuses what it always refused"
        );

        let kinds: Vec<TokenKind> = tokens.iter().map(|t| t.kind.clone()).collect();
        assert_eq!(
            kinds,
            vec![
                TokenKind::Keyword(Keyword::Let),
                TokenKind::Ident("n".into()),
                TokenKind::Eq,
                TokenKind::Int(1),
                TokenKind::Keyword(Keyword::Let),
                TokenKind::Ident("greeting".into()),
                TokenKind::Eq,
                TokenKind::Eof,
            ],
            "everything up to the open quote, and nothing invented for it"
        );
    }

    /// What [`lex`] answers and what [`lex_recovered`] answers are the same
    /// tokens whenever there is nothing to complain about, which is what lets
    /// one be written in terms of the other.
    #[test]
    fn recovery_and_lex_agree_when_there_is_no_error() {
        let source = "export fn main() -> Int { 1 + 2 }";
        let mut sources = SourceMap::new();
        let file = sources.add("test.cove", source);
        let (tokens, diagnostics) = lex_recovered(&sources, file);
        assert!(diagnostics.is_empty());
        assert_eq!(tokens, lex(&sources, file).expect("it lexes"));
    }

    #[test]
    fn duration_example_program_lexes() {
        let tokens =
            kinds("clock.timeout(500ms) { retry(3, attempts) }\nfor attempt in 0..<attempts {}");
        assert!(tokens.contains(&TokenKind::Duration(500_000_000)));
        assert!(tokens.contains(&TokenKind::DotDotLt));
        assert!(tokens.contains(&TokenKind::Keyword(Keyword::For)));
    }
}