// Expect retcode = 0

// 词法分析器：字节流 -> Vec<Token>。镜像 stage1/spl_lexer.c 的行为：
// 行注释作为 token 产出；块注释（可嵌套）整体跳过；换行产出 Endline；EOF 产出 Eof。
// 复用 utils/ascii 做字符分类；整型/字符在词法期解码后放进 TokenKind 载荷。
// Token.text 借用输入字节，不复制。

type .{ .Vec, .VecResult } = @import("utils/vec.spl");
type .{ .ByteView } = @import("utils/byte_view.spl");
type .{ .is_digit, .is_alpha, .is_alnum, .is_hex_digit, .digit_value } = @import("utils/ascii.spl");
type .{
    .Token, .TokenKind, .Radix, .Keyword, .Punct, .keyword_of
} = @import("token.spl");
type .{ .Span, .span_make } = @import("./diag.spl");
type Diag = @import("./diag.spl");

type Lexer = @self;

src: ByteView,
pos: usize,
line: usize,
col: usize,
diag: *Diag, // 借用；可为 null（合法输入不产生诊断）
file: usize,

fn new(src: ByteView, diag: *Diag, file: usize): Lexer {
    ret Lexer { .src = src, .pos = 0, .line = 1, .col = 1, .diag = diag, .file = file };
}

// 记录一条词法诊断（非法输入路径）。合法输入不应触发。
fn report(self: *Lexer, start: usize, end: usize, message: []u8) {
    if self.diag == null { ret; }
    Diag.err(self.diag, span_make(self.file, start, end), .LexUnexpected, message);
}

fn is_ident_start(c: u8): bool { ret is_alpha(c) || c == '_'; }

fn is_ident_cont(c: u8): bool { ret is_alnum(c) || c == '_'; }

fn hex_val(c: u8): u8 {
    match digit_value(c, 16) {
        .Ok(v) => { ret v; },
        .Err(error) => { ret 0; },
    }
    ret 0;
}

// 文本形如 'a' / '\n' / '\x41'；返回解码后的字节。
fn decode_char(text: ByteView): u8 {
    var start: usize = 1;
    var end = text.len;
    if text.len >= 2 && text.ptr[text.len - 1] == '\'' { end = text.len - 1; }
    if end <= start { ret 0; }

    var b = text.ptr[start];
    if b != '\\' { ret b; }

    start = start + 1;
    if start >= end { ret 0; }
    var e = text.ptr[start];
    if e == 'n' { ret '\n'; }
    if e == 't' { ret '\t'; }
    if e == 'r' { ret '\r'; }
    if e == '\\' { ret '\\'; }
    if e == '"' { ret '"'; }
    if e == '\'' { ret '\''; }
    if e == '0' { ret 0; }
    if e == 'x' || e == 'X' {
        var q = start + 1;
        var value: u8 = 0;
        var digits: usize = 0;
        while q < end && digits < 2 && is_hex_digit(text.ptr[q]) {
            value = value * 16 + hex_val(text.ptr[q]);
            q = q + 1;
            digits = digits + 1;
        }
        if digits == 0 { ret e; }
        ret value;
    }
    ret e;
}

// ---- 读取辅助 ----

fn has2(self: *Lexer, ch: u8): bool {
    ret self.pos + 1 < self.src.len && self.src.ptr[self.pos + 1] == ch;
}

fn has3(self: *Lexer, ch1: u8, ch2: u8): bool {
    ret self.pos + 2 < self.src.len && self.src.ptr[self.pos + 1] == ch1 &&
        self.src.ptr[self.pos + 2] == ch2;
}

fn sub(self: *Lexer, start: usize, end: usize): ByteView {
    match ByteView.slice(self.src, start, end) {
        .Ok(v) => { ret v; },
        .Err(error) => { ret ByteView.empty(); },
    }
    ret ByteView.empty();
}

// 产出 token 并把 pos/col 推进到 end。Vec.push 失败视为致命（同 vec.make_vec 风格）。
fn emit_to(self: *Lexer, out: *Vec, kind: TokenKind, start: usize, end: usize, line: usize,
           col: usize) {
    var t = Token.make(kind, sub(self, start, end), line, col, start);
    match Vec.push(out, &t as *u8) {
        .Ok => {},
        .Err(error) => { @assert(false, "lexer: Vec.push failed"); },
    }
    self.pos = end;
    self.col = col + (end - start);
}

fn op1(self: *Lexer, out: *Vec, punct: Punct, start: usize, line: usize, col: usize) {
    emit_to(self, out, .Punct(punct), start, start + 1, line, col);
}

fn op2(self: *Lexer, out: *Vec, punct: Punct, start: usize, line: usize, col: usize) {
    emit_to(self, out, .Punct(punct), start, start + 2, line, col);
}

fn op3(self: *Lexer, out: *Vec, punct: Punct, start: usize, line: usize, col: usize) {
    emit_to(self, out, .Punct(punct), start, start + 3, line, col);
}

// ---- 复合扫描 ----

fn scan_int(self: *Lexer, out: *Vec, start: usize, line: usize, col: usize) {
    var p = self.pos;
    if self.src.ptr[p] == '0' && p + 1 < self.src.len {
        var nc = self.src.ptr[p + 1];
        if nc == 'x' || nc == 'X' {
            p = p + 2;
            while p < self.src.len && (is_hex_digit(self.src.ptr[p]) || self.src.ptr[p] == '_') {
                p = p + 1;
            }
            emit_to(self, out, .Int(.Hex), start, p, line, col);
            ret;
        }
        if nc == 'b' || nc == 'B' {
            p = p + 2;
            while p < self.src.len && (self.src.ptr[p] == '0' || self.src.ptr[p] == '1' ||
                                       self.src.ptr[p] == '_') {
                p = p + 1;
            }
            emit_to(self, out, .Int(.Bin), start, p, line, col);
            ret;
        }
        if nc == 'o' || nc == 'O' {
            p = p + 2;
            while p < self.src.len &&
                   ((self.src.ptr[p] >= '0' && self.src.ptr[p] <= '7') ||
                    self.src.ptr[p] == '_') {
                p = p + 1;
            }
            emit_to(self, out, .Int(.Oct), start, p, line, col);
            ret;
        }
    }

    while p < self.src.len && (is_digit(self.src.ptr[p]) || self.src.ptr[p] == '_') { p = p + 1; }
    if p + 1 < self.src.len && self.src.ptr[p] == '.' && is_digit(self.src.ptr[p + 1]) {
        p = p + 1;
        // 小数部分允许下划线分隔（Rust 风格）。
        while p < self.src.len && (is_digit(self.src.ptr[p]) || self.src.ptr[p] == '_') {
            p = p + 1;
        }
        // 可选指数：e/E [+-] 数字（数字允许下划线）；仅当确有数字时纳入。
        if p < self.src.len && (self.src.ptr[p] == 'e' || self.src.ptr[p] == 'E') {
            var q = p + 1;
            if q < self.src.len && (self.src.ptr[q] == '+' || self.src.ptr[q] == '-') { q = q + 1; }
            if q < self.src.len && is_digit(self.src.ptr[q]) {
                p = q + 1;
                while p < self.src.len && (is_digit(self.src.ptr[p]) || self.src.ptr[p] == '_') {
                    p = p + 1;
                }
            }
        }
        emit_to(self, out, .Float, start, p, line, col);
        ret;
    }
    emit_to(self, out, .Int(.Dec), start, p, line, col);
}

fn scan_char(self: *Lexer, out: *Vec, start: usize, line: usize, col: usize) {
    var p = self.pos + 1;
    var closed = false;
    var escape_ok = true;
    if p < self.src.len {
        if self.src.ptr[p] == '\\' {
            p = p + 1;
            if p < self.src.len && (self.src.ptr[p] == 'x' || self.src.ptr[p] == 'X') {
                p = p + 1;
                var digits: usize = 0;
                while p < self.src.len && digits < 2 && is_hex_digit(self.src.ptr[p]) {
                    p = p + 1;
                    digits = digits + 1;
                }
                if digits == 0 { escape_ok = false; }
            } else if p < self.src.len {
                var e = self.src.ptr[p];
                if !(e == 'n' || e == 't' || e == 'r' || e == '\\' || e == '"' || e == '\'' ||
                     e == '0') {
                    escape_ok = false;
                }
                p = p + 1;
            } else {
                escape_ok = false;
            }
        } else {
            p = p + 1;
        }
    }
    if p < self.src.len && self.src.ptr[p] == '\'' {
        p = p + 1;
        closed = true;
    }
    if !closed {
        report(self, start, p, "unterminated character literal");
    } else if !escape_ok {
        report(self, start, p, "invalid escape in character literal");
    }
    emit_to(self, out, .Char(decode_char(sub(self, start, p))), start, p, line, col);
}

fn scan_string(self: *Lexer, out: *Vec, start: usize, line: usize, col: usize) {
    var p = self.pos + 1;
    var l = line;
    var cc = col + 1;
    var closed = false;
    while p < self.src.len {
        var b = self.src.ptr[p];
        if b == '\\' {
            p = p + 1;
            cc = cc + 1;
            if p < self.src.len {
                p = p + 1;
                cc = cc + 1;
            }
        } else if b == '"' {
            p = p + 1;
            cc = cc + 1;
            closed = true;
            break;
        } else if b == '\n' {
            l = l + 1;
            cc = 1;
            p = p + 1;
        } else {
            p = p + 1;
            cc = cc + 1;
        }
    }
    if !closed { report(self, start, p, "unterminated string literal"); }
    emit_to(self, out, .Str, start, p, line, col);
    self.line = l;
    self.col = cc;
}

fn scan_ident(self: *Lexer, out: *Vec, start: usize, line: usize, col: usize) {
    var p = self.pos;
    while p < self.src.len && is_ident_cont(self.src.ptr[p]) { p = p + 1; }
    var text = sub(self, start, p);
    match keyword_of(text) {
        .Yes(kw) => { emit_to(self, out, .Keyword(kw), start, p, line, col); },
        .No => { emit_to(self, out, .Ident, start, p, line, col); },
    }
}

// 块注释：可嵌套，整体跳过（不产出 token）。未闭合时报诊断。
fn skip_block_comment(self: *Lexer, start: usize) {
    self.pos = self.pos + 2;
    self.col = self.col + 2;
    var depth: usize = 1;
    while self.pos + 1 < self.src.len && depth > 0 {
        if self.src.ptr[self.pos] == '*' && self.src.ptr[self.pos + 1] == '/' {
            depth = depth - 1;
            self.pos = self.pos + 2;
            self.col = self.col + 2;
        } else if self.src.ptr[self.pos] == '/' && self.src.ptr[self.pos + 1] == '*' {
            depth = depth + 1;
            self.pos = self.pos + 2;
            self.col = self.col + 2;
        } else {
            if self.src.ptr[self.pos] == '\n' {
                self.line = self.line + 1;
                self.col = 1;
            } else {
                self.col = self.col + 1;
            }
            self.pos = self.pos + 1;
        }
    }
    if depth > 0 {
        report(self, start, self.src.len, "unterminated block comment");
    }
}

fn run(self: *Lexer, out: *Vec) {
    while self.pos < self.src.len {
        var start = self.pos;
        var line = self.line;
        var col = self.col;
        var c = self.src.ptr[self.pos];

        if c == ' ' || c == '\t' || c == '\r' {
            self.pos = self.pos + 1;
            self.col = self.col + 1;
            continue;
        }
        if c == '\n' {
            emit_to(self, out, .Endline, start, start + 1, line, col);
            self.line = line + 1;
            self.col = 1;
            continue;
        }
        if c == '/' && has2(self, '/') {
            var p = self.pos;
            while p < self.src.len && self.src.ptr[p] != '\n' { p = p + 1; }
            emit_to(self, out, .LineComment, start, p, line, col);
            continue;
        }
        if c == '/' && has2(self, '*') {
            skip_block_comment(self, start);
            continue;
        }
        if c == '\'' {
            scan_char(self, out, start, line, col);
            continue;
        }
        if c == '"' {
            scan_string(self, out, start, line, col);
            continue;
        }
        if is_ident_start(c) {
            scan_ident(self, out, start, line, col);
            continue;
        }
        if is_digit(c) {
            scan_int(self, out, start, line, col);
            continue;
        }

        if c == '<' {
            if has3(self, '-', '>') { op3(self, out, .ArrowBoth, start, line, col); continue; }
            if has3(self, '=', '>') { op3(self, out, .FatArrowBoth, start, line, col); continue; }
            if has3(self, '<', '=') { op3(self, out, .LShAssign, start, line, col); continue; }
            if has2(self, '<') { op2(self, out, .LSh, start, line, col); continue; }
            if has2(self, '-') { op2(self, out, .LeftArrow, start, line, col); continue; }
            if has2(self, '=') { op2(self, out, .Le, start, line, col); continue; }
            op1(self, out, .Lt, start, line, col);
            continue;
        }
        if c == '>' {
            if has3(self, '>', '=') { op3(self, out, .RShAssign, start, line, col); continue; }
            if has2(self, '>') { op2(self, out, .RSh, start, line, col); continue; }
            if has2(self, '=') { op2(self, out, .Ge, start, line, col); continue; }
            op1(self, out, .Gt, start, line, col);
            continue;
        }
        if c == '.' {
            if has3(self, '.', '.') { op3(self, out, .Ellipsis, start, line, col); continue; }
            if has2(self, '.') { op2(self, out, .Range, start, line, col); continue; }
            op1(self, out, .Dot, start, line, col);
            continue;
        }
        if c == '=' {
            if has2(self, '=') { op2(self, out, .Eq, start, line, col); continue; }
            if has2(self, '>') { op2(self, out, .RightFatArrow, start, line, col); continue; }
            op1(self, out, .Assign, start, line, col);
            continue;
        }
        if c == '!' {
            if has2(self, '=') { op2(self, out, .Neq, start, line, col); continue; }
            op1(self, out, .Not, start, line, col);
            continue;
        }
        if c == '&' {
            if has2(self, '&') { op2(self, out, .AndAnd, start, line, col); continue; }
            if has2(self, '=') { op2(self, out, .AndAssign, start, line, col); continue; }
            op1(self, out, .And, start, line, col);
            continue;
        }
        if c == '|' {
            if has2(self, '|') { op2(self, out, .OrOr, start, line, col); continue; }
            if has2(self, '=') { op2(self, out, .OrAssign, start, line, col); continue; }
            op1(self, out, .Or, start, line, col);
            continue;
        }
        if c == '+' {
            if has2(self, '=') { op2(self, out, .AddAssign, start, line, col); continue; }
            op1(self, out, .Add, start, line, col);
            continue;
        }
        if c == '-' {
            if has2(self, '=') { op2(self, out, .SubAssign, start, line, col); continue; }
            if has2(self, '>') { op2(self, out, .RightArrow, start, line, col); continue; }
            op1(self, out, .Sub, start, line, col);
            continue;
        }
        if c == '*' {
            if has2(self, '=') { op2(self, out, .MulAssign, start, line, col); continue; }
            op1(self, out, .Mul, start, line, col);
            continue;
        }
        if c == '/' {
            if has2(self, '=') { op2(self, out, .DivAssign, start, line, col); continue; }
            op1(self, out, .Div, start, line, col);
            continue;
        }
        if c == '%' {
            if has2(self, '=') { op2(self, out, .ModAssign, start, line, col); continue; }
            op1(self, out, .Mod, start, line, col);
            continue;
        }
        if c == '^' {
            if has2(self, '=') { op2(self, out, .XorAssign, start, line, col); continue; }
            op1(self, out, .Xor, start, line, col);
            continue;
        }
        if c == ':' {
            if has2(self, '=') { op2(self, out, .ColonAssign, start, line, col); continue; }
            op1(self, out, .Colon, start, line, col);
            continue;
        }
        if c == '~' { op1(self, out, .BitNot, start, line, col); continue; }
        if c == ',' { op1(self, out, .Comma, start, line, col); continue; }
        if c == ';' { op1(self, out, .Semicolon, start, line, col); continue; }
        if c == '(' { op1(self, out, .LParen, start, line, col); continue; }
        if c == ')' { op1(self, out, .RParen, start, line, col); continue; }
        if c == '[' { op1(self, out, .LBracket, start, line, col); continue; }
        if c == ']' { op1(self, out, .RBracket, start, line, col); continue; }
        if c == '{' { op1(self, out, .LBrace, start, line, col); continue; }
        if c == '}' { op1(self, out, .RBrace, start, line, col); continue; }
        if c == '#' { op1(self, out, .Sharp, start, line, col); continue; }
        if c == '@' { op1(self, out, .At, start, line, col); continue; }
        if c == '$' { op1(self, out, .Dollar, start, line, col); continue; }
        if c == '?' { op1(self, out, .Cond, start, line, col); continue; }

        report(self, start, start + 1, "unexpected character");
        emit_to(self, out, .Unknown(c), start, start + 1, line, col);
    }

    var end = self.src.len;
    emit_to(self, out, .Eof, end, end, self.line, self.col);
}

fn lex(src: ByteView, diag: *Diag, file: usize): VecResult {
    var out = Vec.blank(@sizeof(Token), @alignof(Token));
    match Vec.new(@sizeof(Token), @alignof(Token)) {
        .Ok(v) => { out = v; },
        .Err(error) => { ret .Err(error); },
    }
    var self = new(src, diag, file);
    run(&self, &out);
    ret .Ok(out);
}

// ---- 测试辅助 ----

fn view(text: []u8): ByteView {
    ret ByteView.from_slice(text);
}

fn lex_ok(src: []u8): Vec {
    var out = Vec.blank(@sizeof(Token), @alignof(Token));
    match lex(view(src), null, 0) {
        .Ok(v) => { out = v; },
        .Err(error) => { @assert(false, "lex failed"); },
    }
    ret out;
}

fn at(v: *Vec, index: usize): Token {
    match Vec.get(v, index) {
        .Ok(ptr) => { ret (ptr as *Token).*; },
        .Err(error) => { ret Token.make(.Unknown(0), ByteView.empty(), 0, 0, 0); },
    }
    ret Token.make(.Unknown(0), ByteView.empty(), 0, 0, 0);
}

fn expect_punct(tok: Token, expected: Punct) {
    match tok.kind {
        .Punct(p) => { @assert(@enum2int(p) == @enum2int(expected), "punct kind"); },
        _ => { @assert(false, "expected punct"); },
    }
}

fn expect_keyword(tok: Token, expected: Keyword) {
    match tok.kind {
        .Keyword(k) => { @assert(@enum2int(k) == @enum2int(expected), "keyword kind"); },
        _ => { @assert(false, "expected keyword"); },
    }
}

fn expect_ident(tok: Token) {
    match tok.kind {
        .Ident => {},
        _ => { @assert(false, "expected ident"); },
    }
}

fn expect_int(tok: Token, radix: Radix) {
    match tok.kind {
        .Int(r) => {
            @assert(@enum2int(r) == @enum2int(radix), "int radix");
        },
        _ => { @assert(false, "expected int"); },
    }
}

fn expect_char(tok: Token, value: u8) {
    match tok.kind {
        .Char(v) => { @assert(v == value, "char value"); },
        _ => { @assert(false, "expected char"); },
    }
}

fn expect_endline(tok: Token) {
    match tok.kind {
        .Endline => {},
        _ => { @assert(false, "expected endline"); },
    }
}

fn expect_eof(tok: Token) {
    match tok.kind {
        .Eof => {},
        _ => { @assert(false, "expected eof"); },
    }
}

fn expect_text(tok: Token, literal: []u8) {
    @assert(ByteView.eq(tok.text, view(literal)), "token text");
}

fn expect_pos(tok: Token, line: usize, col: usize) {
    @assert(tok.line == line && tok.col == col, "token position");
}

fn test_basic() {
    var v = lex_ok("fn main() {\n    ret 0;\n}\n");
    defer { Vec.drop(&v); }
    @assert(v.len == 13, "basic token count");

    var t0 = at(&v, 0);
    expect_keyword(t0, .Fn);
    expect_text(t0, "fn");
    expect_pos(t0, 1, 1);
    expect_ident(at(&v, 1));
    expect_text(at(&v, 1), "main");
    expect_punct(at(&v, 2), .LParen);
    expect_punct(at(&v, 3), .RParen);
    expect_punct(at(&v, 4), .LBrace);
    expect_endline(at(&v, 5));
    expect_keyword(at(&v, 6), .Ret);
    expect_pos(at(&v, 6), 2, 5);
    expect_int(at(&v, 7), .Dec);
    expect_punct(at(&v, 8), .Semicolon);
    expect_endline(at(&v, 9));
    expect_punct(at(&v, 10), .RBrace);
    expect_endline(at(&v, 11));
    expect_eof(at(&v, 12));
}

fn test_operators() {
    var v = lex_ok("<-> <- -> <=> <<= << >>= ... .. . := == != < <= > >= && || & | ^ + - * / % ~ ! ? # @ $ , ; : ( ) [ ] { }");
    defer { Vec.drop(&v); }
    @assert(v.len == 43, "operator token count");

    expect_punct(at(&v, 0), .ArrowBoth);
    expect_text(at(&v, 0), "<->");
    expect_punct(at(&v, 1), .LeftArrow);
    expect_punct(at(&v, 2), .RightArrow);
    expect_punct(at(&v, 3), .FatArrowBoth);
    expect_punct(at(&v, 4), .LShAssign);
    expect_punct(at(&v, 5), .LSh);
    expect_text(at(&v, 5), "<<");
    expect_punct(at(&v, 6), .RShAssign);
    expect_punct(at(&v, 7), .Ellipsis);
    expect_punct(at(&v, 8), .Range);
    expect_punct(at(&v, 9), .Dot);
    expect_punct(at(&v, 10), .ColonAssign);
    expect_punct(at(&v, 11), .Eq);
    expect_punct(at(&v, 12), .Neq);
    expect_punct(at(&v, 13), .Lt);
    expect_punct(at(&v, 14), .Le);
    expect_punct(at(&v, 15), .Gt);
    expect_punct(at(&v, 16), .Ge);
    expect_punct(at(&v, 17), .AndAnd);
    expect_punct(at(&v, 18), .OrOr);
    expect_punct(at(&v, 19), .And);
    expect_punct(at(&v, 20), .Or);
    expect_punct(at(&v, 21), .Xor);
    expect_punct(at(&v, 22), .Add);
    expect_punct(at(&v, 23), .Sub);
    expect_punct(at(&v, 24), .Mul);
    expect_punct(at(&v, 25), .Div);
    expect_punct(at(&v, 26), .Mod);
    expect_punct(at(&v, 27), .BitNot);
    expect_punct(at(&v, 28), .Not);
    expect_punct(at(&v, 29), .Cond);
    expect_punct(at(&v, 30), .Sharp);
    expect_punct(at(&v, 31), .At);
    expect_punct(at(&v, 32), .Dollar);
    expect_punct(at(&v, 33), .Comma);
    expect_punct(at(&v, 34), .Semicolon);
    expect_punct(at(&v, 35), .Colon);
    expect_punct(at(&v, 36), .LParen);
    expect_punct(at(&v, 37), .RParen);
    expect_punct(at(&v, 38), .LBracket);
    expect_punct(at(&v, 39), .RBracket);
    expect_punct(at(&v, 40), .LBrace);
    expect_punct(at(&v, 41), .RBrace);
    expect_eof(at(&v, 42));
}

fn test_comments_and_literals() {
    var v = lex_ok("// c\n/* a /* nested */ b */\n'a' '\\n' '\\x41' \"hi\\n\" 0xFF 0b1010 0o17 1_000 3.14");
    defer { Vec.drop(&v); }
    @assert(v.len == 13, "literal token count");

    var comment = at(&v, 0);
    match comment.kind {
        .LineComment => {},
        _ => { @assert(false, "expected line comment"); },
    }
    expect_text(comment, "// c");
    expect_pos(comment, 1, 1);
    expect_endline(at(&v, 1));
    expect_endline(at(&v, 2));

    var ch3 = at(&v, 3);
    expect_char(ch3, 'a');
    @assert(ByteView.eq(ch3.text, view("'a'")), "char text");
    expect_pos(ch3, 3, 1);
    expect_char(at(&v, 4), '\n');
    expect_char(at(&v, 5), 0x41);

    var s = at(&v, 6);
    match s.kind {
        .Str => {},
        _ => { @assert(false, "expected string"); },
    }
    expect_text(s, "\"hi\\n\"");

    expect_int(at(&v, 7), .Hex);
    expect_int(at(&v, 8), .Bin);
    expect_int(at(&v, 9), .Oct);
    expect_int(at(&v, 10), .Dec);

    var fl = at(&v, 11);
    match fl.kind {
        .Float => {},
        _ => { @assert(false, "expected float"); },
    }
    expect_eof(at(&v, 12));
}

// 非法输入走 diag：仍产出 token 以便继续收集错误，但 diag 必须记录。
// Rust 风格数字字面量：分隔符下划线 + 多进制 + 小数/指数字面量。
fn test_number_forms() {
    var v = lex_ok("1_000_000 0xFF_FF 0x_ff 0b_1010 0o_7 1_000.5 1.5_0 1_000.500 1.5e-3 1_0.5_0E2");
    defer { Vec.drop(&v); }
    @assert(v.len == 11, "number token count");

    expect_int(at(&v, 0), .Dec);
    expect_text(at(&v, 0), "1_000_000");
    expect_int(at(&v, 1), .Hex);
    expect_text(at(&v, 1), "0xFF_FF");
    expect_int(at(&v, 2), .Hex);
    expect_text(at(&v, 2), "0x_ff");
    expect_int(at(&v, 3), .Bin);
    expect_text(at(&v, 3), "0b_1010");
    expect_int(at(&v, 4), .Oct);
    expect_text(at(&v, 4), "0o_7");

    var i: usize = 5;
    while i < 10 {
        match at(&v, i).kind {
            .Float => {},
            _ => { @assert(false, "expected float"); },
        }
        i = i + 1;
    }
    expect_text(at(&v, 5), "1_000.5");
    expect_text(at(&v, 6), "1.5_0");
    expect_text(at(&v, 7), "1_000.500");
    expect_text(at(&v, 8), "1.5e-3");
    expect_text(at(&v, 9), "1_0.5_0E2");
    expect_eof(at(&v, 10));
}

fn test_diag_errors() {
    var diag = Diag.new(null);
    defer { Diag.drop(&diag); }

    match lex(view("'\\q' \"abc"), &diag, 0) {
        .Ok(v) => { Vec.drop(&v); },
        .Err(error) => { @assert(false, "lex should return tokens"); },
    }
    @assert(Diag.has_error(diag), "lex errors recorded");

    var clean = Diag.new(null);
    defer { Diag.drop(&clean); }
    match lex(view("fn main() { ret 0; }"), &clean, 0) {
        .Ok(v) => { Vec.drop(&v); },
        .Err(error) => { @assert(false, "legal lex failed"); },
    }
    @assert(!Diag.has_error(clean), "legal input has no diagnostics");
}

fn main() {
    test_basic();
    test_operators();
    test_comments_and_literals();
    test_number_forms();
    test_diag_errors();
}