717 lines
24 KiB
Plaintext
717 lines
24 KiB
Plaintext
// Expect retcode = 0
|
|
|
|
// 词法分析器：字节流 -> Vec<Token>。镜像 stage1/spl_lexer.c 的行为：
|
|
// 行注释作为 token 产出；块注释（可嵌套）整体跳过；换行产出 Endline；EOF 产出 Eof。
|
|
// 复用 utils/ascii 做字符分类；整型/字符在词法期解码后放进 TokenKind 载荷。
|
|
// Token.text 借用输入字节，不复制。
|
|
|
|
type .{ .Vec, .VecResult } = @import("utils/vec.spl");
|
|
type .{ .ByteView } = @import("utils/byte_view.spl");
|
|
type .{ .is_digit, .is_alpha, .is_alnum, .is_hex_digit, .digit_value } = @import("utils/ascii.spl");
|
|
type .{
|
|
.Token, .TokenKind, .Radix, .Keyword, .Punct, .keyword_of
|
|
} = @import("token.spl");
|
|
type .{ .Span, .span_make } = @import("./diag.spl");
|
|
type Diag = @import("./diag.spl");
|
|
|
|
type Lexer = @self;
|
|
|
|
src: ByteView,
|
|
pos: usize,
|
|
line: usize,
|
|
col: usize,
|
|
diag: *Diag, // 借用；可为 null（合法输入不产生诊断）
|
|
file: usize,
|
|
|
|
fn new(src: ByteView, diag: *Diag, file: usize): Lexer {
|
|
ret Lexer { .src = src, .pos = 0, .line = 1, .col = 1, .diag = diag, .file = file };
|
|
}
|
|
|
|
// 记录一条词法诊断（非法输入路径）。合法输入不应触发。
|
|
fn report(self: *Lexer, start: usize, end: usize, message: []u8) {
|
|
if self.diag == null { ret; }
|
|
Diag.err(self.diag, span_make(self.file, start, end), .LexUnexpected, message);
|
|
}
|
|
|
|
fn is_ident_start(c: u8): bool { ret is_alpha(c) || c == '_'; }
|
|
|
|
fn is_ident_cont(c: u8): bool { ret is_alnum(c) || c == '_'; }
|
|
|
|
fn hex_val(c: u8): u8 {
|
|
match digit_value(c, 16) {
|
|
.Ok(v) => { ret v; },
|
|
.Err(error) => { ret 0; },
|
|
}
|
|
ret 0;
|
|
}
|
|
|
|
// 文本形如 'a' / '\n' / '\x41'；返回解码后的字节。
|
|
fn decode_char(text: ByteView): u8 {
|
|
var start: usize = 1;
|
|
var end = text.len;
|
|
if text.len >= 2 && text.ptr[text.len - 1] == '\'' { end = text.len - 1; }
|
|
if end <= start { ret 0; }
|
|
|
|
var b = text.ptr[start];
|
|
if b != '\\' { ret b; }
|
|
|
|
start = start + 1;
|
|
if start >= end { ret 0; }
|
|
var e = text.ptr[start];
|
|
if e == 'n' { ret '\n'; }
|
|
if e == 't' { ret '\t'; }
|
|
if e == 'r' { ret '\r'; }
|
|
if e == '\\' { ret '\\'; }
|
|
if e == '"' { ret '"'; }
|
|
if e == '\'' { ret '\''; }
|
|
if e == '0' { ret 0; }
|
|
if e == 'x' || e == 'X' {
|
|
var q = start + 1;
|
|
var value: u8 = 0;
|
|
var digits: usize = 0;
|
|
while q < end && digits < 2 && is_hex_digit(text.ptr[q]) {
|
|
value = value * 16 + hex_val(text.ptr[q]);
|
|
q = q + 1;
|
|
digits = digits + 1;
|
|
}
|
|
if digits == 0 { ret e; }
|
|
ret value;
|
|
}
|
|
ret e;
|
|
}
|
|
|
|
// ---- 读取辅助 ----
|
|
|
|
fn has2(self: *Lexer, ch: u8): bool {
|
|
ret self.pos + 1 < self.src.len && self.src.ptr[self.pos + 1] == ch;
|
|
}
|
|
|
|
fn has3(self: *Lexer, ch1: u8, ch2: u8): bool {
|
|
ret self.pos + 2 < self.src.len && self.src.ptr[self.pos + 1] == ch1 &&
|
|
self.src.ptr[self.pos + 2] == ch2;
|
|
}
|
|
|
|
fn sub(self: *Lexer, start: usize, end: usize): ByteView {
|
|
if start >= end { ret ByteView.empty(); }
|
|
ret ByteView { .ptr = self.src.ptr + start, .len = end - start };
|
|
}
|
|
|
|
// 产出 token 并把 pos/col 推进到 end。Vec.push 失败视为致命（同 vec.make_vec 风格）。
|
|
fn emit_to(self: *Lexer, out: *Vec, kind: TokenKind, start: usize, end: usize, line: usize,
|
|
col: usize) {
|
|
var t = Token.make(kind, sub(self, start, end), line, col, start);
|
|
match Vec.push(out, &t as *u8) {
|
|
.Ok => {},
|
|
.Err(error) => { @assert(false, "lexer: Vec.push failed"); },
|
|
}
|
|
self.pos = end;
|
|
self.col = col + (end - start);
|
|
}
|
|
|
|
fn op1(self: *Lexer, out: *Vec, punct: Punct, start: usize, line: usize, col: usize) {
|
|
emit_to(self, out, .Punct(punct), start, start + 1, line, col);
|
|
}
|
|
|
|
fn op2(self: *Lexer, out: *Vec, punct: Punct, start: usize, line: usize, col: usize) {
|
|
emit_to(self, out, .Punct(punct), start, start + 2, line, col);
|
|
}
|
|
|
|
fn op3(self: *Lexer, out: *Vec, punct: Punct, start: usize, line: usize, col: usize) {
|
|
emit_to(self, out, .Punct(punct), start, start + 3, line, col);
|
|
}
|
|
|
|
// ---- 复合扫描 ----
|
|
|
|
fn scan_int(self: *Lexer, out: *Vec, start: usize, line: usize, col: usize) {
|
|
var p = self.pos;
|
|
if self.src.ptr[p] == '0' && p + 1 < self.src.len {
|
|
var nc = self.src.ptr[p + 1];
|
|
if nc == 'x' || nc == 'X' {
|
|
p = p + 2;
|
|
while p < self.src.len && (is_hex_digit(self.src.ptr[p]) || self.src.ptr[p] == '_') {
|
|
p = p + 1;
|
|
}
|
|
emit_to(self, out, .Int(.Hex), start, p, line, col);
|
|
ret;
|
|
}
|
|
if nc == 'b' || nc == 'B' {
|
|
p = p + 2;
|
|
while p < self.src.len && (self.src.ptr[p] == '0' || self.src.ptr[p] == '1' ||
|
|
self.src.ptr[p] == '_') {
|
|
p = p + 1;
|
|
}
|
|
emit_to(self, out, .Int(.Bin), start, p, line, col);
|
|
ret;
|
|
}
|
|
if nc == 'o' || nc == 'O' {
|
|
p = p + 2;
|
|
while p < self.src.len &&
|
|
((self.src.ptr[p] >= '0' && self.src.ptr[p] <= '7') ||
|
|
self.src.ptr[p] == '_') {
|
|
p = p + 1;
|
|
}
|
|
emit_to(self, out, .Int(.Oct), start, p, line, col);
|
|
ret;
|
|
}
|
|
}
|
|
|
|
while p < self.src.len && (is_digit(self.src.ptr[p]) || self.src.ptr[p] == '_') { p = p + 1; }
|
|
if p + 1 < self.src.len && self.src.ptr[p] == '.' && is_digit(self.src.ptr[p + 1]) {
|
|
p = p + 1;
|
|
// 小数部分允许下划线分隔（Rust 风格）。
|
|
while p < self.src.len && (is_digit(self.src.ptr[p]) || self.src.ptr[p] == '_') {
|
|
p = p + 1;
|
|
}
|
|
// 可选指数：e/E [+-] 数字（数字允许下划线）；仅当确有数字时纳入。
|
|
if p < self.src.len && (self.src.ptr[p] == 'e' || self.src.ptr[p] == 'E') {
|
|
var q = p + 1;
|
|
if q < self.src.len && (self.src.ptr[q] == '+' || self.src.ptr[q] == '-') { q = q + 1; }
|
|
if q < self.src.len && is_digit(self.src.ptr[q]) {
|
|
p = q + 1;
|
|
while p < self.src.len && (is_digit(self.src.ptr[p]) || self.src.ptr[p] == '_') {
|
|
p = p + 1;
|
|
}
|
|
}
|
|
}
|
|
emit_to(self, out, .Float, start, p, line, col);
|
|
ret;
|
|
}
|
|
emit_to(self, out, .Int(.Dec), start, p, line, col);
|
|
}
|
|
|
|
fn scan_char(self: *Lexer, out: *Vec, start: usize, line: usize, col: usize) {
|
|
var p = self.pos + 1;
|
|
var closed = false;
|
|
var escape_ok = true;
|
|
if p < self.src.len {
|
|
if self.src.ptr[p] == '\\' {
|
|
p = p + 1;
|
|
if p < self.src.len && (self.src.ptr[p] == 'x' || self.src.ptr[p] == 'X') {
|
|
p = p + 1;
|
|
var digits: usize = 0;
|
|
while p < self.src.len && digits < 2 && is_hex_digit(self.src.ptr[p]) {
|
|
p = p + 1;
|
|
digits = digits + 1;
|
|
}
|
|
if digits == 0 { escape_ok = false; }
|
|
} else if p < self.src.len {
|
|
var e = self.src.ptr[p];
|
|
if !(e == 'n' || e == 't' || e == 'r' || e == '\\' || e == '"' || e == '\'' ||
|
|
e == '0') {
|
|
escape_ok = false;
|
|
}
|
|
p = p + 1;
|
|
} else {
|
|
escape_ok = false;
|
|
}
|
|
} else {
|
|
p = p + 1;
|
|
}
|
|
}
|
|
if p < self.src.len && self.src.ptr[p] == '\'' {
|
|
p = p + 1;
|
|
closed = true;
|
|
}
|
|
if !closed {
|
|
report(self, start, p, "unterminated character literal");
|
|
} else if !escape_ok {
|
|
report(self, start, p, "invalid escape in character literal");
|
|
}
|
|
emit_to(self, out, .Char(decode_char(sub(self, start, p))), start, p, line, col);
|
|
}
|
|
|
|
fn scan_string(self: *Lexer, out: *Vec, start: usize, line: usize, col: usize) {
|
|
var p = self.pos + 1;
|
|
var l = line;
|
|
var cc = col + 1;
|
|
var closed = false;
|
|
while p < self.src.len {
|
|
var b = self.src.ptr[p];
|
|
if b == '\\' {
|
|
p = p + 1;
|
|
cc = cc + 1;
|
|
if p < self.src.len {
|
|
p = p + 1;
|
|
cc = cc + 1;
|
|
}
|
|
} else if b == '"' {
|
|
p = p + 1;
|
|
cc = cc + 1;
|
|
closed = true;
|
|
break;
|
|
} else if b == '\n' {
|
|
l = l + 1;
|
|
cc = 1;
|
|
p = p + 1;
|
|
} else {
|
|
p = p + 1;
|
|
cc = cc + 1;
|
|
}
|
|
}
|
|
if !closed { report(self, start, p, "unterminated string literal"); }
|
|
emit_to(self, out, .Str, start, p, line, col);
|
|
self.line = l;
|
|
self.col = cc;
|
|
}
|
|
|
|
fn scan_ident(self: *Lexer, out: *Vec, start: usize, line: usize, col: usize) {
|
|
var p = self.pos;
|
|
while p < self.src.len && is_ident_cont(self.src.ptr[p]) { p = p + 1; }
|
|
var text = sub(self, start, p);
|
|
match keyword_of(text) {
|
|
.Yes(kw) => { emit_to(self, out, .Keyword(kw), start, p, line, col); },
|
|
.No => { emit_to(self, out, .Ident, start, p, line, col); },
|
|
}
|
|
}
|
|
|
|
// 块注释：可嵌套，整体跳过（不产出 token）。未闭合时报诊断。
|
|
fn skip_block_comment(self: *Lexer, start: usize) {
|
|
self.pos = self.pos + 2;
|
|
self.col = self.col + 2;
|
|
var depth: usize = 1;
|
|
while self.pos + 1 < self.src.len && depth > 0 {
|
|
if self.src.ptr[self.pos] == '*' && self.src.ptr[self.pos + 1] == '/' {
|
|
depth = depth - 1;
|
|
self.pos = self.pos + 2;
|
|
self.col = self.col + 2;
|
|
} else if self.src.ptr[self.pos] == '/' && self.src.ptr[self.pos + 1] == '*' {
|
|
depth = depth + 1;
|
|
self.pos = self.pos + 2;
|
|
self.col = self.col + 2;
|
|
} else {
|
|
if self.src.ptr[self.pos] == '\n' {
|
|
self.line = self.line + 1;
|
|
self.col = 1;
|
|
} else {
|
|
self.col = self.col + 1;
|
|
}
|
|
self.pos = self.pos + 1;
|
|
}
|
|
}
|
|
if depth > 0 {
|
|
report(self, start, self.src.len, "unterminated block comment");
|
|
}
|
|
}
|
|
|
|
fn run(self: *Lexer, out: *Vec) {
|
|
while self.pos < self.src.len {
|
|
var start = self.pos;
|
|
var line = self.line;
|
|
var col = self.col;
|
|
var c = self.src.ptr[self.pos];
|
|
|
|
if c == ' ' || c == '\t' || c == '\r' {
|
|
self.pos = self.pos + 1;
|
|
self.col = self.col + 1;
|
|
continue;
|
|
}
|
|
if c == '\n' {
|
|
emit_to(self, out, .Endline, start, start + 1, line, col);
|
|
self.line = line + 1;
|
|
self.col = 1;
|
|
continue;
|
|
}
|
|
if c == '/' && has2(self, '/') {
|
|
var p = self.pos;
|
|
while p < self.src.len && self.src.ptr[p] != '\n' { p = p + 1; }
|
|
emit_to(self, out, .LineComment, start, p, line, col);
|
|
continue;
|
|
}
|
|
if c == '/' && has2(self, '*') {
|
|
skip_block_comment(self, start);
|
|
continue;
|
|
}
|
|
if c == '\'' {
|
|
scan_char(self, out, start, line, col);
|
|
continue;
|
|
}
|
|
if c == '"' {
|
|
scan_string(self, out, start, line, col);
|
|
continue;
|
|
}
|
|
if is_ident_start(c) {
|
|
scan_ident(self, out, start, line, col);
|
|
continue;
|
|
}
|
|
if is_digit(c) {
|
|
scan_int(self, out, start, line, col);
|
|
continue;
|
|
}
|
|
|
|
if c == '<' {
|
|
if has3(self, '-', '>') { op3(self, out, .ArrowBoth, start, line, col); continue; }
|
|
if has3(self, '=', '>') { op3(self, out, .FatArrowBoth, start, line, col); continue; }
|
|
if has3(self, '<', '=') { op3(self, out, .LShAssign, start, line, col); continue; }
|
|
if has2(self, '<') { op2(self, out, .LSh, start, line, col); continue; }
|
|
if has2(self, '-') { op2(self, out, .LeftArrow, start, line, col); continue; }
|
|
if has2(self, '=') { op2(self, out, .Le, start, line, col); continue; }
|
|
op1(self, out, .Lt, start, line, col);
|
|
continue;
|
|
}
|
|
if c == '>' {
|
|
if has3(self, '>', '=') { op3(self, out, .RShAssign, start, line, col); continue; }
|
|
if has2(self, '>') { op2(self, out, .RSh, start, line, col); continue; }
|
|
if has2(self, '=') { op2(self, out, .Ge, start, line, col); continue; }
|
|
op1(self, out, .Gt, start, line, col);
|
|
continue;
|
|
}
|
|
if c == '.' {
|
|
if has3(self, '.', '.') { op3(self, out, .Ellipsis, start, line, col); continue; }
|
|
if has2(self, '.') { op2(self, out, .Range, start, line, col); continue; }
|
|
op1(self, out, .Dot, start, line, col);
|
|
continue;
|
|
}
|
|
if c == '=' {
|
|
if has2(self, '=') { op2(self, out, .Eq, start, line, col); continue; }
|
|
if has2(self, '>') { op2(self, out, .RightFatArrow, start, line, col); continue; }
|
|
op1(self, out, .Assign, start, line, col);
|
|
continue;
|
|
}
|
|
if c == '!' {
|
|
if has2(self, '=') { op2(self, out, .Neq, start, line, col); continue; }
|
|
op1(self, out, .Not, start, line, col);
|
|
continue;
|
|
}
|
|
if c == '&' {
|
|
if has2(self, '&') { op2(self, out, .AndAnd, start, line, col); continue; }
|
|
if has2(self, '=') { op2(self, out, .AndAssign, start, line, col); continue; }
|
|
op1(self, out, .And, start, line, col);
|
|
continue;
|
|
}
|
|
if c == '|' {
|
|
if has2(self, '|') { op2(self, out, .OrOr, start, line, col); continue; }
|
|
if has2(self, '=') { op2(self, out, .OrAssign, start, line, col); continue; }
|
|
op1(self, out, .Or, start, line, col);
|
|
continue;
|
|
}
|
|
if c == '+' {
|
|
if has2(self, '=') { op2(self, out, .AddAssign, start, line, col); continue; }
|
|
op1(self, out, .Add, start, line, col);
|
|
continue;
|
|
}
|
|
if c == '-' {
|
|
if has2(self, '=') { op2(self, out, .SubAssign, start, line, col); continue; }
|
|
if has2(self, '>') { op2(self, out, .RightArrow, start, line, col); continue; }
|
|
op1(self, out, .Sub, start, line, col);
|
|
continue;
|
|
}
|
|
if c == '*' {
|
|
if has2(self, '=') { op2(self, out, .MulAssign, start, line, col); continue; }
|
|
op1(self, out, .Mul, start, line, col);
|
|
continue;
|
|
}
|
|
if c == '/' {
|
|
if has2(self, '=') { op2(self, out, .DivAssign, start, line, col); continue; }
|
|
op1(self, out, .Div, start, line, col);
|
|
continue;
|
|
}
|
|
if c == '%' {
|
|
if has2(self, '=') { op2(self, out, .ModAssign, start, line, col); continue; }
|
|
op1(self, out, .Mod, start, line, col);
|
|
continue;
|
|
}
|
|
if c == '^' {
|
|
if has2(self, '=') { op2(self, out, .XorAssign, start, line, col); continue; }
|
|
op1(self, out, .Xor, start, line, col);
|
|
continue;
|
|
}
|
|
if c == ':' {
|
|
if has2(self, '=') { op2(self, out, .ColonAssign, start, line, col); continue; }
|
|
op1(self, out, .Colon, start, line, col);
|
|
continue;
|
|
}
|
|
if c == '~' { op1(self, out, .BitNot, start, line, col); continue; }
|
|
if c == ',' { op1(self, out, .Comma, start, line, col); continue; }
|
|
if c == ';' { op1(self, out, .Semicolon, start, line, col); continue; }
|
|
if c == '(' { op1(self, out, .LParen, start, line, col); continue; }
|
|
if c == ')' { op1(self, out, .RParen, start, line, col); continue; }
|
|
if c == '[' { op1(self, out, .LBracket, start, line, col); continue; }
|
|
if c == ']' { op1(self, out, .RBracket, start, line, col); continue; }
|
|
if c == '{' { op1(self, out, .LBrace, start, line, col); continue; }
|
|
if c == '}' { op1(self, out, .RBrace, start, line, col); continue; }
|
|
if c == '#' { op1(self, out, .Sharp, start, line, col); continue; }
|
|
if c == '@' { op1(self, out, .At, start, line, col); continue; }
|
|
if c == '$' { op1(self, out, .Dollar, start, line, col); continue; }
|
|
if c == '?' { op1(self, out, .Cond, start, line, col); continue; }
|
|
|
|
report(self, start, start + 1, "unexpected character");
|
|
emit_to(self, out, .Unknown(c), start, start + 1, line, col);
|
|
}
|
|
|
|
var end = self.src.len;
|
|
emit_to(self, out, .Eof, end, end, self.line, self.col);
|
|
}
|
|
|
|
fn lex(src: ByteView, diag: *Diag, file: usize): VecResult {
|
|
var out = Vec.blank(@sizeof(Token), @alignof(Token));
|
|
match Vec.new(@sizeof(Token), @alignof(Token)) {
|
|
.Ok(v) => { out = v; },
|
|
.Err(error) => { ret .Err(error); },
|
|
}
|
|
var self = new(src, diag, file);
|
|
run(&self, &out);
|
|
ret .Ok(out);
|
|
}
|
|
|
|
// ---- 测试辅助 ----
|
|
|
|
fn view(text: []u8): ByteView {
|
|
ret ByteView.from_slice(text);
|
|
}
|
|
|
|
fn lex_ok(src: []u8): Vec {
|
|
var out = Vec.blank(@sizeof(Token), @alignof(Token));
|
|
match lex(view(src), null, 0) {
|
|
.Ok(v) => { out = v; },
|
|
.Err(error) => { @assert(false, "lex failed"); },
|
|
}
|
|
ret out;
|
|
}
|
|
|
|
fn at(v: *Vec, index: usize): Token {
|
|
match Vec.get(v, index) {
|
|
.Ok(ptr) => { ret (ptr as *Token).*; },
|
|
.Err(error) => { ret Token.make(.Unknown(0), ByteView.empty(), 0, 0, 0); },
|
|
}
|
|
ret Token.make(.Unknown(0), ByteView.empty(), 0, 0, 0);
|
|
}
|
|
|
|
fn expect_punct(tok: Token, expected: Punct) {
|
|
match tok.kind {
|
|
.Punct(p) => { @assert(@enum2int(p) == @enum2int(expected), "punct kind"); },
|
|
_ => { @assert(false, "expected punct"); },
|
|
}
|
|
}
|
|
|
|
fn expect_keyword(tok: Token, expected: Keyword) {
|
|
match tok.kind {
|
|
.Keyword(k) => { @assert(@enum2int(k) == @enum2int(expected), "keyword kind"); },
|
|
_ => { @assert(false, "expected keyword"); },
|
|
}
|
|
}
|
|
|
|
fn expect_ident(tok: Token) {
|
|
match tok.kind {
|
|
.Ident => {},
|
|
_ => { @assert(false, "expected ident"); },
|
|
}
|
|
}
|
|
|
|
fn expect_int(tok: Token, radix: Radix) {
|
|
match tok.kind {
|
|
.Int(r) => {
|
|
@assert(@enum2int(r) == @enum2int(radix), "int radix");
|
|
},
|
|
_ => { @assert(false, "expected int"); },
|
|
}
|
|
}
|
|
|
|
fn expect_char(tok: Token, value: u8) {
|
|
match tok.kind {
|
|
.Char(v) => { @assert(v == value, "char value"); },
|
|
_ => { @assert(false, "expected char"); },
|
|
}
|
|
}
|
|
|
|
fn expect_endline(tok: Token) {
|
|
match tok.kind {
|
|
.Endline => {},
|
|
_ => { @assert(false, "expected endline"); },
|
|
}
|
|
}
|
|
|
|
fn expect_eof(tok: Token) {
|
|
match tok.kind {
|
|
.Eof => {},
|
|
_ => { @assert(false, "expected eof"); },
|
|
}
|
|
}
|
|
|
|
fn expect_text(tok: Token, literal: []u8) {
|
|
@assert(ByteView.eq(tok.text, view(literal)), "token text");
|
|
}
|
|
|
|
fn expect_pos(tok: Token, line: usize, col: usize) {
|
|
@assert(tok.line == line && tok.col == col, "token position");
|
|
}
|
|
|
|
fn test_basic() {
|
|
var v = lex_ok("fn main() {\n ret 0;\n}\n");
|
|
defer { Vec.drop(&v); }
|
|
@assert(v.len == 13, "basic token count");
|
|
|
|
var t0 = at(&v, 0);
|
|
expect_keyword(t0, .Fn);
|
|
expect_text(t0, "fn");
|
|
expect_pos(t0, 1, 1);
|
|
expect_ident(at(&v, 1));
|
|
expect_text(at(&v, 1), "main");
|
|
expect_punct(at(&v, 2), .LParen);
|
|
expect_punct(at(&v, 3), .RParen);
|
|
expect_punct(at(&v, 4), .LBrace);
|
|
expect_endline(at(&v, 5));
|
|
expect_keyword(at(&v, 6), .Ret);
|
|
expect_pos(at(&v, 6), 2, 5);
|
|
expect_int(at(&v, 7), .Dec);
|
|
expect_punct(at(&v, 8), .Semicolon);
|
|
expect_endline(at(&v, 9));
|
|
expect_punct(at(&v, 10), .RBrace);
|
|
expect_endline(at(&v, 11));
|
|
expect_eof(at(&v, 12));
|
|
}
|
|
|
|
fn test_operators() {
|
|
var v = lex_ok("<-> <- -> <=> <<= << >>= ... .. . := == != < <= > >= && || & | ^ + - * / % ~ ! ? # @ $ , ; : ( ) [ ] { }");
|
|
defer { Vec.drop(&v); }
|
|
@assert(v.len == 43, "operator token count");
|
|
|
|
expect_punct(at(&v, 0), .ArrowBoth);
|
|
expect_text(at(&v, 0), "<->");
|
|
expect_punct(at(&v, 1), .LeftArrow);
|
|
expect_punct(at(&v, 2), .RightArrow);
|
|
expect_punct(at(&v, 3), .FatArrowBoth);
|
|
expect_punct(at(&v, 4), .LShAssign);
|
|
expect_punct(at(&v, 5), .LSh);
|
|
expect_text(at(&v, 5), "<<");
|
|
expect_punct(at(&v, 6), .RShAssign);
|
|
expect_punct(at(&v, 7), .Ellipsis);
|
|
expect_punct(at(&v, 8), .Range);
|
|
expect_punct(at(&v, 9), .Dot);
|
|
expect_punct(at(&v, 10), .ColonAssign);
|
|
expect_punct(at(&v, 11), .Eq);
|
|
expect_punct(at(&v, 12), .Neq);
|
|
expect_punct(at(&v, 13), .Lt);
|
|
expect_punct(at(&v, 14), .Le);
|
|
expect_punct(at(&v, 15), .Gt);
|
|
expect_punct(at(&v, 16), .Ge);
|
|
expect_punct(at(&v, 17), .AndAnd);
|
|
expect_punct(at(&v, 18), .OrOr);
|
|
expect_punct(at(&v, 19), .And);
|
|
expect_punct(at(&v, 20), .Or);
|
|
expect_punct(at(&v, 21), .Xor);
|
|
expect_punct(at(&v, 22), .Add);
|
|
expect_punct(at(&v, 23), .Sub);
|
|
expect_punct(at(&v, 24), .Mul);
|
|
expect_punct(at(&v, 25), .Div);
|
|
expect_punct(at(&v, 26), .Mod);
|
|
expect_punct(at(&v, 27), .BitNot);
|
|
expect_punct(at(&v, 28), .Not);
|
|
expect_punct(at(&v, 29), .Cond);
|
|
expect_punct(at(&v, 30), .Sharp);
|
|
expect_punct(at(&v, 31), .At);
|
|
expect_punct(at(&v, 32), .Dollar);
|
|
expect_punct(at(&v, 33), .Comma);
|
|
expect_punct(at(&v, 34), .Semicolon);
|
|
expect_punct(at(&v, 35), .Colon);
|
|
expect_punct(at(&v, 36), .LParen);
|
|
expect_punct(at(&v, 37), .RParen);
|
|
expect_punct(at(&v, 38), .LBracket);
|
|
expect_punct(at(&v, 39), .RBracket);
|
|
expect_punct(at(&v, 40), .LBrace);
|
|
expect_punct(at(&v, 41), .RBrace);
|
|
expect_eof(at(&v, 42));
|
|
}
|
|
|
|
fn test_comments_and_literals() {
|
|
var v = lex_ok("// c\n/* a /* nested */ b */\n'a' '\\n' '\\x41' \"hi\\n\" 0xFF 0b1010 0o17 1_000 3.14");
|
|
defer { Vec.drop(&v); }
|
|
@assert(v.len == 13, "literal token count");
|
|
|
|
var comment = at(&v, 0);
|
|
match comment.kind {
|
|
.LineComment => {},
|
|
_ => { @assert(false, "expected line comment"); },
|
|
}
|
|
expect_text(comment, "// c");
|
|
expect_pos(comment, 1, 1);
|
|
expect_endline(at(&v, 1));
|
|
expect_endline(at(&v, 2));
|
|
|
|
var ch3 = at(&v, 3);
|
|
expect_char(ch3, 'a');
|
|
@assert(ByteView.eq(ch3.text, view("'a'")), "char text");
|
|
expect_pos(ch3, 3, 1);
|
|
expect_char(at(&v, 4), '\n');
|
|
expect_char(at(&v, 5), 0x41);
|
|
|
|
var s = at(&v, 6);
|
|
match s.kind {
|
|
.Str => {},
|
|
_ => { @assert(false, "expected string"); },
|
|
}
|
|
expect_text(s, "\"hi\\n\"");
|
|
|
|
expect_int(at(&v, 7), .Hex);
|
|
expect_int(at(&v, 8), .Bin);
|
|
expect_int(at(&v, 9), .Oct);
|
|
expect_int(at(&v, 10), .Dec);
|
|
|
|
var fl = at(&v, 11);
|
|
match fl.kind {
|
|
.Float => {},
|
|
_ => { @assert(false, "expected float"); },
|
|
}
|
|
expect_eof(at(&v, 12));
|
|
}
|
|
|
|
// 非法输入走 diag：仍产出 token 以便继续收集错误，但 diag 必须记录。
|
|
// Rust 风格数字字面量：分隔符下划线 + 多进制 + 小数/指数字面量。
|
|
fn test_number_forms() {
|
|
var v = lex_ok("1_000_000 0xFF_FF 0x_ff 0b_1010 0o_7 1_000.5 1.5_0 1_000.500 1.5e-3 1_0.5_0E2");
|
|
defer { Vec.drop(&v); }
|
|
@assert(v.len == 11, "number token count");
|
|
|
|
expect_int(at(&v, 0), .Dec);
|
|
expect_text(at(&v, 0), "1_000_000");
|
|
expect_int(at(&v, 1), .Hex);
|
|
expect_text(at(&v, 1), "0xFF_FF");
|
|
expect_int(at(&v, 2), .Hex);
|
|
expect_text(at(&v, 2), "0x_ff");
|
|
expect_int(at(&v, 3), .Bin);
|
|
expect_text(at(&v, 3), "0b_1010");
|
|
expect_int(at(&v, 4), .Oct);
|
|
expect_text(at(&v, 4), "0o_7");
|
|
|
|
var i: usize = 5;
|
|
while i < 10 {
|
|
match at(&v, i).kind {
|
|
.Float => {},
|
|
_ => { @assert(false, "expected float"); },
|
|
}
|
|
i = i + 1;
|
|
}
|
|
expect_text(at(&v, 5), "1_000.5");
|
|
expect_text(at(&v, 6), "1.5_0");
|
|
expect_text(at(&v, 7), "1_000.500");
|
|
expect_text(at(&v, 8), "1.5e-3");
|
|
expect_text(at(&v, 9), "1_0.5_0E2");
|
|
expect_eof(at(&v, 10));
|
|
}
|
|
|
|
fn test_diag_errors() {
|
|
var diag = Diag.new(null);
|
|
defer { Diag.drop(&diag); }
|
|
|
|
match lex(view("'\\q' \"abc"), &diag, 0) {
|
|
.Ok(v) => { Vec.drop(&v); },
|
|
.Err(error) => { @assert(false, "lex should return tokens"); },
|
|
}
|
|
@assert(Diag.has_error(diag), "lex errors recorded");
|
|
|
|
var clean = Diag.new(null);
|
|
defer { Diag.drop(&clean); }
|
|
match lex(view("fn main() { ret 0; }"), &clean, 0) {
|
|
.Ok(v) => { Vec.drop(&v); },
|
|
.Err(error) => { @assert(false, "legal lex failed"); },
|
|
}
|
|
@assert(!Diag.has_error(clean), "legal input has no diagnostics");
|
|
}
|
|
|
|
fn main() {
|
|
test_basic();
|
|
test_operators();
|
|
test_comments_and_literals();
|
|
test_number_forms();
|
|
test_diag_errors();
|
|
}
|