stage2 迁移ast解析器 stage1 防止错误死循环

stage2/lexer.spl

@@ -9,7 +9,7 @@ type .{ .Vec, .VecResult } = @import("utils/vec.spl");
 type .{ .ByteView } = @import("utils/byte_view.spl");
 type .{ .is_digit, .is_alpha, .is_alnum, .is_hex_digit, .digit_value } = @import("utils/ascii.spl");
 type .{
-    .Token, .TokenKind, .Radix, .Keyword, .Punct, .int_value, .keyword_of
+    .Token, .TokenKind, .Radix, .Keyword, .Punct, .keyword_of
 } = @import("token.spl");
 type .{ .Span, .span_make } = @import("./diag.spl");
 type Diag = @import("./diag.spl");
@@ -45,23 +45,6 @@ fn hex_val(c: u8): u8 {
     ret 0;
 }
 
-// 跳过下划线，按 radix 折叠；非法字符忽略（前缀 x/b/o 自然被忽略）。
-fn decode_digits(text: ByteView, radix: u8): u64 {
-    var acc: u64 = 0;
-    var i: usize = 0;
-    while i < text.len {
-        var b = text.ptr[i];
-        if b != '_' {
-            match digit_value(b, radix) {
-                .Ok(v) => { acc = acc * (radix as u64) + (v as u64); },
-                .Err(error) => {},
-            }
-        }
-        i = i + 1;
-    }
-    ret acc;
-}
-
 // 文本形如 'a' / '\n' / '\x41'；返回解码后的字节。
 fn decode_char(text: ByteView): u8 {
     var start: usize = 1;
@@ -119,7 +102,7 @@ fn sub(self: *Lexer, start: usize, end: usize): ByteView {
 // 产出 token 并把 pos/col 推进到 end。Vec.push 失败视为致命（同 vec.make_vec 风格）。
 fn emit_to(self: *Lexer, out: *Vec, kind: TokenKind, start: usize, end: usize, line: usize,
            col: usize) {
-    var t = Token.make(kind, sub(self, start, end), line, col);
+    var t = Token.make(kind, sub(self, start, end), line, col, start);
     match Vec.push(out, &t as *u8) {
         .Ok => {},
         .Err(error) => { @assert(false, "lexer: Vec.push failed"); },
@@ -151,8 +134,7 @@ fn scan_int(self: *Lexer, out: *Vec, start: usize, line: usize, col: usize) {
             while p < self.src.len && (is_hex_digit(self.src.ptr[p]) || self.src.ptr[p] == '_') {
                 p = p + 1;
             }
-            emit_to(self, out, .Int(int_value(decode_digits(sub(self, start, p), 16), .Hex)), start,
-                    p, line, col);
+            emit_to(self, out, .Int(.Hex), start, p, line, col);
             ret;
         }
         if nc == 'b' || nc == 'B' {
@@ -161,8 +143,7 @@ fn scan_int(self: *Lexer, out: *Vec, start: usize, line: usize, col: usize) {
                                        self.src.ptr[p] == '_') {
                 p = p + 1;
             }
-            emit_to(self, out, .Int(int_value(decode_digits(sub(self, start, p), 2), .Bin)), start,
-                    p, line, col);
+            emit_to(self, out, .Int(.Bin), start, p, line, col);
             ret;
         }
         if nc == 'o' || nc == 'O' {
@@ -172,8 +153,7 @@ fn scan_int(self: *Lexer, out: *Vec, start: usize, line: usize, col: usize) {
                     self.src.ptr[p] == '_') {
                 p = p + 1;
             }
-            emit_to(self, out, .Int(int_value(decode_digits(sub(self, start, p), 8), .Oct)), start,
-                    p, line, col);
+            emit_to(self, out, .Int(.Oct), start, p, line, col);
             ret;
         }
     }
@@ -181,12 +161,25 @@ fn scan_int(self: *Lexer, out: *Vec, start: usize, line: usize, col: usize) {
     while p < self.src.len && (is_digit(self.src.ptr[p]) || self.src.ptr[p] == '_') { p = p + 1; }
     if p + 1 < self.src.len && self.src.ptr[p] == '.' && is_digit(self.src.ptr[p + 1]) {
         p = p + 1;
-        while p < self.src.len && is_digit(self.src.ptr[p]) { p = p + 1; }
+        // 小数部分允许下划线分隔（Rust 风格）。
+        while p < self.src.len && (is_digit(self.src.ptr[p]) || self.src.ptr[p] == '_') {
+            p = p + 1;
+        }
+        // 可选指数：e/E [+-] 数字（数字允许下划线）；仅当确有数字时纳入。
+        if p < self.src.len && (self.src.ptr[p] == 'e' || self.src.ptr[p] == 'E') {
+            var q = p + 1;
+            if q < self.src.len && (self.src.ptr[q] == '+' || self.src.ptr[q] == '-') { q = q + 1; }
+            if q < self.src.len && is_digit(self.src.ptr[q]) {
+                p = q + 1;
+                while p < self.src.len && (is_digit(self.src.ptr[p]) || self.src.ptr[p] == '_') {
+                    p = p + 1;
+                }
+            }
+        }
         emit_to(self, out, .Float, start, p, line, col);
         ret;
     }
-    emit_to(self, out, .Int(int_value(decode_digits(sub(self, start, p), 10), .Dec)), start, p, line,
-            col);
+    emit_to(self, out, .Int(.Dec), start, p, line, col);
 }
 
 fn scan_char(self: *Lexer, out: *Vec, start: usize, line: usize, col: usize) {
@@ -481,9 +474,9 @@ fn lex_ok(src: []u8): Vec {
 fn at(v: *Vec, index: usize): Token {
     match Vec.get(v, index) {
         .Ok(ptr) => { ret (ptr as *Token).*; },
-        .Err(error) => { ret Token.make(.Unknown(0), ByteView.empty(), 0, 0); },
+        .Err(error) => { ret Token.make(.Unknown(0), ByteView.empty(), 0, 0, 0); },
     }
-    ret Token.make(.Unknown(0), ByteView.empty(), 0, 0);
+    ret Token.make(.Unknown(0), ByteView.empty(), 0, 0, 0);
 }
 
 fn expect_punct(tok: Token, expected: Punct) {
@@ -507,11 +500,10 @@ fn expect_ident(tok: Token) {
     }
 }
 
-fn expect_int(tok: Token, value: u64, radix: Radix) {
+fn expect_int(tok: Token, radix: Radix) {
     match tok.kind {
-        .Int(iv) => {
-            @assert(iv.value == value, "int value");
-            @assert(@enum2int(iv.radix) == @enum2int(radix), "int radix");
+        .Int(r) => {
+            @assert(@enum2int(r) == @enum2int(radix), "int radix");
         },
         _ => { @assert(false, "expected int"); },
     }
@@ -563,7 +555,7 @@ fn test_basic() {
     expect_endline(at(&v, 5));
     expect_keyword(at(&v, 6), .Ret);
     expect_pos(at(&v, 6), 2, 5);
-    expect_int(at(&v, 7), 0, .Dec);
+    expect_int(at(&v, 7), .Dec);
     expect_punct(at(&v, 8), .Semicolon);
     expect_endline(at(&v, 9));
     expect_punct(at(&v, 10), .RBrace);
@@ -652,10 +644,10 @@ fn test_comments_and_literals() {
     }
     expect_text(s, "\"hi\\n\"");
 
-    expect_int(at(&v, 7), 255, .Hex);
-    expect_int(at(&v, 8), 10, .Bin);
-    expect_int(at(&v, 9), 15, .Oct);
-    expect_int(at(&v, 10), 1000, .Dec);
+    expect_int(at(&v, 7), .Hex);
+    expect_int(at(&v, 8), .Bin);
+    expect_int(at(&v, 9), .Oct);
+    expect_int(at(&v, 10), .Dec);
 
     var fl = at(&v, 11);
     match fl.kind {
@@ -666,6 +658,39 @@ fn test_comments_and_literals() {
 }
 
 // 非法输入走 diag：仍产出 token 以便继续收集错误，但 diag 必须记录。
+// Rust 风格数字字面量：分隔符下划线 + 多进制 + 小数/指数字面量。
+fn test_number_forms() {
+    var v = lex_ok("1_000_000 0xFF_FF 0x_ff 0b_1010 0o_7 1_000.5 1.5_0 1_000.500 1.5e-3 1_0.5_0E2");
+    defer { Vec.drop(&v); }
+    @assert(v.len == 11, "number token count");
+
+    expect_int(at(&v, 0), .Dec);
+    expect_text(at(&v, 0), "1_000_000");
+    expect_int(at(&v, 1), .Hex);
+    expect_text(at(&v, 1), "0xFF_FF");
+    expect_int(at(&v, 2), .Hex);
+    expect_text(at(&v, 2), "0x_ff");
+    expect_int(at(&v, 3), .Bin);
+    expect_text(at(&v, 3), "0b_1010");
+    expect_int(at(&v, 4), .Oct);
+    expect_text(at(&v, 4), "0o_7");
+
+    var i: usize = 5;
+    while i < 10 {
+        match at(&v, i).kind {
+            .Float => {},
+            _ => { @assert(false, "expected float"); },
+        }
+        i = i + 1;
+    }
+    expect_text(at(&v, 5), "1_000.5");
+    expect_text(at(&v, 6), "1.5_0");
+    expect_text(at(&v, 7), "1_000.500");
+    expect_text(at(&v, 8), "1.5e-3");
+    expect_text(at(&v, 9), "1_0.5_0E2");
+    expect_eof(at(&v, 10));
+}
+
 fn test_diag_errors() {
     var diag = Diag.new(null);
     defer { Diag.drop(&diag); }
@@ -689,5 +714,6 @@ fn main() {
     test_basic();
     test_operators();
     test_comments_and_literals();
+    test_number_forms();
     test_diag_errors();
 }