stage2 提供词法分析器通过与stage1的差异测试 stage1提供数组到切片的隐式转换

stage2/lexer.spl

@@ -11,6 +11,8 @@ type .{ .is_digit, .is_alpha, .is_alnum, .is_hex_digit, .digit_value } = @import
 type .{
     .Token, .TokenKind, .Radix, .Keyword, .Punct, .int_value, .keyword_of
 } = @import("token.spl");
+type .{ .Span, .span_make } = @import("./diag.spl");
+type Diag = @import("./diag.spl");
 
 type Lexer = @self;
 
@@ -18,9 +20,17 @@ src: ByteView,
 pos: usize,
 line: usize,
 col: usize,
+diag: *Diag, // 借用；可为 null（合法输入不产生诊断）
+file: usize,
 
-fn new(src: ByteView): Lexer {
-    ret Lexer { .src = src, .pos = 0, .line = 1, .col = 1 };
+fn new(src: ByteView, diag: *Diag, file: usize): Lexer {
+    ret Lexer { .src = src, .pos = 0, .line = 1, .col = 1, .diag = diag, .file = file };
+}
+
+// 记录一条词法诊断（非法输入路径）。合法输入不应触发。
+fn report(self: *Lexer, start: usize, end: usize, message: []u8) {
+    if self.diag == null { ret; }
+    Diag.err(self.diag, span_make(self.file, start, end), .LexUnexpected, message);
 }
 
 fn is_ident_start(c: u8): bool { ret is_alpha(c) || c == '_'; }
@@ -181,6 +191,8 @@ fn scan_int(self: *Lexer, out: *Vec, start: usize, line: usize, col: usize) {
 
 fn scan_char(self: *Lexer, out: *Vec, start: usize, line: usize, col: usize) {
     var p = self.pos + 1;
+    var closed = false;
+    var escape_ok = true;
     if p < self.src.len {
         if self.src.ptr[p] == '\\' {
             p = p + 1;
@@ -191,14 +203,30 @@ fn scan_char(self: *Lexer, out: *Vec, start: usize, line: usize, col: usize) {
                     p = p + 1;
                     digits = digits + 1;
                 }
+                if digits == 0 { escape_ok = false; }
             } else if p < self.src.len {
+                var e = self.src.ptr[p];
+                if !(e == 'n' || e == 't' || e == 'r' || e == '\\' || e == '"' || e == '\'' ||
+                     e == '0') {
+                    escape_ok = false;
+                }
                 p = p + 1;
+            } else {
+                escape_ok = false;
             }
         } else {
             p = p + 1;
         }
     }
-    if p < self.src.len && self.src.ptr[p] == '\'' { p = p + 1; }
+    if p < self.src.len && self.src.ptr[p] == '\'' {
+        p = p + 1;
+        closed = true;
+    }
+    if !closed {
+        report(self, start, p, "unterminated character literal");
+    } else if !escape_ok {
+        report(self, start, p, "invalid escape in character literal");
+    }
     emit_to(self, out, .Char(decode_char(sub(self, start, p))), start, p, line, col);
 }
 
@@ -206,6 +234,7 @@ fn scan_string(self: *Lexer, out: *Vec, start: usize, line: usize, col: usize) {
     var p = self.pos + 1;
     var l = line;
     var cc = col + 1;
+    var closed = false;
     while p < self.src.len {
         var b = self.src.ptr[p];
         if b == '\\' {
@@ -218,6 +247,7 @@ fn scan_string(self: *Lexer, out: *Vec, start: usize, line: usize, col: usize) {
         } else if b == '"' {
             p = p + 1;
             cc = cc + 1;
+            closed = true;
             break;
         } else if b == '\n' {
             l = l + 1;
@@ -228,6 +258,7 @@ fn scan_string(self: *Lexer, out: *Vec, start: usize, line: usize, col: usize) {
             cc = cc + 1;
         }
     }
+    if !closed { report(self, start, p, "unterminated string literal"); }
     emit_to(self, out, .Str, start, p, line, col);
     self.line = l;
     self.col = cc;
@@ -243,8 +274,8 @@ fn scan_ident(self: *Lexer, out: *Vec, start: usize, line: usize, col: usize) {
     }
 }
 
-// 块注释：可嵌套，整体跳过（不产出 token）。
-fn skip_block_comment(self: *Lexer) {
+// 块注释：可嵌套，整体跳过（不产出 token）。未闭合时报诊断。
+fn skip_block_comment(self: *Lexer, start: usize) {
     self.pos = self.pos + 2;
     self.col = self.col + 2;
     var depth: usize = 1;
@@ -267,6 +298,9 @@ fn skip_block_comment(self: *Lexer) {
             self.pos = self.pos + 1;
         }
     }
+    if depth > 0 {
+        report(self, start, self.src.len, "unterminated block comment");
+    }
 }
 
 fn run(self: *Lexer, out: *Vec) {
@@ -294,7 +328,7 @@ fn run(self: *Lexer, out: *Vec) {
             continue;
         }
         if c == '/' && has2(self, '*') {
-            skip_block_comment(self);
+            skip_block_comment(self, start);
             continue;
         }
         if c == '\'' {
@@ -318,6 +352,7 @@ fn run(self: *Lexer, out: *Vec) {
             if has3(self, '-', '>') { op3(self, out, .ArrowBoth, start, line, col); continue; }
             if has3(self, '=', '>') { op3(self, out, .FatArrowBoth, start, line, col); continue; }
             if has3(self, '<', '=') { op3(self, out, .LShAssign, start, line, col); continue; }
+            if has2(self, '<') { op2(self, out, .LSh, start, line, col); continue; }
             if has2(self, '-') { op2(self, out, .LeftArrow, start, line, col); continue; }
             if has2(self, '=') { op2(self, out, .Le, start, line, col); continue; }
             op1(self, out, .Lt, start, line, col);
@@ -409,6 +444,7 @@ fn run(self: *Lexer, out: *Vec) {
         if c == '$' { op1(self, out, .Dollar, start, line, col); continue; }
         if c == '?' { op1(self, out, .Cond, start, line, col); continue; }
 
+        report(self, start, start + 1, "unexpected character");
         emit_to(self, out, .Unknown(c), start, start + 1, line, col);
     }
 
@@ -416,13 +452,13 @@ fn run(self: *Lexer, out: *Vec) {
     emit_to(self, out, .Eof, end, end, self.line, self.col);
 }
 
-fn lex(src: ByteView): VecResult {
+fn lex(src: ByteView, diag: *Diag, file: usize): VecResult {
     var out = Vec.blank(@sizeof(Token), @alignof(Token));
     match Vec.new(@sizeof(Token), @alignof(Token)) {
         .Ok(v) => { out = v; },
         .Err(error) => { ret .Err(error); },
     }
-    var self = new(src);
+    var self = new(src, diag, file);
     run(&self, &out);
     ret .Ok(out);
 }
@@ -435,7 +471,7 @@ fn view(text: []u8): ByteView {
 
 fn lex_ok(src: []u8): Vec {
     var out = Vec.blank(@sizeof(Token), @alignof(Token));
-    match lex(view(src)) {
+    match lex(view(src), null, 0) {
         .Ok(v) => { out = v; },
         .Err(error) => { @assert(false, "lex failed"); },
     }
@@ -536,9 +572,9 @@ fn test_basic() {
 }
 
 fn test_operators() {
-    var v = lex_ok("<-> <- -> <=> <<= >>= ... .. . := == != < <= > >= && || & | ^ + - * / % ~ ! ? # @ $ , ; : ( ) [ ] { }");
+    var v = lex_ok("<-> <- -> <=> <<= << >>= ... .. . := == != < <= > >= && || & | ^ + - * / % ~ ! ? # @ $ , ; : ( ) [ ] { }");
     defer { Vec.drop(&v); }
-    @assert(v.len == 42, "operator token count");
+    @assert(v.len == 43, "operator token count");
 
     expect_punct(at(&v, 0), .ArrowBoth);
     expect_text(at(&v, 0), "<->");
@@ -546,43 +582,45 @@ fn test_operators() {
     expect_punct(at(&v, 2), .RightArrow);
     expect_punct(at(&v, 3), .FatArrowBoth);
     expect_punct(at(&v, 4), .LShAssign);
-    expect_punct(at(&v, 5), .RShAssign);
-    expect_punct(at(&v, 6), .Ellipsis);
-    expect_punct(at(&v, 7), .Range);
-    expect_punct(at(&v, 8), .Dot);
-    expect_punct(at(&v, 9), .ColonAssign);
-    expect_punct(at(&v, 10), .Eq);
-    expect_punct(at(&v, 11), .Neq);
-    expect_punct(at(&v, 12), .Lt);
-    expect_punct(at(&v, 13), .Le);
-    expect_punct(at(&v, 14), .Gt);
-    expect_punct(at(&v, 15), .Ge);
-    expect_punct(at(&v, 16), .AndAnd);
-    expect_punct(at(&v, 17), .OrOr);
-    expect_punct(at(&v, 18), .And);
-    expect_punct(at(&v, 19), .Or);
-    expect_punct(at(&v, 20), .Xor);
-    expect_punct(at(&v, 21), .Add);
-    expect_punct(at(&v, 22), .Sub);
-    expect_punct(at(&v, 23), .Mul);
-    expect_punct(at(&v, 24), .Div);
-    expect_punct(at(&v, 25), .Mod);
-    expect_punct(at(&v, 26), .BitNot);
-    expect_punct(at(&v, 27), .Not);
-    expect_punct(at(&v, 28), .Cond);
-    expect_punct(at(&v, 29), .Sharp);
-    expect_punct(at(&v, 30), .At);
-    expect_punct(at(&v, 31), .Dollar);
-    expect_punct(at(&v, 32), .Comma);
-    expect_punct(at(&v, 33), .Semicolon);
-    expect_punct(at(&v, 34), .Colon);
-    expect_punct(at(&v, 35), .LParen);
-    expect_punct(at(&v, 36), .RParen);
-    expect_punct(at(&v, 37), .LBracket);
-    expect_punct(at(&v, 38), .RBracket);
-    expect_punct(at(&v, 39), .LBrace);
-    expect_punct(at(&v, 40), .RBrace);
-    expect_eof(at(&v, 41));
+    expect_punct(at(&v, 5), .LSh);
+    expect_text(at(&v, 5), "<<");
+    expect_punct(at(&v, 6), .RShAssign);
+    expect_punct(at(&v, 7), .Ellipsis);
+    expect_punct(at(&v, 8), .Range);
+    expect_punct(at(&v, 9), .Dot);
+    expect_punct(at(&v, 10), .ColonAssign);
+    expect_punct(at(&v, 11), .Eq);
+    expect_punct(at(&v, 12), .Neq);
+    expect_punct(at(&v, 13), .Lt);
+    expect_punct(at(&v, 14), .Le);
+    expect_punct(at(&v, 15), .Gt);
+    expect_punct(at(&v, 16), .Ge);
+    expect_punct(at(&v, 17), .AndAnd);
+    expect_punct(at(&v, 18), .OrOr);
+    expect_punct(at(&v, 19), .And);
+    expect_punct(at(&v, 20), .Or);
+    expect_punct(at(&v, 21), .Xor);
+    expect_punct(at(&v, 22), .Add);
+    expect_punct(at(&v, 23), .Sub);
+    expect_punct(at(&v, 24), .Mul);
+    expect_punct(at(&v, 25), .Div);
+    expect_punct(at(&v, 26), .Mod);
+    expect_punct(at(&v, 27), .BitNot);
+    expect_punct(at(&v, 28), .Not);
+    expect_punct(at(&v, 29), .Cond);
+    expect_punct(at(&v, 30), .Sharp);
+    expect_punct(at(&v, 31), .At);
+    expect_punct(at(&v, 32), .Dollar);
+    expect_punct(at(&v, 33), .Comma);
+    expect_punct(at(&v, 34), .Semicolon);
+    expect_punct(at(&v, 35), .Colon);
+    expect_punct(at(&v, 36), .LParen);
+    expect_punct(at(&v, 37), .RParen);
+    expect_punct(at(&v, 38), .LBracket);
+    expect_punct(at(&v, 39), .RBracket);
+    expect_punct(at(&v, 40), .LBrace);
+    expect_punct(at(&v, 41), .RBrace);
+    expect_eof(at(&v, 42));
 }
 
 fn test_comments_and_literals() {
@@ -627,8 +665,29 @@ fn test_comments_and_literals() {
     expect_eof(at(&v, 12));
 }
 
+// 非法输入走 diag：仍产出 token 以便继续收集错误，但 diag 必须记录。
+fn test_diag_errors() {
+    var diag = Diag.new(null);
+    defer { Diag.drop(&diag); }
+
+    match lex(view("'\\q' \"abc"), &diag, 0) {
+        .Ok(v) => { Vec.drop(&v); },
+        .Err(error) => { @assert(false, "lex should return tokens"); },
+    }
+    @assert(Diag.has_error(diag), "lex errors recorded");
+
+    var clean = Diag.new(null);
+    defer { Diag.drop(&clean); }
+    match lex(view("fn main() { ret 0; }"), &clean, 0) {
+        .Ok(v) => { Vec.drop(&v); },
+        .Err(error) => { @assert(false, "legal lex failed"); },
+    }
+    @assert(!Diag.has_error(clean), "legal input has no diagnostics");
+}
+
 fn main() {
     test_basic();
     test_operators();
     test_comments_and_literals();
+    test_diag_errors();
 }