stage2 更多的工具库 stage1 修复问题

stage2/token.spl

@@ -0,0 +1,190 @@
+// Expect retcode = 0
+
+// 词法单元：用带载荷 ADT 表达 TokenKind；关键词/算符各自独立 enum。
+// 整型与字符在词法期解码并随载荷携带；文本一律借用源切片（不复制）。
+
+type .{ .ByteView } = @import("utils/byte_view.spl");
+
+type Keyword = enum {
+    Any, As, Bool, Break, Catch, Comptime, Const, Continue, Defer, Else, Enum,
+    Errdefer, False, Fn, For, If, Loop, Match, Null, Ret, Shape, Struct, True,
+    Try, Type, Undefined, Union, Var, While, Underscore,
+}
+
+type KeywordResult = enum {
+    Yes: Keyword,
+    No,
+}
+
+type Punct = enum {
+    Eq, Assign, AddAssign, Add, SubAssign, Sub, MulAssign, Mul, DivAssign, Div,
+    ModAssign, Mod, RightArrow, LeftArrow, ArrowBoth, FatArrowBoth, RightFatArrow,
+    AndAnd, AndAssign, And, OrOr, OrAssign, Or, XorAssign, Xor,
+    LShAssign, LSh, Le, Lt, RShAssign, RSh, Ge, Gt, Not, Neq, BitNot,
+    LBracket, RBracket, LParen, RParen, LBrace, RBrace,
+    Semicolon, Comma, Colon, ColonAssign, Dot, Range, Ellipsis, Cond, Sharp, At, Dollar,
+}
+
+type Radix = enum {
+    Dec,
+    Bin,
+    Oct,
+    Hex,
+}
+
+type IntValue = struct {
+    value: u64,
+    radix: Radix,
+}
+
+type TokenKind = enum {
+    Keyword: Keyword,
+    Punct: Punct,
+    Ident,
+    Int: IntValue,
+    Float,
+    Char: u8,
+    Str,
+    LineComment,
+    Endline,
+    Eof,
+    Unknown: u8,
+}
+
+type Token = @self;
+
+kind: TokenKind,
+text: ByteView,
+line: usize,
+col: usize,
+
+fn make(kind: TokenKind, text: ByteView, line: usize, col: usize): Token {
+    ret Token { .kind = kind, .text = text, .line = line, .col = col };
+}
+
+fn int_value(value: u64, radix: Radix): IntValue {
+    ret IntValue { .value = value, .radix = radix };
+}
+
+fn is_trivia(kind: TokenKind): bool {
+    match kind {
+        .Endline => { ret true; },
+        .LineComment => { ret true; },
+        _ => { ret false; },
+    }
+}
+
+fn keyword_of(text: ByteView): KeywordResult {
+    if is(text, "any") { ret .Yes(.Any); }
+    if is(text, "as") { ret .Yes(.As); }
+    if is(text, "bool") { ret .Yes(.Bool); }
+    if is(text, "break") { ret .Yes(.Break); }
+    if is(text, "catch") { ret .Yes(.Catch); }
+    if is(text, "comptime") { ret .Yes(.Comptime); }
+    if is(text, "const") { ret .Yes(.Const); }
+    if is(text, "continue") { ret .Yes(.Continue); }
+    if is(text, "defer") { ret .Yes(.Defer); }
+    if is(text, "else") { ret .Yes(.Else); }
+    if is(text, "enum") { ret .Yes(.Enum); }
+    if is(text, "errdefer") { ret .Yes(.Errdefer); }
+    if is(text, "false") { ret .Yes(.False); }
+    if is(text, "fn") { ret .Yes(.Fn); }
+    if is(text, "for") { ret .Yes(.For); }
+    if is(text, "if") { ret .Yes(.If); }
+    if is(text, "loop") { ret .Yes(.Loop); }
+    if is(text, "match") { ret .Yes(.Match); }
+    if is(text, "null") { ret .Yes(.Null); }
+    if is(text, "ret") { ret .Yes(.Ret); }
+    if is(text, "shape") { ret .Yes(.Shape); }
+    if is(text, "struct") { ret .Yes(.Struct); }
+    if is(text, "true") { ret .Yes(.True); }
+    if is(text, "try") { ret .Yes(.Try); }
+    if is(text, "type") { ret .Yes(.Type); }
+    if is(text, "undefined") { ret .Yes(.Undefined); }
+    if is(text, "union") { ret .Yes(.Union); }
+    if is(text, "var") { ret .Yes(.Var); }
+    if is(text, "while") { ret .Yes(.While); }
+    if is(text, "_") { ret .Yes(.Underscore); }
+    ret .No;
+}
+
+// ---- 测试辅助 ----
+
+fn view(text: []u8): ByteView {
+    ret ByteView.from_slice(text);
+}
+
+fn is(text: ByteView, literal: []u8): bool {
+    ret ByteView.eq(text, view(literal));
+}
+
+fn expect_keyword(result: KeywordResult, expected: Keyword) {
+    match result {
+        .Yes(kw) => { @assert(@enum2int(kw) == @enum2int(expected), "keyword"); },
+        .No => { @assert(false, "expected keyword"); },
+    }
+}
+
+fn test_keywords() {
+    expect_keyword(keyword_of(view("fn")), .Fn);
+    expect_keyword(keyword_of(view("while")), .While);
+    expect_keyword(keyword_of(view("_")), .Underscore);
+    match keyword_of(view("foo")) {
+        .Yes(kw) => { @assert(false, "foo is not a keyword"); },
+        .No => {},
+    }
+    match keyword_of(view("")) {
+        .Yes(kw) => { @assert(false, "empty is not a keyword"); },
+        .No => {},
+    }
+}
+
+fn test_adt_payloads() {
+    var t = make(.Int(int_value(255, .Hex)), view("0xFF"), 3, 5);
+    @assert(t.line == 3 && t.col == 5, "position");
+    match t.kind {
+        .Int(iv) => {
+            @assert(iv.value == 255, "int value");
+            var hex: Radix = .Hex;
+            @assert(@enum2int(iv.radix) == @enum2int(hex), "int radix");
+        },
+        _ => { @assert(false, "expected int"); },
+    }
+
+    var c = make(.Char('a'), view("'a'"), 1, 1);
+    match c.kind {
+        .Char(value) => { @assert(value == 'a', "char value"); },
+        _ => { @assert(false, "expected char"); },
+    }
+
+    var p = make(.Punct(.ArrowBoth), view("<->"), 1, 1);
+    match p.kind {
+        .Punct(punct) => {
+            var both: Punct = .ArrowBoth;
+            @assert(@enum2int(punct) == @enum2int(both), "punct");
+        },
+        _ => { @assert(false, "expected punct"); },
+    }
+
+    var k = make(.Keyword(.Ret), view("ret"), 1, 1);
+    match k.kind {
+        .Keyword(kw) => {
+            var ret_kw: Keyword = .Ret;
+            @assert(@enum2int(kw) == @enum2int(ret_kw), "keyword kind");
+        },
+        _ => { @assert(false, "expected keyword"); },
+    }
+}
+
+fn test_trivia() {
+    @assert(is_trivia(.Endline), "endline trivia");
+    @assert(is_trivia(.LineComment), "comment trivia");
+    @assert(!is_trivia(.Ident), "ident not trivia");
+    @assert(!is_trivia(.Eof), "eof not trivia");
+}
+
+fn main() {
+    test_keywords();
+    test_adt_payloads();
+    test_trivia();
+}