spl/stage2/token.spl

// Expect retcode = 0

// 词法单元：用带载荷 ADT 表达 TokenKind；关键词/算符各自独立 enum。
// 整型与字符在词法期解码并随载荷携带；文本一律借用源切片（不复制）。

type .{ .ByteView } = @import("utils/byte_view.spl");

type Keyword = enum {
    Any, As, Bool, Break, Catch, Comptime, Const, Continue, Defer, Else, Enum,
    Errdefer, False, Fn, For, If, Loop, Match, Null, Ret, Shape, Struct, True,
    Try, Type, Undefined, Union, Var, While, Underscore,
}

type KeywordResult = enum {
    Yes: Keyword,
    No,
}

type Punct = enum {
    Eq, Assign, AddAssign, Add, SubAssign, Sub, MulAssign, Mul, DivAssign, Div,
    ModAssign, Mod, RightArrow, LeftArrow, ArrowBoth, FatArrowBoth, RightFatArrow,
    AndAnd, AndAssign, And, OrOr, OrAssign, Or, XorAssign, Xor,
    LShAssign, LSh, Le, Lt, RShAssign, RSh, Ge, Gt, Not, Neq, BitNot,
    LBracket, RBracket, LParen, RParen, LBrace, RBrace,
    Semicolon, Comma, Colon, ColonAssign, Dot, Range, Ellipsis, Cond, Sharp, At, Dollar,
}

type Radix = enum {
    Dec,
    Bin,
    Oct,
    Hex,
}

type IntValue = struct {
    value: u64,
    radix: Radix,
}

type TokenKind = enum {
    Keyword: Keyword,
    Punct: Punct,
    Ident,
    Int: IntValue,
    Float,
    Char: u8,
    Str,
    LineComment,
    Endline,
    Eof,
    Unknown: u8,
}

type Token = @self;

kind: TokenKind,
text: ByteView,
line: usize,
col: usize,

fn make(kind: TokenKind, text: ByteView, line: usize, col: usize): Token {
    ret Token { .kind = kind, .text = text, .line = line, .col = col };
}

fn int_value(value: u64, radix: Radix): IntValue {
    ret IntValue { .value = value, .radix = radix };
}

fn is_trivia(kind: TokenKind): bool {
    match kind {
        .Endline => { ret true; },
        .LineComment => { ret true; },
        _ => { ret false; },
    }
}

fn keyword_of(text: ByteView): KeywordResult {
    if is(text, "any") { ret .Yes(.Any); }
    if is(text, "as") { ret .Yes(.As); }
    if is(text, "bool") { ret .Yes(.Bool); }
    if is(text, "break") { ret .Yes(.Break); }
    if is(text, "catch") { ret .Yes(.Catch); }
    if is(text, "comptime") { ret .Yes(.Comptime); }
    if is(text, "const") { ret .Yes(.Const); }
    if is(text, "continue") { ret .Yes(.Continue); }
    if is(text, "defer") { ret .Yes(.Defer); }
    if is(text, "else") { ret .Yes(.Else); }
    if is(text, "enum") { ret .Yes(.Enum); }
    if is(text, "errdefer") { ret .Yes(.Errdefer); }
    if is(text, "false") { ret .Yes(.False); }
    if is(text, "fn") { ret .Yes(.Fn); }
    if is(text, "for") { ret .Yes(.For); }
    if is(text, "if") { ret .Yes(.If); }
    if is(text, "loop") { ret .Yes(.Loop); }
    if is(text, "match") { ret .Yes(.Match); }
    if is(text, "null") { ret .Yes(.Null); }
    if is(text, "ret") { ret .Yes(.Ret); }
    if is(text, "shape") { ret .Yes(.Shape); }
    if is(text, "struct") { ret .Yes(.Struct); }
    if is(text, "true") { ret .Yes(.True); }
    if is(text, "try") { ret .Yes(.Try); }
    if is(text, "type") { ret .Yes(.Type); }
    if is(text, "undefined") { ret .Yes(.Undefined); }
    if is(text, "union") { ret .Yes(.Union); }
    if is(text, "var") { ret .Yes(.Var); }
    if is(text, "while") { ret .Yes(.While); }
    if is(text, "_") { ret .Yes(.Underscore); }
    ret .No;
}

// ---- 测试辅助 ----

fn view(text: []u8): ByteView {
    ret ByteView.from_slice(text);
}

fn is(text: ByteView, literal: []u8): bool {
    ret ByteView.eq(text, view(literal));
}

fn expect_keyword(result: KeywordResult, expected: Keyword) {
    match result {
        .Yes(kw) => { @assert(@enum2int(kw) == @enum2int(expected), "keyword"); },
        .No => { @assert(false, "expected keyword"); },
    }
}

fn test_keywords() {
    expect_keyword(keyword_of(view("fn")), .Fn);
    expect_keyword(keyword_of(view("while")), .While);
    expect_keyword(keyword_of(view("_")), .Underscore);
    match keyword_of(view("foo")) {
        .Yes(kw) => { @assert(false, "foo is not a keyword"); },
        .No => {},
    }
    match keyword_of(view("")) {
        .Yes(kw) => { @assert(false, "empty is not a keyword"); },
        .No => {},
    }
}

fn test_adt_payloads() {
    var t = make(.Int(int_value(255, .Hex)), view("0xFF"), 3, 5);
    @assert(t.line == 3 && t.col == 5, "position");
    match t.kind {
        .Int(iv) => {
            @assert(iv.value == 255, "int value");
            var hex: Radix = .Hex;
            @assert(@enum2int(iv.radix) == @enum2int(hex), "int radix");
        },
        _ => { @assert(false, "expected int"); },
    }

    var c = make(.Char('a'), view("'a'"), 1, 1);
    match c.kind {
        .Char(value) => { @assert(value == 'a', "char value"); },
        _ => { @assert(false, "expected char"); },
    }

    var p = make(.Punct(.ArrowBoth), view("<->"), 1, 1);
    match p.kind {
        .Punct(punct) => {
            var both: Punct = .ArrowBoth;
            @assert(@enum2int(punct) == @enum2int(both), "punct");
        },
        _ => { @assert(false, "expected punct"); },
    }

    var k = make(.Keyword(.Ret), view("ret"), 1, 1);
    match k.kind {
        .Keyword(kw) => {
            var ret_kw: Keyword = .Ret;
            @assert(@enum2int(kw) == @enum2int(ret_kw), "keyword kind");
        },
        _ => { @assert(false, "expected keyword"); },
    }
}

fn test_trivia() {
    @assert(is_trivia(.Endline), "endline trivia");
    @assert(is_trivia(.LineComment), "comment trivia");
    @assert(!is_trivia(.Ident), "ident not trivia");
    @assert(!is_trivia(.Eof), "eof not trivia");
}

fn main() {
    test_keywords();
    test_adt_payloads();
    test_trivia();
}