426 lines
15 KiB
Plaintext
426 lines
15 KiB
Plaintext
// Expect retcode = 0
|
|
|
|
// 词法单元：TokenKind 为带载荷 ADT，关键词/算符各自独立 enum；整型/字符在词法期
|
|
// 解码，文本一律借用源切片。
|
|
|
|
type .{ .ByteView } = @import("utils/byte_view.spl");
|
|
|
|
type Keyword = enum {
|
|
Any, As, Bool, Break, Catch, Comptime, Const, Continue, Defer, Else, Enum,
|
|
Errdefer, False, Fn, For, If, Loop, Match, Null, Ret, Shape, Struct, True,
|
|
Try, Type, Undefined, Union, Var, While, Underscore,
|
|
}
|
|
|
|
type KeywordResult = enum {
|
|
Yes: Keyword,
|
|
No,
|
|
}
|
|
|
|
type Punct = enum {
|
|
Eq, Assign, AddAssign, Add, SubAssign, Sub, MulAssign, Mul, DivAssign, Div,
|
|
ModAssign, Mod, RightArrow, LeftArrow, ArrowBoth, FatArrowBoth, RightFatArrow,
|
|
AndAnd, AndAssign, And, OrOr, OrAssign, Or, XorAssign, Xor,
|
|
LShAssign, LSh, Le, Lt, RShAssign, RSh, Ge, Gt, Not, Neq, BitNot,
|
|
LBracket, RBracket, LParen, RParen, LBrace, RBrace,
|
|
Semicolon, Comma, Colon, ColonAssign, Dot, Range, Ellipsis, Cond, Sharp, At, Dollar,
|
|
}
|
|
|
|
type Radix = enum {
|
|
Dec,
|
|
Bin,
|
|
Oct,
|
|
Hex,
|
|
}
|
|
|
|
type TokenKind = enum {
|
|
Keyword: Keyword,
|
|
Punct: Punct,
|
|
Ident,
|
|
Int: Radix, // 仅记进制；数值由 parser 按任意宽度（mp）解析
|
|
Float,
|
|
Char: u8,
|
|
Str,
|
|
LineComment,
|
|
Endline,
|
|
Eof,
|
|
Unknown: u8,
|
|
}
|
|
|
|
type Token = @self;
|
|
|
|
kind: TokenKind,
|
|
text: ByteView,
|
|
line: usize,
|
|
col: usize,
|
|
offset: usize, // 源文件字节偏移（供 AST Span 使用）
|
|
|
|
fn make(kind: TokenKind, text: ByteView, line: usize, col: usize, offset: usize): Token {
|
|
ret Token { .kind = kind, .text = text, .line = line, .col = col, .offset = offset };
|
|
}
|
|
|
|
fn is_trivia(kind: TokenKind): bool {
|
|
match kind {
|
|
.Endline => { ret true; },
|
|
.LineComment => { ret true; },
|
|
_ => { ret false; },
|
|
}
|
|
}
|
|
|
|
// 先按长度与首字节分派，再做完整比较，避免对每个标识符做全表串比较。
|
|
fn keyword_of(text: ByteView): KeywordResult {
|
|
var n = text.len;
|
|
if n == 0 { ret .No; }
|
|
var c0 = text.ptr[0];
|
|
if n == 2 {
|
|
if c0 == 'a' && is(text, "as") { ret .Yes(.As); }
|
|
if c0 == 'f' && is(text, "fn") { ret .Yes(.Fn); }
|
|
if c0 == 'i' && is(text, "if") { ret .Yes(.If); }
|
|
ret .No;
|
|
}
|
|
if n == 3 {
|
|
if c0 == 'a' && is(text, "any") { ret .Yes(.Any); }
|
|
if c0 == 'f' && is(text, "for") { ret .Yes(.For); }
|
|
if c0 == 'r' && is(text, "ret") { ret .Yes(.Ret); }
|
|
if c0 == 't' && is(text, "try") { ret .Yes(.Try); }
|
|
if c0 == 'v' && is(text, "var") { ret .Yes(.Var); }
|
|
ret .No;
|
|
}
|
|
if n == 4 {
|
|
if c0 == 'b' && is(text, "bool") { ret .Yes(.Bool); }
|
|
if c0 == 'e' && is(text, "else") { ret .Yes(.Else); }
|
|
if c0 == 'e' && is(text, "enum") { ret .Yes(.Enum); }
|
|
if c0 == 'l' && is(text, "loop") { ret .Yes(.Loop); }
|
|
if c0 == 'n' && is(text, "null") { ret .Yes(.Null); }
|
|
if c0 == 't' && is(text, "true") { ret .Yes(.True); }
|
|
if c0 == 't' && is(text, "type") { ret .Yes(.Type); }
|
|
ret .No;
|
|
}
|
|
if n == 5 {
|
|
if c0 == 'b' && is(text, "break") { ret .Yes(.Break); }
|
|
if c0 == 'c' && is(text, "catch") { ret .Yes(.Catch); }
|
|
if c0 == 'c' && is(text, "const") { ret .Yes(.Const); }
|
|
if c0 == 'd' && is(text, "defer") { ret .Yes(.Defer); }
|
|
if c0 == 'f' && is(text, "false") { ret .Yes(.False); }
|
|
if c0 == 'm' && is(text, "match") { ret .Yes(.Match); }
|
|
if c0 == 's' && is(text, "shape") { ret .Yes(.Shape); }
|
|
if c0 == 'u' && is(text, "union") { ret .Yes(.Union); }
|
|
if c0 == 'w' && is(text, "while") { ret .Yes(.While); }
|
|
ret .No;
|
|
}
|
|
if n == 6 {
|
|
if c0 == 's' && is(text, "struct") { ret .Yes(.Struct); }
|
|
ret .No;
|
|
}
|
|
if n == 8 {
|
|
if c0 == 'c' && is(text, "comptime") { ret .Yes(.Comptime); }
|
|
if c0 == 'c' && is(text, "continue") { ret .Yes(.Continue); }
|
|
if c0 == 'e' && is(text, "errdefer") { ret .Yes(.Errdefer); }
|
|
ret .No;
|
|
}
|
|
if n == 9 {
|
|
if c0 == 'u' && is(text, "undefined") { ret .Yes(.Undefined); }
|
|
ret .No;
|
|
}
|
|
if n == 1 {
|
|
if c0 == '_' && is(text, "_") { ret .Yes(.Underscore); }
|
|
ret .No;
|
|
}
|
|
ret .No;
|
|
}
|
|
|
|
|
|
fn keyword_name(kw: Keyword): []u8 {
|
|
match kw {
|
|
.Any => { ret "KW_ANY"; },
|
|
.As => { ret "KW_AS"; },
|
|
.Bool => { ret "KW_BOOL"; },
|
|
.Break => { ret "KW_BREAK"; },
|
|
.Catch => { ret "KW_CATCH"; },
|
|
.Comptime => { ret "KW_COMPTIME"; },
|
|
.Const => { ret "KW_CONST"; },
|
|
.Continue => { ret "KW_CONTINUE"; },
|
|
.Defer => { ret "KW_DEFER"; },
|
|
.Else => { ret "KW_ELSE"; },
|
|
.Enum => { ret "KW_ENUM"; },
|
|
.Errdefer => { ret "KW_ERRDEFER"; },
|
|
.False => { ret "KW_FALSE"; },
|
|
.Fn => { ret "KW_FN"; },
|
|
.For => { ret "KW_FOR"; },
|
|
.If => { ret "KW_IF"; },
|
|
.Loop => { ret "KW_LOOP"; },
|
|
.Match => { ret "KW_MATCH"; },
|
|
.Null => { ret "KW_NULL"; },
|
|
.Ret => { ret "KW_RET"; },
|
|
.Shape => { ret "KW_SHAPE"; },
|
|
.Struct => { ret "KW_STRUCT"; },
|
|
.True => { ret "KW_TRUE"; },
|
|
.Try => { ret "KW_TRY"; },
|
|
.Type => { ret "KW_TYPE"; },
|
|
.Undefined => { ret "KW_UNDEFINED"; },
|
|
.Union => { ret "KW_UNION"; },
|
|
.Var => { ret "KW_VAR"; },
|
|
.While => { ret "KW_WHILE"; },
|
|
.Underscore => { ret "KW_UNDERSCORE"; },
|
|
}
|
|
ret "?";
|
|
}
|
|
|
|
fn punct_name(punct: Punct): []u8 {
|
|
match punct {
|
|
.Eq => { ret "TOK_EQ"; },
|
|
.Assign => { ret "TOK_ASSIGN"; },
|
|
.AddAssign => { ret "TOK_ASSIGN_ADD"; },
|
|
.Add => { ret "TOK_ADD"; },
|
|
.SubAssign => { ret "TOK_ASSIGN_SUB"; },
|
|
.Sub => { ret "TOK_SUB"; },
|
|
.MulAssign => { ret "TOK_ASSIGN_MUL"; },
|
|
.Mul => { ret "TOK_MUL"; },
|
|
.DivAssign => { ret "TOK_ASSIGN_DIV"; },
|
|
.Div => { ret "TOK_DIV"; },
|
|
.ModAssign => { ret "TOK_ASSIGN_MOD"; },
|
|
.Mod => { ret "TOK_MOD"; },
|
|
.RightArrow => { ret "TOK_RIGHT_ARROW"; },
|
|
.LeftArrow => { ret "TOK_LEFT_ARROW"; },
|
|
.ArrowBoth => { ret "TOK_ARROW_BOTH"; },
|
|
.FatArrowBoth => { ret "TOK_FAT_ARROW_BOTH"; },
|
|
.RightFatArrow => { ret "TOK_RIGHT_FAT_ARROW"; },
|
|
.AndAnd => { ret "TOK_AND_AND"; },
|
|
.AndAssign => { ret "TOK_ASSIGN_AND"; },
|
|
.And => { ret "TOK_AND"; },
|
|
.OrOr => { ret "TOK_OR_OR"; },
|
|
.OrAssign => { ret "TOK_ASSIGN_OR"; },
|
|
.Or => { ret "TOK_OR"; },
|
|
.XorAssign => { ret "TOK_ASSIGN_XOR"; },
|
|
.Xor => { ret "TOK_XOR"; },
|
|
.LShAssign => { ret "TOK_ASSIGN_L_SH"; },
|
|
.LSh => { ret "TOK_L_SH"; },
|
|
.Le => { ret "TOK_LE"; },
|
|
.Lt => { ret "TOK_LT"; },
|
|
.RShAssign => { ret "TOK_ASSIGN_R_SH"; },
|
|
.RSh => { ret "TOK_R_SH"; },
|
|
.Ge => { ret "TOK_GE"; },
|
|
.Gt => { ret "TOK_GT"; },
|
|
.Not => { ret "TOK_NOT"; },
|
|
.Neq => { ret "TOK_NEQ"; },
|
|
.BitNot => { ret "TOK_BIT_NOT"; },
|
|
.LBracket => { ret "TOK_L_BRACKET"; },
|
|
.RBracket => { ret "TOK_R_BRACKET"; },
|
|
.LParen => { ret "TOK_L_PAREN"; },
|
|
.RParen => { ret "TOK_R_PAREN"; },
|
|
.LBrace => { ret "TOK_L_BRACE"; },
|
|
.RBrace => { ret "TOK_R_BRACE"; },
|
|
.Semicolon => { ret "TOK_SEMICOLON"; },
|
|
.Comma => { ret "TOK_COMMA"; },
|
|
.Colon => { ret "TOK_COLON"; },
|
|
.ColonAssign => { ret "TOK_COLON_ASSIGN"; },
|
|
.Dot => { ret "TOK_DOT"; },
|
|
.Range => { ret "TOK_RANGE"; },
|
|
.Ellipsis => { ret "TOK_ELLIPSIS"; },
|
|
.Cond => { ret "TOK_COND"; },
|
|
.Sharp => { ret "TOK_SHARP"; },
|
|
.At => { ret "TOK_AT"; },
|
|
.Dollar => { ret "TOK_DOLLAR"; },
|
|
}
|
|
ret "?";
|
|
}
|
|
|
|
fn kind_name(kind: TokenKind): []u8 {
|
|
match kind {
|
|
.Keyword(kw) => { ret keyword_name(kw); },
|
|
.Punct(punct) => { ret punct_name(punct); },
|
|
.Ident => { ret "TOK_IDENT"; },
|
|
.Int(value) => { ret "TOK_INT_LITERAL"; },
|
|
.Float => { ret "TOK_FLOAT_LITERAL"; },
|
|
.Char(value) => { ret "TOK_CHAR_LITERAL"; },
|
|
.Str => { ret "TOK_STRING_LITERAL"; },
|
|
.LineComment => { ret "TOK_LINE_COMMENT"; },
|
|
.Endline => { ret "TOK_ENDLINE"; },
|
|
.Eof => { ret "TOK_EOF"; },
|
|
.Unknown(value) => { ret "TOK_UNKNOWN"; },
|
|
}
|
|
ret "?";
|
|
}
|
|
|
|
|
|
fn view(text: []u8): ByteView {
|
|
ret ByteView.from_slice(text);
|
|
}
|
|
|
|
fn is(text: ByteView, literal: []u8): bool {
|
|
ret ByteView.eq(text, view(literal));
|
|
}
|
|
|
|
fn expect_keyword(result: KeywordResult, expected: Keyword) {
|
|
match result {
|
|
.Yes(kw) => { @assert(@enum2int(kw) == @enum2int(expected), "keyword"); },
|
|
.No => { @assert(false, "expected keyword"); },
|
|
}
|
|
}
|
|
|
|
fn test_keywords() {
|
|
expect_keyword(keyword_of(view("fn")), .Fn);
|
|
expect_keyword(keyword_of(view("while")), .While);
|
|
expect_keyword(keyword_of(view("_")), .Underscore);
|
|
match keyword_of(view("foo")) {
|
|
.Yes(kw) => { @assert(false, "foo is not a keyword"); },
|
|
.No => {},
|
|
}
|
|
match keyword_of(view("")) {
|
|
.Yes(kw) => { @assert(false, "empty is not a keyword"); },
|
|
.No => {},
|
|
}
|
|
}
|
|
|
|
fn test_adt_payloads() {
|
|
var t = make(.Int(.Hex), view("0xFF"), 3, 5, 0);
|
|
@assert(t.line == 3 && t.col == 5, "position");
|
|
match t.kind {
|
|
.Int(radix) => {
|
|
var hex: Radix = .Hex;
|
|
@assert(@enum2int(radix) == @enum2int(hex), "int radix");
|
|
},
|
|
_ => { @assert(false, "expected int"); },
|
|
}
|
|
|
|
var c = make(.Char('a'), view("'a'"), 1, 1, 0);
|
|
match c.kind {
|
|
.Char(value) => { @assert(value == 'a', "char value"); },
|
|
_ => { @assert(false, "expected char"); },
|
|
}
|
|
|
|
var p = make(.Punct(.ArrowBoth), view("<->"), 1, 1, 0);
|
|
match p.kind {
|
|
.Punct(punct) => {
|
|
var both: Punct = .ArrowBoth;
|
|
@assert(@enum2int(punct) == @enum2int(both), "punct");
|
|
},
|
|
_ => { @assert(false, "expected punct"); },
|
|
}
|
|
|
|
var k = make(.Keyword(.Ret), view("ret"), 1, 1, 0);
|
|
match k.kind {
|
|
.Keyword(kw) => {
|
|
var ret_kw: Keyword = .Ret;
|
|
@assert(@enum2int(kw) == @enum2int(ret_kw), "keyword kind");
|
|
},
|
|
_ => { @assert(false, "expected keyword"); },
|
|
}
|
|
}
|
|
|
|
fn test_trivia() {
|
|
@assert(is_trivia(.Endline), "endline trivia");
|
|
@assert(is_trivia(.LineComment), "comment trivia");
|
|
@assert(!is_trivia(.Ident), "ident not trivia");
|
|
@assert(!is_trivia(.Eof), "eof not trivia");
|
|
}
|
|
|
|
fn expect_kind_name(kind: TokenKind, expected: []u8) {
|
|
@assert(ByteView.eq(ByteView.from_slice(kind_name(kind)),
|
|
ByteView.from_slice(expected)),
|
|
"token kind name");
|
|
}
|
|
|
|
fn test_kind_names() {
|
|
expect_kind_name(.Keyword(.Any), "KW_ANY");
|
|
expect_kind_name(.Keyword(.As), "KW_AS");
|
|
expect_kind_name(.Keyword(.Bool), "KW_BOOL");
|
|
expect_kind_name(.Keyword(.Break), "KW_BREAK");
|
|
expect_kind_name(.Keyword(.Catch), "KW_CATCH");
|
|
expect_kind_name(.Keyword(.Comptime), "KW_COMPTIME");
|
|
expect_kind_name(.Keyword(.Const), "KW_CONST");
|
|
expect_kind_name(.Keyword(.Continue), "KW_CONTINUE");
|
|
expect_kind_name(.Keyword(.Defer), "KW_DEFER");
|
|
expect_kind_name(.Keyword(.Else), "KW_ELSE");
|
|
expect_kind_name(.Keyword(.Enum), "KW_ENUM");
|
|
expect_kind_name(.Keyword(.Errdefer), "KW_ERRDEFER");
|
|
expect_kind_name(.Keyword(.False), "KW_FALSE");
|
|
expect_kind_name(.Keyword(.Fn), "KW_FN");
|
|
expect_kind_name(.Keyword(.For), "KW_FOR");
|
|
expect_kind_name(.Keyword(.If), "KW_IF");
|
|
expect_kind_name(.Keyword(.Loop), "KW_LOOP");
|
|
expect_kind_name(.Keyword(.Match), "KW_MATCH");
|
|
expect_kind_name(.Keyword(.Null), "KW_NULL");
|
|
expect_kind_name(.Keyword(.Ret), "KW_RET");
|
|
expect_kind_name(.Keyword(.Shape), "KW_SHAPE");
|
|
expect_kind_name(.Keyword(.Struct), "KW_STRUCT");
|
|
expect_kind_name(.Keyword(.True), "KW_TRUE");
|
|
expect_kind_name(.Keyword(.Try), "KW_TRY");
|
|
expect_kind_name(.Keyword(.Type), "KW_TYPE");
|
|
expect_kind_name(.Keyword(.Undefined), "KW_UNDEFINED");
|
|
expect_kind_name(.Keyword(.Union), "KW_UNION");
|
|
expect_kind_name(.Keyword(.Var), "KW_VAR");
|
|
expect_kind_name(.Keyword(.While), "KW_WHILE");
|
|
expect_kind_name(.Keyword(.Underscore), "KW_UNDERSCORE");
|
|
|
|
expect_kind_name(.Punct(.Eq), "TOK_EQ");
|
|
expect_kind_name(.Punct(.Assign), "TOK_ASSIGN");
|
|
expect_kind_name(.Punct(.AddAssign), "TOK_ASSIGN_ADD");
|
|
expect_kind_name(.Punct(.Add), "TOK_ADD");
|
|
expect_kind_name(.Punct(.SubAssign), "TOK_ASSIGN_SUB");
|
|
expect_kind_name(.Punct(.Sub), "TOK_SUB");
|
|
expect_kind_name(.Punct(.MulAssign), "TOK_ASSIGN_MUL");
|
|
expect_kind_name(.Punct(.Mul), "TOK_MUL");
|
|
expect_kind_name(.Punct(.DivAssign), "TOK_ASSIGN_DIV");
|
|
expect_kind_name(.Punct(.Div), "TOK_DIV");
|
|
expect_kind_name(.Punct(.ModAssign), "TOK_ASSIGN_MOD");
|
|
expect_kind_name(.Punct(.Mod), "TOK_MOD");
|
|
expect_kind_name(.Punct(.RightArrow), "TOK_RIGHT_ARROW");
|
|
expect_kind_name(.Punct(.LeftArrow), "TOK_LEFT_ARROW");
|
|
expect_kind_name(.Punct(.ArrowBoth), "TOK_ARROW_BOTH");
|
|
expect_kind_name(.Punct(.FatArrowBoth), "TOK_FAT_ARROW_BOTH");
|
|
expect_kind_name(.Punct(.RightFatArrow), "TOK_RIGHT_FAT_ARROW");
|
|
expect_kind_name(.Punct(.AndAnd), "TOK_AND_AND");
|
|
expect_kind_name(.Punct(.AndAssign), "TOK_ASSIGN_AND");
|
|
expect_kind_name(.Punct(.And), "TOK_AND");
|
|
expect_kind_name(.Punct(.OrOr), "TOK_OR_OR");
|
|
expect_kind_name(.Punct(.OrAssign), "TOK_ASSIGN_OR");
|
|
expect_kind_name(.Punct(.Or), "TOK_OR");
|
|
expect_kind_name(.Punct(.XorAssign), "TOK_ASSIGN_XOR");
|
|
expect_kind_name(.Punct(.Xor), "TOK_XOR");
|
|
expect_kind_name(.Punct(.LShAssign), "TOK_ASSIGN_L_SH");
|
|
expect_kind_name(.Punct(.LSh), "TOK_L_SH");
|
|
expect_kind_name(.Punct(.Le), "TOK_LE");
|
|
expect_kind_name(.Punct(.Lt), "TOK_LT");
|
|
expect_kind_name(.Punct(.RShAssign), "TOK_ASSIGN_R_SH");
|
|
expect_kind_name(.Punct(.RSh), "TOK_R_SH");
|
|
expect_kind_name(.Punct(.Ge), "TOK_GE");
|
|
expect_kind_name(.Punct(.Gt), "TOK_GT");
|
|
expect_kind_name(.Punct(.Not), "TOK_NOT");
|
|
expect_kind_name(.Punct(.Neq), "TOK_NEQ");
|
|
expect_kind_name(.Punct(.BitNot), "TOK_BIT_NOT");
|
|
expect_kind_name(.Punct(.LBracket), "TOK_L_BRACKET");
|
|
expect_kind_name(.Punct(.RBracket), "TOK_R_BRACKET");
|
|
expect_kind_name(.Punct(.LParen), "TOK_L_PAREN");
|
|
expect_kind_name(.Punct(.RParen), "TOK_R_PAREN");
|
|
expect_kind_name(.Punct(.LBrace), "TOK_L_BRACE");
|
|
expect_kind_name(.Punct(.RBrace), "TOK_R_BRACE");
|
|
expect_kind_name(.Punct(.Semicolon), "TOK_SEMICOLON");
|
|
expect_kind_name(.Punct(.Comma), "TOK_COMMA");
|
|
expect_kind_name(.Punct(.Colon), "TOK_COLON");
|
|
expect_kind_name(.Punct(.ColonAssign), "TOK_COLON_ASSIGN");
|
|
expect_kind_name(.Punct(.Dot), "TOK_DOT");
|
|
expect_kind_name(.Punct(.Range), "TOK_RANGE");
|
|
expect_kind_name(.Punct(.Ellipsis), "TOK_ELLIPSIS");
|
|
expect_kind_name(.Punct(.Cond), "TOK_COND");
|
|
expect_kind_name(.Punct(.Sharp), "TOK_SHARP");
|
|
expect_kind_name(.Punct(.At), "TOK_AT");
|
|
expect_kind_name(.Punct(.Dollar), "TOK_DOLLAR");
|
|
|
|
expect_kind_name(.Ident, "TOK_IDENT");
|
|
expect_kind_name(.Int(.Dec), "TOK_INT_LITERAL");
|
|
expect_kind_name(.Float, "TOK_FLOAT_LITERAL");
|
|
expect_kind_name(.Char('a'), "TOK_CHAR_LITERAL");
|
|
expect_kind_name(.Str, "TOK_STRING_LITERAL");
|
|
expect_kind_name(.LineComment, "TOK_LINE_COMMENT");
|
|
expect_kind_name(.Endline, "TOK_ENDLINE");
|
|
expect_kind_name(.Eof, "TOK_EOF");
|
|
expect_kind_name(.Unknown(0), "TOK_UNKNOWN");
|
|
}
|
|
|
|
fn main() {
|
|
test_keywords();
|
|
test_adt_payloads();
|
|
test_trivia();
|
|
test_kind_names();
|
|
}
|