#ifndef __TOKEN_H__ #define __TOKEN_H__ enum CSTD_KEYWORD { CSTD_C89, CSTD_C99, CEXT_ASM, }; // Using Binary Search To Fast Find Keyword #define KEYWORD_TABLE \ X(asm , CEXT_ASM, TOKEN_ASM) \ X(break , CSTD_C89, TOKEN_BREAK) \ X(case , CSTD_C89, TOKEN_CASE) \ X(char , CSTD_C89, TOKEN_CHAR) \ X(const , CSTD_C89, TOKEN_CONST) \ X(continue , CSTD_C89, TOKEN_CONTINUE) \ X(default , CSTD_C89, TOKEN_DEFAULT) \ X(do , CSTD_C89, TOKEN_DO) \ X(double , CSTD_C89, TOKEN_DOUBLE) \ X(else , CSTD_C89, TOKEN_ELSE) \ X(enum , CSTD_C89, TOKEN_ENUM) \ X(extern , CSTD_C89, TOKEN_EXTERN) \ X(float , CSTD_C89, TOKEN_FLOAT) \ X(for , CSTD_C89, TOKEN_FOR) \ X(goto , CSTD_C89, TOKEN_GOTO) \ X(if , CSTD_C89, TOKEN_IF) \ X(inline , CSTD_C99, TOKEN_INLINE) \ X(int , CSTD_C89, TOKEN_INT) \ X(long , CSTD_C89, TOKEN_LONG) \ X(register , CSTD_C89, TOKEN_REGISTER) \ X(restrict , CSTD_C99, TOKEN_RESTRICT) \ X(return , CSTD_C89, TOKEN_RETURN) \ X(short , CSTD_C89, TOKEN_SHORT) \ X(signed , CSTD_C89, TOKEN_SIGNED) \ X(sizeof , CSTD_C89, TOKEN_SIZEOF) \ X(static , CSTD_C89, TOKEN_STATIC) \ X(struct , CSTD_C89, TOKEN_STRUCT) \ X(switch , CSTD_C89, TOKEN_SWITCH) \ X(typedef , CSTD_C89, TOKEN_TYPEDEF) \ X(union , CSTD_C89, TOKEN_UNION) \ X(unsigned , CSTD_C89, TOKEN_UNSIGNED) \ X(void , CSTD_C89, TOKEN_VOID) \ X(volatile , CSTD_C89, TOKEN_VOLATILE) \ X(while , CSTD_C89, TOKEN_WHILE) \ // KEYWORD_TABLE #define TOKEN_TABLE \ X(EOF , TOKEN_EOF) \ X(init , TOKEN_INIT) \ X(flush , TOKEN_FLUSH) \ X("==" , TOKEN_EQ) \ X("=" , TOKEN_ASSIGN) \ X("++" , TOKEN_ADD_ADD) \ X("+=" , TOKEN_ASSIGN_ADD) \ X("+" , TOKEN_ADD) \ X("--" , TOKEN_SUB_SUB) \ X("-=" , TOKEN_ASSIGN_SUB) \ X("->" , TOKEN_DEREF) \ X("-" , TOKEN_SUB) \ X("*=" , TOKEN_ASSIGN_MUL) \ X("*" , TOKEN_MUL) \ X("/=" , TOKEN_ASSIGN_DIV) \ X("/" , TOKEN_DIV) \ X("//" , TOKEN_LINE_COMMENT) \ X("/* */" , TOKEN_BLOCK_COMMENT) \ X("%=" , TOKEN_ASSIGN_MOD) \ X("%" , TOKEN_MOD) \ X("&&" , TOKEN_AND_AND) \ X("&=" , TOKEN_ASSIGN_AND) \ X("&" , TOKEN_AND) \ X("||" , TOKEN_OR_OR) \ X("|=" , TOKEN_ASSIGN_OR) \ X("|" , TOKEN_OR) \ X("^=" , TOKEN_ASSIGN_XOR) \ X("^" , TOKEN_XOR) \ X("<<=" , TOKEN_ASSIGN_L_SH) \ X("<<" , TOKEN_L_SH) \ X("<=" , TOKEN_LE) \ X("<" , TOKEN_LT) \ X(">>=" , TOKEN_ASSIGN_R_SH) \ X(">>" , TOKEN_R_SH) \ X(">=" , TOKEN_GE) \ X(">" , TOKEN_GT) \ X("!" , TOKEN_NOT) \ X("!=" , TOKEN_NEQ) \ X("~" , TOKEN_BIT_NOT) \ X("[" , TOKEN_L_BRACKET) \ X("]" , TOKEN_R_BRACKET) \ X("(" , TOKEN_L_PAREN) \ X(")" , TOKEN_R_PAREN) \ X("{" , TOKEN_L_BRACE) \ X("}" , TOKEN_R_BRACE) \ X(";" , TOKEN_SEMICOLON) \ X("," , TOKEN_COMMA) \ X(":" , TOKEN_COLON) \ X("." , TOKEN_DOT) \ X("..." , TOKEN_ELLIPSIS) \ X("?" , TOKEN_COND) \ X(identifier , TOKEN_IDENT) \ X(int_literal , TOKEN_INT_LITERAL) \ X(float_literal , TOKEN_FLOAT_LITERAL) \ X(char_literal , TOKEN_CHAR_LITERAL) \ X(string_literal , TOKEN_STRING_LITERAL) \ // END // 定义TokenType枚举 typedef enum tok_type { // 处理普通token #define X(str, tok) tok, TOKEN_TABLE #undef X // 处理关键字(保持原有格式) #define X(name, std, tok) tok, KEYWORD_TABLE #undef X } tok_type_t; typedef struct tok_val { int have; union { char ch; int i; float f; double d; long long ll; char* str; }; } tok_val_t; typedef struct tok { tok_type_t type; tok_val_t val; } tok_t; typedef struct tok_buf { int cur; int end; int peek; int size; int cap; tok_t* buf; void* stream; void (*gettok)(void* stream, tok_t* token); } tok_buf_t; typedef void(*get_tokbuf_func)(void* stream, tok_t* token); void init_tokbuf(tok_buf_t* tokbuf, void* stream, get_tokbuf_func gettok); tok_t* peek_tok(tok_buf_t* tokbuf); tok_t* pop_tok(tok_buf_t* tokbuf); void flush_peek_tok(tok_buf_t* tokbuf); tok_type_t peek_tok_type(tok_buf_t* tokbuf); int expect_pop_tok(tok_buf_t* tokbuf, tok_type_t type); const char* get_tok_name(tok_type_t type); #endif