zzy/smcc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat add func call and rewrite codes

Browse Source
This commit is contained in:
ZZY
2025-03-07 12:29:53 +08:00
parent 09299e339c
commit 95bf44eb3f
37 changed files with 3369 additions and 1063 deletions
Download Patch File Download Diff File Expand all files Collapse all files

76
ccompiler/frontend/lexer/lexer.c
View File

@ -26,13 +26,15 @@ the distribution and installation instructions.
Chris Fraser / cwf@aya.yale.edu
David Hanson / drh@drhanson.net
*/
#define FRONTEND_IMPLEMENTATION
#include "../frontend.h"
#include "token.h"
#include "lexer.h"
static const struct {
const char* name;
enum CSTD_KEYWORD std_type;
enum TokenType tok;
tok_type_t tok;
} keywords[] = {
#define X(name, std_type, tok, ...) { #name, std_type, tok },
KEYWORD_TABLE
@ -72,7 +74,7 @@ static inline int keyword_cmp(const char* name, int len) {
return -1; // Not a keyword.
}
void init_lexer(struct Lexer* lexer, const char* file_name, void* stream, lexer_sread_fn sread)
void init_lexer(lexer_t* lexer, const char* file_name, void* stream, lexer_sread_fn sread)
{
lexer->cur_ptr = lexer->end_ptr = (unsigned char*)&(lexer->buffer);
lexer->index = 1;
@ -86,12 +88,12 @@ void init_lexer(struct Lexer* lexer, const char* file_name, void* stream, lexer_
}
}
static void flush_buffer(struct Lexer* lexer) {
static void flush_buffer(lexer_t* lexer) {
int num = lexer->end_ptr - lexer->cur_ptr;
for (int i = 0; i < num; i++) {
lexer->buffer[i] = lexer->cur_ptr[i];
}
lexer->cur_ptr = lexer->buffer;
lexer->cur_ptr = (unsigned char*)lexer->buffer;
int read_size = LEXER_BUFFER_SIZE - num;
// TODO size_t to int maybe lose precision
@ -109,7 +111,7 @@ static void flush_buffer(struct Lexer* lexer) {
}
}
static void goto_newline(struct Lexer* lexer) {
static void goto_newline(lexer_t* lexer) {
do {
if (lexer->cur_ptr == lexer->end_ptr) {
flush_buffer(lexer);
@ -119,7 +121,7 @@ static void goto_newline(struct Lexer* lexer) {
} while (*lexer->cur_ptr != '\n' && *lexer->cur_ptr != '\0');
}
static void goto_block_comment(struct Lexer* lexer) {
static void goto_block_comment(lexer_t* lexer) {
while (1) {
if (lexer->end_ptr - lexer->cur_ptr < 2) {
flush_buffer(lexer);
@ -155,7 +157,7 @@ static char got_slash(unsigned char* peek) {
}
}
static void parse_char_literal(struct Lexer* lexer, struct Token* token) {
static void parse_char_literal(lexer_t* lexer, tok_t* token) {
char val = 0;
unsigned char* peek = lexer->cur_ptr + 1;
if (*peek == '\\') {
@ -166,16 +168,16 @@ static void parse_char_literal(struct Lexer* lexer, struct Token* token) {
}
if (*peek != '\'') error("Unclosed character literal");
token->constant.ch = val;
token->val.ch = val;
lexer->cur_ptr = peek + 1;
token->constant.have = 1;
token->val.have = 1;
token->type = TOKEN_CHAR_LITERAL;
}
static void parse_string_literal(struct Lexer* lexer, struct Token* token) {
static void parse_string_literal(lexer_t* lexer, tok_t* token) {
unsigned char* peek = lexer->cur_ptr + 1;
// TODO string literal size check
char* dest = token->constant.str = xmalloc(LEXER_MAX_TOKEN_SIZE + 1);
char* dest = token->val.str = xmalloc(LEXER_MAX_TOKEN_SIZE + 1);
int len = 0;
while (*peek != '"') {
@ -191,12 +193,12 @@ static void parse_string_literal(struct Lexer* lexer, struct Token* token) {
}
dest[len] = '\0';
lexer->cur_ptr = peek + 1;
token->constant.have = 1;
token->val.have = 1;
token->type = TOKEN_STRING_LITERAL;
}
// FIXME it write by AI maybe error
static void parse_number(struct Lexer* lexer, struct Token* token) {
static void parse_number(lexer_t* lexer, tok_t* token) {
unsigned char* peek = lexer->cur_ptr;
int base = 10;
int is_float = 0;
@ -255,12 +257,12 @@ static void parse_number(struct Lexer* lexer, struct Token* token) {
if ((*peek == 'e' || *peek == 'E') && base == 10) {
is_float = 1;
peek++;
int exp_sign = 1;
// int exp_sign = 1;
int exponent = 0;
if (*peek == '+') peek++;
else if (*peek == '-') {
exp_sign = -1;
// exp_sign = -1;
peek++;
}
@ -273,19 +275,19 @@ static void parse_number(struct Lexer* lexer, struct Token* token) {
// 存储结果
lexer->cur_ptr = peek;
token->constant.have = 1;
token->val.have = 1;
if (is_float) {
token->constant.d = float_val;
token->val.d = float_val;
token->type = TOKEN_FLOAT_LITERAL;
} else {
token->constant.ll = int_val;
token->val.ll = int_val;
token->type = TOKEN_INT_LITERAL;
}
}
#define GOT_ONE_TOKEN_BUF_SIZE 64
// /zh/c/language/operator_arithmetic.html
void get_token(struct Lexer* lexer, struct Token* token) {
void get_token(lexer_t* lexer, tok_t* token) {
// 需要保证缓冲区始终可读
if (lexer->end_ptr - lexer->cur_ptr < GOT_ONE_TOKEN_BUF_SIZE) {
flush_buffer(lexer);
@ -305,8 +307,8 @@ void get_token(struct Lexer* lexer, struct Token* token) {
token->type = TOKEN_FLUSH;
}
enum TokenType tok = TOKEN_INIT;
struct TokenConstant constant;
tok_type_t tok = TOKEN_INIT;
tok_val_t constant;
constant.have = 0;
// once step
@ -392,7 +394,7 @@ void get_token(struct Lexer* lexer, struct Token* token) {
switch (*peek++) {
case '=': tok = TOKEN_NEQ; break;
default: peek--, tok = TOKEN_NOT; break;
}
} break;
case '[':
tok = TOKEN_L_BRACKET; break;
case ']':
@ -454,7 +456,7 @@ void get_token(struct Lexer* lexer, struct Token* token) {
case 'S': case 'T': case 'U': case 'V': case 'W': case 'X':case 'Y': case 'Z':
case '_':
// TOKEN_IDENT
if (*peek == 'L' && *peek == '\'' || *peek == 'L' && *peek == '"') {
if ((*peek == 'L' && *peek == '\'') || (*peek == 'L' && *peek == '"')) {
error("unsupport wide-character char literal by `L` format");
}
while (1) {
@ -469,18 +471,18 @@ void get_token(struct Lexer* lexer, struct Token* token) {
break;
}
int res = keyword_cmp(lexer->cur_ptr, peek - (lexer->cur_ptr));
int res = keyword_cmp((const char*)lexer->cur_ptr, peek - (lexer->cur_ptr));
if (res == -1) {
int strlen = peek - lexer->cur_ptr;
unsigned char* str = xmalloc(strlen + 1);
constant.have = 1;
constant.str = str;
constant.str = (char*)str;
for (int i = 0; i < strlen; i++) {
str[i] = lexer->cur_ptr[i];
}
str[strlen] = '\0';
constant.have = 1;
constant.str = str;
constant.str = (char*)str;
tok = TOKEN_IDENT; break;
} else {
tok = keywords[res].tok; break;
@ -492,32 +494,16 @@ void get_token(struct Lexer* lexer, struct Token* token) {
lexer->cur_ptr = peek;
END:
token->constant = constant;
token->val = constant;
token->type = tok;
}
// get_token maybe got invalid (with parser)
void get_valid_token(struct Lexer* lexer, struct Token* token) {
enum TokenType type;
void get_valid_token(lexer_t* lexer, tok_t* token) {
tok_type_t type;
do {
get_token(lexer, token);
type = token->type;
} while (type == TOKEN_FLUSH || type == TOKEN_LINE_COMMENT || type == TOKEN_BLOCK_COMMENT);
}
// 生成字符串映射（根据需求选择#str或#name）
static const char* token_strings[] = {
// 普通token使用#str
#define X(str, tok) [tok] = #str,
TOKEN_TABLE
#undef X
// 关键字使用#name
#define X(name, std, tok) [tok] = #name,
KEYWORD_TABLE
#undef X
};
const char* get_token_name(enum TokenType type) {
return token_strings[type];
}

23
ccompiler/frontend/lexer/lexer.h
View File

@ -2,13 +2,17 @@
#define __LEXER_H__
#include "token.h"
#ifndef LEXER_MAX_TOKEN_SIZE
#define LEXER_MAX_TOKEN_SIZE 63
#endif
#ifndef LEXER_BUFFER_SIZE
#define LEXER_BUFFER_SIZE 4095
#endif
typedef int (*lexer_sread_fn)(void *dst_buf, int dst_size,
int elem_size, int count, void *stream);
struct Lexer {
typedef struct lexer {
int line;
int index;
// const char current_file_name[LEXER_BUFFER_SIZE+1];
@ -19,22 +23,15 @@ struct Lexer {
lexer_sread_fn sread;
void* stream;
};
} lexer_t;
struct Token {
enum TokenType type;
struct TokenConstant constant;
};
void init_lexer(struct Lexer* lexer, const char* file_name, void* stream,
void init_lexer(lexer_t* lexer, const char* file_name, void* stream,
lexer_sread_fn sread);
//
void get_token(struct Lexer* lexer, struct Token* token);
// pure token getter it will included empty token like TOKEN_FLUSH
void get_token(lexer_t* lexer, tok_t* token);
// get_token maybe got invalid (with parser as TOKEN_FLUSH)
void get_valid_token(struct Lexer* lexer, struct Token* token);
const char* get_token_name(enum TokenType token);
void get_valid_token(lexer_t* lexer, tok_t* token);
#endif

17
ccompiler/frontend/lexer/tests/Makefile Normal file
View File

@ -0,0 +1,17 @@
CC = gcc
CFLAGS = -g -Wall
SRC = ../lexer.c ../token.c
all = test_all
test_all: test
./test
run:
$(CC) $(CFLAGS) $(SRC) run.c -o run
test:
$(CC) $(CFLAGS) $(SRC) -o test test.c
clean:
rm -f test run

10
ccompiler/frontend/lexer/tests/test_lexer.c → ccompiler/frontend/lexer/tests/run.c
View File

@ -1,8 +1,8 @@
#include "../lexer.h"
#include <stdio.h>
// gcc -g ../lexer.c test_lexer.c -o test_lexer
// gcc -g ../lexer.c ../token.c test_lexer.c -o test_lexer
/*
struct TokenConstant {
tok_tConstant {
int have;
union {
char ch;
@ -31,9 +31,9 @@ int main(int argc, char* argv[]) {
}
printf("open file success\n");
struct Lexer lexer;
lexer_t lexer;
init_lexer(&lexer, "test_lexter.c", fp, (lexer_sread_fn)fread_s);
struct Token tok;
tok_t tok;
while (1) {
get_valid_token(&lexer, &tok);
@ -41,6 +41,6 @@ int main(int argc, char* argv[]) {
break;
}
printf("line: %d, column: %d, type: %3d, typename: %s\n",
lexer.line, lexer.index, tok.type, get_token_name(tok.type));
lexer.line, lexer.index, tok.type, get_tok_name(tok.type));
}
}

178
ccompiler/frontend/lexer/tests/test.c Normal file
View File

@ -0,0 +1,178 @@
// test_lexer.c
#include "../../../../libcore/acutest.h"
#include "../lexer.h"
#include <string.h>
int test_read(void *dst_buf, int dst_size, int elem_size, int count, void *stream) {
if (stream == NULL) {
return 0;
}
int size = dst_size > elem_size * count ? elem_size * count : dst_size;
memcpy(dst_buf, stream, size);
return size;
}
// 测试辅助函数
static inline void test_lexer_string(const char* input, tok_type_t expected_type) {
lexer_t lexer;
tok_t token;
init_lexer(&lexer, "test.c", (void*)input, test_read);
get_valid_token(&lexer, &token);
TEST_CHECK(token.type == expected_type);
TEST_MSG("Expected: %s", get_tok_name(expected_type));
TEST_MSG("Got: %s", get_tok_name(token.type));
}
// 基础运算符测试
void test_operators() {
TEST_CASE("Arithmetic operators"); {
test_lexer_string("+", TOKEN_ADD);
test_lexer_string("++", TOKEN_ADD_ADD);
test_lexer_string("+=", TOKEN_ASSIGN_ADD);
test_lexer_string("-", TOKEN_SUB);
test_lexer_string("--", TOKEN_SUB_SUB);
test_lexer_string("-=", TOKEN_ASSIGN_SUB);
test_lexer_string("*", TOKEN_MUL);
test_lexer_string("*=", TOKEN_ASSIGN_MUL);
test_lexer_string("/", TOKEN_DIV);
test_lexer_string("/=", TOKEN_ASSIGN_DIV);
test_lexer_string("%", TOKEN_MOD);
test_lexer_string("%=", TOKEN_ASSIGN_MOD);
}
TEST_CASE("Bitwise operators"); {
test_lexer_string("&", TOKEN_AND);
test_lexer_string("&&", TOKEN_AND_AND);
test_lexer_string("&=", TOKEN_ASSIGN_AND);
test_lexer_string("|", TOKEN_OR);
test_lexer_string("||", TOKEN_OR_OR);
test_lexer_string("|=", TOKEN_ASSIGN_OR);
test_lexer_string("^", TOKEN_XOR);
test_lexer_string("^=", TOKEN_ASSIGN_XOR);
test_lexer_string("~", TOKEN_BIT_NOT);
test_lexer_string("<<", TOKEN_L_SH);
test_lexer_string("<<=", TOKEN_ASSIGN_L_SH);
test_lexer_string(">>", TOKEN_R_SH);
test_lexer_string(">>=", TOKEN_ASSIGN_R_SH);
}
TEST_CASE("Comparison operators"); {
test_lexer_string("==", TOKEN_EQ);
test_lexer_string("!=", TOKEN_NEQ);
test_lexer_string("<", TOKEN_LT);
test_lexer_string("<=", TOKEN_LE);
test_lexer_string(">", TOKEN_GT);
test_lexer_string(">=", TOKEN_GE);
}
TEST_CASE("Special symbols"); {
test_lexer_string("(", TOKEN_L_PAREN);
test_lexer_string(")", TOKEN_R_PAREN);
test_lexer_string("[", TOKEN_L_BRACKET);
test_lexer_string("]", TOKEN_R_BRACKET);
test_lexer_string("{", TOKEN_L_BRACE);
test_lexer_string("}", TOKEN_R_BRACE);
test_lexer_string(";", TOKEN_SEMICOLON);
test_lexer_string(",", TOKEN_COMMA);
test_lexer_string(":", TOKEN_COLON);
test_lexer_string(".", TOKEN_DOT);
test_lexer_string("...", TOKEN_ELLIPSIS);
test_lexer_string("->", TOKEN_DEREF);
test_lexer_string("?", TOKEN_COND);
}
}
// 关键字测试
void test_keywords() {
TEST_CASE("C89 keywords");
test_lexer_string("while", TOKEN_WHILE);
test_lexer_string("sizeof", TOKEN_SIZEOF);
// TEST_CASE("C99 keywords");
// test_lexer_string("restrict", TOKEN_RESTRICT);
// test_lexer_string("_Bool", TOKEN_INT); // 需确认你的类型定义
}
// 字面量测试
void test_literals() {
TEST_CASE("Integer literals"); {
// 十进制
test_lexer_string("0", TOKEN_INT_LITERAL);
test_lexer_string("123", TOKEN_INT_LITERAL);
// test_lexer_string("2147483647", TOKEN_INT_LITERAL);
// // 十六进制
// test_lexer_string("0x0", TOKEN_INT_LITERAL);
// test_lexer_string("0x1A3F", TOKEN_INT_LITERAL);
// test_lexer_string("0XABCDEF", TOKEN_INT_LITERAL);
// // 八进制
// test_lexer_string("0123", TOKEN_INT_LITERAL);
// test_lexer_string("0777", TOKEN_INT_LITERAL);
// // 边界值测试
// test_lexer_string("2147483647", TOKEN_INT_LITERAL); // INT_MAX
// test_lexer_string("4294967295", TOKEN_INT_LITERAL); // UINT_MAX
}
// TEST_CASE("Character literals"); {
// test_lexer_string("'a'", TOKEN_CHAR_LITERAL);
// test_lexer_string("'\\n'", TOKEN_CHAR_LITERAL);
// test_lexer_string("'\\t'", TOKEN_CHAR_LITERAL);
// test_lexer_string("'\\\\'", TOKEN_CHAR_LITERAL);
// test_lexer_string("'\\0'", TOKEN_CHAR_LITERAL);
// }
TEST_CASE("String literals"); {
test_lexer_string("\"hello\"", TOKEN_STRING_LITERAL);
test_lexer_string("\"multi-line\\nstring\"", TOKEN_STRING_LITERAL);
test_lexer_string("\"escape\\\"quote\"", TOKEN_STRING_LITERAL);
}
// TEST_CASE("Integer literals");
// test_lexer_string("123", TOKEN_INT_LITERAL);
// test_lexer_string("0x1F", TOKEN_INT_LITERAL);
// TEST_CASE("Floating literals");
// test_lexer_string("3.14e-5", TOKEN_FLOAT_LITERAL);
// TEST_CASE("Character literals");
// test_lexer_string("'\\n'", TOKEN_CHAR_LITERAL);
}
// 边界测试
void test_edge_cases() {
// TEST_CASE("Long identifiers");
// char long_id[LEXER_MAX_TOKEN_SIZE+2] = {0};
// memset(long_id, 'a', LEXER_MAX_TOKEN_SIZE+1);
// test_lexer_string(long_id, TOKEN_IDENT);
// TEST_CASE("Buffer boundary");
// char boundary[LEXER_BUFFER_SIZE*2] = {0};
// memset(boundary, '+', LEXER_BUFFER_SIZE*2-1);
// test_lexer_string(boundary, TOKEN_ADD);
}
// 错误处理测试
void test_error_handling() {
TEST_CASE("Invalid characters");
lexer_t lexer;
tok_t token;
init_lexer(&lexer, "test.c", NULL, test_read);
get_valid_token(&lexer, &token);
TEST_CHECK(token.type == TOKEN_EOF); // 应触发错误处理
}
// 测试列表
TEST_LIST = {
{"operators", test_operators},
{"keywords", test_keywords},
{"literals", test_literals},
{"edge_cases", test_edge_cases},
{"error_handling", test_error_handling},
{NULL, NULL}
};

86
ccompiler/frontend/lexer/token.c Normal file
View File

@ -0,0 +1,86 @@
#define FRONTEND_IMPLEMENTATION
#include "../frontend.h"
#include "token.h"
#define ROUND_IDX(idx) ((idx) % tokbuf->cap)
tok_t* pop_tok(tok_buf_t* tokbuf) {
if (tokbuf->size == 0) {
error("no token to pop");
return NULL;
}
int idx = tokbuf->cur;
tokbuf->cur = ROUND_IDX(idx + 1);
tokbuf->size -= 1;
return tokbuf->buf + idx;
}
void flush_peek_tok(tok_buf_t* tokbuf) {
tokbuf->peek = tokbuf->cur;
}
void init_tokbuf(tok_buf_t *tokbuf, void *stream, get_tokbuf_func gettok) {
tokbuf->cur = 0;
tokbuf->end = 0;
tokbuf->peek = 0;
tokbuf->size = 0;
tokbuf->stream = stream;
tokbuf->gettok = gettok;
tokbuf->buf = NULL;
tokbuf->cap = 0;
}
tok_t *peek_tok(tok_buf_t *tokbuf)
{
int idx = tokbuf->peek;
idx = ROUND_IDX(idx + 1);
if (tokbuf->size >= tokbuf->cap) {
error("peek too deep, outof array size");
}
if (tokbuf->peek == tokbuf->end) {
if (tokbuf->size == tokbuf->cap) {
error("peek_tok buffer overflow");
}
if (tokbuf->gettok == NULL) {
error("peek_tok can not got tok");
}
tokbuf->gettok(tokbuf->stream, &(tokbuf->buf[idx]));
tokbuf->size++;
tokbuf->end = idx;
}
tokbuf->peek = idx;
return &(tokbuf->buf[idx]);
}
tok_type_t peek_tok_type(tok_buf_t* tokbuf) {
return peek_tok(tokbuf)->type;
}
int expect_pop_tok(tok_buf_t* tokbuf, tok_type_t type) {
flush_peek_tok(tokbuf);
tok_t* tok = peek_tok(tokbuf);
if (tok->type != type) {
error("expected tok: %s, got %s", get_tok_name(type), get_tok_name(tok->type));
} else {
pop_tok(tokbuf);
}
return 0;
}
// 生成字符串映射（根据需求选择#str或#name）
static const char* token_strings[] = {
// 普通token使用#str
#define X(str, tok) [tok] = #str,
TOKEN_TABLE
#undef X
// 关键字使用#name
#define X(name, std, tok) [tok] = #name,
KEYWORD_TABLE
#undef X
};
const char* get_tok_name(tok_type_t type) {
return token_strings[type];
}

145
ccompiler/frontend/lexer/token.h
View File

@ -105,7 +105,7 @@ enum CSTD_KEYWORD {
// END
// 定义TokenType枚举
enum TokenType {
typedef enum tok_type {
// 处理普通token
#define X(str, tok) tok,
TOKEN_TABLE
@ -115,9 +115,9 @@ enum TokenType {
#define X(name, std, tok) tok,
KEYWORD_TABLE
#undef X
};
} tok_type_t;
struct TokenConstant {
typedef struct tok_val {
int have;
union {
char ch;
@ -127,124 +127,31 @@ struct TokenConstant {
long long ll;
char* str;
};
};
} tok_val_t;
// "break"
// "case"
// "char"
// "const"
// "continue"
// "default"
// "do"
// "double"
// "else"
// "enum"
// "extern"
// "float"
// "for"
// "goto"
// "if"
// "inline (C99)"
// "int"
// "long"
// "register"
// "restrict (C99)"
// "return"
// "short"
// "signed"
// "sizeof"
// "static"
// "struct"
// "switch"
// "typedef"
// "union"
// "unsigned"
// "void"
// "volatile"
// "while"
typedef struct tok {
tok_type_t type;
tok_val_t val;
} tok_t;
// alignas (C23)
// alignof (C23)
// auto
// bool (C23)
// constexpr (C23)
// false (C23)
// nullptr (C23)
// static_assert (C23)
// thread_local (C23)
// true (C23)
// typeof (C23)
// typeof_unqual (C23)
// _Alignas (C11)
// _Alignof (C11)
// _Atomic (C11)
// _BitInt (C23)
// _Bool (C99)
// _Complex (C99)
// _Decimal128 (C23)
// _Decimal32 (C23)
// _Decimal64 (C23)
// _Generic (C11)
// _Imaginary (C99)
// _Noreturn (C11)
// _Static_assert (C11)
// _Thread_local (C11)
typedef struct tok_buf {
int cur;
int end;
int peek;
int size;
int cap;
tok_t* buf;
void* stream;
void (*gettok)(void* stream, tok_t* token);
} tok_buf_t;
// a = b
// a += b
// a -= b
// a *= b
// a /= b
// a %= b
// a &= b
// a |= b
// a ^= b
// a <<= b
// a >>= b
// ++a
// --a
// a++
// a--
// +a
// -a
// a + b
// a - b
// a * b
// a / b
// a % b
// ~a
// a & b
// a | b
// a ^ b
// a << b
// a >> b
// !a
// a && b
// a || b
// a == b
// a != b
// a < b
// a > b
// a <= b
// a >= b
// a[b]
// *a
// &a
// a->b
// a.b
// a(...)
// a, b
// (type) a
// a ? b : c
// sizeof
// _Alignof
// (C11)
typedef void(*get_tokbuf_func)(void* stream, tok_t* token);
void init_tokbuf(tok_buf_t* tokbuf, void* stream, get_tokbuf_func gettok);
tok_t* peek_tok(tok_buf_t* tokbuf);
tok_t* pop_tok(tok_buf_t* tokbuf);
void flush_peek_tok(tok_buf_t* tokbuf);
tok_type_t peek_tok_type(tok_buf_t* tokbuf);
int expect_pop_tok(tok_buf_t* tokbuf, tok_type_t type);
const char* get_tok_name(tok_type_t type);
#endif