This commit is contained in:
ZZY
2025-04-01 00:13:21 +08:00
parent 2b4857001c
commit 74f43a1ab7
79 changed files with 2271 additions and 2861 deletions

View File

@ -7,7 +7,6 @@ CFLAGS = -g -Wall -I../..
LEXER_DIR = ./lexer
PARSER_DIR = ./parser
AST_DIR = ./parser/ast
SYMTAB_DIR = ./parser/symtab
# 源文件列表
SRCS = \
@ -23,9 +22,6 @@ SRCS = \
$(AST_DIR)/program.c \
$(AST_DIR)/stmt.c \
$(AST_DIR)/term.c \
$(SYMTAB_DIR)/hashmap.c \
$(SYMTAB_DIR)/scope.c \
$(SYMTAB_DIR)/symtab.c \
# 生成目标文件列表
OBJS = $(SRCS:.c=.o)

View File

@ -1,17 +1,18 @@
#include <lib/core.h>
#include "frontend.h"
#include "parser/symtab/symtab.h"
ast_node_t* frontend(const char* file, void* stream, sread_fn sread) {
ast_node_t* cc_frontend(const char* file, void* stream, sread_fn sread) {
init_lib_core();
strpool_t strpool;
init_strpool(&strpool);
lexer_t lexer;
cc_lexer_t lexer;
init_lexer(&lexer, file, stream, sread, &strpool);
symtab_t symtab;
init_symtab(&symtab);
// TODO global scope
symtab_enter_scope(&symtab);
parser_t parser;
init_parser(&parser, &lexer, &symtab);

View File

@ -1,9 +1,9 @@
#ifndef __SMCC_FRONTEND_H__
#define __SMCC_FRONTEND_H__
#ifndef __SMCC_CC_FRONTEND_H__
#define __SMCC_CC_FRONTEND_H__
#include "lexer/lexer.h"
#include "parser/parser.h"
typedef int (*sread_fn)(void *dst_buf, int dst_size, int elem_size, int count, void *stream);
ast_node_t* frontend(const char* file, void* stream, sread_fn sread);
typedef int (*sread_fn)(void *dst_buf, int elem_size, int count, void *stream);
ast_node_t* cc_frontend(const char* file, void* stream, sread_fn sread);
#endif

View File

@ -74,7 +74,7 @@ static inline int keyword_cmp(const char* name, int len) {
return -1; // Not a keyword.
}
void init_lexer(lexer_t* lexer, const char* file_name, void* stream, lexer_sread_fn sread, strpool_t* strpool) {
void init_lexer(cc_lexer_t* lexer, const char* file_name, void* stream, lexer_sread_fn sread, strpool_t* strpool) {
lexer->strpool = strpool;
lexer->cur_ptr = lexer->end_ptr = (char*)&(lexer->buffer);
lexer->loc.fname = strpool_intern(lexer->strpool, file_name);
@ -87,7 +87,7 @@ void init_lexer(lexer_t* lexer, const char* file_name, void* stream, lexer_sread
rt_memset(lexer->buffer, 0, sizeof(lexer->buffer));
}
static void flush_buffer(lexer_t* lexer) {
static void flush_buffer(cc_lexer_t* lexer) {
int num = lexer->end_ptr - lexer->cur_ptr;
for (int i = 0; i < num; i++) {
lexer->buffer[i] = lexer->cur_ptr[i];
@ -96,7 +96,7 @@ static void flush_buffer(lexer_t* lexer) {
int read_size = LEXER_BUFFER_SIZE - num;
// TODO rt_size_t to int maybe lose precision
int got_size = lexer->sread(lexer->buffer + num, read_size, 1, read_size, lexer->stream);
int got_size = lexer->sread(lexer->buffer + num, 1, read_size, lexer->stream);
if (got_size < 0) {
LEX_ERROR("lexer read error");
} else if (got_size < read_size) {
@ -110,7 +110,7 @@ static void flush_buffer(lexer_t* lexer) {
}
}
static void goto_newline(lexer_t* lexer) {
static void goto_newline(cc_lexer_t* lexer) {
do {
if (lexer->cur_ptr == lexer->end_ptr) {
flush_buffer(lexer);
@ -120,7 +120,7 @@ static void goto_newline(lexer_t* lexer) {
} while (*lexer->cur_ptr != '\n' && *lexer->cur_ptr != '\0');
}
static void goto_block_comment(lexer_t* lexer) {
static void goto_block_comment(cc_lexer_t* lexer) {
while (1) {
if (lexer->end_ptr - lexer->cur_ptr < 2) {
flush_buffer(lexer);
@ -159,7 +159,7 @@ static char got_slash(char* peek) {
return -1;
}
static void parse_char_literal(lexer_t* lexer, tok_t* token) {
static void parse_char_literal(cc_lexer_t* lexer, tok_t* token) {
char val = 0;
char* peek = lexer->cur_ptr + 1;
if (*peek == '\\') {
@ -175,7 +175,7 @@ static void parse_char_literal(lexer_t* lexer, tok_t* token) {
token->val.ch = val;
}
static void parse_string_literal(lexer_t* lexer, tok_t* token) {
static void parse_string_literal(cc_lexer_t* lexer, tok_t* token) {
char* peek = lexer->cur_ptr + 1;
// TODO string literal size check
static char dest[LEXER_MAX_TOKEN_SIZE + 1];
@ -200,7 +200,7 @@ static void parse_string_literal(lexer_t* lexer, tok_t* token) {
}
// FIXME it write by AI maybe error
static void parse_number(lexer_t* lexer, tok_t* token) {
static void parse_number(cc_lexer_t* lexer, tok_t* token) {
char* peek = lexer->cur_ptr;
int base = 10;
int is_float = 0;
@ -290,7 +290,7 @@ static void parse_number(lexer_t* lexer, tok_t* token) {
#define GOT_ONE_TOKEN_BUF_SIZE 64
// /zh/c/language/operator_arithmetic.html
void get_token(lexer_t* lexer, tok_t* token) {
void get_token(cc_lexer_t* lexer, tok_t* token) {
// 需要保证缓冲区始终可读
if (lexer->end_ptr - lexer->cur_ptr < GOT_ONE_TOKEN_BUF_SIZE) {
flush_buffer(lexer);
@ -515,7 +515,7 @@ static const tok_basic_type_t tok_type_map[] = {
}
// get_token maybe got invalid (with parser)
void get_valid_token(lexer_t* lexer, tok_t* token) {
void get_valid_token(cc_lexer_t* lexer, tok_t* token) {
tok_basic_type_t type;
do {
get_token(lexer, token);

View File

@ -10,10 +10,9 @@
#define LEXER_BUFFER_SIZE 4095
#endif
typedef int (*lexer_sread_fn)(void *dst_buf, int dst_size,
int elem_size, int count, void *stream);
typedef int (*lexer_sread_fn)(void *dst_buf, int elem_size, int count, void *stream);
typedef struct lexer {
typedef struct cc_lexer {
loc_t loc;
char* cur_ptr; // 当前扫描的字符,但是还没有开始扫描
@ -24,15 +23,15 @@ typedef struct lexer {
void* stream;
strpool_t* strpool;
} lexer_t;
} cc_lexer_t;
void init_lexer(lexer_t* lexer, const char* file_name, void* stream,
void init_lexer(cc_lexer_t* lexer, const char* file_name, void* stream,
lexer_sread_fn sread, strpool_t* strpool);
// pure token getter it will included empty token like TOKEN_BLANK
void get_token(lexer_t* lexer, tok_t* token);
void get_token(cc_lexer_t* lexer, tok_t* token);
// get_token maybe got invalid (with parser as TOKEN_BLANK)
void get_valid_token(lexer_t* lexer, tok_t* token);
void get_valid_token(cc_lexer_t* lexer, tok_t* token);
#endif

View File

@ -1,18 +0,0 @@
CC = gcc
CFLAGS = -g -Wall -I../../../.. -DLEX_LOG_LEVEL=4
SRC = ../lexer.c ../token.c
LIB = -L../../../../lib -lcore
all = test_all
test_all: test
./test
run:
$(CC) $(CFLAGS) $(SRC) run.c $(LIB) -o run
test:
$(CC) $(CFLAGS) $(SRC) $(LIB) -o test test.c
clean:
rm -f test run

View File

@ -1,56 +0,0 @@
#include "../lexer.h"
#include <stdio.h>
#include <string.h>
// gcc -g ../lexer.c ../token.c test_lexer.c -o test_lexer
/*
tok_tConstant {
int have;
union {
char ch;
int i;
float f;
double d;
long long ll;
char* str;
};
};
*/
int g_num;
int g_num_arr[3];
int main(int argc, char* argv[]) {
// int num = 0;
// You Must Be Call
init_lib_core();
if (argc == 3 && strcmp(argv[2], "-nodebug") == 0) {
log_set_level(NULL, LOG_LEVEL_ALL & ~LOG_LEVEL_DEBUG);
}
const char* file_name = "run.c";
if (argc == 2) {
file_name = argv[1];
}
FILE* fp = fopen(file_name, "r");
if (fp == NULL) {
perror("open file failed");
return 1;
}
printf("open file success\n");
lexer_t lexer;
strpool_t strpool;
init_strpool(&strpool);
init_lexer(&lexer, file_name, fp, (lexer_sread_fn)fread_s, &strpool);
tok_t tok;
while (1) {
get_valid_token(&lexer, &tok);
if (tok.sub_type == TOKEN_EOF) {
break;
}
LOG_DEBUG("tk type `%s` in %s:%d:%d", get_tok_name(tok.sub_type), tok.loc.fname, tok.loc.line, tok.loc.col);
// LOG_DEBUG("%s", tok.val.str);
// printf("line: %d, column: %d, type: %3d, typename: %s\n",
// lexer.line, lexer.index, tok.type, get_tok_name(tok.type));
}
}

View File

@ -1,172 +0,0 @@
// test_lexer.c
#include <lib/acutest.h>
#include "../lexer.h"
#include <string.h>
int test_read(void *dst_buf, int dst_size, int elem_size, int count, void *stream) {
if (stream == NULL) {
return 0;
}
int size = dst_size > elem_size * count ? elem_size * count : dst_size;
memcpy(dst_buf, stream, size);
return size;
}
// 测试辅助函数
static inline void test_lexer_string(const char* input, cc_tktype_t expected_type) {
lexer_t lexer;
tok_t token;
init_lexer(&lexer, "test.c", (void*)input, test_read);
get_valid_token(&lexer, &token);
TEST_CHECK(token.type == expected_type);
TEST_MSG("Expected: %s", get_tok_name(expected_type));
TEST_MSG("Got: %s", get_tok_name(token.type));
}
// 基础运算符测试
void test_operators() {
TEST_CASE("Arithmetic operators"); {
test_lexer_string("+", TOKEN_ADD);
test_lexer_string("++", TOKEN_ADD_ADD);
test_lexer_string("+=", TOKEN_ASSIGN_ADD);
test_lexer_string("-", TOKEN_SUB);
test_lexer_string("--", TOKEN_SUB_SUB);
test_lexer_string("-=", TOKEN_ASSIGN_SUB);
test_lexer_string("*", TOKEN_MUL);
test_lexer_string("*=", TOKEN_ASSIGN_MUL);
test_lexer_string("/", TOKEN_DIV);
test_lexer_string("/=", TOKEN_ASSIGN_DIV);
test_lexer_string("%", TOKEN_MOD);
test_lexer_string("%=", TOKEN_ASSIGN_MOD);
}
TEST_CASE("Bitwise operators"); {
test_lexer_string("&", TOKEN_AND);
test_lexer_string("&&", TOKEN_AND_AND);
test_lexer_string("&=", TOKEN_ASSIGN_AND);
test_lexer_string("|", TOKEN_OR);
test_lexer_string("||", TOKEN_OR_OR);
test_lexer_string("|=", TOKEN_ASSIGN_OR);
test_lexer_string("^", TOKEN_XOR);
test_lexer_string("^=", TOKEN_ASSIGN_XOR);
test_lexer_string("~", TOKEN_BIT_NOT);
test_lexer_string("<<", TOKEN_L_SH);
test_lexer_string("<<=", TOKEN_ASSIGN_L_SH);
test_lexer_string(">>", TOKEN_R_SH);
test_lexer_string(">>=", TOKEN_ASSIGN_R_SH);
}
TEST_CASE("Comparison operators"); {
test_lexer_string("==", TOKEN_EQ);
test_lexer_string("!=", TOKEN_NEQ);
test_lexer_string("<", TOKEN_LT);
test_lexer_string("<=", TOKEN_LE);
test_lexer_string(">", TOKEN_GT);
test_lexer_string(">=", TOKEN_GE);
}
TEST_CASE("Special symbols"); {
test_lexer_string("(", TOKEN_L_PAREN);
test_lexer_string(")", TOKEN_R_PAREN);
test_lexer_string("[", TOKEN_L_BRACKET);
test_lexer_string("]", TOKEN_R_BRACKET);
test_lexer_string("{", TOKEN_L_BRACE);
test_lexer_string("}", TOKEN_R_BRACE);
test_lexer_string(";", TOKEN_SEMICOLON);
test_lexer_string(",", TOKEN_COMMA);
test_lexer_string(":", TOKEN_COLON);
test_lexer_string(".", TOKEN_DOT);
test_lexer_string("...", TOKEN_ELLIPSIS);
test_lexer_string("->", TOKEN_DEREF);
test_lexer_string("?", TOKEN_COND);
}
}
// 关键字测试
void test_keywords() {
TEST_CASE("C89 keywords");
test_lexer_string("while", TOKEN_WHILE);
test_lexer_string("sizeof", TOKEN_SIZEOF);
// TEST_CASE("C99 keywords");
// test_lexer_string("restrict", TOKEN_RESTRICT);
// test_lexer_string("_Bool", TOKEN_INT); // 需确认你的类型定义
}
// 字面量测试
void test_literals() {
TEST_CASE("Integer literals"); {
// 十进制
test_lexer_string("0", TOKEN_INT_LITERAL);
test_lexer_string("123", TOKEN_INT_LITERAL);
// test_lexer_string("2147483647", TOKEN_INT_LITERAL);
// // 十六进制
// test_lexer_string("0x0", TOKEN_INT_LITERAL);
// test_lexer_string("0x1A3F", TOKEN_INT_LITERAL);
// test_lexer_string("0XABCDEF", TOKEN_INT_LITERAL);
// // 八进制
// test_lexer_string("0123", TOKEN_INT_LITERAL);
// test_lexer_string("0777", TOKEN_INT_LITERAL);
// // 边界值测试
// test_lexer_string("2147483647", TOKEN_INT_LITERAL); // INT_MAX
// test_lexer_string("4294967295", TOKEN_INT_LITERAL); // UINT_MAX
}
TEST_CASE("Character literals"); {
test_lexer_string("'a'", TOKEN_CHAR_LITERAL);
test_lexer_string("'\\n'", TOKEN_CHAR_LITERAL);
test_lexer_string("'\\t'", TOKEN_CHAR_LITERAL);
test_lexer_string("'\\\\'", TOKEN_CHAR_LITERAL);
test_lexer_string("'\\0'", TOKEN_CHAR_LITERAL);
}
TEST_CASE("String literals"); {
test_lexer_string("\"hello\"", TOKEN_STRING_LITERAL);
test_lexer_string("\"multi-line\\nstring\"", TOKEN_STRING_LITERAL);
test_lexer_string("\"escape\\\"quote\"", TOKEN_STRING_LITERAL);
}
// TEST_CASE("Floating literals");
// test_lexer_string("3.14e-5", TOKEN_FLOAT_LITERAL);
}
// 边界测试
void test_edge_cases() {
// TEST_CASE("Long identifiers");
// char long_id[LEXER_MAX_TOKEN_SIZE+2] = {0};
// memset(long_id, 'a', LEXER_MAX_TOKEN_SIZE+1);
// test_lexer_string(long_id, TOKEN_IDENT);
// TEST_CASE("Buffer boundary");
// char boundary[LEXER_BUFFER_SIZE*2] = {0};
// memset(boundary, '+', LEXER_BUFFER_SIZE*2-1);
// test_lexer_string(boundary, TOKEN_ADD);
}
// 错误处理测试
void test_error_handling() {
TEST_CASE("Invalid characters");
lexer_t lexer;
tok_t token;
init_lexer(&lexer, "test.c", NULL, test_read);
get_valid_token(&lexer, &token);
TEST_CHECK(token.type == TOKEN_EOF); // 应触发错误处理
}
// 测试列表
TEST_LIST = {
{"operators", test_operators},
{"keywords", test_keywords},
{"literals", test_literals},
{"edge_cases", test_edge_cases},
{"error_handling", test_error_handling},
{NULL, NULL}
};

View File

@ -89,6 +89,7 @@ typedef struct ast_node {
vector_header(children, struct ast_node *);
} block;
struct {
symtab_key_t key;
struct ast_node * decl_node;
tok_t tok;
} syms;

View File

@ -1,8 +1,6 @@
#include "../ast.h"
#include "../parser.h"
#include "../symtab/symtab.h"
#ifndef BLOCK_MAX_NODE
#define BLOCK_MAX_NODE (1024)

View File

@ -1,6 +1,5 @@
#include "../ast.h"
#include "../parser.h"
#include "../symtab/symtab.h"
/**
* 0 false
@ -49,7 +48,10 @@ ast_node_t* parse_decl_val(parser_t* parser) {
node->decl_val.type = type_node;
node->decl_val.name = name_node;
node->type = NT_DECL_VAR;
symtab_add_symbol(parser->symtab, name_node->syms.tok.val.str, node, 0);
type_node->syms.key.uid = parser->symtab->cur_scope->uid;
type_node->syms.key.strp_name = name_node->syms.tok.val.str;
symtab_add(parser->symtab, &type_node->syms.key, node);
ttype = peek_tok_type(tokbuf);
if (ttype == TOKEN_ASSIGN) {

View File

@ -1,6 +1,5 @@
#include "../ast.h"
#include "../parser.h"
#include "../symtab/symtab.h"
// Copy from `CParse`
/**
@ -253,11 +252,10 @@ static ast_node_t* parse_call(tok_stream_t* tokbuf, symtab_t *symtab, ast_node_t
}
pop_tok(tokbuf); // 跳过 ')'
const char* name = ident->syms.tok.val.str;
ast_node_t* sym = symtab_lookup_symbol(symtab, name);
ast_node_t* sym = symtab_get(symtab, &ident->syms.key);
// TODO check func is match
if (sym == NULL || sym->type != NT_DECL_FUNC) {
LOG_ERROR("function not decl %s", name);
LOG_FATAL("function not decl %s", ident->syms.key.strp_name);
}
node->call.name = ident;
node->call.func_decl = sym;
@ -345,10 +343,12 @@ static ast_node_t *parse_primary_expression(tok_stream_t* tokbuf, symtab_t *symt
case TOKEN_IDENT:
node = expect_pop_ident(tokbuf);
cc_tktype_t ttype = peek_tok_type(tokbuf);
node->syms.key.uid = 0;
node->syms.key.strp_name = tok->val.str;
if (ttype == TOKEN_L_PAREN) {
node = parse_call(tokbuf, symtab, node);
} else {
void *sym = symtab_lookup_symbol(symtab, tok->val.str);
void *sym = symtab_get(symtab, &node->syms.key);
if (sym == NULL) {
LOG_ERROR("undefined symbol but use %s", tok->val.str);
}

View File

@ -1,6 +1,5 @@
#include "../ast.h"
#include "../parser.h"
#include "../symtab/symtab.h"
#ifndef FUNC_PARAM_CACHE_SIZE
#define FUNC_PARAM_CACHE_SIZE 32 // 合理初始值可覆盖99%常见情况
@ -40,7 +39,10 @@ static void parse_params(parser_t* parser, tok_stream_t* cache, ast_node_t* node
node->decl_val.expr_stmt = NULL;
node->decl_val.data = NULL;
vector_push(params->params.params, node);
symtab_add_symbol(parser->symtab, id_node->syms.tok.val.str, node, 0);
id_node->syms.key.uid = parser->symtab->cur_scope->uid;
id_node->syms.key.strp_name = id_node->syms.tok.val.str;
symtab_add(parser->symtab, &id_node->syms.key, node);
break;
case TOKEN_L_PAREN: {
depth++;
@ -129,7 +131,11 @@ void parse_func(parser_t* parser) {
ast_type_t type = check_is_func_decl(&(parser->tokbuf), &cache);
ast_node_t* prev = symtab_add_symbol(parser->symtab, func_name, decl, 1);
name_node->syms.key.uid = parser->symtab->cur_scope->uid;
name_node->syms.key.strp_name = func_name;
ast_node_t* prev = symtab_get(parser->symtab, &name_node->syms.key);
// TODO Change something
if (prev != NULL) {
if (prev->type != NT_DECL_FUNC) {
LOG_ERROR("the symbol duplicate old is %d, new is func", prev->type);
@ -143,6 +149,7 @@ void parse_func(parser_t* parser) {
}
return;
}
symtab_add(parser->symtab, &name_node->syms.key, decl);
vector_push(parser->root->root.children, decl);
if (type == NT_DECL_FUNC) {
return;

View File

@ -2,7 +2,7 @@
#include "parser.h"
#include "type.h"
void init_parser(parser_t* parser, lexer_t* lexer, symtab_t* symtab) {
void init_parser(parser_t* parser, cc_lexer_t* lexer, symtab_t* symtab) {
init_lib_core();
parser->cur_node = NULL;

View File

@ -2,23 +2,22 @@
#define __PARSER_H__
#include "../lexer/lexer.h"
#include <lib/utils/symtab/symtab.h>
#include "ast.h"
typedef struct lexer lexer_t;
typedef struct symtab symtab_t;
#define PARSER_MAX_TOKEN_QUEUE 16
typedef struct parser {
ast_node_t* root;
ast_node_t* cur_node;
lexer_t* lexer;
cc_lexer_t* lexer;
symtab_t* symtab;
tok_stream_t tokbuf;
tok_t TokenBuffer[PARSER_MAX_TOKEN_QUEUE];
int err_level;
} parser_t;
void init_parser(parser_t* parser, lexer_t* lexer, symtab_t* symtab);
void init_parser(parser_t* parser, cc_lexer_t* lexer, symtab_t* symtab);
void run_parser(parser_t* parser);
#endif

View File

@ -1,4 +0,0 @@
extern int _print_str(const char* str);
int main(void) {
_print_str("Hello, world!\n");
}

View File

@ -1,14 +0,0 @@
// int __print_str(char* str);
int f(void);
int main(void) {
int a;
// f();
// a = 1 + 2 * 3 + 4;
// __print_str("Hello, world!\n");
a = 3 - f() * (3 + 2) % 6;
// 测试用例:
// if (a) if (2) 3; else b;
// 是否正确解析为 if (a) { if (b) c else d }
}

View File

@ -1,37 +0,0 @@
#include "../parser.h"
#include "../ast/ast.h"
#include "../symtab/symtab.h"
#include <stdio.h>
// gcc -g ../parser.c ../../lexer/lexer.c ../ast/ast.c ../ast/block.c ../ast/decl.c ../ast/expr.c ../ast/func.c ../ast/program.c ../ast/stmt.c ../ast/term.c ../symtab/hashmap.c ../symtab/scope.c ../symtab/symtab.c test_parser.c -o test_parser
// gcc -g test_parser.c -L../.. -lfrontend -o test_parser
int main(int argc, char** argv) {
init_lib_core();
const char* file_name = "test_file.c";
if (argc == 2) {
file_name = argv[1];
}
FILE* fp = fopen(file_name, "r");
if (fp == NULL) {
perror("open file failed");
return 1;
}
printf("open file success\n");
lexer_t lexer;
strpool_t strpool;
init_strpool(&strpool);
init_lexer(&lexer, file_name, fp, (lexer_sread_fn)fread_s, &strpool);
struct SymbolTable symtab;
init_symtab(&symtab);
struct parser parser;
init_parser(&parser, &lexer, &symtab);
parse_prog(&parser);
printf("parse_end\n");
pnt_ast(parser.root, 0);
return 0;
}