refactor(lex_parser): 移除旧的词法解析器实现并更新依赖
移除了 libs/lex_parser 目录下的所有头文件和源文件,包括: - lex_parser.h 和 lex_parser.c 核心解析功能 - 所有测试文件(test_char.c, test_identifier.c, test_number.c, test_skip_block_comment.c, test_skip_line.c, test_string.c) 更新了 lexer 模块的依赖配置,将 lex_parser 替换为 sstream, 同时更新了 lexer.h 中的相关包含头文件和数据结构定义, 简化了 scc_lexer_t 结构体的字段。
This commit is contained in:
51
libs/sstream/src/main.c
Normal file
51
libs/sstream/src/main.c
Normal file
@@ -0,0 +1,51 @@
|
||||
#include "scc_sstream.h"
|
||||
#include <stdio.h>
|
||||
|
||||
int main(int argc, char **argv) {
|
||||
const char *filename = (argc > 1) ? argv[1] : __FILE__; // 默认读取自身
|
||||
scc_sstream_t stream;
|
||||
scc_sstream_ring_t *ring;
|
||||
|
||||
// 初始化
|
||||
if (scc_sstream_init(&stream, filename, 16) != 0) {
|
||||
fprintf(stderr, "Failed to initialize stream for %s\n", filename);
|
||||
return 1;
|
||||
}
|
||||
ring = scc_sstream_ref_ring(&stream);
|
||||
Assert(ring != null);
|
||||
|
||||
printf("Reading file: %s\n", filename);
|
||||
|
||||
scc_sstream_char_t elem;
|
||||
cbool ok;
|
||||
int char_count = 0;
|
||||
int line_count = 0;
|
||||
|
||||
// 循环读取所有字符
|
||||
while (1) {
|
||||
scc_ring_next_consume(*ring, elem, ok);
|
||||
if (!ok)
|
||||
break; // 文件结束或错误
|
||||
|
||||
char_count++;
|
||||
if (elem.character == '\n')
|
||||
line_count++;
|
||||
|
||||
// 打印前 200 个字符的位置信息(避免刷屏)
|
||||
if (char_count <= 200) {
|
||||
printf("char[%d]: '%c' (line %zu, col %zu)\n", char_count,
|
||||
elem.character == '\n' ? ' '
|
||||
: elem.character, // 换行符显示为空格
|
||||
elem.pos.line, elem.pos.col);
|
||||
}
|
||||
}
|
||||
|
||||
printf("\nSummary:\n");
|
||||
printf(" Total characters: %d\n", char_count);
|
||||
printf(" Total lines: %d\n", line_count);
|
||||
|
||||
// 释放资源
|
||||
scc_sstream_drop_ring(ring);
|
||||
scc_sstream_drop(&stream);
|
||||
return 0;
|
||||
}
|
||||
145
libs/sstream/src/scc_sstream.c
Normal file
145
libs/sstream/src/scc_sstream.c
Normal file
@@ -0,0 +1,145 @@
|
||||
#include <scc_sstream.h>
|
||||
|
||||
// 内部扫描函数:从指定位置扫描下一个有效字符
|
||||
static int sstream_scan_at(scc_sstream_t *stream, scc_pos_t scan_pos,
|
||||
scc_pos_t *out_char_pos, scc_pos_t *out_next_pos) {
|
||||
while (1) {
|
||||
if (scan_pos.offset >= stream->len)
|
||||
return -1; // EOF
|
||||
|
||||
scc_pos_t start = scan_pos;
|
||||
char c = stream->src[scan_pos.offset];
|
||||
|
||||
// 处理反斜杠换行
|
||||
if (c == '\\') {
|
||||
usize next_off = scan_pos.offset + 1;
|
||||
if (next_off < stream->len) {
|
||||
char n = stream->src[next_off];
|
||||
if (n == '\n') {
|
||||
// 跳过 '\' 和 '\n'
|
||||
scan_pos.offset += 2;
|
||||
scan_pos.line++;
|
||||
scan_pos.col = 1;
|
||||
continue;
|
||||
} else if (n == '\r' && next_off + 1 < stream->len &&
|
||||
stream->src[next_off + 1] == '\n') {
|
||||
// 跳过 '\' + '\r' + '\n'
|
||||
scan_pos.offset += 3;
|
||||
scan_pos.line++;
|
||||
scan_pos.col = 1;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 处理 \r\n 转换为 \n
|
||||
if (c == '\r') {
|
||||
usize next_off = scan_pos.offset + 1;
|
||||
if (next_off < stream->len && stream->src[next_off] == '\n') {
|
||||
if (out_char_pos)
|
||||
*out_char_pos = start;
|
||||
// 下一个位置:偏移+2,行+1,列=1
|
||||
scan_pos.offset += 2;
|
||||
scan_pos.line++;
|
||||
scan_pos.col = 1;
|
||||
if (out_next_pos)
|
||||
*out_next_pos = scan_pos;
|
||||
return '\n';
|
||||
}
|
||||
}
|
||||
|
||||
// 普通字符(包括单独的 '\n'、'\r' 等)
|
||||
if (out_char_pos)
|
||||
*out_char_pos = start;
|
||||
// 计算下一个位置
|
||||
scan_pos.offset++;
|
||||
if (c == '\n') {
|
||||
scan_pos.line++;
|
||||
scan_pos.col = 1;
|
||||
} else {
|
||||
scan_pos.col++;
|
||||
}
|
||||
if (out_next_pos)
|
||||
*out_next_pos = scan_pos;
|
||||
return c;
|
||||
}
|
||||
}
|
||||
|
||||
// 环形缓冲区填充回调(通过 userdata 获取流对象)
|
||||
static cbool fill_func(scc_sstream_char_t *out, void *userdata) {
|
||||
scc_sstream_t *stream = (scc_sstream_t *)userdata;
|
||||
if (stream->fill_pos.offset >= stream->len)
|
||||
return false; // 已到文件尾
|
||||
|
||||
int ch = sstream_scan_at(stream, stream->fill_pos, &out->pos,
|
||||
&(stream->fill_pos));
|
||||
if (ch == -1)
|
||||
return false;
|
||||
out->character = ch;
|
||||
return true;
|
||||
}
|
||||
|
||||
int scc_sstream_init(scc_sstream_t *stream, const char *fname, int ring_size) {
|
||||
Assert(stream != null && fname != null);
|
||||
scc_file_t file = scc_fopen(fname, SCC_FILE_READ);
|
||||
usize fsize = scc_fsize(file);
|
||||
if (fsize == 0) {
|
||||
LOG_WARN("file size is 0");
|
||||
scc_fclose(file);
|
||||
return 0;
|
||||
}
|
||||
char *buffer = (char *)scc_malloc(fsize);
|
||||
scc_memset(buffer, 0, fsize);
|
||||
usize read_ret = scc_fread(file, buffer, fsize);
|
||||
Assert(read_ret == fsize); /* read bytes assert it */
|
||||
scc_fclose(file);
|
||||
|
||||
scc_sstream_init_by_buffer(stream, buffer, read_ret, 1, ring_size);
|
||||
stream->fname = fname;
|
||||
stream->fill_pos.name = stream->fname;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int scc_sstream_init_by_buffer(scc_sstream_t *stream, const char *buffer,
|
||||
usize len, int owned, int ring_size) {
|
||||
stream->fname = "<buffer>";
|
||||
stream->fill_pos = scc_pos_create();
|
||||
stream->fill_pos.name = stream->fname;
|
||||
stream->src = buffer;
|
||||
stream->len = len;
|
||||
stream->owned_src = owned;
|
||||
|
||||
scc_ring_init(stream->ring, ring_size <= 0 ? 64 : ring_size, fill_func,
|
||||
stream);
|
||||
stream->used = 0;
|
||||
return 0;
|
||||
}
|
||||
|
||||
scc_sstream_ring_t *scc_sstream_ref_ring(scc_sstream_t *stream) {
|
||||
Assert(stream != null);
|
||||
stream->used++;
|
||||
return &stream->ring;
|
||||
}
|
||||
|
||||
void scc_sstream_drop_ring(scc_sstream_ring_t *ring) {
|
||||
Assert(ring != null && ring->userdata != null);
|
||||
scc_sstream_t *stream = (scc_sstream_t *)ring->userdata;
|
||||
if (stream->used > 0) {
|
||||
stream->used--;
|
||||
} else {
|
||||
LOG_WARN("double drop sstream ring");
|
||||
}
|
||||
}
|
||||
|
||||
void scc_sstream_drop(scc_sstream_t *stream) {
|
||||
Assert(stream != null);
|
||||
if (stream->used) {
|
||||
LOG_FATAL("drop sstream must be drop ring before ref [%d]",
|
||||
stream->used);
|
||||
}
|
||||
if (stream->src && stream->owned_src) {
|
||||
scc_free((void *)stream->src);
|
||||
stream->src = null;
|
||||
}
|
||||
scc_ring_free(stream->ring);
|
||||
}
|
||||
Reference in New Issue
Block a user