zzy/scc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(lex_parser, pprocessor): replace consume with next and remove stream resets

Browse Source 
- Replace `scc_probe_stream_consume` with `scc_probe_stream_next` for consistent stream advancement
- Remove redundant `scc_probe_stream_reset` calls before peeking, as `next` and `peek` handle state
- Update `scc_cstring_new` to `scc_cstring_create` and `scc_pos_init` to `scc_pos_create` for naming consistency
- Change `scc_pp_macro_get` parameter to `const scc_cstring_t*` for better const-correctness
- Improves code clarity and maintains proper stream position tracking
This commit is contained in:
zzy
2025-12-28 10:49:29 +08:00
parent 07f5d9331b
commit 09f4ac8de0
20 changed files with 445 additions and 262 deletions
Download Patch File Download Diff File Expand all files Collapse all files

73
libs/lex_parser/src/lex_parser.c
View File

@@ -2,17 +2,17 @@
void scc_lex_parse_skip_endline(scc_probe_stream_t *input, scc_pos_t *pos) {
Assert(input != null && pos != null);
scc_probe_stream_reset(input);
// scc_probe_stream_reset(input);
int ch = scc_probe_stream_peek(input);
if (ch == '\r') {
scc_probe_stream_consume(input);
scc_probe_stream_next(input);
ch = scc_probe_stream_peek(input);
if (ch == '\n') {
scc_probe_stream_consume(input);
scc_probe_stream_next(input);
}
scc_pos_next_line(pos);
} else if (ch == '\n') {
scc_probe_stream_consume(input);
scc_probe_stream_next(input);
scc_pos_next_line(pos);
} else {
LOG_WARN("not a newline character");
@@ -60,7 +60,7 @@ static inline int got_simple_escape(int ch) {
void scc_lex_parse_skip_line(scc_probe_stream_t *input, scc_pos_t *pos) {
scc_probe_stream_t *stream = input;
Assert(stream != null && pos != null);
scc_probe_stream_reset(stream);
// scc_probe_stream_reset(stream);
while (1) {
int ch = scc_probe_stream_peek(stream);
@@ -73,7 +73,7 @@ void scc_lex_parse_skip_line(scc_probe_stream_t *input, scc_pos_t *pos) {
scc_lex_parse_skip_endline(stream, pos);
return;
} else {
scc_probe_stream_consume(stream);
scc_probe_stream_next(stream);
scc_pos_next(pos);
}
}
@@ -84,18 +84,18 @@ void scc_lex_parse_skip_block_comment(scc_probe_stream_t *input,
scc_probe_stream_t *stream = input;
Assert(stream != null && pos != null);
int ch;
scc_probe_stream_reset(stream);
ch = scc_probe_stream_consume(stream);
// scc_probe_stream_reset(stream);
ch = scc_probe_stream_next(stream);
scc_pos_next(pos);
// FIXME Assertion
Assert(ch == '/');
ch = scc_probe_stream_consume(stream);
ch = scc_probe_stream_next(stream);
scc_pos_next(pos);
Assert(ch == '*');
// all ready match `/*`
while (1) {
scc_probe_stream_reset(stream);
// scc_probe_stream_reset(stream);
ch = scc_probe_stream_peek(stream);
if (ch == scc_stream_eof) {
@@ -107,12 +107,12 @@ void scc_lex_parse_skip_block_comment(scc_probe_stream_t *input,
scc_lex_parse_skip_endline(stream, pos);
continue;
}
scc_probe_stream_consume(stream);
scc_probe_stream_next(stream);
scc_pos_next(pos);
if (ch == '*') {
ch = scc_probe_stream_peek(stream);
if (ch == '/') {
scc_probe_stream_consume(stream);
scc_probe_stream_next(stream);
scc_pos_next(pos);
return;
}
@@ -123,7 +123,7 @@ void scc_lex_parse_skip_block_comment(scc_probe_stream_t *input,
void scc_lex_parse_skip_whitespace(scc_probe_stream_t *input, scc_pos_t *pos) {
scc_probe_stream_t *stream = input;
Assert(stream != null && pos != null);
scc_probe_stream_reset(stream);
// scc_probe_stream_reset(stream);
while (1) {
int ch = scc_probe_stream_peek(stream);
@@ -131,7 +131,7 @@ void scc_lex_parse_skip_whitespace(scc_probe_stream_t *input, scc_pos_t *pos) {
return;
}
scc_probe_stream_consume(stream);
scc_probe_stream_next(stream);
scc_pos_next(pos);
}
}
@@ -143,7 +143,7 @@ static inline cbool _lex_parse_uint(scc_probe_stream_t *input, scc_pos_t *pos,
return false;
}
Assert(base == 2 || base == 8 || base == 10 || base == 16);
scc_probe_stream_reset(input);
// scc_probe_stream_reset(input);
int ch, tmp;
usize n = 0;
usize offset = pos->offset;
@@ -167,7 +167,7 @@ static inline cbool _lex_parse_uint(scc_probe_stream_t *input, scc_pos_t *pos,
return false;
}
scc_probe_stream_consume(input);
scc_probe_stream_next(input);
scc_pos_next(pos);
n = n * base + tmp;
// TODO number overflow
@@ -191,8 +191,8 @@ static inline cbool _lex_parse_uint(scc_probe_stream_t *input, scc_pos_t *pos,
int scc_lex_parse_char(scc_probe_stream_t *input, scc_pos_t *pos) {
scc_probe_stream_t *stream = input;
Assert(stream != null && pos != null);
scc_probe_stream_reset(stream);
int ch = scc_probe_stream_peek(stream);
int ch = scc_probe_stream_next(stream);
scc_pos_next(pos);
int ret = scc_stream_eof;
if (ch == scc_stream_eof) {
@@ -202,17 +202,16 @@ int scc_lex_parse_char(scc_probe_stream_t *input, scc_pos_t *pos) {
LOG_WARN("Unexpected character '%c' at begin", ch);
goto ERR;
}
scc_probe_stream_consume(stream);
scc_pos_next(pos);
// scc_probe_stream_next(stream);
ch = scc_probe_stream_consume(stream);
ch = scc_probe_stream_next(stream);
scc_pos_next(pos);
if (ch == scc_stream_eof) {
LOG_WARN("Unexpected EOF at middle");
goto ERR;
} else if (ch == '\\') {
ch = scc_probe_stream_consume(stream);
ch = scc_probe_stream_next(stream);
scc_pos_next(pos);
if (ch == '0') {
// 数字转义序列
@@ -238,7 +237,7 @@ int scc_lex_parse_char(scc_probe_stream_t *input, scc_pos_t *pos) {
} else {
ret = ch;
}
if ((ch = scc_probe_stream_consume(stream)) != '\'') {
if ((ch = scc_probe_stream_next(stream)) != '\'') {
LOG_ERROR("Unclosed character literal '%c' at end, expect `'`", ch);
scc_pos_next(pos);
goto ERR;
@@ -262,7 +261,7 @@ cbool scc_lex_parse_string(scc_probe_stream_t *input, scc_pos_t *pos,
scc_cstring_t *output) {
scc_probe_stream_t *stream = input;
Assert(stream != null && pos != null && output != null);
scc_probe_stream_reset(stream);
// scc_probe_stream_reset(stream);
int ch = scc_probe_stream_peek(stream);
Assert(scc_cstring_is_empty(output));
@@ -273,7 +272,7 @@ cbool scc_lex_parse_string(scc_probe_stream_t *input, scc_pos_t *pos,
LOG_WARN("Unexpected character '%c' at begin", ch);
goto ERR;
}
scc_probe_stream_consume(stream);
scc_probe_stream_next(stream);
scc_pos_next(pos);
scc_cstring_t str = scc_cstring_from_cstr("");
@@ -288,8 +287,8 @@ cbool scc_lex_parse_string(scc_probe_stream_t *input, scc_pos_t *pos,
goto ERR;
} else if (ch == '\\') {
// TODO bad practice and maybe bugs here
scc_probe_stream_consume(stream);
ch = scc_probe_stream_consume(stream);
scc_probe_stream_next(stream);
ch = scc_probe_stream_next(stream);
int val = got_simple_escape(ch);
if (val == -1) {
LOG_ERROR("Invalid escape character it is \\%c [%d]", ch, ch);
@@ -298,12 +297,12 @@ cbool scc_lex_parse_string(scc_probe_stream_t *input, scc_pos_t *pos,
continue;
}
} else if (ch == '"') {
scc_probe_stream_consume(stream);
scc_probe_stream_next(stream);
scc_pos_next(pos);
break;
}
scc_probe_stream_consume(stream);
scc_probe_stream_next(stream);
scc_pos_next(pos);
scc_cstring_append_ch(&str, ch);
}
@@ -328,7 +327,7 @@ cbool scc_lex_parse_number(scc_probe_stream_t *input, scc_pos_t *pos,
usize *output) {
scc_probe_stream_t *stream = input;
Assert(stream != null && pos != null && output != null);
scc_probe_stream_reset(stream);
// scc_probe_stream_reset(stream);
int ch = scc_probe_stream_peek(stream);
int base = 10; // 默认十进制
@@ -339,7 +338,7 @@ cbool scc_lex_parse_number(scc_probe_stream_t *input, scc_pos_t *pos,
if (ch == '0') {
// 消费 '0'
scc_probe_stream_consume(stream);
scc_probe_stream_next(stream);
scc_pos_next(pos);
// 查看下一个字符
@@ -347,12 +346,12 @@ cbool scc_lex_parse_number(scc_probe_stream_t *input, scc_pos_t *pos,
if (ch == 'x' || ch == 'X') {
// 十六进制
base = 16;
scc_probe_stream_consume(stream);
scc_probe_stream_next(stream);
scc_pos_next(pos);
} else if (ch == 'b' || ch == 'B') {
// 二进制 (C23扩展)
base = 2;
scc_probe_stream_consume(stream);
scc_probe_stream_next(stream);
scc_pos_next(pos);
} else if (ch >= '0' && ch <= '7') {
// 八进制
@@ -375,7 +374,7 @@ cbool scc_lex_parse_number(scc_probe_stream_t *input, scc_pos_t *pos,
}
// 解析整数部分
scc_probe_stream_reset(stream);
// scc_probe_stream_reset(stream);
usize n;
if (_lex_parse_uint(stream, pos, base, &n) == false) {
// 如果没有匹配任何数字，但输入是 '0'，已经处理过了
@@ -384,7 +383,7 @@ cbool scc_lex_parse_number(scc_probe_stream_t *input, scc_pos_t *pos,
// 单个数字的情况，例如 "1"
// 我们需要消费这个数字并返回它的值
if (ch >= '1' && ch <= '9') {
scc_probe_stream_consume(stream);
scc_probe_stream_next(stream);
scc_pos_next(pos);
*output = ch - '0';
return true;
@@ -412,7 +411,7 @@ cbool scc_lex_parse_identifier(scc_probe_stream_t *input, scc_pos_t *pos,
Assert(input != null && pos != null && output != null);
Assert(scc_cstring_is_empty(output));
scc_probe_stream_t *stream = input;
scc_probe_stream_reset(stream);
// scc_probe_stream_reset(stream);
int ch = scc_probe_stream_peek(stream);
if (ch == scc_stream_eof) {
@@ -420,7 +419,7 @@ cbool scc_lex_parse_identifier(scc_probe_stream_t *input, scc_pos_t *pos,
} else if (scc_lex_parse_is_identifier_prefix(ch)) {
while (1) {
scc_cstring_append_ch(output, ch);
scc_probe_stream_consume(stream);
scc_probe_stream_next(stream);
scc_pos_next(pos);
ch = scc_probe_stream_peek(stream);
if ((ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') ||

8
libs/lex_parser/tests/test_char.c
View File

@@ -4,12 +4,16 @@
cbool check_char(const char *str, int expect, int *output) {
log_set_level(&__default_logger_root, 0);
scc_pos_t pos = scc_pos_init();
scc_pos_t pos = scc_pos_create();
scc_mem_probe_stream_t mem_stream;
scc_probe_stream_t *stream =
scc_mem_probe_stream_init(&mem_stream, str, scc_strlen(str), false);
*output = scc_lex_parse_char(stream, &pos);
return *output == expect;
cbool ret1 = *output == expect;
scc_probe_stream_reset(stream);
*output = scc_lex_parse_char(stream, &pos);
cbool ret2 = *output == expect;
return ret1 && ret2;
}
#define CHECK_CHAR_VALID(str, expect) \

6
libs/lex_parser/tests/test_identifier.c
View File

@@ -5,7 +5,7 @@
cbool check_identifier(const char *str, const char *expect,
scc_cstring_t *output) {
log_set_level(&__default_logger_root, 0);
scc_pos_t pos = scc_pos_init();
scc_pos_t pos = scc_pos_create();
scc_mem_probe_stream_t mem_stream;
scc_probe_stream_t *stream =
scc_mem_probe_stream_init(&mem_stream, str, scc_strlen(str), false);
@@ -19,7 +19,7 @@ cbool check_identifier(const char *str, const char *expect,
#define CHECK_IDENTIFIER_VALID(str, expect) \
do { \
scc_cstring_t _output = scc_cstring_new(); \
scc_cstring_t _output = scc_cstring_create(); \
cbool ret = check_identifier(str, expect, &_output); \
TEST_CHECK(ret == true); \
TEST_CHECK(strcmp(_output.data, expect) == 0); \
@@ -28,7 +28,7 @@ cbool check_identifier(const char *str, const char *expect,
#define CHECK_IDENTIFIER_INVALID(str) \
do { \
scc_cstring_t _output = scc_cstring_new(); \
scc_cstring_t _output = scc_cstring_create(); \
cbool ret = check_identifier(str, NULL, &_output); \
TEST_CHECK(ret == false); \
scc_cstring_free(&_output); \

2
libs/lex_parser/tests/test_number.c
View File

@@ -5,7 +5,7 @@ cbool check(const char *str, usize expect, usize *output) {
// TODO maybe have other logger
(void)(expect);
log_set_level(&__default_logger_root, 0);
scc_pos_t pos = scc_pos_init();
scc_pos_t pos = scc_pos_create();
scc_mem_probe_stream_t mem_stream;
scc_probe_stream_t *stream =
scc_mem_probe_stream_init(&mem_stream, str, scc_strlen(str), false);

3
libs/lex_parser/tests/test_skip_block_comment.c
View File

@@ -4,12 +4,13 @@
void check_skip_block_comment(const char *str, const char *expect_remaining) {
log_set_level(&__default_logger_root, 0);
scc_pos_t pos = scc_pos_init();
scc_pos_t pos = scc_pos_create();
scc_mem_probe_stream_t mem_stream;
scc_probe_stream_t *stream =
scc_mem_probe_stream_init(&mem_stream, str, scc_strlen(str), false);
scc_lex_parse_skip_block_comment(stream, &pos);
scc_probe_stream_sync(stream);
// Check remaining content
char buffer[256] = {0};

3
libs/lex_parser/tests/test_skip_line.c
View File

@@ -4,12 +4,13 @@
void check_skip_line(const char *str, const char *expect_remaining) {
log_set_level(&__default_logger_root, 0);
scc_pos_t pos = scc_pos_init();
scc_pos_t pos = scc_pos_create();
scc_mem_probe_stream_t mem_stream;
scc_probe_stream_t *stream =
scc_mem_probe_stream_init(&mem_stream, str, scc_strlen(str), false);
scc_lex_parse_skip_line(stream, &pos);
scc_probe_stream_sync(stream);
// Check remaining content
char buffer[256] = {0};

6
libs/lex_parser/tests/test_string.c
View File

@@ -4,7 +4,7 @@
cbool check_string(const char *str, const char *expect, scc_cstring_t *output) {
log_set_level(&__default_logger_root, 0);
scc_pos_t pos = scc_pos_init();
scc_pos_t pos = scc_pos_create();
scc_mem_probe_stream_t mem_stream;
scc_probe_stream_t *stream =
scc_mem_probe_stream_init(&mem_stream, str, scc_strlen(str), false);
@@ -18,7 +18,7 @@ cbool check_string(const char *str, const char *expect, scc_cstring_t *output) {
#define CHECK_STRING_VALID(str, expect) \
do { \
scc_cstring_t _output = scc_cstring_new(); \
scc_cstring_t _output = scc_cstring_create(); \
cbool ret = check_string(str, expect, &_output); \
TEST_CHECK(ret == true); \
TEST_CHECK(strcmp(_output.data, expect) == 0); \
@@ -27,7 +27,7 @@ cbool check_string(const char *str, const char *expect, scc_cstring_t *output) {
#define CHECK_STRING_INVALID(str) \
do { \
scc_cstring_t _output = scc_cstring_new(); \
scc_cstring_t _output = scc_cstring_create(); \
cbool ret = check_string(str, NULL, &_output); \
TEST_CHECK(ret == false); \
scc_cstring_free(&_output); \