init basic

ccompiler/frontend/lexer/lexer.h

@@ -0,0 +1,40 @@
+#ifndef __LEXER_H__
+#define __LEXER_H__
+
+#include "token.h"
+#define LEXER_MAX_TOKEN_SIZE 63
+#define LEXER_BUFFER_SIZE 4095
+
+typedef int (*lexer_sread_fn)(void *dst_buf, int dst_size,
+        int elem_size, int count, void *stream);
+
+struct Lexer {
+    int line;
+    int index;
+    // const char current_file_name[LEXER_BUFFER_SIZE+1];
+
+    unsigned char* cur_ptr; // 当前扫描的字符，但是还没有开始扫描
+    unsigned char* end_ptr; // 缓冲区最后一个字符的下一个位置
+    char buffer[LEXER_BUFFER_SIZE+1];
+
+    lexer_sread_fn sread;
+    void* stream;
+};
+
+struct Token {
+    enum TokenType type;
+    struct TokenConstant constant;
+};
+
+void init_lexer(struct Lexer* lexer, const char* file_name, void* stream,
+    lexer_sread_fn sread);
+
+// 
+void get_token(struct Lexer* lexer, struct Token* token);
+
+// get_token maybe got invalid (with parser as TOKEN_FLUSH)
+void get_valid_token(struct Lexer* lexer, struct Token* token);
+
+const char* get_token_name(enum TokenType token);
+
+#endif