stage2 更多的工具库
This commit is contained in:
zzy committed
1 parent
4141e7d574
commit
16a1f030d4
15 files changed
+1914
-109
No files matched your search
@@ -0,0 +1,251 @@
|
||||
// Expect retcode = 0
|
||||
// UTF-8 工具：解码/编码、校验、码点数、以及 UTF-8 字节偏移 <-> UTF-16 code unit 偏移
|
||||
// （LSP 位置基于 UTF-16）。严格校验过长编码与代理区。
|
||||
|
||||
type .{ .Error, .Status, .make } = @import("error.spl");
|
||||
type .{ .ByteBuffer } = @import("byte_buffer.spl");
|
||||
|
||||
// 由首字节推断序列长度；非法/续字节返回 0。
|
||||
fn seq_len(b: u8): usize {
|
||||
if b < 0x80 { ret 1; }
|
||||
if b < 0xC0 { ret 0; } // 续字节
|
||||
if b < 0xE0 { ret 2; }
|
||||
if b < 0xF0 { ret 3; }
|
||||
if b < 0xF8 { ret 4; }
|
||||
ret 0;
|
||||
}
|
||||
|
||||
// 从 index 解码一个码点到 out_cp，返回消耗字节数；非法返回 0。
|
||||
fn decode(data: *u8, len: usize, index: usize, out_cp: *u32): usize {
|
||||
if index >= len { ret 0; }
|
||||
var b0 = data[index];
|
||||
|
||||
if b0 < 0x80 {
|
||||
out_cp.* = b0 as u32;
|
||||
ret 1;
|
||||
}
|
||||
if b0 < 0xC2 { ret 0; } // 续字节或过长编码的前导
|
||||
|
||||
if b0 < 0xE0 {
|
||||
if index + 1 >= len { ret 0; }
|
||||
var b1 = data[index + 1];
|
||||
if (b1 & 0xC0) != 0x80 { ret 0; }
|
||||
out_cp.* = (((b0 & 0x1F) as u32) << 6) | ((b1 & 0x3F) as u32);
|
||||
ret 2;
|
||||
}
|
||||
if b0 < 0xF0 {
|
||||
if index + 2 >= len { ret 0; }
|
||||
var b1 = data[index + 1];
|
||||
var b2 = data[index + 2];
|
||||
if (b1 & 0xC0) != 0x80 || (b2 & 0xC0) != 0x80 { ret 0; }
|
||||
var cp = (((b0 & 0x0F) as u32) << 12) | (((b1 & 0x3F) as u32) << 6) | ((b2 & 0x3F) as u32);
|
||||
if cp < 0x800 { ret 0; } // 过长
|
||||
if cp >= 0xD800 && cp <= 0xDFFF { ret 0; } // 代理区
|
||||
out_cp.* = cp;
|
||||
ret 3;
|
||||
}
|
||||
if b0 < 0xF5 {
|
||||
if index + 3 >= len { ret 0; }
|
||||
var b1 = data[index + 1];
|
||||
var b2 = data[index + 2];
|
||||
var b3 = data[index + 3];
|
||||
if (b1 & 0xC0) != 0x80 || (b2 & 0xC0) != 0x80 || (b3 & 0xC0) != 0x80 { ret 0; }
|
||||
var cp = (((b0 & 0x07) as u32) << 18) | (((b1 & 0x3F) as u32) << 12) |
|
||||
(((b2 & 0x3F) as u32) << 6) | ((b3 & 0x3F) as u32);
|
||||
if cp < 0x10000 || cp > 0x10FFFF { ret 0; }
|
||||
out_cp.* = cp;
|
||||
ret 4;
|
||||
}
|
||||
ret 0;
|
||||
}
|
||||
|
||||
fn valid(data: *u8, len: usize): bool {
|
||||
var i: usize = 0;
|
||||
var cp: u32 = 0;
|
||||
while i < len {
|
||||
var n = decode(data, len, i, &cp);
|
||||
if n == 0 { ret false; }
|
||||
i = i + n;
|
||||
}
|
||||
ret true;
|
||||
}
|
||||
|
||||
fn encode_into(out: *ByteBuffer, cp: u32): Status {
|
||||
if cp < 0x80 {
|
||||
ret ByteBuffer.append_byte(out, cp as u8);
|
||||
}
|
||||
if cp < 0x800 {
|
||||
var b0 = (0xC0 | ((cp >> 6) as u8));
|
||||
var b1 = (0x80 | ((cp & 0x3F) as u8));
|
||||
match ByteBuffer.append_byte(out, b0) { .Ok => {}, .Err(e) => { ret .Err(e); } }
|
||||
ret ByteBuffer.append_byte(out, b1);
|
||||
}
|
||||
if cp >= 0xD800 && cp <= 0xDFFF { ret .Err(make(.InvalidArgument)); }
|
||||
if cp < 0x10000 {
|
||||
var b0 = (0xE0 | ((cp >> 12) as u8));
|
||||
var b1 = (0x80 | (((cp >> 6) & 0x3F) as u8));
|
||||
var b2 = (0x80 | ((cp & 0x3F) as u8));
|
||||
match ByteBuffer.append_byte(out, b0) { .Ok => {}, .Err(e) => { ret .Err(e); } }
|
||||
match ByteBuffer.append_byte(out, b1) { .Ok => {}, .Err(e) => { ret .Err(e); } }
|
||||
ret ByteBuffer.append_byte(out, b2);
|
||||
}
|
||||
if cp <= 0x10FFFF {
|
||||
var b0 = (0xF0 | ((cp >> 18) as u8));
|
||||
var b1 = (0x80 | (((cp >> 12) & 0x3F) as u8));
|
||||
var b2 = (0x80 | (((cp >> 6) & 0x3F) as u8));
|
||||
var b3 = (0x80 | ((cp & 0x3F) as u8));
|
||||
match ByteBuffer.append_byte(out, b0) { .Ok => {}, .Err(e) => { ret .Err(e); } }
|
||||
match ByteBuffer.append_byte(out, b1) { .Ok => {}, .Err(e) => { ret .Err(e); } }
|
||||
match ByteBuffer.append_byte(out, b2) { .Ok => {}, .Err(e) => { ret .Err(e); } }
|
||||
ret ByteBuffer.append_byte(out, b3);
|
||||
}
|
||||
ret .Err(make(.InvalidArgument));
|
||||
}
|
||||
|
||||
fn count_codepoints(data: *u8, len: usize): usize {
|
||||
var i: usize = 0;
|
||||
var n: usize = 0;
|
||||
var cp: u32 = 0;
|
||||
while i < len {
|
||||
var used = decode(data, len, i, &cp);
|
||||
if used == 0 { ret n; }
|
||||
i = i + used;
|
||||
n = n + 1;
|
||||
}
|
||||
ret n;
|
||||
}
|
||||
|
||||
// 码点占用的 UTF-16 code unit 数：BMP 为 1，补充平面为 2。
|
||||
fn utf16_units(data: *u8, len: usize): usize {
|
||||
var i: usize = 0;
|
||||
var n: usize = 0;
|
||||
var cp: u32 = 0;
|
||||
while i < len {
|
||||
var used = decode(data, len, i, &cp);
|
||||
if used == 0 { ret n; }
|
||||
if cp >= 0x10000 { n = n + 2; } else { n = n + 1; }
|
||||
i = i + used;
|
||||
}
|
||||
ret n;
|
||||
}
|
||||
|
||||
// 字节偏移 -> UTF-16 偏移（夹取到 [0, utf16_units]）。
|
||||
fn byte_to_utf16(data: *u8, len: usize, byte_off: usize): usize {
|
||||
var i: usize = 0;
|
||||
var n: usize = 0;
|
||||
var cp: u32 = 0;
|
||||
while i < len && i < byte_off {
|
||||
var used = decode(data, len, i, &cp);
|
||||
if used == 0 { ret n; }
|
||||
if cp >= 0x10000 { n = n + 2; } else { n = n + 1; }
|
||||
i = i + used;
|
||||
}
|
||||
ret n;
|
||||
}
|
||||
|
||||
// UTF-16 偏移 -> 字节偏移（夹取；落在码点内部时返回该码点起始字节）。
|
||||
fn utf16_to_byte(data: *u8, len: usize, utf16_off: usize): usize {
|
||||
var i: usize = 0;
|
||||
var n: usize = 0;
|
||||
var cp: u32 = 0;
|
||||
while i < len {
|
||||
if n >= utf16_off { ret i; }
|
||||
var used = decode(data, len, i, &cp);
|
||||
if used == 0 { ret i; }
|
||||
var units: usize = 1;
|
||||
if cp >= 0x10000 { units = 2; }
|
||||
if n + units > utf16_off { ret i; }
|
||||
n = n + units;
|
||||
i = i + used;
|
||||
}
|
||||
ret i;
|
||||
}
|
||||
|
||||
// ---- 测试辅助 ----
|
||||
|
||||
fn expect_ok(result: Status) {
|
||||
match result {
|
||||
.Ok => {},
|
||||
.Err(error) => { @assert(false, "unexpected utf8 error"); },
|
||||
}
|
||||
}
|
||||
|
||||
fn test_decode_encode_roundtrip() {
|
||||
var out = ByteBuffer.new();
|
||||
expect_ok(encode_into(&out, 0x41)); // 'A'
|
||||
expect_ok(encode_into(&out, 0xE9)); // é
|
||||
expect_ok(encode_into(&out, 0x20AC)); // €
|
||||
expect_ok(encode_into(&out, 0x1F600)); // 😀
|
||||
var bytes = ByteBuffer.as_slice(out);
|
||||
@assert(bytes.len == 1 + 2 + 3 + 4, "encoded length");
|
||||
@assert(bytes.ptr[0] == 0x41, "A");
|
||||
@assert(bytes.ptr[1] == 0xC3 && bytes.ptr[2] == 0xA9, "e-acute");
|
||||
@assert(bytes.ptr[3] == 0xE2 && bytes.ptr[4] == 0x82 && bytes.ptr[5] == 0xAC, "euro");
|
||||
@assert(bytes.ptr[6] == 0xF0 && bytes.ptr[9] == 0x80, "emoji head/tail");
|
||||
|
||||
@assert(valid(bytes.ptr, bytes.len), "valid");
|
||||
@assert(count_codepoints(bytes.ptr, bytes.len) == 4, "codepoints");
|
||||
@assert(utf16_units(bytes.ptr, bytes.len) == 1 + 1 + 1 + 2, "utf16 units");
|
||||
|
||||
var cp: u32 = 0;
|
||||
@assert(decode(bytes.ptr, bytes.len, 0, &cp) == 1 && cp == 0x41, "dec A");
|
||||
@assert(decode(bytes.ptr, bytes.len, 1, &cp) == 2 && cp == 0xE9, "dec e");
|
||||
@assert(decode(bytes.ptr, bytes.len, 3, &cp) == 3 && cp == 0x20AC, "dec euro");
|
||||
@assert(decode(bytes.ptr, bytes.len, 6, &cp) == 4 && cp == 0x1F600, "dec emoji");
|
||||
ByteBuffer.drop(&out);
|
||||
}
|
||||
|
||||
fn test_invalid() {
|
||||
var bad1: [2]u8 = .[0xC0, 0x80]; // 过长
|
||||
var bad2: [3]u8 = .[0xED, 0xA0, 0x80]; // 代理区
|
||||
var bad3: [2]u8 = .[0xC3, 0x28]; // 坏续字节
|
||||
var bad4: [1]u8 = .[0x80]; // 孤立续字节
|
||||
var bad5: [3]u8 = .[0xE2, 0x82]; // 截断
|
||||
var s1 = bad1[0..2];
|
||||
var s2 = bad2[0..3];
|
||||
var s3 = bad3[0..2];
|
||||
var s4 = bad4[0..1];
|
||||
var s5 = bad5[0..2];
|
||||
@assert(!valid(s1.ptr, s1.len), "overlong");
|
||||
@assert(!valid(s2.ptr, s2.len), "surrogate");
|
||||
@assert(!valid(s3.ptr, s3.len), "bad cont");
|
||||
@assert(!valid(s4.ptr, s4.len), "lone cont");
|
||||
@assert(!valid(s5.ptr, s5.len), "truncated");
|
||||
|
||||
var out = ByteBuffer.new();
|
||||
match encode_into(&out, 0xD800) {
|
||||
.Ok => { @assert(false, "surrogate encode"); },
|
||||
.Err(error) => {},
|
||||
}
|
||||
match encode_into(&out, 0x110000) {
|
||||
.Ok => { @assert(false, "out of range encode"); },
|
||||
.Err(error) => {},
|
||||
}
|
||||
ByteBuffer.drop(&out);
|
||||
}
|
||||
|
||||
fn test_offsets() {
|
||||
var out = ByteBuffer.new();
|
||||
expect_ok(encode_into(&out, 0x41)); // byte 0, utf16 0
|
||||
expect_ok(encode_into(&out, 0x1F600)); // bytes 1..4, utf16 1..2
|
||||
expect_ok(encode_into(&out, 0x42)); // byte 5, utf16 3
|
||||
var b = ByteBuffer.as_slice(out);
|
||||
|
||||
@assert(byte_to_utf16(b.ptr, b.len, 0) == 0, "b0");
|
||||
@assert(byte_to_utf16(b.ptr, b.len, 1) == 1, "b1");
|
||||
@assert(byte_to_utf16(b.ptr, b.len, 5) == 3, "b5");
|
||||
@assert(byte_to_utf16(b.ptr, b.len, 6) == 4, "clamp");
|
||||
|
||||
@assert(utf16_to_byte(b.ptr, b.len, 0) == 0, "u0");
|
||||
@assert(utf16_to_byte(b.ptr, b.len, 1) == 1, "u1");
|
||||
@assert(utf16_to_byte(b.ptr, b.len, 2) == 1, "u2 stays in emoji");
|
||||
@assert(utf16_to_byte(b.ptr, b.len, 3) == 5, "u3");
|
||||
@assert(utf16_to_byte(b.ptr, b.len, 99) == 6, "clamp");
|
||||
ByteBuffer.drop(&out);
|
||||
}
|
||||
|
||||
fn main() {
|
||||
test_decode_encode_roundtrip();
|
||||
test_invalid();
|
||||
test_offsets();
|
||||
}
|
||||
Reference in new issue
Block a user