290 lines
9.9 KiB
Plaintext
290 lines
9.9 KiB
Plaintext
// Expect retcode = 0
|
||
|
||
// UTF-8 工具：解码/编码、校验、码点数、以及 UTF-8 字节偏移 <-> UTF-16 code unit 偏移
|
||
// （LSP 位置基于 UTF-16）。严格校验过长编码与代理区。
|
||
|
||
type .{ .Error, .Status, .make } = @import("error.spl");
|
||
type .{ .ByteBuffer } = @import("byte_buffer.spl");
|
||
|
||
// 解码结果：Ok 携带消耗字节数；明确区分越界/截断/非法，避免 0 哨兵歧义。
|
||
type DecodeResult = enum {
|
||
Ok: usize,
|
||
Eof, // index 已越过输入
|
||
Truncated, // 序列尾部字节不足
|
||
Invalid, // 续字节/过长编码/代理区/超出码点范围
|
||
}
|
||
|
||
// 从 index 解码一个码点到 out_cp；返回 DecodeResult。
|
||
fn decode(data: *u8, len: usize, index: usize, out_cp: *u32): DecodeResult {
|
||
if index >= len { ret .Eof; }
|
||
var b0 = data[index];
|
||
|
||
if b0 < 0x80 {
|
||
out_cp.* = b0 as u32;
|
||
ret .Ok(1);
|
||
}
|
||
if b0 < 0xC2 { ret .Invalid; } // 续字节或过长编码的前导
|
||
|
||
if b0 < 0xE0 {
|
||
if index + 1 >= len { ret .Truncated; }
|
||
var b1 = data[index + 1];
|
||
if (b1 & 0xC0) != 0x80 { ret .Invalid; }
|
||
out_cp.* = (((b0 & 0x1F) as u32) << 6) | ((b1 & 0x3F) as u32);
|
||
ret .Ok(2);
|
||
}
|
||
if b0 < 0xF0 {
|
||
if index + 2 >= len { ret .Truncated; }
|
||
var b1 = data[index + 1];
|
||
var b2 = data[index + 2];
|
||
if (b1 & 0xC0) != 0x80 || (b2 & 0xC0) != 0x80 { ret .Invalid; }
|
||
var cp = (((b0 & 0x0F) as u32) << 12) | (((b1 & 0x3F) as u32) << 6) | ((b2 & 0x3F) as u32);
|
||
if cp < 0x800 { ret .Invalid; } // 过长
|
||
if cp >= 0xD800 && cp <= 0xDFFF { ret .Invalid; } // 代理区
|
||
out_cp.* = cp;
|
||
ret .Ok(3);
|
||
}
|
||
if b0 < 0xF5 {
|
||
if index + 3 >= len { ret .Truncated; }
|
||
var b1 = data[index + 1];
|
||
var b2 = data[index + 2];
|
||
var b3 = data[index + 3];
|
||
if (b1 & 0xC0) != 0x80 || (b2 & 0xC0) != 0x80 || (b3 & 0xC0) != 0x80 { ret .Invalid; }
|
||
var cp = (((b0 & 0x07) as u32) << 18) | (((b1 & 0x3F) as u32) << 12) |
|
||
(((b2 & 0x3F) as u32) << 6) | ((b3 & 0x3F) as u32);
|
||
if cp < 0x10000 || cp > 0x10FFFF { ret .Invalid; }
|
||
out_cp.* = cp;
|
||
ret .Ok(4);
|
||
}
|
||
ret .Invalid;
|
||
}
|
||
|
||
// 测试辅助：把 DecodeResult 归约为“消耗字节数”，非 Ok 视为 0。
|
||
fn decode_used(data: *u8, len: usize, index: usize, out_cp: *u32): usize {
|
||
match decode(data, len, index, out_cp) {
|
||
.Ok(used) => { ret used; },
|
||
.Eof => { ret 0; },
|
||
.Truncated => { ret 0; },
|
||
.Invalid => { ret 0; },
|
||
}
|
||
ret 0;
|
||
}
|
||
|
||
fn valid(data: *u8, len: usize): bool {
|
||
var i: usize = 0;
|
||
var cp: u32 = 0;
|
||
while i < len {
|
||
match decode(data, len, i, &cp) {
|
||
.Ok(used) => { i = i + used; },
|
||
_ => { ret false; },
|
||
}
|
||
}
|
||
ret true;
|
||
}
|
||
|
||
fn encode_into(out: *ByteBuffer, cp: u32): Status {
|
||
if cp < 0x80 {
|
||
ret ByteBuffer.append_byte(out, cp as u8);
|
||
}
|
||
if cp < 0x800 {
|
||
var b0 = (0xC0 | ((cp >> 6) as u8));
|
||
var b1 = (0x80 | ((cp & 0x3F) as u8));
|
||
match ByteBuffer.append_byte(out, b0) { .Ok => {}, .Err(e) => { ret .Err(e); } }
|
||
ret ByteBuffer.append_byte(out, b1);
|
||
}
|
||
if cp >= 0xD800 && cp <= 0xDFFF { ret .Err(make(.InvalidArgument)); }
|
||
if cp < 0x10000 {
|
||
var b0 = (0xE0 | ((cp >> 12) as u8));
|
||
var b1 = (0x80 | (((cp >> 6) & 0x3F) as u8));
|
||
var b2 = (0x80 | ((cp & 0x3F) as u8));
|
||
match ByteBuffer.append_byte(out, b0) { .Ok => {}, .Err(e) => { ret .Err(e); } }
|
||
match ByteBuffer.append_byte(out, b1) { .Ok => {}, .Err(e) => { ret .Err(e); } }
|
||
ret ByteBuffer.append_byte(out, b2);
|
||
}
|
||
if cp <= 0x10FFFF {
|
||
var b0 = (0xF0 | ((cp >> 18) as u8));
|
||
var b1 = (0x80 | (((cp >> 12) & 0x3F) as u8));
|
||
var b2 = (0x80 | (((cp >> 6) & 0x3F) as u8));
|
||
var b3 = (0x80 | ((cp & 0x3F) as u8));
|
||
match ByteBuffer.append_byte(out, b0) { .Ok => {}, .Err(e) => { ret .Err(e); } }
|
||
match ByteBuffer.append_byte(out, b1) { .Ok => {}, .Err(e) => { ret .Err(e); } }
|
||
match ByteBuffer.append_byte(out, b2) { .Ok => {}, .Err(e) => { ret .Err(e); } }
|
||
ret ByteBuffer.append_byte(out, b3);
|
||
}
|
||
ret .Err(make(.InvalidArgument));
|
||
}
|
||
|
||
fn count_codepoints(data: *u8, len: usize): usize {
|
||
var i: usize = 0;
|
||
var n: usize = 0;
|
||
var cp: u32 = 0;
|
||
while i < len {
|
||
match decode(data, len, i, &cp) {
|
||
.Ok(used) => { i = i + used; n = n + 1; },
|
||
_ => { ret n; },
|
||
}
|
||
}
|
||
ret n;
|
||
}
|
||
|
||
// 码点占用的 UTF-16 code unit 数：BMP 为 1，补充平面为 2。
|
||
fn utf16_units(data: *u8, len: usize): usize {
|
||
var i: usize = 0;
|
||
var n: usize = 0;
|
||
var cp: u32 = 0;
|
||
while i < len {
|
||
match decode(data, len, i, &cp) {
|
||
.Ok(used) => {
|
||
if cp >= 0x10000 { n = n + 2; } else { n = n + 1; }
|
||
i = i + used;
|
||
},
|
||
_ => { ret n; },
|
||
}
|
||
}
|
||
ret n;
|
||
}
|
||
|
||
// 字节偏移 -> UTF-16 偏移（夹取到 [0, utf16_units]）。
|
||
fn byte_to_utf16(data: *u8, len: usize, byte_off: usize): usize {
|
||
var i: usize = 0;
|
||
var n: usize = 0;
|
||
var cp: u32 = 0;
|
||
while i < len && i < byte_off {
|
||
match decode(data, len, i, &cp) {
|
||
.Ok(used) => {
|
||
if cp >= 0x10000 { n = n + 2; } else { n = n + 1; }
|
||
i = i + used;
|
||
},
|
||
_ => { ret n; },
|
||
}
|
||
}
|
||
ret n;
|
||
}
|
||
|
||
// UTF-16 偏移 -> 字节偏移（夹取；落在码点内部时返回该码点起始字节）。
|
||
fn utf16_to_byte(data: *u8, len: usize, utf16_off: usize): usize {
|
||
var i: usize = 0;
|
||
var n: usize = 0;
|
||
var cp: u32 = 0;
|
||
while i < len {
|
||
if n >= utf16_off { ret i; }
|
||
var used: usize = 0;
|
||
match decode(data, len, i, &cp) {
|
||
.Ok(value) => { used = value; },
|
||
_ => { ret i; },
|
||
}
|
||
var units: usize = 1;
|
||
if cp >= 0x10000 { units = 2; }
|
||
if n + units > utf16_off { ret i; }
|
||
n = n + units;
|
||
i = i + used;
|
||
}
|
||
ret i;
|
||
}
|
||
|
||
|
||
fn expect_ok(result: Status) {
|
||
match result {
|
||
.Ok => {},
|
||
.Err(error) => { @assert(false, "unexpected utf8 error"); },
|
||
}
|
||
}
|
||
|
||
fn test_decode_encode_roundtrip() {
|
||
var out = ByteBuffer.new();
|
||
defer { ByteBuffer.drop(&out); }
|
||
expect_ok(encode_into(&out, 0x41)); // 'A'
|
||
expect_ok(encode_into(&out, 0xE9)); // é
|
||
expect_ok(encode_into(&out, 0x20AC)); // €
|
||
expect_ok(encode_into(&out, 0x1F600)); // 😀
|
||
var bytes = ByteBuffer.as_slice(out);
|
||
@assert(bytes.len == 1 + 2 + 3 + 4, "encoded length");
|
||
@assert(bytes.ptr[0] == 0x41, "A");
|
||
@assert(bytes.ptr[1] == 0xC3 && bytes.ptr[2] == 0xA9, "e-acute");
|
||
@assert(bytes.ptr[3] == 0xE2 && bytes.ptr[4] == 0x82 && bytes.ptr[5] == 0xAC, "euro");
|
||
@assert(bytes.ptr[6] == 0xF0 && bytes.ptr[9] == 0x80, "emoji head/tail");
|
||
|
||
@assert(valid(bytes.ptr, bytes.len), "valid");
|
||
@assert(count_codepoints(bytes.ptr, bytes.len) == 4, "codepoints");
|
||
@assert(utf16_units(bytes.ptr, bytes.len) == 1 + 1 + 1 + 2, "utf16 units");
|
||
|
||
var cp: u32 = 0;
|
||
@assert(decode_used(bytes.ptr, bytes.len, 0, &cp) == 1 && cp == 0x41, "dec A");
|
||
@assert(decode_used(bytes.ptr, bytes.len, 1, &cp) == 2 && cp == 0xE9, "dec e");
|
||
@assert(decode_used(bytes.ptr, bytes.len, 3, &cp) == 3 && cp == 0x20AC, "dec euro");
|
||
@assert(decode_used(bytes.ptr, bytes.len, 6, &cp) == 4 && cp == 0x1F600, "dec emoji");
|
||
}
|
||
|
||
fn test_invalid() {
|
||
var bad1: [2]u8 = .[0xC0, 0x80]; // 过长
|
||
var bad2: [3]u8 = .[0xED, 0xA0, 0x80]; // 代理区
|
||
var bad3: [2]u8 = .[0xC3, 0x28]; // 坏续字节
|
||
var bad4: [1]u8 = .[0x80]; // 孤立续字节
|
||
var bad5: [3]u8 = .[0xE2, 0x82]; // 截断
|
||
var s1 = bad1[0..2];
|
||
var s2 = bad2[0..3];
|
||
var s3 = bad3[0..2];
|
||
var s4 = bad4[0..1];
|
||
var s5 = bad5[0..2];
|
||
@assert(!valid(s1.ptr, s1.len), "overlong");
|
||
@assert(!valid(s2.ptr, s2.len), "surrogate");
|
||
@assert(!valid(s3.ptr, s3.len), "bad cont");
|
||
@assert(!valid(s4.ptr, s4.len), "lone cont");
|
||
@assert(!valid(s5.ptr, s5.len), "truncated");
|
||
|
||
// 新 ADT：越界/截断/非法可区分。
|
||
var cp: u32 = 0;
|
||
match decode(s5.ptr, s5.len, 0, &cp) {
|
||
.Truncated => {},
|
||
_ => { @assert(false, "truncated sequence should be Truncated"); },
|
||
}
|
||
match decode(s5.ptr, s5.len, s5.len, &cp) {
|
||
.Eof => {},
|
||
_ => { @assert(false, "past end should be Eof"); },
|
||
}
|
||
match decode(s1.ptr, s1.len, 0, &cp) {
|
||
.Invalid => {},
|
||
_ => { @assert(false, "overlong should be Invalid"); },
|
||
}
|
||
match decode(s4.ptr, s4.len, 0, &cp) {
|
||
.Invalid => {},
|
||
_ => { @assert(false, "lone continuation should be Invalid"); },
|
||
}
|
||
|
||
var out = ByteBuffer.new();
|
||
defer { ByteBuffer.drop(&out); }
|
||
match encode_into(&out, 0xD800) {
|
||
.Ok => { @assert(false, "surrogate encode"); },
|
||
.Err(error) => {},
|
||
}
|
||
match encode_into(&out, 0x110000) {
|
||
.Ok => { @assert(false, "out of range encode"); },
|
||
.Err(error) => {},
|
||
}
|
||
}
|
||
|
||
fn test_offsets() {
|
||
var out = ByteBuffer.new();
|
||
defer { ByteBuffer.drop(&out); }
|
||
expect_ok(encode_into(&out, 0x41)); // byte 0, utf16 0
|
||
expect_ok(encode_into(&out, 0x1F600)); // bytes 1..4, utf16 1..2
|
||
expect_ok(encode_into(&out, 0x42)); // byte 5, utf16 3
|
||
var b = ByteBuffer.as_slice(out);
|
||
|
||
@assert(byte_to_utf16(b.ptr, b.len, 0) == 0, "b0");
|
||
@assert(byte_to_utf16(b.ptr, b.len, 1) == 1, "b1");
|
||
@assert(byte_to_utf16(b.ptr, b.len, 5) == 3, "b5");
|
||
@assert(byte_to_utf16(b.ptr, b.len, 6) == 4, "clamp");
|
||
|
||
@assert(utf16_to_byte(b.ptr, b.len, 0) == 0, "u0");
|
||
@assert(utf16_to_byte(b.ptr, b.len, 1) == 1, "u1");
|
||
@assert(utf16_to_byte(b.ptr, b.len, 2) == 1, "u2 stays in emoji");
|
||
@assert(utf16_to_byte(b.ptr, b.len, 3) == 5, "u3");
|
||
@assert(utf16_to_byte(b.ptr, b.len, 99) == 6, "clamp");
|
||
}
|
||
|
||
fn main() {
|
||
test_decode_encode_roundtrip();
|
||
test_invalid();
|
||
test_offsets();
|
||
}
|