summaryrefslogtreecommitdiff
path: root/src/Utf8Decoder.zig
diff options
context:
space:
mode:
Diffstat (limited to 'src/Utf8Decoder.zig')
-rw-r--r--src/Utf8Decoder.zig142
1 files changed, 0 insertions, 142 deletions
diff --git a/src/Utf8Decoder.zig b/src/Utf8Decoder.zig
deleted file mode 100644
index 6bb0d98..0000000
--- a/src/Utf8Decoder.zig
+++ /dev/null
@@ -1,142 +0,0 @@
1//! DFA-based non-allocating error-replacing UTF-8 decoder.
2//!
3//! This implementation is based largely on the excellent work of
4//! Bjoern Hoehrmann, with slight modifications to support error-
5//! replacement.
6//!
7//! For details on Bjoern's DFA-based UTF-8 decoder, see
8//! http://bjoern.hoehrmann.de/utf-8/decoder/dfa (MIT licensed)
9const UTF8Decoder = @This();
10
11const std = @import("std");
12const testing = std.testing;
13
14const log = std.log.scoped(.utf8decoder);
15
16// zig fmt: off
17const char_classes = [_]u4{
18 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
19 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
20 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
21 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
22 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,
23 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7, 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,
24 8,8,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,
25 10,3,3,3,3,3,3,3,3,3,3,3,3,4,3,3, 11,6,6,6,5,8,8,8,8,8,8,8,8,8,8,8,
26};
27
28const transitions = [_]u8 {
29 0,12,24,36,60,96,84,12,12,12,48,72, 12,12,12,12,12,12,12,12,12,12,12,12,
30 12, 0,12,12,12,12,12, 0,12, 0,12,12, 12,24,12,12,12,12,12,24,12,24,12,12,
31 12,12,12,12,12,12,12,24,12,12,12,12, 12,24,12,12,12,12,12,12,12,24,12,12,
32 12,12,12,12,12,12,12,36,12,36,12,12, 12,36,12,12,12,12,12,36,12,36,12,12,
33 12,36,12,12,12,12,12,12,12,12,12,12,
34};
35// zig fmt: on
36
37// DFA states
38const ACCEPT_STATE = 0;
39const REJECT_STATE = 12;
40
41// This is where we accumulate our current codepoint.
42accumulator: u21 = 0,
43// The internal state of the DFA.
44state: u8 = ACCEPT_STATE,
45
46/// Takes the next byte in the utf-8 sequence and emits a tuple of
47/// - The codepoint that was generated, if there is one.
48/// - A boolean that indicates whether the provided byte was consumed.
49///
50/// The only case where the byte is not consumed is if an ill-formed
51/// sequence is reached, in which case a replacement character will be
52/// emitted and the byte will not be consumed.
53///
54/// If the byte is not consumed, the caller is responsible for calling
55/// again with the same byte before continuing.
56pub inline fn next(self: *UTF8Decoder, byte: u8) struct { ?u21, bool } {
57 const char_class = char_classes[byte];
58
59 const initial_state = self.state;
60
61 if (self.state != ACCEPT_STATE) {
62 self.accumulator <<= 6;
63 self.accumulator |= (byte & 0x3F);
64 } else {
65 self.accumulator = (@as(u21, 0xFF) >> char_class) & (byte);
66 }
67
68 self.state = transitions[self.state + char_class];
69
70 if (self.state == ACCEPT_STATE) {
71 defer self.accumulator = 0;
72
73 // Emit the fully decoded codepoint.
74 return .{ self.accumulator, true };
75 } else if (self.state == REJECT_STATE) {
76 self.accumulator = 0;
77 self.state = ACCEPT_STATE;
78 // Emit a replacement character. If we rejected the first byte
79 // in a sequence, then it was consumed, otherwise it was not.
80 return .{ 0xFFFD, initial_state == ACCEPT_STATE };
81 } else {
82 // Emit nothing, we're in the middle of a sequence.
83 return .{ null, true };
84 }
85}
86
87test "ASCII" {
88 var d: UTF8Decoder = .{};
89 var out: [13]u8 = undefined;
90 for ("Hello, World!", 0..) |byte, i| {
91 const res = d.next(byte);
92 try testing.expect(res[1]);
93 if (res[0]) |codepoint| {
94 out[i] = @intCast(codepoint);
95 }
96 }
97
98 try testing.expect(std.mem.eql(u8, &out, "Hello, World!"));
99}
100
101test "Well formed utf-8" {
102 var d: UTF8Decoder = .{};
103 var out: [4]u21 = undefined;
104 var i: usize = 0;
105 // 4 bytes, 3 bytes, 2 bytes, 1 byte
106 for ("๐Ÿ˜„โœครA") |byte| {
107 var consumed = false;
108 while (!consumed) {
109 const res = d.next(byte);
110 consumed = res[1];
111 // There are no errors in this sequence, so
112 // every byte should be consumed first try.
113 try testing.expect(consumed == true);
114 if (res[0]) |codepoint| {
115 out[i] = codepoint;
116 i += 1;
117 }
118 }
119 }
120
121 try testing.expect(std.mem.eql(u21, &out, &[_]u21{ 0x1F604, 0x2724, 0xC1, 0x41 }));
122}
123
124test "Partially invalid utf-8" {
125 var d: UTF8Decoder = .{};
126 var out: [5]u21 = undefined;
127 var i: usize = 0;
128 // Illegally terminated sequence, valid sequence, illegal surrogate pair.
129 for ("\xF0\x9F๐Ÿ˜„\xED\xA0\x80") |byte| {
130 var consumed = false;
131 while (!consumed) {
132 const res = d.next(byte);
133 consumed = res[1];
134 if (res[0]) |codepoint| {
135 out[i] = codepoint;
136 i += 1;
137 }
138 }
139 }
140
141 try testing.expect(std.mem.eql(u21, &out, &[_]u21{ 0xFFFD, 0x1F604, 0xFFFD, 0xFFFD, 0xFFFD }));
142}
HIC SVNT DRACONES