Merge branch 'develop-next'HEADv0.14.1master

1 files changed, 188 insertions, 12 deletions

diff --git a/src/code_point.zig b/src/code_point.zig
index fe7ad6e..7a638af 100644
--- a/src/code_point.zig
+++ b/src/code_point.zig
@@ -4,18 +4,33 @@
 //! Represents invalid data according to the Replacement of Maximal
 //! Subparts algorithm.
 
+pub const uoffset = if (@import("config").fat_offset) u64 else u32;
+
 /// `CodePoint` represents a Unicode code point by its code,
 /// length, and offset in the source bytes.
 pub const CodePoint = struct {
     code: u21,
     len: u3,
-    offset: u32,
+    offset: uoffset,
+
+    /// Return the slice of this codepoint, given the original string.
+    pub inline fn bytes(cp: CodePoint, str: []const u8) []const u8 {
+        return str[cp.offset..][0..cp.len];
+    }
+
+    pub fn format(cp: CodePoint, _: []const u8, _: std.fmt.FormatOptions, writer: anytype) !void {
+        try writer.print("CodePoint '{u}' .{{ ", .{cp.code});
+        try writer.print(
+            ".code = 0x{x}, .offset = {d}, .len = {d} }}",
+            .{ cp.code, cp.offset, cp.len },
+        );
+    }
 };
 
 /// This function is deprecated and will be removed in a later release.
 /// Use `decodeAtIndex` or `decodeAtCursor`.
-pub fn decode(bytes: []const u8, offset: u32) ?CodePoint {
-    var off: u32 = 0;
+pub fn decode(bytes: []const u8, offset: uoffset) ?CodePoint {
+    var off: uoffset = 0;
     var maybe_code = decodeAtCursor(bytes, &off);
     if (maybe_code) |*code| {
         code.offset = offset;
@@ -24,15 +39,23 @@ pub fn decode(bytes: []const u8, offset: u32) ?CodePoint {
     return null;
 }
 
+/// Return the codepoint at `index`, even if `index` is in the middle
+/// of that codepoint.
+pub fn codepointAtIndex(bytes: []const u8, index: uoffset) ?CodePoint {
+    var idx = index;
+    while (idx > 0 and 0x80 <= bytes[idx] and bytes[idx] <= 0xbf) : (idx -= 1) {}
+    return decodeAtIndex(bytes, idx);
+}
+
 /// Decode the CodePoint, if any, at `bytes[idx]`.
-pub fn decodeAtIndex(bytes: []const u8, idx: u32) ?CodePoint {
-    var off = idx;
+pub fn decodeAtIndex(bytes: []const u8, index: uoffset) ?CodePoint {
+    var off = index;
     return decodeAtCursor(bytes, &off);
 }
 
 /// Decode the CodePoint, if any, at `bytes[cursor.*]`.  After, the
 /// cursor will point at the next potential codepoint index.
-pub fn decodeAtCursor(bytes: []const u8, cursor: *u32) ?CodePoint {
+pub fn decodeAtCursor(bytes: []const u8, cursor: *uoffset) ?CodePoint {
     // EOS
     if (cursor.* >= bytes.len) return null;
 
@@ -98,6 +121,9 @@ pub fn decodeAtCursor(bytes: []const u8, cursor: *u32) ?CodePoint {
     }
     if (st == RUNE_REJECT or cursor.* == bytes.len) {
         @branchHint(.cold);
+        // This, and the branch below, detect truncation, the
+        // only invalid state handled differently by the Maximal
+        // Subparts algorithm.
         if (state_dfa[@intCast(u8dfa[byte])] == RUNE_REJECT) {
             cursor.* -= 2; // +1
             return .{
@@ -148,7 +174,7 @@ pub fn decodeAtCursor(bytes: []const u8, cursor: *u32) ?CodePoint {
 /// `Iterator` iterates a string one `CodePoint` at-a-time.
 pub const Iterator = struct {
     bytes: []const u8,
-    i: u32 = 0,
+    i: uoffset = 0,
 
     pub fn init(bytes: []const u8) Iterator {
         return .{ .bytes = bytes, .i = 0 };
@@ -158,10 +184,19 @@ pub const Iterator = struct {
         return decodeAtCursor(self.bytes, &self.i);
     }
 
-    pub fn peek(self: *Iterator) ?CodePoint {
-        const saved_i = self.i;
-        defer self.i = saved_i;
-        return self.next();
+    pub fn peek(iter: *Iterator) ?CodePoint {
+        const saved_i = iter.i;
+        defer iter.i = saved_i;
+        return iter.next();
+    }
+
+    /// Create a backward iterator at this point.  It will repeat
+    /// the last CodePoint seen.
+    pub fn reverseIterator(iter: *const Iterator) ReverseIterator {
+        if (iter.i == iter.bytes.len) {
+            return .init(iter.bytes);
+        }
+        return .{ .i = iter.i, .bytes = iter.bytes };
     }
 };
 
@@ -233,6 +268,55 @@ const class_mask: [12]u8 = .{
     0,
 };
 
+pub const ReverseIterator = struct {
+    bytes: []const u8,
+    i: ?uoffset,
+
+    pub fn init(str: []const u8) ReverseIterator {
+        var r_iter: ReverseIterator = undefined;
+        r_iter.bytes = str;
+        r_iter.i = if (str.len == 0) 0 else @intCast(str.len - 1);
+        return r_iter;
+    }
+
+    pub fn prev(iter: *ReverseIterator) ?CodePoint {
+        if (iter.i == null) return null;
+        var i_prev = iter.i.?;
+
+        while (i_prev > 0) : (i_prev -= 1) {
+            if (!followbyte(iter.bytes[i_prev])) break;
+        }
+
+        if (i_prev > 0)
+            iter.i = i_prev - 1
+        else
+            iter.i = null;
+
+        return decode(iter.bytes[i_prev..], i_prev);
+    }
+
+    pub fn peek(iter: *ReverseIterator) ?CodePoint {
+        const saved_i = iter.i;
+        defer iter.i = saved_i;
+        return iter.prev();
+    }
+
+    /// Create a forward iterator at this point.  It will repeat the
+    /// last CodePoint seen.
+    pub fn forwardIterator(iter: *const ReverseIterator) Iterator {
+        if (iter.i) |i| {
+            var fwd: Iterator = .{ .i = i, .bytes = iter.bytes };
+            _ = fwd.next();
+            return fwd;
+        }
+        return .{ .i = 0, .bytes = iter.bytes };
+    }
+};
+
+inline fn followbyte(b: u8) bool {
+    return 0x80 <= b and b <= 0xbf;
+}
+
 test "decode" {
     const bytes = "🌩️";
     const res = decode(bytes, 0);
@@ -246,7 +330,7 @@ test "decode" {
     }
 }
 
-test "peek" {
+test Iterator {
     var iter = Iterator{ .bytes = "Hi" };
 
     try expectEqual(@as(u21, 'H'), iter.next().?.code);
@@ -256,6 +340,54 @@ test "peek" {
     try expectEqual(@as(?CodePoint, null), iter.next());
 }
 
+const code_point = @This();
+
+// Keep this in sync with the README
+test "Code point iterator" {
+    const str = "Hi 😊";
+    var iter: code_point.Iterator = .init(str);
+    var i: usize = 0;
+
+    while (iter.next()) |cp| : (i += 1) {
+        // The `code` field is the actual code point scalar as a `u21`.
+        if (i == 0) try expect(cp.code == 'H');
+        if (i == 1) try expect(cp.code == 'i');
+        if (i == 2) try expect(cp.code == ' ');
+
+        if (i == 3) {
+            try expect(cp.code == '😊');
+            // The `offset` field is the byte offset in the
+            // source string.
+            try expect(cp.offset == 3);
+            try expectEqual(cp, code_point.decodeAtIndex(str, cp.offset).?);
+            // The `len` field is the length in bytes of the
+            // code point in the source string.
+            try expect(cp.len == 4);
+            // There is also a 'cursor' decode, like so:
+            {
+                var cursor = cp.offset;
+                try expectEqual(cp, code_point.decodeAtCursor(str, &cursor).?);
+                // Which advances the cursor variable to the next possible
+                // offset, in this case, `str.len`.  Don't forget to account
+                // for this possibility!
+                try expectEqual(cp.offset + cp.len, cursor);
+            }
+            // There's also this, for when you aren't sure if you have the
+            // correct start for a code point:
+            try expectEqual(cp, code_point.codepointAtIndex(str, cp.offset + 1).?);
+        }
+        // Reverse iteration is also an option:
+        var r_iter: code_point.ReverseIterator = .init(str);
+        // Both iterators can be peeked:
+        try expectEqual('😊', r_iter.peek().?.code);
+        try expectEqual('😊', r_iter.prev().?.code);
+        // Both kinds of iterators can be reversed:
+        var fwd_iter = r_iter.forwardIterator(); // or iter.reverseIterator();
+        // This will always return the last codepoint from
+        // the prior iterator, _if_ it yielded one:
+        try expectEqual('😊', fwd_iter.next().?.code);
+    }
+}
 test "overlongs" {
     // None of these should equal `/`, all should be byte-for-byte
     // handled as replacement characters.
@@ -346,6 +478,50 @@ test "truncation" {
     }
 }
 
+test ReverseIterator {
+    {
+        var r_iter: ReverseIterator = .init("ABC");
+        try testing.expectEqual(@as(u21, 'C'), r_iter.prev().?.code);
+        try testing.expectEqual(@as(u21, 'B'), r_iter.peek().?.code);
+        try testing.expectEqual(@as(u21, 'B'), r_iter.prev().?.code);
+        try testing.expectEqual(@as(u21, 'A'), r_iter.prev().?.code);
+        try testing.expectEqual(@as(?CodePoint, null), r_iter.peek());
+        try testing.expectEqual(@as(?CodePoint, null), r_iter.prev());
+        try testing.expectEqual(@as(?CodePoint, null), r_iter.prev());
+    }
+    {
+        var r_iter: ReverseIterator = .init("∅δq🦾ă");
+        try testing.expectEqual(@as(u21, 'ă'), r_iter.prev().?.code);
+        try testing.expectEqual(@as(u21, '🦾'), r_iter.prev().?.code);
+        try testing.expectEqual(@as(u21, 'q'), r_iter.prev().?.code);
+        try testing.expectEqual(@as(u21, 'δ'), r_iter.peek().?.code);
+        try testing.expectEqual(@as(u21, 'δ'), r_iter.prev().?.code);
+        try testing.expectEqual(@as(u21, '∅'), r_iter.peek().?.code);
+        try testing.expectEqual(@as(u21, '∅'), r_iter.peek().?.code);
+        try testing.expectEqual(@as(u21, '∅'), r_iter.prev().?.code);
+        try testing.expectEqual(@as(?CodePoint, null), r_iter.peek());
+        try testing.expectEqual(@as(?CodePoint, null), r_iter.prev());
+        try testing.expectEqual(@as(?CodePoint, null), r_iter.prev());
+    }
+    {
+        var r_iter: ReverseIterator = .init("123");
+        try testing.expectEqual(@as(u21, '3'), r_iter.prev().?.code);
+        try testing.expectEqual(@as(u21, '2'), r_iter.prev().?.code);
+        try testing.expectEqual(@as(u21, '1'), r_iter.prev().?.code);
+        var iter = r_iter.forwardIterator();
+        try testing.expectEqual(@as(u21, '1'), iter.next().?.code);
+        try testing.expectEqual(@as(u21, '2'), iter.next().?.code);
+        try testing.expectEqual(@as(u21, '3'), iter.next().?.code);
+        r_iter = iter.reverseIterator();
+        try testing.expectEqual(@as(u21, '3'), r_iter.prev().?.code);
+        try testing.expectEqual(@as(u21, '2'), r_iter.prev().?.code);
+        iter = r_iter.forwardIterator();
+        r_iter = iter.reverseIterator();
+        try testing.expectEqual(@as(u21, '2'), iter.next().?.code);
+        try testing.expectEqual(@as(u21, '2'), r_iter.prev().?.code);
+    }
+}
+
 const std = @import("std");
 const testing = std.testing;
 const expect = testing.expect;