Merge pull request 'grapheme-peek' (#18) from atman/zg:grapheme-peek into master

Reviewed-on: https://codeberg.org/dude_the_builder/zg/pulls/18

2 files changed, 103 insertions, 8 deletions

diff --git a/src/grapheme.zig b/src/grapheme.zig
index 911c856..7538f5b 100644
--- a/src/grapheme.zig
+++ b/src/grapheme.zig
@@ -77,6 +77,75 @@ pub const Iterator = struct {
 
         return Grapheme{ .len = gc_len, .offset = gc_start };
     }
+
+    pub fn peek(self: *Self) ?Grapheme {
+        const saved_cp_iter = self.cp_iter;
+        const s0 = self.buf[0];
+        const s1 = self.buf[1];
+
+        self.advance();
+
+        // If no more
+        if (self.buf[0] == null) {
+            self.cp_iter = saved_cp_iter;
+            self.buf[0] = s0;
+            self.buf[1] = s1;
+            return null;
+        }
+        // If last one
+        if (self.buf[1] == null) {
+            const len = self.buf[0].?.len;
+            const offset = self.buf[0].?.offset;
+            self.cp_iter = saved_cp_iter;
+            self.buf[0] = s0;
+            self.buf[1] = s1;
+            return Grapheme{ .len = len, .offset = offset };
+        }
+        // If ASCII
+        if (self.buf[0].?.code != '\r' and self.buf[0].?.code < 128 and self.buf[1].?.code < 128) {
+            const len = self.buf[0].?.len;
+            const offset = self.buf[0].?.offset;
+            self.cp_iter = saved_cp_iter;
+            self.buf[0] = s0;
+            self.buf[1] = s1;
+            return Grapheme{ .len = len, .offset = offset };
+        }
+
+        const gc_start = self.buf[0].?.offset;
+        var gc_len: u8 = self.buf[0].?.len;
+        var state = State{};
+
+        if (graphemeBreak(
+            self.buf[0].?.code,
+            self.buf[1].?.code,
+            self.data,
+            &state,
+        )) {
+            self.cp_iter = saved_cp_iter;
+            self.buf[0] = s0;
+            self.buf[1] = s1;
+            return Grapheme{ .len = gc_len, .offset = gc_start };
+        }
+
+        while (true) {
+            self.advance();
+            if (self.buf[0] == null) break;
+
+            gc_len += self.buf[0].?.len;
+
+            if (graphemeBreak(
+                self.buf[0].?.code,
+                if (self.buf[1]) |ncp| ncp.code else 0,
+                self.data,
+                &state,
+            )) break;
+        }
+        self.cp_iter = saved_cp_iter;
+        self.buf[0] = s0;
+        self.buf[1] = s1;
+
+        return Grapheme{ .len = gc_len, .offset = gc_start };
+    }
 };
 
 // Predicates
diff --git a/src/unicode_tests.zig b/src/unicode_tests.zig
index 448ce41..245c03f 100644
--- a/src/unicode_tests.zig
+++ b/src/unicode_tests.zig
@@ -7,11 +7,37 @@ const mem = std.mem;
 const testing = std.testing;
 const unicode = std.unicode;
 
+const grapheme = @import("grapheme");
 const Grapheme = @import("grapheme").Grapheme;
 const GraphemeData = @import("grapheme").GraphemeData;
 const GraphemeIterator = @import("grapheme").Iterator;
 const Normalize = @import("Normalize");
 
+comptime {
+    testing.refAllDecls(grapheme);
+}
+test "Iterator.peek" {
+    const peek_seq = "aΔ👨🏻‍🌾→";
+    const data = try GraphemeData.init(std.testing.allocator);
+    defer data.deinit();
+
+    var iter = grapheme.Iterator.init(peek_seq, &data);
+    const peek_a = iter.peek().?;
+    const next_a = iter.next().?;
+    try std.testing.expectEqual(peek_a, next_a);
+    try std.testing.expectEqualStrings("a", peek_a.bytes(peek_seq));
+    const peek_d1 = iter.peek().?;
+    const peek_d2 = iter.peek().?;
+    try std.testing.expectEqual(peek_d1, peek_d2);
+    const next_d = iter.next().?;
+    try std.testing.expectEqual(peek_d2, next_d);
+    try std.testing.expectEqual(iter.peek(), iter.next());
+    try std.testing.expectEqual(iter.peek(), iter.next());
+    try std.testing.expectEqual(null, iter.peek());
+    try std.testing.expectEqual(null, iter.peek());
+    try std.testing.expectEqual(iter.peek(), iter.next());
+}
+
 test "Unicode normalization tests" {
     var arena = heap.ArenaAllocator.init(testing.allocator);
     defer arena.deinit();
@@ -35,7 +61,7 @@ test "Unicode normalization tests" {
         // Skip comments or empty lines.
         if (line.len == 0 or line[0] == '#' or line[0] == '@') continue;
         // Iterate over fields.
-        var fields = mem.split(u8, line, ";");
+        var fields = mem.splitScalar(u8, line, ';');
         var field_index: usize = 0;
         var input: []u8 = undefined;
         defer allocator.free(input);
@@ -45,7 +71,7 @@ test "Unicode normalization tests" {
                 var i_buf = std.ArrayList(u8).init(allocator);
                 defer i_buf.deinit();
 
-                var i_fields = mem.split(u8, field, " ");
+                var i_fields = mem.splitScalar(u8, field, ' ');
                 while (i_fields.next()) |s| {
                     const icp = try fmt.parseInt(u21, s, 16);
                     const len = try unicode.utf8Encode(icp, &cp_buf);
@@ -59,7 +85,7 @@ test "Unicode normalization tests" {
                 var w_buf = std.ArrayList(u8).init(allocator);
                 defer w_buf.deinit();
 
-                var w_fields = mem.split(u8, field, " ");
+                var w_fields = mem.splitScalar(u8, field, ' ');
                 while (w_fields.next()) |s| {
                     const wcp = try fmt.parseInt(u21, s, 16);
                     const len = try unicode.utf8Encode(wcp, &cp_buf);
@@ -76,7 +102,7 @@ test "Unicode normalization tests" {
                 var w_buf = std.ArrayList(u8).init(allocator);
                 defer w_buf.deinit();
 
-                var w_fields = mem.split(u8, field, " ");
+                var w_fields = mem.splitScalar(u8, field, ' ');
                 while (w_fields.next()) |s| {
                     const wcp = try fmt.parseInt(u21, s, 16);
                     const len = try unicode.utf8Encode(wcp, &cp_buf);
@@ -93,7 +119,7 @@ test "Unicode normalization tests" {
                 var w_buf = std.ArrayList(u8).init(allocator);
                 defer w_buf.deinit();
 
-                var w_fields = mem.split(u8, field, " ");
+                var w_fields = mem.splitScalar(u8, field, ' ');
                 while (w_fields.next()) |s| {
                     const wcp = try fmt.parseInt(u21, s, 16);
                     const len = try unicode.utf8Encode(wcp, &cp_buf);
@@ -110,7 +136,7 @@ test "Unicode normalization tests" {
                 var w_buf = std.ArrayList(u8).init(allocator);
                 defer w_buf.deinit();
 
-                var w_fields = mem.split(u8, field, " ");
+                var w_fields = mem.splitScalar(u8, field, ' ');
                 while (w_fields.next()) |s| {
                     const wcp = try fmt.parseInt(u21, s, 16);
                     const len = try unicode.utf8Encode(wcp, &cp_buf);
@@ -158,11 +184,11 @@ test "Segmentation GraphemeIterator" {
         var all_bytes = std.ArrayList(u8).init(allocator);
         defer all_bytes.deinit();
 
-        var graphemes = std.mem.split(u8, line, " ÷ ");
+        var graphemes = std.mem.splitSequence(u8, line, " ÷ ");
         var bytes_index: u32 = 0;
 
         while (graphemes.next()) |field| {
-            var code_points = std.mem.split(u8, field, " ");
+            var code_points = std.mem.splitScalar(u8, field, ' ');
             var cp_buf: [4]u8 = undefined;
             var cp_index: u32 = 0;
             var gc_len: u8 = 0;