Merge branch 'develop-next'HEADv0.14.1master

1 files changed, 331 insertions, 67 deletions

diff --git a/src/unicode_tests.zig b/src/unicode_tests.zig
index 2249007..ae177a9 100644
--- a/src/unicode_tests.zig
+++ b/src/unicode_tests.zig
@@ -1,43 +1,4 @@
-const std = @import("std");
-const fmt = std.fmt;
-const fs = std.fs;
-const io = std.io;
-const heap = std.heap;
-const mem = std.mem;
-const testing = std.testing;
-const unicode = std.unicode;
-
-const grapheme = @import("Graphemes");
-const Grapheme = @import("Graphemes").Grapheme;
-const Graphemes = @import("Graphemes");
-const GraphemeIterator = @import("Graphemes").Iterator;
-const Normalize = @import("Normalize");
-
-comptime {
-    testing.refAllDecls(grapheme);
-}
-
-test "Iterator.peek" {
-    const peek_seq = "aΔ👨🏻‍🌾→";
-    const data = try Graphemes.init(std.testing.allocator);
-    defer data.deinit(std.testing.allocator);
-
-    var iter = data.iterator(peek_seq);
-    const peek_a = iter.peek().?;
-    const next_a = iter.next().?;
-    try std.testing.expectEqual(peek_a, next_a);
-    try std.testing.expectEqualStrings("a", peek_a.bytes(peek_seq));
-    const peek_d1 = iter.peek().?;
-    const peek_d2 = iter.peek().?;
-    try std.testing.expectEqual(peek_d1, peek_d2);
-    const next_d = iter.next().?;
-    try std.testing.expectEqual(peek_d2, next_d);
-    try std.testing.expectEqual(iter.peek(), iter.next());
-    try std.testing.expectEqual(iter.peek(), iter.next());
-    try std.testing.expectEqual(null, iter.peek());
-    try std.testing.expectEqual(null, iter.peek());
-    try std.testing.expectEqual(iter.peek(), iter.next());
-}
+const dbg_print = false;
 
 test "Unicode normalization tests" {
     var arena = heap.ArenaAllocator.init(testing.allocator);
@@ -50,16 +11,14 @@ test "Unicode normalization tests" {
     var file = try fs.cwd().openFile("data/unicode/NormalizationTest.txt", .{});
     defer file.close();
     var buf_reader = io.bufferedReader(file.reader());
-    const input_stream = buf_reader.reader();
+    var input_stream = buf_reader.reader();
 
-    var line_no: usize = 0;
     var buf: [4096]u8 = undefined;
     var cp_buf: [4]u8 = undefined;
 
-    while (try input_stream.readUntilDelimiterOrEof(&buf, '\n')) |line| {
-        line_no += 1;
-        // Skip comments or empty lines.
-        if (line.len == 0 or line[0] == '#' or line[0] == '@') continue;
+    var line_iter: IterRead = .{ .read = &input_stream };
+
+    while (try line_iter.next(&buf)) |line| {
         // Iterate over fields.
         var fields = mem.splitScalar(u8, line, ';');
         var field_index: usize = 0;
@@ -80,7 +39,7 @@ test "Unicode normalization tests" {
 
                 input = try i_buf.toOwnedSlice();
             } else if (field_index == 1) {
-                //debug.print("\n*** {s} ***\n", .{line});
+                if (dbg_print) debug.print("\n*** {s} ***\n", .{line});
                 // NFC, time to test.
                 var w_buf = std.ArrayList(u8).init(allocator);
                 defer w_buf.deinit();
@@ -162,20 +121,17 @@ test "Segmentation GraphemeIterator" {
     var buf_reader = std.io.bufferedReader(file.reader());
     var input_stream = buf_reader.reader();
 
-    const data = try Graphemes.init(allocator);
-    defer data.deinit(allocator);
+    const graph = try Graphemes.init(allocator);
+    defer graph.deinit(allocator);
 
     var buf: [4096]u8 = undefined;
-    var line_no: usize = 1;
-
-    while (try input_stream.readUntilDelimiterOrEof(&buf, '\n')) |raw| : (line_no += 1) {
-        // Skip comments or empty lines.
-        if (raw.len == 0 or raw[0] == '#' or raw[0] == '@') continue;
+    var line_iter: IterRead = .{ .read = &input_stream };
 
+    while (try line_iter.next(&buf)) |raw| {
         // Clean up.
         var line = std.mem.trimLeft(u8, raw, "÷ ");
-        if (std.mem.indexOf(u8, line, " ÷\t#")) |octo| {
-            line = line[0..octo];
+        if (std.mem.indexOf(u8, line, " ÷\t")) |final| {
+            line = line[0..final];
         }
         // Iterate over fields.
         var want = std.ArrayList(Grapheme).init(allocator);
@@ -185,12 +141,12 @@ test "Segmentation GraphemeIterator" {
         defer all_bytes.deinit();
 
         var graphemes = std.mem.splitSequence(u8, line, " ÷ ");
-        var bytes_index: u32 = 0;
+        var bytes_index: uoffset = 0;
 
         while (graphemes.next()) |field| {
             var code_points = std.mem.splitScalar(u8, field, ' ');
             var cp_buf: [4]u8 = undefined;
-            var cp_index: u32 = 0;
+            var cp_index: uoffset = 0;
             var gc_len: u8 = 0;
 
             while (code_points.next()) |code_point| {
@@ -206,16 +162,324 @@ test "Segmentation GraphemeIterator" {
             bytes_index += cp_index;
         }
 
-        // std.debug.print("\nline {}: {s}\n", .{ line_no, all_bytes.items });
-        var iter = data.iterator(all_bytes.items);
+        const this_str = all_bytes.items;
+
+        {
+            var iter = graph.iterator(this_str);
+
+            // Check.
+            for (want.items, 1..) |want_gc, idx| {
+                const got_gc = (iter.next()).?;
+                try std.testing.expectEqualStrings(
+                    want_gc.bytes(this_str),
+                    got_gc.bytes(this_str),
+                );
+                for (got_gc.offset..got_gc.offset + got_gc.len) |i| {
+                    const this_gc = graph.graphemeAtIndex(this_str, i);
+                    std.testing.expectEqualSlices(
+                        u8,
+                        got_gc.bytes(this_str),
+                        this_gc.bytes(this_str),
+                    ) catch |err| {
+                        debug.print("Wrong grapheme on line {d} #{d} offset {d}\n", .{ line_iter.line, idx, i });
+                        return err;
+                    };
+                }
+                var after_iter = graph.iterateAfterGrapheme(this_str, got_gc);
+                if (after_iter.next()) |next_gc| {
+                    if (iter.peek()) |next_peek| {
+                        std.testing.expectEqualSlices(
+                            u8,
+                            next_gc.bytes(this_str),
+                            next_peek.bytes(this_str),
+                        ) catch |err| {
+                            debug.print("Peeks differ on line {d} #{d} \n", .{ line_iter.line, idx });
+                            return err;
+                        };
+                    } else {
+                        debug.print("Mismatch: peek missing, next found, line {d} #{d}\n", .{ line_iter.line, idx });
+                        try testing.expect(false);
+                    }
+                } else {
+                    try testing.expectEqual(null, iter.peek());
+                }
+            }
+        }
+        {
+            var iter = graph.reverseIterator(this_str);
+
+            // Check.
+            var i: usize = want.items.len;
+            while (i > 0) {
+                i -= 1;
+                const want_gc = want.items[i];
+                const got_gc = iter.prev() orelse {
+                    std.debug.print(
+                        "line {d} grapheme {d}: expected {any} found null\n",
+                        .{ line_iter.line, i, want_gc },
+                    );
+                    return error.TestExpectedEqual;
+                };
+                std.testing.expectEqualStrings(
+                    want_gc.bytes(this_str),
+                    got_gc.bytes(this_str),
+                ) catch |err| {
+                    std.debug.print(
+                        "line {d} grapheme {d}: expected {any} found {any}\n",
+                        .{ line_iter.line, i, want_gc, got_gc },
+                    );
+                    return err;
+                };
+                var before_iter = graph.iterateBeforeGrapheme(this_str, got_gc);
+                if (before_iter.prev()) |prev_gc| {
+                    if (iter.peek()) |prev_peek| {
+                        std.testing.expectEqualSlices(
+                            u8,
+                            prev_gc.bytes(this_str),
+                            prev_peek.bytes(this_str),
+                        ) catch |err| {
+                            debug.print("Peeks differ on line {d} #{d} \n", .{ line_iter.line, i });
+                            return err;
+                        };
+                    } else {
+                        debug.print("Mismatch: peek missing, prev found, line {d} #{d}\n", .{ line_iter.line, i });
+                        try testing.expect(false);
+                    }
+                } else {
+                    try testing.expectEqual(null, iter.peek());
+                }
+            }
+        }
+    }
+}
+
+test "Segmentation Word Iterator" {
+    const allocator = std.testing.allocator;
+    var file = try std.fs.cwd().openFile("data/unicode/auxiliary/WordBreakTest.txt", .{});
+    defer file.close();
+    var buf_reader = std.io.bufferedReader(file.reader());
+    var input_stream = buf_reader.reader();
+
+    const wb = try Words.init(allocator);
+    defer wb.deinit(allocator);
+
+    var buf: [4096]u8 = undefined;
+    var line_iter: IterRead = .{ .read = &input_stream };
+
+    while (try line_iter.next(&buf)) |raw| {
+        // Clean up.
+        var line = std.mem.trimLeft(u8, raw, "÷ ");
+        if (std.mem.indexOf(u8, line, " ÷\t")) |final| {
+            line = line[0..final];
+        }
+        // Iterate over fields.
+        var want = std.ArrayList(Word).init(allocator);
+        defer want.deinit();
+
+        var all_bytes = std.ArrayList(u8).init(allocator);
+        defer all_bytes.deinit();
+
+        var words = std.mem.splitSequence(u8, line, " ÷ ");
+        var bytes_index: uoffset = 0;
+
+        while (words.next()) |field| {
+            var code_points = std.mem.splitScalar(u8, field, ' ');
+            var cp_buf: [4]u8 = undefined;
+            var cp_index: uoffset = 0;
+            var gc_len: u8 = 0;
 
-        // Check.
-        for (want.items) |want_gc| {
-            const got_gc = (iter.next()).?;
-            try std.testing.expectEqualStrings(
-                want_gc.bytes(all_bytes.items),
-                got_gc.bytes(all_bytes.items),
-            );
+            while (code_points.next()) |code_point| {
+                if (std.mem.eql(u8, code_point, "×")) continue;
+                const cp: u21 = try std.fmt.parseInt(u21, code_point, 16);
+                const len = try unicode.utf8Encode(cp, &cp_buf);
+                try all_bytes.appendSlice(cp_buf[0..len]);
+                cp_index += len;
+                gc_len += len;
+            }
+
+            try want.append(Word{ .len = gc_len, .offset = bytes_index });
+            bytes_index += cp_index;
+        }
+        const this_str = all_bytes.items;
+
+        {
+            var iter = wb.iterator(this_str);
+            var peeked: ?Word = iter.peek();
+
+            // Check.
+            for (want.items, 1..) |want_word, idx| {
+                const got_word = (iter.next()).?;
+                std.testing.expectEqualStrings(
+                    want_word.bytes(this_str),
+                    got_word.bytes(this_str),
+                ) catch |err| {
+                    debug.print("Error on line {d}, #{d}\n", .{ line_iter.line, idx });
+                    return err;
+                };
+                std.testing.expectEqualStrings(
+                    peeked.?.bytes(this_str),
+                    got_word.bytes(this_str),
+                ) catch |err| {
+                    debug.print("Peek != word on line {d} #{d}\n", .{ line_iter.line, idx });
+                    return err;
+                };
+                var r_iter = iter.reverseIterator();
+                const if_r_word = r_iter.prev();
+                if (if_r_word) |r_word| {
+                    std.testing.expectEqualStrings(
+                        want_word.bytes(this_str),
+                        r_word.bytes(this_str),
+                    ) catch |err| {
+                        debug.print("Reversal Error on line {d}, #{d}\n", .{ line_iter.line, idx });
+                        return err;
+                    };
+                } else {
+                    try testing.expect(false);
+                }
+                var peek_iter = wb.iterateAfterWord(this_str, got_word);
+                const peek_1 = peek_iter.next();
+                if (peek_1) |p1| {
+                    const peek_2 = iter.peek();
+                    if (peek_2) |p2| {
+                        std.testing.expectEqualSlices(
+                            u8,
+                            p1.bytes(this_str),
+                            p2.bytes(this_str),
+                        ) catch |err| {
+                            debug.print("Bad peek on line {d} #{d} offset {d}\n", .{ line_iter.line, idx + 1, idx });
+                            return err;
+                        };
+                    } else {
+                        try testing.expect(false);
+                    }
+                } else {
+                    try testing.expectEqual(null, iter.peek());
+                }
+                for (got_word.offset..got_word.offset + got_word.len) |i| {
+                    const this_word = wb.wordAtIndex(this_str, i);
+                    std.testing.expectEqualSlices(
+                        u8,
+                        got_word.bytes(this_str),
+                        this_word.bytes(this_str),
+                    ) catch |err| {
+                        debug.print("Wrong word on line {d} #{d} offset {d}\n", .{ line_iter.line, idx, i });
+                        return err;
+                    };
+                }
+                peeked = iter.peek();
+            }
+        }
+        {
+            var r_iter = wb.reverseIterator(this_str);
+            var peeked: ?Word = r_iter.peek();
+            var idx = want.items.len - 1;
+
+            while (true) : (idx -= 1) {
+                const want_word = want.items[idx];
+                const got_word = r_iter.prev().?;
+                std.testing.expectEqualSlices(
+                    u8,
+                    want_word.bytes(this_str),
+                    got_word.bytes(this_str),
+                ) catch |err| {
+                    debug.print("Error on line {d}, #{d}\n", .{ line_iter.line, idx + 1 });
+                    return err;
+                };
+                std.testing.expectEqualStrings(
+                    peeked.?.bytes(this_str),
+                    got_word.bytes(this_str),
+                ) catch |err| {
+                    debug.print("Peek != word on line {d} #{d}\n", .{ line_iter.line, idx + 1 });
+                    return err;
+                };
+                var f_iter = r_iter.forwardIterator();
+                const if_f_word = f_iter.next();
+                if (if_f_word) |f_word| {
+                    std.testing.expectEqualStrings(
+                        want_word.bytes(this_str),
+                        f_word.bytes(this_str),
+                    ) catch |err| {
+                        debug.print("Reversal Error on line {d}, #{d}\n", .{ line_iter.line, idx });
+                        return err;
+                    };
+                } else {
+                    try testing.expect(false);
+                }
+                var peek_iter = wb.iterateBeforeWord(this_str, got_word);
+                const peek_1 = peek_iter.prev();
+                if (peek_1) |p1| {
+                    const peek_2 = r_iter.peek();
+                    if (peek_2) |p2| {
+                        std.testing.expectEqualSlices(
+                            u8,
+                            p1.bytes(this_str),
+                            p2.bytes(this_str),
+                        ) catch |err| {
+                            debug.print("Bad peek on line {d} #{d} offset {d}\n", .{ line_iter.line, idx + 1, idx });
+                            return err;
+                        };
+                    } else {
+                        try testing.expect(false);
+                    }
+                } else {
+                    try testing.expectEqual(null, r_iter.peek());
+                }
+                for (got_word.offset..got_word.offset + got_word.len) |i| {
+                    const this_word = wb.wordAtIndex(this_str, i);
+                    std.testing.expectEqualSlices(
+                        u8,
+                        got_word.bytes(this_str),
+                        this_word.bytes(this_str),
+                    ) catch |err| {
+                        debug.print("Wrong word on line {d} #{d} offset {d}\n", .{ line_iter.line, idx + 1, i });
+                        return err;
+                    };
+                }
+                peeked = r_iter.peek();
+                if (idx == 0) break;
+            }
         }
     }
 }
+
+const IterRead = struct {
+    read: *Reader,
+    line: usize = 0,
+
+    pub fn next(iter: *IterRead, buf: []u8) !?[]const u8 {
+        defer iter.line += 1;
+        const maybe_line = try iter.read.readUntilDelimiterOrEof(buf, '#');
+        if (maybe_line) |this_line| {
+            try iter.read.skipUntilDelimiterOrEof('\n');
+            if (this_line.len == 0 or this_line[0] == '@') {
+                // comment, next line
+                return iter.next(buf);
+            } else {
+                return this_line;
+            }
+        } else {
+            return null;
+        }
+    }
+};
+
+const std = @import("std");
+const fmt = std.fmt;
+const fs = std.fs;
+const io = std.io;
+const Reader = io.BufferedReader(4096, fs.File.Reader).Reader;
+const heap = std.heap;
+const mem = std.mem;
+const debug = std.debug;
+const testing = std.testing;
+const unicode = std.unicode;
+
+const uoffset = @FieldType(Word, "offset");
+
+const Grapheme = @import("Graphemes").Grapheme;
+const Graphemes = @import("Graphemes");
+const GraphemeIterator = @import("Graphemes").Iterator;
+const Normalize = @import("Normalize");
+
+const Words = @import("Words");
+const Word = Words.Word;