Add WordBreakPropertyData

Passes some simple lookup tests.

3 files changed, 276 insertions, 0 deletions

diff --git a/build.zig b/build.zig
index 58fd3e7..f89e90c 100644
--- a/build.zig
+++ b/build.zig
@@ -22,6 +22,15 @@ pub fn build(b: *std.Build) void {
     const run_gbp_gen_exe = b.addRunArtifact(gbp_gen_exe);
     const gbp_gen_out = run_gbp_gen_exe.addOutputFileArg("gbp.bin.z");
 
+    const wbp_gen_exe = b.addExecutable(.{
+        .name = "wbp",
+        .root_source_file = b.path("codegen/wbp.zig"),
+        .target = b.graph.host,
+        .optimize = .Debug,
+    });
+    const run_wbp_gen_exe = b.addRunArtifact(wbp_gen_exe);
+    const wbp_gen_out = run_wbp_gen_exe.addOutputFileArg("wbp.bin.z");
+
     // Display width
     const cjk = b.option(bool, "cjk", "Ambiguous code points are wide (display width: 2).") orelse false;
     const options = b.addOptions();
@@ -183,6 +192,7 @@ pub fn build(b: *std.Build) void {
     const props_gen_out = run_props_gen_exe.addOutputFileArg("props.bin.z");
 
     // Modules we provide
+
     // Code points
     const code_point = b.addModule("code_point", .{
         .root_source_file = b.path("src/code_point.zig"),
@@ -215,6 +225,23 @@ pub fn build(b: *std.Build) void {
     });
     const grapheme_tr = b.addRunArtifact(grapheme_t);
 
+    // Word Breaking
+    const word_break = b.addModule("WordBreak", .{
+        .root_source_file = b.path("src/WordBreak.zig"),
+        .target = target,
+        .optimize = optimize,
+    });
+    word_break.addAnonymousImport("wbp", .{ .root_source_file = wbp_gen_out });
+    word_break.addImport("code_point", code_point);
+
+    const word_break_t = b.addTest(.{
+        .name = "WordBreak",
+        .root_module = word_break,
+        .target = target,
+        .optimize = optimize,
+    });
+    const word_break_tr = b.addRunArtifact(word_break_t);
+
     // ASCII utilities
     const ascii = b.addModule("ascii", .{
         .root_source_file = b.path("src/ascii.zig"),
@@ -452,6 +479,7 @@ pub fn build(b: *std.Build) void {
     test_step.dependOn(&code_point_tr.step);
     test_step.dependOn(&display_width_tr.step);
     test_step.dependOn(&grapheme_tr.step);
+    test_step.dependOn(&word_break_tr.step);
     test_step.dependOn(&ascii_tr.step);
     test_step.dependOn(&ccc_data_tr.step);
     test_step.dependOn(&canon_data_tr.step);
diff --git a/codegen/wbp.zig b/codegen/wbp.zig
new file mode 100644
index 0000000..741103e
--- /dev/null
+++ b/codegen/wbp.zig
@@ -0,0 +1,146 @@
+const std = @import("std");
+const builtin = @import("builtin");
+
+const WordBreakProperty = enum(u5) {
+    none,
+    Double_Quote,
+    Single_Quote,
+    Hebrew_Letter,
+    CR,
+    LF,
+    Newline,
+    Extend,
+    Regional_Indicator,
+    Format,
+    Katakana,
+    ALetter,
+    MidLetter,
+    MidNum,
+    MidNumLet,
+    Numeric,
+    ExtendNumLet,
+    ZWJ,
+    WSegSpace,
+};
+
+const block_size = 256;
+const Block = [block_size]u5;
+
+const BlockMap = std.HashMap(
+    Block,
+    u16,
+    struct {
+        pub fn hash(_: @This(), k: Block) u64 {
+            var hasher = std.hash.Wyhash.init(0);
+            std.hash.autoHashStrat(&hasher, k, .DeepRecursive);
+            return hasher.final();
+        }
+
+        pub fn eql(_: @This(), a: Block, b: Block) bool {
+            return std.mem.eql(u5, &a, &b);
+        }
+    },
+    std.hash_map.default_max_load_percentage,
+);
+
+pub fn main() !void {
+    var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
+    defer arena.deinit();
+    const allocator = arena.allocator();
+
+    var flat_map = std.AutoHashMap(u21, u5).init(allocator);
+    defer flat_map.deinit();
+
+    var line_buf: [4096]u8 = undefined;
+
+    // Process HangulSyllableType.txt
+    var in_file = try std.fs.cwd().openFile("data/unicode/auxiliary/WordBreakProperty.txt", .{});
+    defer in_file.close();
+    var in_buf = std.io.bufferedReader(in_file.reader());
+    const in_reader = in_buf.reader();
+
+    while (try in_reader.readUntilDelimiterOrEof(&line_buf, '\n')) |line| {
+        if (line.len == 0 or line[0] == '#') continue;
+
+        const no_comment = if (std.mem.indexOfScalar(u8, line, '#')) |octo| line[0..octo] else line;
+
+        var field_iter = std.mem.tokenizeAny(u8, no_comment, "; ");
+        var current_code: [2]u21 = undefined;
+
+        var i: usize = 0;
+        while (field_iter.next()) |field| : (i += 1) {
+            switch (i) {
+                0 => {
+                    // Code point(s)
+                    if (std.mem.indexOf(u8, field, "..")) |dots| {
+                        current_code = .{
+                            try std.fmt.parseInt(u21, field[0..dots], 16),
+                            try std.fmt.parseInt(u21, field[dots + 2 ..], 16),
+                        };
+                    } else {
+                        const code = try std.fmt.parseInt(u21, field, 16);
+                        current_code = .{ code, code };
+                    }
+                },
+                1 => {
+                    // WordBreak type
+                    const st: WordBreakProperty = std.meta.stringToEnum(WordBreakProperty, field) orelse .none;
+                    for (current_code[0]..current_code[1] + 1) |cp| try flat_map.put(@intCast(cp), @intFromEnum(st));
+                },
+                else => {},
+            }
+        }
+    }
+
+    var blocks_map = BlockMap.init(allocator);
+    defer blocks_map.deinit();
+
+    var stage1 = std.ArrayList(u16).init(allocator);
+    defer stage1.deinit();
+
+    var stage2 = std.ArrayList(u5).init(allocator);
+    defer stage2.deinit();
+
+    var block: Block = [_]u5{0} ** block_size;
+    var block_len: u16 = 0;
+
+    for (0..0x110000) |i| {
+        const cp: u21 = @intCast(i);
+        const st = flat_map.get(cp) orelse 0;
+
+        // Process block
+        block[block_len] = st;
+        block_len += 1;
+
+        if (block_len < block_size and cp != 0x10ffff) continue;
+
+        const gop = try blocks_map.getOrPut(block);
+        if (!gop.found_existing) {
+            gop.value_ptr.* = @intCast(stage2.items.len);
+            try stage2.appendSlice(&block);
+        }
+
+        try stage1.append(gop.value_ptr.*);
+        block_len = 0;
+    }
+
+    var args_iter = try std.process.argsWithAllocator(allocator);
+    defer args_iter.deinit();
+    _ = args_iter.skip();
+    const output_path = args_iter.next() orelse @panic("No output file arg!");
+
+    const compressor = std.compress.flate.deflate.compressor;
+    var out_file = try std.fs.cwd().createFile(output_path, .{});
+    defer out_file.close();
+    var out_comp = try compressor(.raw, out_file.writer(), .{ .level = .best });
+    const writer = out_comp.writer();
+
+    const endian = builtin.cpu.arch.endian();
+    try writer.writeInt(u16, @intCast(stage1.items.len), endian);
+    for (stage1.items) |i| try writer.writeInt(u16, i, endian);
+
+    try writer.writeInt(u16, @intCast(stage2.items.len), endian);
+    for (stage2.items) |i| try writer.writeInt(u8, i, endian);
+
+    try out_comp.flush();
+}
diff --git a/src/WordBreak.zig b/src/WordBreak.zig
new file mode 100644
index 0000000..9044740
--- /dev/null
+++ b/src/WordBreak.zig
@@ -0,0 +1,102 @@
+//! Word Breaking Algorithm.
+
+const WordBreakProperty = enum(u5) {
+    none,
+    Double_Quote,
+    Single_Quote,
+    Hebrew_Letter,
+    CR,
+    LF,
+    Newline,
+    Extend,
+    Regional_Indicator,
+    Format,
+    Katakana,
+    ALetter,
+    MidLetter,
+    MidNum,
+    MidNumLet,
+    Numeric,
+    ExtendNumLet,
+    ZWJ,
+    WSegSpace,
+};
+
+s1: []u16 = undefined,
+s2: []u5 = undefined,
+
+const WordBreak = @This();
+
+pub fn init(allocator: Allocator) Allocator.Error!WordBreak {
+    var wb: WordBreak = undefined;
+    try wb.setup(allocator);
+    return wb;
+}
+
+pub fn setup(wb: *WordBreak, allocator: Allocator) Allocator.Error!void {
+    wb.setupImpl(allocator) catch |err| {
+        switch (err) {
+            error.OutOfMemory => |e| return e,
+            else => unreachable,
+        }
+    };
+}
+
+inline fn setupImpl(wb: *WordBreak, allocator: Allocator) !void {
+    const decompressor = compress.flate.inflate.decompressor;
+    const in_bytes = @embedFile("wbp");
+    var in_fbs = std.io.fixedBufferStream(in_bytes);
+    var in_decomp = decompressor(.raw, in_fbs.reader());
+    var reader = in_decomp.reader();
+
+    const endian = builtin.cpu.arch.endian();
+
+    const stage_1_len: u16 = try reader.readInt(u16, endian);
+    wb.s1 = try allocator.alloc(u16, stage_1_len);
+    errdefer allocator.free(wb.s1);
+    for (0..stage_1_len) |i| wb.s1[i] = try reader.readInt(u16, endian);
+
+    const stage_2_len: u16 = try reader.readInt(u16, endian);
+    wb.s2 = try allocator.alloc(u5, stage_2_len);
+    errdefer allocator.free(wb.s2);
+    for (0..stage_2_len) |i| wb.s2[i] = @intCast(try reader.readInt(u8, endian));
+    var count_0: usize = 0;
+    for (wb.s2) |nyb| {
+        if (nyb == 0) count_0 += 1;
+    }
+}
+
+pub fn deinit(wordbreak: *const WordBreak, allocator: mem.Allocator) void {
+    allocator.free(wordbreak.s1);
+    allocator.free(wordbreak.s2);
+}
+
+/// Returns the word break property type for `cp`.
+pub fn breakProperty(wordbreak: *const WordBreak, cp: u21) WordBreakProperty {
+    return @enumFromInt(wordbreak.s2[wordbreak.s1[cp >> 8] + (cp & 0xff)]);
+}
+
+test "Word Break Properties" {
+    const wb = try WordBreak.init(testing.allocator);
+    defer wb.deinit(testing.allocator);
+    try testing.expectEqual(.CR, wb.breakProperty('\r'));
+    try testing.expectEqual(.LF, wb.breakProperty('\n'));
+    try testing.expectEqual(.Hebrew_Letter, wb.breakProperty('ש'));
+    try testing.expectEqual(.Katakana, wb.breakProperty('\u{30ff}'));
+}
+
+fn testAllocations(allocator: Allocator) !void {
+    const wb = try WordBreak.init(allocator);
+    wb.deinit(allocator);
+}
+
+test "allocation safety" {
+    try testing.checkAllAllocationFailures(testing.allocator, testAllocations, .{});
+}
+
+const std = @import("std");
+const builtin = @import("builtin");
+const compress = std.compress;
+const mem = std.mem;
+const Allocator = mem.Allocator;
+const testing = std.testing;