From 7a212f5ec5aabf016d17d3ed28649e7982b810ef Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Wed, 30 Apr 2025 12:02:17 -0400 Subject: grapheme now Graphemes, Data files gone --- src/GraphemeData.zig | 87 ----------- src/Graphemes.zig | 421 ++++++++++++++++++++++++++++++++++++++++++++++++++ src/WidthData.zig | 102 ------------ src/grapheme.zig | 421 -------------------------------------------------- src/unicode_tests.zig | 8 +- 5 files changed, 425 insertions(+), 614 deletions(-) delete mode 100644 src/GraphemeData.zig create mode 100644 src/Graphemes.zig delete mode 100644 src/WidthData.zig delete mode 100644 src/grapheme.zig (limited to 'src') diff --git a/src/GraphemeData.zig b/src/GraphemeData.zig deleted file mode 100644 index df025cb..0000000 --- a/src/GraphemeData.zig +++ /dev/null @@ -1,87 +0,0 @@ -const std = @import("std"); -const builtin = @import("builtin"); -const compress = std.compress; -const mem = std.mem; - -/// Indic syllable type. -pub const Indic = enum { - none, - - Consonant, - Extend, - Linker, -}; - -/// Grapheme break property. -pub const Gbp = enum { - none, - Control, - CR, - Extend, - L, - LF, - LV, - LVT, - Prepend, - Regional_Indicator, - SpacingMark, - T, - V, - ZWJ, -}; - -s1: []u16 = undefined, -s2: []u16 = undefined, -s3: []u8 = undefined, - -const Self = @This(); - -pub inline fn init(allocator: mem.Allocator) mem.Allocator.Error!Self { - const decompressor = compress.flate.inflate.decompressor; - const in_bytes = @embedFile("gbp"); - var in_fbs = std.io.fixedBufferStream(in_bytes); - var in_decomp = decompressor(.raw, in_fbs.reader()); - var reader = in_decomp.reader(); - - const endian = builtin.cpu.arch.endian(); - - var self = Self{}; - - const s1_len: u16 = reader.readInt(u16, endian) catch unreachable; - self.s1 = try allocator.alloc(u16, s1_len); - errdefer allocator.free(self.s1); - for (0..s1_len) |i| self.s1[i] = reader.readInt(u16, endian) catch unreachable; - - const s2_len: u16 = reader.readInt(u16, endian) catch unreachable; - self.s2 = try allocator.alloc(u16, s2_len); - errdefer allocator.free(self.s2); - for (0..s2_len) |i| self.s2[i] = reader.readInt(u16, endian) catch unreachable; - - const s3_len: u16 = reader.readInt(u16, endian) catch unreachable; - self.s3 = try allocator.alloc(u8, s3_len); - errdefer allocator.free(self.s3); - _ = reader.readAll(self.s3) catch unreachable; - - return self; -} - -pub inline fn deinit(self: *const Self, allocator: mem.Allocator) void { - allocator.free(self.s1); - allocator.free(self.s2); - allocator.free(self.s3); -} - -/// Lookup the grapheme break property for a code point. -pub inline fn gbp(self: Self, cp: u21) Gbp { - return @enumFromInt(self.s3[self.s2[self.s1[cp >> 8] + (cp & 0xff)]] >> 4); -} - -/// Lookup the indic syllable type for a code point. -pub inline fn indic(self: Self, cp: u21) Indic { - return @enumFromInt((self.s3[self.s2[self.s1[cp >> 8] + (cp & 0xff)]] >> 1) & 0x7); -} - -/// Lookup the emoji property for a code point. -pub inline fn isEmoji(self: Self, cp: u21) bool { - return self.s3[self.s2[self.s1[cp >> 8] + (cp & 0xff)]] & 1 == 1; -} diff --git a/src/Graphemes.zig b/src/Graphemes.zig new file mode 100644 index 0000000..79cd2c6 --- /dev/null +++ b/src/Graphemes.zig @@ -0,0 +1,421 @@ +const std = @import("std"); +const builtin = @import("builtin"); +const mem = std.mem; +const Allocator = mem.Allocator; +const compress = std.compress; +const unicode = std.unicode; + +const CodePoint = @import("code_point").CodePoint; +const CodePointIterator = @import("code_point").Iterator; + +s1: []u16 = undefined, +s2: []u16 = undefined, +s3: []u8 = undefined, + +const Graphemes = @This(); + +pub inline fn init(allocator: mem.Allocator) mem.Allocator.Error!Graphemes { + const decompressor = compress.flate.inflate.decompressor; + const in_bytes = @embedFile("gbp"); + var in_fbs = std.io.fixedBufferStream(in_bytes); + var in_decomp = decompressor(.raw, in_fbs.reader()); + var reader = in_decomp.reader(); + + const endian = builtin.cpu.arch.endian(); + + var self = Graphemes{}; + + const s1_len: u16 = reader.readInt(u16, endian) catch unreachable; + self.s1 = try allocator.alloc(u16, s1_len); + errdefer allocator.free(self.s1); + for (0..s1_len) |i| self.s1[i] = reader.readInt(u16, endian) catch unreachable; + + const s2_len: u16 = reader.readInt(u16, endian) catch unreachable; + self.s2 = try allocator.alloc(u16, s2_len); + errdefer allocator.free(self.s2); + for (0..s2_len) |i| self.s2[i] = reader.readInt(u16, endian) catch unreachable; + + const s3_len: u16 = reader.readInt(u16, endian) catch unreachable; + self.s3 = try allocator.alloc(u8, s3_len); + errdefer allocator.free(self.s3); + _ = reader.readAll(self.s3) catch unreachable; + + return self; +} + +pub fn deinit(graphemes: *const Graphemes, allocator: mem.Allocator) void { + allocator.free(graphemes.s1); + allocator.free(graphemes.s2); + allocator.free(graphemes.s3); +} + +/// Lookup the grapheme break property for a code point. +pub fn gbp(graphemes: Graphemes, cp: u21) Gbp { + return @enumFromInt(graphemes.s3[graphemes.s2[graphemes.s1[cp >> 8] + (cp & 0xff)]] >> 4); +} + +/// Lookup the indic syllable type for a code point. +pub fn indic(graphemes: Graphemes, cp: u21) Indic { + return @enumFromInt((graphemes.s3[graphemes.s2[graphemes.s1[cp >> 8] + (cp & 0xff)]] >> 1) & 0x7); +} + +/// Lookup the emoji property for a code point. +pub fn isEmoji(graphemes: Graphemes, cp: u21) bool { + return graphemes.s3[graphemes.s2[graphemes.s1[cp >> 8] + (cp & 0xff)]] & 1 == 1; +} + +pub fn iterator(graphemes: *const Graphemes, string: []const u8) Iterator { + return Iterator.init(string, graphemes); +} + +/// Indic syllable type. +pub const Indic = enum { + none, + + Consonant, + Extend, + Linker, +}; + +/// Grapheme break property. +pub const Gbp = enum { + none, + Control, + CR, + Extend, + L, + LF, + LV, + LVT, + Prepend, + Regional_Indicator, + SpacingMark, + T, + V, + ZWJ, +}; + +/// `Grapheme` represents a Unicode grapheme cluster by its length and offset in the source bytes. +pub const Grapheme = struct { + len: u8, + offset: u32, + + /// `bytes` returns the slice of bytes that correspond to + /// this grapheme cluster in `src`. + pub fn bytes(self: Grapheme, src: []const u8) []const u8 { + return src[self.offset..][0..self.len]; + } +}; + +/// `Iterator` iterates a sting of UTF-8 encoded bytes one grapheme cluster at-a-time. +pub const Iterator = struct { + buf: [2]?CodePoint = .{ null, null }, + cp_iter: CodePointIterator, + data: *const Graphemes, + + const Self = @This(); + + /// Assumes `src` is valid UTF-8. + pub fn init(str: []const u8, data: *const Graphemes) Self { + var self = Self{ .cp_iter = .{ .bytes = str }, .data = data }; + self.advance(); + return self; + } + + fn advance(self: *Self) void { + self.buf[0] = self.buf[1]; + self.buf[1] = self.cp_iter.next(); + } + + pub fn next(self: *Self) ?Grapheme { + self.advance(); + + // If no more + if (self.buf[0] == null) return null; + // If last one + if (self.buf[1] == null) return Grapheme{ .len = self.buf[0].?.len, .offset = self.buf[0].?.offset }; + // If ASCII + if (self.buf[0].?.code != '\r' and self.buf[0].?.code < 128 and self.buf[1].?.code < 128) { + return Grapheme{ .len = self.buf[0].?.len, .offset = self.buf[0].?.offset }; + } + + const gc_start = self.buf[0].?.offset; + var gc_len: u8 = self.buf[0].?.len; + var state = State{}; + + if (graphemeBreak( + self.buf[0].?.code, + self.buf[1].?.code, + self.data, + &state, + )) return Grapheme{ .len = gc_len, .offset = gc_start }; + + while (true) { + self.advance(); + if (self.buf[0] == null) break; + + gc_len += self.buf[0].?.len; + + if (graphemeBreak( + self.buf[0].?.code, + if (self.buf[1]) |ncp| ncp.code else 0, + self.data, + &state, + )) break; + } + + return Grapheme{ .len = gc_len, .offset = gc_start }; + } + + pub fn peek(self: *Self) ?Grapheme { + const saved_cp_iter = self.cp_iter; + const s0 = self.buf[0]; + const s1 = self.buf[1]; + + self.advance(); + + // If no more + if (self.buf[0] == null) { + self.cp_iter = saved_cp_iter; + self.buf[0] = s0; + self.buf[1] = s1; + return null; + } + // If last one + if (self.buf[1] == null) { + const len = self.buf[0].?.len; + const offset = self.buf[0].?.offset; + self.cp_iter = saved_cp_iter; + self.buf[0] = s0; + self.buf[1] = s1; + return Grapheme{ .len = len, .offset = offset }; + } + // If ASCII + if (self.buf[0].?.code != '\r' and self.buf[0].?.code < 128 and self.buf[1].?.code < 128) { + const len = self.buf[0].?.len; + const offset = self.buf[0].?.offset; + self.cp_iter = saved_cp_iter; + self.buf[0] = s0; + self.buf[1] = s1; + return Grapheme{ .len = len, .offset = offset }; + } + + const gc_start = self.buf[0].?.offset; + var gc_len: u8 = self.buf[0].?.len; + var state = State{}; + + if (graphemeBreak( + self.buf[0].?.code, + self.buf[1].?.code, + self.data, + &state, + )) { + self.cp_iter = saved_cp_iter; + self.buf[0] = s0; + self.buf[1] = s1; + return Grapheme{ .len = gc_len, .offset = gc_start }; + } + + while (true) { + self.advance(); + if (self.buf[0] == null) break; + + gc_len += self.buf[0].?.len; + + if (graphemeBreak( + self.buf[0].?.code, + if (self.buf[1]) |ncp| ncp.code else 0, + self.data, + &state, + )) break; + } + self.cp_iter = saved_cp_iter; + self.buf[0] = s0; + self.buf[1] = s1; + + return Grapheme{ .len = gc_len, .offset = gc_start }; + } +}; + +// Predicates +fn isBreaker(cp: u21, data: *const Graphemes) bool { + // Extract relevant properties. + const cp_gbp_prop = data.gbp(cp); + return cp == '\x0d' or cp == '\x0a' or cp_gbp_prop == .Control; +} + +// Grapheme break state. +pub const State = struct { + bits: u3 = 0, + + // Extended Pictographic (emoji) + fn hasXpic(self: State) bool { + return self.bits & 1 == 1; + } + fn setXpic(self: *State) void { + self.bits |= 1; + } + fn unsetXpic(self: *State) void { + self.bits ^= 1; + } + + // Regional Indicatior (flags) + fn hasRegional(self: State) bool { + return self.bits & 2 == 2; + } + fn setRegional(self: *State) void { + self.bits |= 2; + } + fn unsetRegional(self: *State) void { + self.bits ^= 2; + } + + // Indic Conjunct + fn hasIndic(self: State) bool { + return self.bits & 4 == 4; + } + fn setIndic(self: *State) void { + self.bits |= 4; + } + fn unsetIndic(self: *State) void { + self.bits ^= 4; + } +}; + +/// `graphemeBreak` returns true only if a grapheme break point is required +/// between `cp1` and `cp2`. `state` should start out as 0. If calling +/// iteratively over a sequence of code points, this function must be called +/// IN ORDER on ALL potential breaks in a string. +/// Modeled after the API of utf8proc's `utf8proc_grapheme_break_stateful`. +/// https://github.com/JuliaStrings/utf8proc/blob/2bbb1ba932f727aad1fab14fafdbc89ff9dc4604/utf8proc.h#L599-L617 +pub fn graphemeBreak( + cp1: u21, + cp2: u21, + data: *const Graphemes, + state: *State, +) bool { + // Extract relevant properties. + const cp1_gbp_prop = data.gbp(cp1); + const cp1_indic_prop = data.indic(cp1); + const cp1_is_emoji = data.isEmoji(cp1); + + const cp2_gbp_prop = data.gbp(cp2); + const cp2_indic_prop = data.indic(cp2); + const cp2_is_emoji = data.isEmoji(cp2); + + // GB11: Emoji Extend* ZWJ x Emoji + if (!state.hasXpic() and cp1_is_emoji) state.setXpic(); + // GB9c: Indic Conjunct Break + if (!state.hasIndic() and cp1_indic_prop == .Consonant) state.setIndic(); + + // GB3: CR x LF + if (cp1 == '\r' and cp2 == '\n') return false; + + // GB4: Control + if (isBreaker(cp1, data)) return true; + + // GB11: Emoji Extend* ZWJ x Emoji + if (state.hasXpic() and + cp1_gbp_prop == .ZWJ and + cp2_is_emoji) + { + state.unsetXpic(); + return false; + } + + // GB9b: x (Extend | ZWJ) + if (cp2_gbp_prop == .Extend or cp2_gbp_prop == .ZWJ) return false; + + // GB9a: x Spacing + if (cp2_gbp_prop == .SpacingMark) return false; + + // GB9b: Prepend x + if (cp1_gbp_prop == .Prepend and !isBreaker(cp2, data)) return false; + + // GB12, GB13: RI x RI + if (cp1_gbp_prop == .Regional_Indicator and cp2_gbp_prop == .Regional_Indicator) { + if (state.hasRegional()) { + state.unsetRegional(); + return true; + } else { + state.setRegional(); + return false; + } + } + + // GB6: Hangul L x (L|V|LV|VT) + if (cp1_gbp_prop == .L) { + if (cp2_gbp_prop == .L or + cp2_gbp_prop == .V or + cp2_gbp_prop == .LV or + cp2_gbp_prop == .LVT) return false; + } + + // GB7: Hangul (LV | V) x (V | T) + if (cp1_gbp_prop == .LV or cp1_gbp_prop == .V) { + if (cp2_gbp_prop == .V or + cp2_gbp_prop == .T) return false; + } + + // GB8: Hangul (LVT | T) x T + if (cp1_gbp_prop == .LVT or cp1_gbp_prop == .T) { + if (cp2_gbp_prop == .T) return false; + } + + // GB9c: Indic Conjunct Break + if (state.hasIndic() and + cp1_indic_prop == .Consonant and + (cp2_indic_prop == .Extend or cp2_indic_prop == .Linker)) + { + return false; + } + + if (state.hasIndic() and + cp1_indic_prop == .Extend and + cp2_indic_prop == .Linker) + { + return false; + } + + if (state.hasIndic() and + (cp1_indic_prop == .Linker or cp1_gbp_prop == .ZWJ) and + cp2_indic_prop == .Consonant) + { + state.unsetIndic(); + return false; + } + + return true; +} + +test "Segmentation ZWJ and ZWSP emoji sequences" { + const seq_1 = "\u{1F43B}\u{200D}\u{2744}\u{FE0F}"; + const seq_2 = "\u{1F43B}\u{200D}\u{2744}\u{FE0F}"; + const with_zwj = seq_1 ++ "\u{200D}" ++ seq_2; + const with_zwsp = seq_1 ++ "\u{200B}" ++ seq_2; + const no_joiner = seq_1 ++ seq_2; + + const graphemes = try Graphemes.init(std.testing.allocator); + defer graphemes.deinit(std.testing.allocator); + + { + var iter = graphemes.iterator(with_zwj); + var i: usize = 0; + while (iter.next()) |_| : (i += 1) {} + try std.testing.expectEqual(@as(usize, 1), i); + } + + { + var iter = graphemes.iterator(with_zwsp); + var i: usize = 0; + while (iter.next()) |_| : (i += 1) {} + try std.testing.expectEqual(@as(usize, 3), i); + } + + { + var iter = graphemes.iterator(no_joiner); + var i: usize = 0; + while (iter.next()) |_| : (i += 1) {} + try std.testing.expectEqual(@as(usize, 2), i); + } +} diff --git a/src/WidthData.zig b/src/WidthData.zig deleted file mode 100644 index ca7eaf0..0000000 --- a/src/WidthData.zig +++ /dev/null @@ -1,102 +0,0 @@ -const std = @import("std"); -const builtin = @import("builtin"); -const compress = std.compress; -const mem = std.mem; -const testing = std.testing; - -const Graphemes = @import("Graphemes"); - -g_data: Graphemes, -s1: []u16 = undefined, -s2: []i4 = undefined, -owns_gdata: bool, - -const Self = @This(); - -pub fn init(allocator: mem.Allocator) mem.Allocator.Error!Self { - var self: Self = try Self.setup(allocator); - errdefer { - allocator.free(self.s1); - allocator.free(self.s2); - } - self.owns_gdata = true; - self.g_data = try Graphemes.init(allocator); - errdefer self.g_data.deinit(allocator); - return self; -} - -pub fn initWithGraphemeData(allocator: mem.Allocator, g_data: Graphemes) mem.Allocator.Error!Self { - var self = try Self.setup(allocator); - self.g_data = g_data; - self.owns_gdata = false; - return self; -} - -// Sets up the DisplayWidthData, leaving the GraphemeData undefined. -fn setup(allocator: mem.Allocator) mem.Allocator.Error!Self { - const decompressor = compress.flate.inflate.decompressor; - const in_bytes = @embedFile("dwp"); - var in_fbs = std.io.fixedBufferStream(in_bytes); - var in_decomp = decompressor(.raw, in_fbs.reader()); - var reader = in_decomp.reader(); - - const endian = builtin.cpu.arch.endian(); - - var self: Self = undefined; - - const stage_1_len: u16 = reader.readInt(u16, endian) catch unreachable; - self.s1 = try allocator.alloc(u16, stage_1_len); - errdefer allocator.free(self.s1); - for (0..stage_1_len) |i| self.s1[i] = reader.readInt(u16, endian) catch unreachable; - - const stage_2_len: u16 = reader.readInt(u16, endian) catch unreachable; - self.s2 = try allocator.alloc(i4, stage_2_len); - errdefer allocator.free(self.s2); - for (0..stage_2_len) |i| self.s2[i] = @intCast(reader.readInt(i8, endian) catch unreachable); - - return self; -} - -pub fn deinit(self: *const Self, allocator: mem.Allocator) void { - allocator.free(self.s1); - allocator.free(self.s2); - if (self.owns_gdata) self.g_data.deinit(allocator); -} - -/// codePointWidth returns the number of cells `cp` requires when rendered -/// in a fixed-pitch font (i.e. a terminal screen). This can range from -1 to -/// 3, where BACKSPACE and DELETE return -1 and 3-em-dash returns 3. C0/C1 -/// control codes return 0. If `cjk` is true, ambiguous code points return 2, -/// otherwise they return 1. -pub fn codePointWidth(self: Self, cp: u21) i4 { - return self.s2[self.s1[cp >> 8] + (cp & 0xff)]; -} - -test "codePointWidth" { - const wd = try Self.init(std.testing.allocator); - defer wd.deinit(std.testing.allocator); - try testing.expectEqual(@as(i4, 0), wd.codePointWidth(0x0000)); // null - try testing.expectEqual(@as(i4, -1), wd.codePointWidth(0x8)); // \b - try testing.expectEqual(@as(i4, -1), wd.codePointWidth(0x7f)); // DEL - try testing.expectEqual(@as(i4, 0), wd.codePointWidth(0x0005)); // Cf - try testing.expectEqual(@as(i4, 0), wd.codePointWidth(0x0007)); // \a BEL - try testing.expectEqual(@as(i4, 0), wd.codePointWidth(0x000A)); // \n LF - try testing.expectEqual(@as(i4, 0), wd.codePointWidth(0x000B)); // \v VT - try testing.expectEqual(@as(i4, 0), wd.codePointWidth(0x000C)); // \f FF - try testing.expectEqual(@as(i4, 0), wd.codePointWidth(0x000D)); // \r CR - try testing.expectEqual(@as(i4, 0), wd.codePointWidth(0x000E)); // SQ - try testing.expectEqual(@as(i4, 0), wd.codePointWidth(0x000F)); // SI - - try testing.expectEqual(@as(i4, 0), wd.codePointWidth(0x070F)); // Cf - try testing.expectEqual(@as(i4, 1), wd.codePointWidth(0x0603)); // Cf Arabic - - try testing.expectEqual(@as(i4, 1), wd.codePointWidth(0x00AD)); // soft-hyphen - try testing.expectEqual(@as(i4, 2), wd.codePointWidth(0x2E3A)); // two-em dash - try testing.expectEqual(@as(i4, 3), wd.codePointWidth(0x2E3B)); // three-em dash - - try testing.expectEqual(@as(i4, 1), wd.codePointWidth(0x00BD)); // ambiguous halfwidth - - try testing.expectEqual(@as(i4, 1), wd.codePointWidth('é')); - try testing.expectEqual(@as(i4, 2), wd.codePointWidth('😊')); - try testing.expectEqual(@as(i4, 2), wd.codePointWidth('统')); -} diff --git a/src/grapheme.zig b/src/grapheme.zig deleted file mode 100644 index 79cd2c6..0000000 --- a/src/grapheme.zig +++ /dev/null @@ -1,421 +0,0 @@ -const std = @import("std"); -const builtin = @import("builtin"); -const mem = std.mem; -const Allocator = mem.Allocator; -const compress = std.compress; -const unicode = std.unicode; - -const CodePoint = @import("code_point").CodePoint; -const CodePointIterator = @import("code_point").Iterator; - -s1: []u16 = undefined, -s2: []u16 = undefined, -s3: []u8 = undefined, - -const Graphemes = @This(); - -pub inline fn init(allocator: mem.Allocator) mem.Allocator.Error!Graphemes { - const decompressor = compress.flate.inflate.decompressor; - const in_bytes = @embedFile("gbp"); - var in_fbs = std.io.fixedBufferStream(in_bytes); - var in_decomp = decompressor(.raw, in_fbs.reader()); - var reader = in_decomp.reader(); - - const endian = builtin.cpu.arch.endian(); - - var self = Graphemes{}; - - const s1_len: u16 = reader.readInt(u16, endian) catch unreachable; - self.s1 = try allocator.alloc(u16, s1_len); - errdefer allocator.free(self.s1); - for (0..s1_len) |i| self.s1[i] = reader.readInt(u16, endian) catch unreachable; - - const s2_len: u16 = reader.readInt(u16, endian) catch unreachable; - self.s2 = try allocator.alloc(u16, s2_len); - errdefer allocator.free(self.s2); - for (0..s2_len) |i| self.s2[i] = reader.readInt(u16, endian) catch unreachable; - - const s3_len: u16 = reader.readInt(u16, endian) catch unreachable; - self.s3 = try allocator.alloc(u8, s3_len); - errdefer allocator.free(self.s3); - _ = reader.readAll(self.s3) catch unreachable; - - return self; -} - -pub fn deinit(graphemes: *const Graphemes, allocator: mem.Allocator) void { - allocator.free(graphemes.s1); - allocator.free(graphemes.s2); - allocator.free(graphemes.s3); -} - -/// Lookup the grapheme break property for a code point. -pub fn gbp(graphemes: Graphemes, cp: u21) Gbp { - return @enumFromInt(graphemes.s3[graphemes.s2[graphemes.s1[cp >> 8] + (cp & 0xff)]] >> 4); -} - -/// Lookup the indic syllable type for a code point. -pub fn indic(graphemes: Graphemes, cp: u21) Indic { - return @enumFromInt((graphemes.s3[graphemes.s2[graphemes.s1[cp >> 8] + (cp & 0xff)]] >> 1) & 0x7); -} - -/// Lookup the emoji property for a code point. -pub fn isEmoji(graphemes: Graphemes, cp: u21) bool { - return graphemes.s3[graphemes.s2[graphemes.s1[cp >> 8] + (cp & 0xff)]] & 1 == 1; -} - -pub fn iterator(graphemes: *const Graphemes, string: []const u8) Iterator { - return Iterator.init(string, graphemes); -} - -/// Indic syllable type. -pub const Indic = enum { - none, - - Consonant, - Extend, - Linker, -}; - -/// Grapheme break property. -pub const Gbp = enum { - none, - Control, - CR, - Extend, - L, - LF, - LV, - LVT, - Prepend, - Regional_Indicator, - SpacingMark, - T, - V, - ZWJ, -}; - -/// `Grapheme` represents a Unicode grapheme cluster by its length and offset in the source bytes. -pub const Grapheme = struct { - len: u8, - offset: u32, - - /// `bytes` returns the slice of bytes that correspond to - /// this grapheme cluster in `src`. - pub fn bytes(self: Grapheme, src: []const u8) []const u8 { - return src[self.offset..][0..self.len]; - } -}; - -/// `Iterator` iterates a sting of UTF-8 encoded bytes one grapheme cluster at-a-time. -pub const Iterator = struct { - buf: [2]?CodePoint = .{ null, null }, - cp_iter: CodePointIterator, - data: *const Graphemes, - - const Self = @This(); - - /// Assumes `src` is valid UTF-8. - pub fn init(str: []const u8, data: *const Graphemes) Self { - var self = Self{ .cp_iter = .{ .bytes = str }, .data = data }; - self.advance(); - return self; - } - - fn advance(self: *Self) void { - self.buf[0] = self.buf[1]; - self.buf[1] = self.cp_iter.next(); - } - - pub fn next(self: *Self) ?Grapheme { - self.advance(); - - // If no more - if (self.buf[0] == null) return null; - // If last one - if (self.buf[1] == null) return Grapheme{ .len = self.buf[0].?.len, .offset = self.buf[0].?.offset }; - // If ASCII - if (self.buf[0].?.code != '\r' and self.buf[0].?.code < 128 and self.buf[1].?.code < 128) { - return Grapheme{ .len = self.buf[0].?.len, .offset = self.buf[0].?.offset }; - } - - const gc_start = self.buf[0].?.offset; - var gc_len: u8 = self.buf[0].?.len; - var state = State{}; - - if (graphemeBreak( - self.buf[0].?.code, - self.buf[1].?.code, - self.data, - &state, - )) return Grapheme{ .len = gc_len, .offset = gc_start }; - - while (true) { - self.advance(); - if (self.buf[0] == null) break; - - gc_len += self.buf[0].?.len; - - if (graphemeBreak( - self.buf[0].?.code, - if (self.buf[1]) |ncp| ncp.code else 0, - self.data, - &state, - )) break; - } - - return Grapheme{ .len = gc_len, .offset = gc_start }; - } - - pub fn peek(self: *Self) ?Grapheme { - const saved_cp_iter = self.cp_iter; - const s0 = self.buf[0]; - const s1 = self.buf[1]; - - self.advance(); - - // If no more - if (self.buf[0] == null) { - self.cp_iter = saved_cp_iter; - self.buf[0] = s0; - self.buf[1] = s1; - return null; - } - // If last one - if (self.buf[1] == null) { - const len = self.buf[0].?.len; - const offset = self.buf[0].?.offset; - self.cp_iter = saved_cp_iter; - self.buf[0] = s0; - self.buf[1] = s1; - return Grapheme{ .len = len, .offset = offset }; - } - // If ASCII - if (self.buf[0].?.code != '\r' and self.buf[0].?.code < 128 and self.buf[1].?.code < 128) { - const len = self.buf[0].?.len; - const offset = self.buf[0].?.offset; - self.cp_iter = saved_cp_iter; - self.buf[0] = s0; - self.buf[1] = s1; - return Grapheme{ .len = len, .offset = offset }; - } - - const gc_start = self.buf[0].?.offset; - var gc_len: u8 = self.buf[0].?.len; - var state = State{}; - - if (graphemeBreak( - self.buf[0].?.code, - self.buf[1].?.code, - self.data, - &state, - )) { - self.cp_iter = saved_cp_iter; - self.buf[0] = s0; - self.buf[1] = s1; - return Grapheme{ .len = gc_len, .offset = gc_start }; - } - - while (true) { - self.advance(); - if (self.buf[0] == null) break; - - gc_len += self.buf[0].?.len; - - if (graphemeBreak( - self.buf[0].?.code, - if (self.buf[1]) |ncp| ncp.code else 0, - self.data, - &state, - )) break; - } - self.cp_iter = saved_cp_iter; - self.buf[0] = s0; - self.buf[1] = s1; - - return Grapheme{ .len = gc_len, .offset = gc_start }; - } -}; - -// Predicates -fn isBreaker(cp: u21, data: *const Graphemes) bool { - // Extract relevant properties. - const cp_gbp_prop = data.gbp(cp); - return cp == '\x0d' or cp == '\x0a' or cp_gbp_prop == .Control; -} - -// Grapheme break state. -pub const State = struct { - bits: u3 = 0, - - // Extended Pictographic (emoji) - fn hasXpic(self: State) bool { - return self.bits & 1 == 1; - } - fn setXpic(self: *State) void { - self.bits |= 1; - } - fn unsetXpic(self: *State) void { - self.bits ^= 1; - } - - // Regional Indicatior (flags) - fn hasRegional(self: State) bool { - return self.bits & 2 == 2; - } - fn setRegional(self: *State) void { - self.bits |= 2; - } - fn unsetRegional(self: *State) void { - self.bits ^= 2; - } - - // Indic Conjunct - fn hasIndic(self: State) bool { - return self.bits & 4 == 4; - } - fn setIndic(self: *State) void { - self.bits |= 4; - } - fn unsetIndic(self: *State) void { - self.bits ^= 4; - } -}; - -/// `graphemeBreak` returns true only if a grapheme break point is required -/// between `cp1` and `cp2`. `state` should start out as 0. If calling -/// iteratively over a sequence of code points, this function must be called -/// IN ORDER on ALL potential breaks in a string. -/// Modeled after the API of utf8proc's `utf8proc_grapheme_break_stateful`. -/// https://github.com/JuliaStrings/utf8proc/blob/2bbb1ba932f727aad1fab14fafdbc89ff9dc4604/utf8proc.h#L599-L617 -pub fn graphemeBreak( - cp1: u21, - cp2: u21, - data: *const Graphemes, - state: *State, -) bool { - // Extract relevant properties. - const cp1_gbp_prop = data.gbp(cp1); - const cp1_indic_prop = data.indic(cp1); - const cp1_is_emoji = data.isEmoji(cp1); - - const cp2_gbp_prop = data.gbp(cp2); - const cp2_indic_prop = data.indic(cp2); - const cp2_is_emoji = data.isEmoji(cp2); - - // GB11: Emoji Extend* ZWJ x Emoji - if (!state.hasXpic() and cp1_is_emoji) state.setXpic(); - // GB9c: Indic Conjunct Break - if (!state.hasIndic() and cp1_indic_prop == .Consonant) state.setIndic(); - - // GB3: CR x LF - if (cp1 == '\r' and cp2 == '\n') return false; - - // GB4: Control - if (isBreaker(cp1, data)) return true; - - // GB11: Emoji Extend* ZWJ x Emoji - if (state.hasXpic() and - cp1_gbp_prop == .ZWJ and - cp2_is_emoji) - { - state.unsetXpic(); - return false; - } - - // GB9b: x (Extend | ZWJ) - if (cp2_gbp_prop == .Extend or cp2_gbp_prop == .ZWJ) return false; - - // GB9a: x Spacing - if (cp2_gbp_prop == .SpacingMark) return false; - - // GB9b: Prepend x - if (cp1_gbp_prop == .Prepend and !isBreaker(cp2, data)) return false; - - // GB12, GB13: RI x RI - if (cp1_gbp_prop == .Regional_Indicator and cp2_gbp_prop == .Regional_Indicator) { - if (state.hasRegional()) { - state.unsetRegional(); - return true; - } else { - state.setRegional(); - return false; - } - } - - // GB6: Hangul L x (L|V|LV|VT) - if (cp1_gbp_prop == .L) { - if (cp2_gbp_prop == .L or - cp2_gbp_prop == .V or - cp2_gbp_prop == .LV or - cp2_gbp_prop == .LVT) return false; - } - - // GB7: Hangul (LV | V) x (V | T) - if (cp1_gbp_prop == .LV or cp1_gbp_prop == .V) { - if (cp2_gbp_prop == .V or - cp2_gbp_prop == .T) return false; - } - - // GB8: Hangul (LVT | T) x T - if (cp1_gbp_prop == .LVT or cp1_gbp_prop == .T) { - if (cp2_gbp_prop == .T) return false; - } - - // GB9c: Indic Conjunct Break - if (state.hasIndic() and - cp1_indic_prop == .Consonant and - (cp2_indic_prop == .Extend or cp2_indic_prop == .Linker)) - { - return false; - } - - if (state.hasIndic() and - cp1_indic_prop == .Extend and - cp2_indic_prop == .Linker) - { - return false; - } - - if (state.hasIndic() and - (cp1_indic_prop == .Linker or cp1_gbp_prop == .ZWJ) and - cp2_indic_prop == .Consonant) - { - state.unsetIndic(); - return false; - } - - return true; -} - -test "Segmentation ZWJ and ZWSP emoji sequences" { - const seq_1 = "\u{1F43B}\u{200D}\u{2744}\u{FE0F}"; - const seq_2 = "\u{1F43B}\u{200D}\u{2744}\u{FE0F}"; - const with_zwj = seq_1 ++ "\u{200D}" ++ seq_2; - const with_zwsp = seq_1 ++ "\u{200B}" ++ seq_2; - const no_joiner = seq_1 ++ seq_2; - - const graphemes = try Graphemes.init(std.testing.allocator); - defer graphemes.deinit(std.testing.allocator); - - { - var iter = graphemes.iterator(with_zwj); - var i: usize = 0; - while (iter.next()) |_| : (i += 1) {} - try std.testing.expectEqual(@as(usize, 1), i); - } - - { - var iter = graphemes.iterator(with_zwsp); - var i: usize = 0; - while (iter.next()) |_| : (i += 1) {} - try std.testing.expectEqual(@as(usize, 3), i); - } - - { - var iter = graphemes.iterator(no_joiner); - var i: usize = 0; - while (iter.next()) |_| : (i += 1) {} - try std.testing.expectEqual(@as(usize, 2), i); - } -} diff --git a/src/unicode_tests.zig b/src/unicode_tests.zig index de1b9ec..3cb5df5 100644 --- a/src/unicode_tests.zig +++ b/src/unicode_tests.zig @@ -7,10 +7,10 @@ const mem = std.mem; const testing = std.testing; const unicode = std.unicode; -const grapheme = @import("grapheme"); -const Grapheme = @import("grapheme").Grapheme; -const Graphemes = @import("grapheme"); -const GraphemeIterator = @import("grapheme").Iterator; +const grapheme = @import("Graphemes"); +const Grapheme = @import("Graphemes").Grapheme; +const Graphemes = @import("Graphemes"); +const GraphemeIterator = @import("Graphemes").Iterator; const Normalize = @import("Normalize"); comptime { -- cgit v1.2.3