From ad306724ae574b1a22abbcb6de37e65a69db82e4 Mon Sep 17 00:00:00 2001 From: Jose Colon Rodriguez Date: Mon, 26 Feb 2024 20:22:41 -0400 Subject: Using NormData nfkd --- src/CanonData.zig | 56 +++++++++ src/Canonical.zig | 56 --------- src/CombiningClassData.zig | 48 -------- src/CombiningData.zig | 48 ++++++++ src/CompatData.zig | 50 ++++++++ src/DisplayWidthData.zig | 82 ------------- src/NormData.zig | 8 +- src/Normalizer.zig | 129 ++++++--------------- src/WidthData.zig | 82 +++++++++++++ .../compatibility_decompositions.txt.deflate | Bin 15332 -> 0 bytes src/main.zig | 11 +- 11 files changed, 283 insertions(+), 287 deletions(-) create mode 100644 src/CanonData.zig delete mode 100644 src/Canonical.zig delete mode 100644 src/CombiningClassData.zig create mode 100644 src/CombiningData.zig create mode 100644 src/CompatData.zig delete mode 100644 src/DisplayWidthData.zig create mode 100644 src/WidthData.zig delete mode 100644 src/autogen/compatibility_decompositions.txt.deflate (limited to 'src') diff --git a/src/CanonData.zig b/src/CanonData.zig new file mode 100644 index 0000000..81d3eec --- /dev/null +++ b/src/CanonData.zig @@ -0,0 +1,56 @@ +const std = @import("std"); +const builtin = @import("builtin"); +const compress = std.compress; +const mem = std.mem; + +allocator: mem.Allocator, +nfc: std.AutoHashMap([2]u21, u21), +nfd: [][2]u21 = undefined, + +const Self = @This(); + +pub fn init(allocator: mem.Allocator) !Self { + const decompressor = compress.deflate.decompressor; + const in_bytes = @embedFile("canon"); + var in_fbs = std.io.fixedBufferStream(in_bytes); + var in_decomp = try decompressor(allocator, in_fbs.reader(), null); + defer in_decomp.deinit(); + var reader = in_decomp.reader(); + + const endian = builtin.cpu.arch.endian(); + var self = Self{ + .allocator = allocator, + .nfc = std.AutoHashMap([2]u21, u21).init(allocator), + .nfd = try allocator.alloc([2]u21, 0x110000), + }; + + for (0..0x110000) |i| self.nfd[i] = .{ @intCast(i), 0 }; + + while (true) { + const len: u8 = try reader.readInt(u8, endian); + if (len == 0) break; + const cp = try reader.readInt(u24, endian); + self.nfd[cp][0] = @intCast(try reader.readInt(u24, endian)); + if (len == 3) { + self.nfd[cp][1] = @intCast(try reader.readInt(u24, endian)); + try self.nfc.put(self.nfd[cp], @intCast(cp)); + } + } + + return self; +} + +pub fn deinit(self: *Self) void { + self.nfc.deinit(); + self.allocator.free(self.nfd); +} + +/// Returns canonical decomposition for `cp`. +pub inline fn toNfd(self: Self, cp: u21) [2]u21 { + return self.nfd[cp]; +} + +// Returns the primary composite for the codepoints in `cp`. +pub inline fn toNfc(self: Self, cps: [2]u21) ?u21 { + return self.nfc.get(cps); +} diff --git a/src/Canonical.zig b/src/Canonical.zig deleted file mode 100644 index 81d3eec..0000000 --- a/src/Canonical.zig +++ /dev/null @@ -1,56 +0,0 @@ -const std = @import("std"); -const builtin = @import("builtin"); -const compress = std.compress; -const mem = std.mem; - -allocator: mem.Allocator, -nfc: std.AutoHashMap([2]u21, u21), -nfd: [][2]u21 = undefined, - -const Self = @This(); - -pub fn init(allocator: mem.Allocator) !Self { - const decompressor = compress.deflate.decompressor; - const in_bytes = @embedFile("canon"); - var in_fbs = std.io.fixedBufferStream(in_bytes); - var in_decomp = try decompressor(allocator, in_fbs.reader(), null); - defer in_decomp.deinit(); - var reader = in_decomp.reader(); - - const endian = builtin.cpu.arch.endian(); - var self = Self{ - .allocator = allocator, - .nfc = std.AutoHashMap([2]u21, u21).init(allocator), - .nfd = try allocator.alloc([2]u21, 0x110000), - }; - - for (0..0x110000) |i| self.nfd[i] = .{ @intCast(i), 0 }; - - while (true) { - const len: u8 = try reader.readInt(u8, endian); - if (len == 0) break; - const cp = try reader.readInt(u24, endian); - self.nfd[cp][0] = @intCast(try reader.readInt(u24, endian)); - if (len == 3) { - self.nfd[cp][1] = @intCast(try reader.readInt(u24, endian)); - try self.nfc.put(self.nfd[cp], @intCast(cp)); - } - } - - return self; -} - -pub fn deinit(self: *Self) void { - self.nfc.deinit(); - self.allocator.free(self.nfd); -} - -/// Returns canonical decomposition for `cp`. -pub inline fn toNfd(self: Self, cp: u21) [2]u21 { - return self.nfd[cp]; -} - -// Returns the primary composite for the codepoints in `cp`. -pub inline fn toNfc(self: Self, cps: [2]u21) ?u21 { - return self.nfc.get(cps); -} diff --git a/src/CombiningClassData.zig b/src/CombiningClassData.zig deleted file mode 100644 index 95c947d..0000000 --- a/src/CombiningClassData.zig +++ /dev/null @@ -1,48 +0,0 @@ -const std = @import("std"); -const builtin = @import("builtin"); -const compress = std.compress; -const mem = std.mem; - -allocator: mem.Allocator, -s1: []u16 = undefined, -s2: []u8 = undefined, - -const Self = @This(); - -pub fn init(allocator: mem.Allocator) !Self { - const decompressor = compress.deflate.decompressor; - const in_bytes = @embedFile("ccc"); - var in_fbs = std.io.fixedBufferStream(in_bytes); - var in_decomp = try decompressor(allocator, in_fbs.reader(), null); - defer in_decomp.deinit(); - var reader = in_decomp.reader(); - - const endian = builtin.cpu.arch.endian(); - - var self = Self{ .allocator = allocator }; - - const stage_1_len: u16 = try reader.readInt(u16, endian); - self.s1 = try allocator.alloc(u16, stage_1_len); - for (0..stage_1_len) |i| self.s1[i] = try reader.readInt(u16, endian); - - const stage_2_len: u16 = try reader.readInt(u16, endian); - self.s2 = try allocator.alloc(u8, stage_2_len); - _ = try reader.readAll(self.s2); - - return self; -} - -pub fn deinit(self: *Self) void { - self.allocator.free(self.s1); - self.allocator.free(self.s2); -} - -/// Returns the canonical combining class for a code point. -pub inline fn ccc(self: Self, cp: u21) u8 { - return self.s2[self.s1[cp >> 8] + (cp & 0xff)]; -} - -/// True if `cp` is a starter code point, not a combining character. -pub inline fn isStarter(self: Self, cp: u21) bool { - return self.s2[self.s1[cp >> 8] + (cp & 0xff)] == 0; -} diff --git a/src/CombiningData.zig b/src/CombiningData.zig new file mode 100644 index 0000000..95c947d --- /dev/null +++ b/src/CombiningData.zig @@ -0,0 +1,48 @@ +const std = @import("std"); +const builtin = @import("builtin"); +const compress = std.compress; +const mem = std.mem; + +allocator: mem.Allocator, +s1: []u16 = undefined, +s2: []u8 = undefined, + +const Self = @This(); + +pub fn init(allocator: mem.Allocator) !Self { + const decompressor = compress.deflate.decompressor; + const in_bytes = @embedFile("ccc"); + var in_fbs = std.io.fixedBufferStream(in_bytes); + var in_decomp = try decompressor(allocator, in_fbs.reader(), null); + defer in_decomp.deinit(); + var reader = in_decomp.reader(); + + const endian = builtin.cpu.arch.endian(); + + var self = Self{ .allocator = allocator }; + + const stage_1_len: u16 = try reader.readInt(u16, endian); + self.s1 = try allocator.alloc(u16, stage_1_len); + for (0..stage_1_len) |i| self.s1[i] = try reader.readInt(u16, endian); + + const stage_2_len: u16 = try reader.readInt(u16, endian); + self.s2 = try allocator.alloc(u8, stage_2_len); + _ = try reader.readAll(self.s2); + + return self; +} + +pub fn deinit(self: *Self) void { + self.allocator.free(self.s1); + self.allocator.free(self.s2); +} + +/// Returns the canonical combining class for a code point. +pub inline fn ccc(self: Self, cp: u21) u8 { + return self.s2[self.s1[cp >> 8] + (cp & 0xff)]; +} + +/// True if `cp` is a starter code point, not a combining character. +pub inline fn isStarter(self: Self, cp: u21) bool { + return self.s2[self.s1[cp >> 8] + (cp & 0xff)] == 0; +} diff --git a/src/CompatData.zig b/src/CompatData.zig new file mode 100644 index 0000000..a1f5de6 --- /dev/null +++ b/src/CompatData.zig @@ -0,0 +1,50 @@ +const std = @import("std"); +const builtin = @import("builtin"); +const compress = std.compress; +const mem = std.mem; + +allocator: mem.Allocator, +nfkd: [][]u21 = undefined, + +const Self = @This(); + +pub fn init(allocator: mem.Allocator) !Self { + const decompressor = compress.deflate.decompressor; + const in_bytes = @embedFile("compat"); + var in_fbs = std.io.fixedBufferStream(in_bytes); + var in_decomp = try decompressor(allocator, in_fbs.reader(), null); + defer in_decomp.deinit(); + var reader = in_decomp.reader(); + + const endian = builtin.cpu.arch.endian(); + var self = Self{ + .allocator = allocator, + .nfkd = try allocator.alloc([]u21, 0x110000), + }; + + for (0..0x110000) |i| self.nfkd[i] = &.{}; + + while (true) { + const len: u8 = try reader.readInt(u8, endian); + if (len == 0) break; + const cp = try reader.readInt(u24, endian); + self.nfkd[cp] = try allocator.alloc(u21, len - 1); + for (0..len - 1) |i| { + self.nfkd[cp][i] = @intCast(try reader.readInt(u24, endian)); + } + } + + return self; +} + +pub fn deinit(self: *Self) void { + for (self.nfkd) |slice| { + if (slice.len != 0) self.allocator.free(slice); + } + self.allocator.free(self.nfkd); +} + +/// Returns compatibility decomposition for `cp`. +pub inline fn toNfkd(self: Self, cp: u21) []u21 { + return self.nfkd[cp]; +} diff --git a/src/DisplayWidthData.zig b/src/DisplayWidthData.zig deleted file mode 100644 index 32f8658..0000000 --- a/src/DisplayWidthData.zig +++ /dev/null @@ -1,82 +0,0 @@ -const std = @import("std"); -const builtin = @import("builtin"); -const compress = std.compress; -const mem = std.mem; -const testing = std.testing; - -const GraphemeData = @import("GraphemeData"); - -allocator: mem.Allocator, -g_data: GraphemeData, -s1: []u16 = undefined, -s2: []i3 = undefined, - -const Self = @This(); - -pub fn init(allocator: mem.Allocator) !Self { - const decompressor = compress.deflate.decompressor; - const in_bytes = @embedFile("dwp"); - var in_fbs = std.io.fixedBufferStream(in_bytes); - var in_decomp = try decompressor(allocator, in_fbs.reader(), null); - defer in_decomp.deinit(); - var reader = in_decomp.reader(); - - const endian = builtin.cpu.arch.endian(); - - var self = Self{ - .allocator = allocator, - .g_data = try GraphemeData.init(allocator), - }; - - const stage_1_len: u16 = try reader.readInt(u16, endian); - self.s1 = try allocator.alloc(u16, stage_1_len); - for (0..stage_1_len) |i| self.s1[i] = try reader.readInt(u16, endian); - - const stage_2_len: u16 = try reader.readInt(u16, endian); - self.s2 = try allocator.alloc(i3, stage_2_len); - for (0..stage_2_len) |i| self.s2[i] = @intCast(try reader.readInt(i8, endian)); - - return self; -} - -pub fn deinit(self: *Self) void { - self.allocator.free(self.s1); - self.allocator.free(self.s2); - self.g_data.deinit(); -} - -/// codePointWidth returns the number of cells `cp` requires when rendered -/// in a fixed-pitch font (i.e. a terminal screen). This can range from -1 to -/// 3, where BACKSPACE and DELETE return -1 and 3-em-dash returns 3. C0/C1 -/// control codes return 0. If `cjk` is true, ambiguous code points return 2, -/// otherwise they return 1. -pub inline fn codePointWidth(self: Self, cp: u21) i3 { - return self.s2[self.s1[cp >> 8] + (cp & 0xff)]; -} - -test "codePointWidth" { - try testing.expectEqual(@as(i3, 0), codePointWidth(0x0000)); // null - try testing.expectEqual(@as(i3, -1), codePointWidth(0x8)); // \b - try testing.expectEqual(@as(i3, -1), codePointWidth(0x7f)); // DEL - try testing.expectEqual(@as(i3, 0), codePointWidth(0x0005)); // Cf - try testing.expectEqual(@as(i3, 0), codePointWidth(0x0007)); // \a BEL - try testing.expectEqual(@as(i3, 0), codePointWidth(0x000A)); // \n LF - try testing.expectEqual(@as(i3, 0), codePointWidth(0x000B)); // \v VT - try testing.expectEqual(@as(i3, 0), codePointWidth(0x000C)); // \f FF - try testing.expectEqual(@as(i3, 0), codePointWidth(0x000D)); // \r CR - try testing.expectEqual(@as(i3, 0), codePointWidth(0x000E)); // SQ - try testing.expectEqual(@as(i3, 0), codePointWidth(0x000F)); // SI - - try testing.expectEqual(@as(i3, 0), codePointWidth(0x070F)); // Cf - try testing.expectEqual(@as(i3, 1), codePointWidth(0x0603)); // Cf Arabic - - try testing.expectEqual(@as(i3, 1), codePointWidth(0x00AD)); // soft-hyphen - try testing.expectEqual(@as(i3, 2), codePointWidth(0x2E3A)); // two-em dash - try testing.expectEqual(@as(i3, 3), codePointWidth(0x2E3B)); // three-em dash - - try testing.expectEqual(@as(i3, 1), codePointWidth(0x00BD)); // ambiguous halfwidth - - try testing.expectEqual(@as(i3, 1), codePointWidth('é')); - try testing.expectEqual(@as(i3, 2), codePointWidth('😊')); - try testing.expectEqual(@as(i3, 2), codePointWidth('统')); -} diff --git a/src/NormData.zig b/src/NormData.zig index c6fa8e8..83110f0 100644 --- a/src/NormData.zig +++ b/src/NormData.zig @@ -1,11 +1,13 @@ const std = @import("std"); const mem = std.mem; -const CanonData = @import("CanonicalData"); -const CccData = @import("CombiningClassData"); +const CanonData = @import("CanonData"); +const CccData = @import("CombiningData"); +const CompatData = @import("CompatData"); canon_data: CanonData, ccc_data: CccData, +compat_data: CompatData, const Self = @This(); @@ -13,10 +15,12 @@ pub fn init(allocator: std.mem.Allocator) !Self { return Self{ .canon_data = try CanonData.init(allocator), .ccc_data = try CccData.init(allocator), + .compat_data = try CompatData.init(allocator), }; } pub fn deinit(self: *Self) void { self.canon_data.deinit(); self.ccc_data.deinit(); + self.compat_data.deinit(); } diff --git a/src/Normalizer.zig b/src/Normalizer.zig index 2e2e6e4..1434043 100644 --- a/src/Normalizer.zig +++ b/src/Normalizer.zig @@ -12,57 +12,10 @@ const norm_props = @import("ziglyph").normalization_props; pub const NormData = @import("NormData"); -nfkd_map: std.AutoHashMap(u21, [18]u21), norm_data: *NormData, const Self = @This(); -pub fn init(allocator: std.mem.Allocator, norm_data: *NormData) !Self { - var self = Self{ - .nfkd_map = std.AutoHashMap(u21, [18]u21).init(allocator), - .norm_data = norm_data, - }; - errdefer self.deinit(); - - // Compatibility decompositions - const dekomp_file = @embedFile("autogen/compatibility_decompositions.txt.deflate"); - var dekomp_stream = std.io.fixedBufferStream(dekomp_file); - var dekomp_decomp = try std.compress.deflate.decompressor(allocator, dekomp_stream.reader(), null); - defer dekomp_decomp.deinit(); - - var dekomp_buf = std.io.bufferedReader(dekomp_decomp.reader()); - const dekomp_reader = dekomp_buf.reader(); - var buf: [4096]u8 = undefined; - - while (try dekomp_reader.readUntilDelimiterOrEof(&buf, '\n')) |line| { - if (line.len == 0) continue; - var fields = std.mem.split(u8, line, ";"); - const cp_a = try std.fmt.parseInt(u21, fields.next().?, 16); - var cps = [_]u21{0} ** 18; - var i: usize = 0; - - while (fields.next()) |cp| : (i += 1) { - cps[i] = try std.fmt.parseInt(u21, cp, 16); - } - - try self.nfkd_map.put(cp_a, cps); - } - - return self; -} - -pub fn deinit(self: *Self) void { - self.nfkd_map.deinit(); -} - -test "init / deinit" { - const allocator = testing.allocator; - var norm_data = try NormData.init(allocator); - defer norm_data.deinit(); - var n = try init(allocator, &norm_data); - defer n.deinit(); -} - // Hangul processing utilities. fn isHangulPrecomposed(cp: u21) bool { if (hangul_map.syllableType(cp)) |kind| return kind == .LV or kind == .LVT; @@ -140,10 +93,11 @@ pub fn mapping(self: Self, cp: u21, form: Form) Decomp { @memcpy(dc.cps[0..len], canon_dc[0..len]); } - if (self.nfkd_map.get(cp)) |array| { + const compat_dc = self.norm_data.compat_data.toNfkd(cp); + if (compat_dc.len != 0) { if (form != .nfd) { dc.form = .nfkd; - @memcpy(dc.cps[0..array.len], &array); + @memcpy(dc.cps[0..compat_dc.len], compat_dc); } } @@ -210,10 +164,9 @@ pub fn decompose(self: Self, cp: u21, form: Form) Decomp { test "decompose" { const allocator = testing.allocator; - var norm_data = try NormData.init(allocator); - defer norm_data.deinit(); - var n = try init(allocator, &norm_data); - defer n.deinit(); + var data = try NormData.init(allocator); + defer data.deinit(); + var n = Self{ .norm_data = &data }; var dc = n.decompose('é', .nfd); try std.testing.expect(dc.form == .nfd); @@ -334,10 +287,9 @@ fn nfxd(self: Self, allocator: std.mem.Allocator, str: []const u8, form: Form) ! test "nfd ASCII / no-alloc" { const allocator = testing.allocator; - var norm_data = try NormData.init(allocator); - defer norm_data.deinit(); - var n = try init(allocator, &norm_data); - defer n.deinit(); + var data = try NormData.init(allocator); + defer data.deinit(); + var n = Self{ .norm_data = &data }; var result = try n.nfd(allocator, "Hello World!"); defer result.deinit(); @@ -347,10 +299,9 @@ test "nfd ASCII / no-alloc" { test "nfd !ASCII / alloc" { const allocator = testing.allocator; - var norm_data = try NormData.init(allocator); - defer norm_data.deinit(); - var n = try init(allocator, &norm_data); - defer n.deinit(); + var data = try NormData.init(allocator); + defer data.deinit(); + var n = Self{ .norm_data = &data }; var result = try n.nfd(allocator, "Héllo World! \u{3d3}"); defer result.deinit(); @@ -360,10 +311,9 @@ test "nfd !ASCII / alloc" { test "nfkd ASCII / no-alloc" { const allocator = testing.allocator; - var norm_data = try NormData.init(allocator); - defer norm_data.deinit(); - var n = try init(allocator, &norm_data); - defer n.deinit(); + var data = try NormData.init(allocator); + defer data.deinit(); + var n = Self{ .norm_data = &data }; var result = try n.nfkd(allocator, "Hello World!"); defer result.deinit(); @@ -373,10 +323,9 @@ test "nfkd ASCII / no-alloc" { test "nfkd !ASCII / alloc" { const allocator = testing.allocator; - var norm_data = try NormData.init(allocator); - defer norm_data.deinit(); - var n = try init(allocator, &norm_data); - defer n.deinit(); + var data = try NormData.init(allocator); + defer data.deinit(); + var n = Self{ .norm_data = &data }; var result = try n.nfkd(allocator, "Héllo World! \u{3d3}"); defer result.deinit(); @@ -529,10 +478,9 @@ fn nfxc(self: Self, allocator: std.mem.Allocator, str: []const u8, form: Form) ! test "nfc" { const allocator = testing.allocator; - var norm_data = try NormData.init(allocator); - defer norm_data.deinit(); - var n = try init(allocator, &norm_data); - defer n.deinit(); + var data = try NormData.init(allocator); + defer data.deinit(); + var n = Self{ .norm_data = &data }; var result = try n.nfc(allocator, "Complex char: \u{3D2}\u{301}"); defer result.deinit(); @@ -542,10 +490,9 @@ test "nfc" { test "nfkc" { const allocator = testing.allocator; - var norm_data = try NormData.init(allocator); - defer norm_data.deinit(); - var n = try init(allocator, &norm_data); - defer n.deinit(); + var data = try NormData.init(allocator); + defer data.deinit(); + var n = Self{ .norm_data = &data }; var result = try n.nfkc(allocator, "Complex char: \u{03A5}\u{0301}"); defer result.deinit(); @@ -603,10 +550,9 @@ pub fn eql(self: Self, allocator: std.mem.Allocator, a: []const u8, b: []const u test "eql" { const allocator = testing.allocator; - var norm_data = try NormData.init(allocator); - defer norm_data.deinit(); - var n = try init(allocator, &norm_data); - defer n.deinit(); + var data = try NormData.init(allocator); + defer data.deinit(); + var n = Self{ .norm_data = &data }; try std.testing.expect(try n.eql(allocator, "foé", "foe\u{0301}")); try std.testing.expect(try n.eql(allocator, "foϓ", "fo\u{03D2}\u{0301}")); @@ -672,10 +618,9 @@ pub fn eqlCaseless(self: Self, allocator: std.mem.Allocator, a: []const u8, b: [ test "eqlCaseless" { const allocator = testing.allocator; - var norm_data = try NormData.init(allocator); - defer norm_data.deinit(); - var n = try init(allocator, &norm_data); - defer n.deinit(); + var data = try NormData.init(allocator); + defer data.deinit(); + var n = Self{ .norm_data = &data }; try std.testing.expect(try n.eqlCaseless(allocator, "Foϓ", "fo\u{03D2}\u{0301}")); try std.testing.expect(try n.eqlCaseless(allocator, "FOÉ", "foe\u{0301}")); // foÉ == foé @@ -709,10 +654,9 @@ pub fn isFcd(self: Self, str: []const u8) bool { test "isFcd" { const allocator = testing.allocator; - var norm_data = try NormData.init(allocator); - defer norm_data.deinit(); - var n = try init(allocator, &norm_data); - defer n.deinit(); + var data = try NormData.init(allocator); + defer data.deinit(); + var n = Self{ .norm_data = &data }; const is_nfc = "José \u{3D3}"; try std.testing.expect(n.isFcd(is_nfc)); @@ -729,10 +673,9 @@ test "Unicode normalization tests" { defer arena.deinit(); var allocator = arena.allocator(); - var norm_data = try NormData.init(allocator); - defer norm_data.deinit(); - var n = try init(allocator, &norm_data); - defer n.deinit(); + var data = try NormData.init(allocator); + defer data.deinit(); + var n = Self{ .norm_data = &data }; var file = try std.fs.cwd().openFile("data/unicode/NormalizationTest.txt", .{}); defer file.close(); diff --git a/src/WidthData.zig b/src/WidthData.zig new file mode 100644 index 0000000..32f8658 --- /dev/null +++ b/src/WidthData.zig @@ -0,0 +1,82 @@ +const std = @import("std"); +const builtin = @import("builtin"); +const compress = std.compress; +const mem = std.mem; +const testing = std.testing; + +const GraphemeData = @import("GraphemeData"); + +allocator: mem.Allocator, +g_data: GraphemeData, +s1: []u16 = undefined, +s2: []i3 = undefined, + +const Self = @This(); + +pub fn init(allocator: mem.Allocator) !Self { + const decompressor = compress.deflate.decompressor; + const in_bytes = @embedFile("dwp"); + var in_fbs = std.io.fixedBufferStream(in_bytes); + var in_decomp = try decompressor(allocator, in_fbs.reader(), null); + defer in_decomp.deinit(); + var reader = in_decomp.reader(); + + const endian = builtin.cpu.arch.endian(); + + var self = Self{ + .allocator = allocator, + .g_data = try GraphemeData.init(allocator), + }; + + const stage_1_len: u16 = try reader.readInt(u16, endian); + self.s1 = try allocator.alloc(u16, stage_1_len); + for (0..stage_1_len) |i| self.s1[i] = try reader.readInt(u16, endian); + + const stage_2_len: u16 = try reader.readInt(u16, endian); + self.s2 = try allocator.alloc(i3, stage_2_len); + for (0..stage_2_len) |i| self.s2[i] = @intCast(try reader.readInt(i8, endian)); + + return self; +} + +pub fn deinit(self: *Self) void { + self.allocator.free(self.s1); + self.allocator.free(self.s2); + self.g_data.deinit(); +} + +/// codePointWidth returns the number of cells `cp` requires when rendered +/// in a fixed-pitch font (i.e. a terminal screen). This can range from -1 to +/// 3, where BACKSPACE and DELETE return -1 and 3-em-dash returns 3. C0/C1 +/// control codes return 0. If `cjk` is true, ambiguous code points return 2, +/// otherwise they return 1. +pub inline fn codePointWidth(self: Self, cp: u21) i3 { + return self.s2[self.s1[cp >> 8] + (cp & 0xff)]; +} + +test "codePointWidth" { + try testing.expectEqual(@as(i3, 0), codePointWidth(0x0000)); // null + try testing.expectEqual(@as(i3, -1), codePointWidth(0x8)); // \b + try testing.expectEqual(@as(i3, -1), codePointWidth(0x7f)); // DEL + try testing.expectEqual(@as(i3, 0), codePointWidth(0x0005)); // Cf + try testing.expectEqual(@as(i3, 0), codePointWidth(0x0007)); // \a BEL + try testing.expectEqual(@as(i3, 0), codePointWidth(0x000A)); // \n LF + try testing.expectEqual(@as(i3, 0), codePointWidth(0x000B)); // \v VT + try testing.expectEqual(@as(i3, 0), codePointWidth(0x000C)); // \f FF + try testing.expectEqual(@as(i3, 0), codePointWidth(0x000D)); // \r CR + try testing.expectEqual(@as(i3, 0), codePointWidth(0x000E)); // SQ + try testing.expectEqual(@as(i3, 0), codePointWidth(0x000F)); // SI + + try testing.expectEqual(@as(i3, 0), codePointWidth(0x070F)); // Cf + try testing.expectEqual(@as(i3, 1), codePointWidth(0x0603)); // Cf Arabic + + try testing.expectEqual(@as(i3, 1), codePointWidth(0x00AD)); // soft-hyphen + try testing.expectEqual(@as(i3, 2), codePointWidth(0x2E3A)); // two-em dash + try testing.expectEqual(@as(i3, 3), codePointWidth(0x2E3B)); // three-em dash + + try testing.expectEqual(@as(i3, 1), codePointWidth(0x00BD)); // ambiguous halfwidth + + try testing.expectEqual(@as(i3, 1), codePointWidth('é')); + try testing.expectEqual(@as(i3, 2), codePointWidth('😊')); + try testing.expectEqual(@as(i3, 2), codePointWidth('统')); +} diff --git a/src/autogen/compatibility_decompositions.txt.deflate b/src/autogen/compatibility_decompositions.txt.deflate deleted file mode 100644 index 0370b4c..0000000 Binary files a/src/autogen/compatibility_decompositions.txt.deflate and /dev/null differ diff --git a/src/main.zig b/src/main.zig index 05c2ea4..2c2cf8c 100644 --- a/src/main.zig +++ b/src/main.zig @@ -16,9 +16,9 @@ const std = @import("std"); // const ascii = @import("ascii"); // const ascii = std.ascii; -// const norm = @import("ziglyph").Normalizer; +// const Normalizer = @import("ziglyph").Normalizer; const NormData = @import("Normalizer").NormData; -const norm = @import("Normalizer"); +const Normalizer = @import("Normalizer"); pub fn main() !void { var args_iter = std.process.args(); @@ -32,10 +32,9 @@ pub fn main() !void { const input = try std.fs.cwd().readFileAlloc(allocator, in_path, std.math.maxInt(u32)); defer allocator.free(input); - var norm_data = try NormData.init(allocator); - defer norm_data.deinit(); - var n = try norm.init(allocator, &norm_data); - defer n.deinit(); + var data = try NormData.init(allocator); + defer data.deinit(); + var n = Normalizer{ .norm_data = &data }; // var n = try norm.init(allocator); // defer n.deinit(); -- cgit v1.2.3