8 files changed, 97 insertions, 101 deletions

diff --git a/src/Canonical.zig b/src/CanonData.zig
index 81d3eec..81d3eec 100644
--- a/src/Canonical.zig
+++ b/src/CanonData.zig
diff --git a/src/CombiningClassData.zig b/src/CombiningData.zig
index 95c947d..95c947d 100644
--- a/src/CombiningClassData.zig
+++ b/src/CombiningData.zig
diff --git a/src/CompatData.zig b/src/CompatData.zig
new file mode 100644
index 0000000..a1f5de6
--- /dev/null
+++ b/src/CompatData.zig
@@ -0,0 +1,50 @@
+const std = @import("std");
+const builtin = @import("builtin");
+const compress = std.compress;
+const mem = std.mem;
+
+allocator: mem.Allocator,
+nfkd: [][]u21 = undefined,
+
+const Self = @This();
+
+pub fn init(allocator: mem.Allocator) !Self {
+    const decompressor = compress.deflate.decompressor;
+    const in_bytes = @embedFile("compat");
+    var in_fbs = std.io.fixedBufferStream(in_bytes);
+    var in_decomp = try decompressor(allocator, in_fbs.reader(), null);
+    defer in_decomp.deinit();
+    var reader = in_decomp.reader();
+
+    const endian = builtin.cpu.arch.endian();
+    var self = Self{
+        .allocator = allocator,
+        .nfkd = try allocator.alloc([]u21, 0x110000),
+    };
+
+    for (0..0x110000) |i| self.nfkd[i] = &.{};
+
+    while (true) {
+        const len: u8 = try reader.readInt(u8, endian);
+        if (len == 0) break;
+        const cp = try reader.readInt(u24, endian);
+        self.nfkd[cp] = try allocator.alloc(u21, len - 1);
+        for (0..len - 1) |i| {
+            self.nfkd[cp][i] = @intCast(try reader.readInt(u24, endian));
+        }
+    }
+
+    return self;
+}
+
+pub fn deinit(self: *Self) void {
+    for (self.nfkd) |slice| {
+        if (slice.len != 0) self.allocator.free(slice);
+    }
+    self.allocator.free(self.nfkd);
+}
+
+/// Returns compatibility decomposition for `cp`.
+pub inline fn toNfkd(self: Self, cp: u21) []u21 {
+    return self.nfkd[cp];
+}
diff --git a/src/NormData.zig b/src/NormData.zig
index c6fa8e8..83110f0 100644
--- a/src/NormData.zig
+++ b/src/NormData.zig
@@ -1,11 +1,13 @@
 const std = @import("std");
 const mem = std.mem;
 
-const CanonData = @import("CanonicalData");
-const CccData = @import("CombiningClassData");
+const CanonData = @import("CanonData");
+const CccData = @import("CombiningData");
+const CompatData = @import("CompatData");
 
 canon_data: CanonData,
 ccc_data: CccData,
+compat_data: CompatData,
 
 const Self = @This();
 
@@ -13,10 +15,12 @@ pub fn init(allocator: std.mem.Allocator) !Self {
     return Self{
         .canon_data = try CanonData.init(allocator),
         .ccc_data = try CccData.init(allocator),
+        .compat_data = try CompatData.init(allocator),
     };
 }
 
 pub fn deinit(self: *Self) void {
     self.canon_data.deinit();
     self.ccc_data.deinit();
+    self.compat_data.deinit();
 }
diff --git a/src/Normalizer.zig b/src/Normalizer.zig
index 2e2e6e4..1434043 100644
--- a/src/Normalizer.zig
+++ b/src/Normalizer.zig
@@ -12,57 +12,10 @@ const norm_props = @import("ziglyph").normalization_props;
 
 pub const NormData = @import("NormData");
 
-nfkd_map: std.AutoHashMap(u21, [18]u21),
 norm_data: *NormData,
 
 const Self = @This();
 
-pub fn init(allocator: std.mem.Allocator, norm_data: *NormData) !Self {
-    var self = Self{
-        .nfkd_map = std.AutoHashMap(u21, [18]u21).init(allocator),
-        .norm_data = norm_data,
-    };
-    errdefer self.deinit();
-
-    // Compatibility decompositions
-    const dekomp_file = @embedFile("autogen/compatibility_decompositions.txt.deflate");
-    var dekomp_stream = std.io.fixedBufferStream(dekomp_file);
-    var dekomp_decomp = try std.compress.deflate.decompressor(allocator, dekomp_stream.reader(), null);
-    defer dekomp_decomp.deinit();
-
-    var dekomp_buf = std.io.bufferedReader(dekomp_decomp.reader());
-    const dekomp_reader = dekomp_buf.reader();
-    var buf: [4096]u8 = undefined;
-
-    while (try dekomp_reader.readUntilDelimiterOrEof(&buf, '\n')) |line| {
-        if (line.len == 0) continue;
-        var fields = std.mem.split(u8, line, ";");
-        const cp_a = try std.fmt.parseInt(u21, fields.next().?, 16);
-        var cps = [_]u21{0} ** 18;
-        var i: usize = 0;
-
-        while (fields.next()) |cp| : (i += 1) {
-            cps[i] = try std.fmt.parseInt(u21, cp, 16);
-        }
-
-        try self.nfkd_map.put(cp_a, cps);
-    }
-
-    return self;
-}
-
-pub fn deinit(self: *Self) void {
-    self.nfkd_map.deinit();
-}
-
-test "init / deinit" {
-    const allocator = testing.allocator;
-    var norm_data = try NormData.init(allocator);
-    defer norm_data.deinit();
-    var n = try init(allocator, &norm_data);
-    defer n.deinit();
-}
-
 // Hangul processing utilities.
 fn isHangulPrecomposed(cp: u21) bool {
     if (hangul_map.syllableType(cp)) |kind| return kind == .LV or kind == .LVT;
@@ -140,10 +93,11 @@ pub fn mapping(self: Self, cp: u21, form: Form) Decomp {
         @memcpy(dc.cps[0..len], canon_dc[0..len]);
     }
 
-    if (self.nfkd_map.get(cp)) |array| {
+    const compat_dc = self.norm_data.compat_data.toNfkd(cp);
+    if (compat_dc.len != 0) {
         if (form != .nfd) {
             dc.form = .nfkd;
-            @memcpy(dc.cps[0..array.len], &array);
+            @memcpy(dc.cps[0..compat_dc.len], compat_dc);
         }
     }
 
@@ -210,10 +164,9 @@ pub fn decompose(self: Self, cp: u21, form: Form) Decomp {
 
 test "decompose" {
     const allocator = testing.allocator;
-    var norm_data = try NormData.init(allocator);
-    defer norm_data.deinit();
-    var n = try init(allocator, &norm_data);
-    defer n.deinit();
+    var data = try NormData.init(allocator);
+    defer data.deinit();
+    var n = Self{ .norm_data = &data };
 
     var dc = n.decompose('é', .nfd);
     try std.testing.expect(dc.form == .nfd);
@@ -334,10 +287,9 @@ fn nfxd(self: Self, allocator: std.mem.Allocator, str: []const u8, form: Form) !
 
 test "nfd ASCII / no-alloc" {
     const allocator = testing.allocator;
-    var norm_data = try NormData.init(allocator);
-    defer norm_data.deinit();
-    var n = try init(allocator, &norm_data);
-    defer n.deinit();
+    var data = try NormData.init(allocator);
+    defer data.deinit();
+    var n = Self{ .norm_data = &data };
 
     var result = try n.nfd(allocator, "Hello World!");
     defer result.deinit();
@@ -347,10 +299,9 @@ test "nfd ASCII / no-alloc" {
 
 test "nfd !ASCII / alloc" {
     const allocator = testing.allocator;
-    var norm_data = try NormData.init(allocator);
-    defer norm_data.deinit();
-    var n = try init(allocator, &norm_data);
-    defer n.deinit();
+    var data = try NormData.init(allocator);
+    defer data.deinit();
+    var n = Self{ .norm_data = &data };
 
     var result = try n.nfd(allocator, "Héllo World! \u{3d3}");
     defer result.deinit();
@@ -360,10 +311,9 @@ test "nfd !ASCII / alloc" {
 
 test "nfkd ASCII / no-alloc" {
     const allocator = testing.allocator;
-    var norm_data = try NormData.init(allocator);
-    defer norm_data.deinit();
-    var n = try init(allocator, &norm_data);
-    defer n.deinit();
+    var data = try NormData.init(allocator);
+    defer data.deinit();
+    var n = Self{ .norm_data = &data };
 
     var result = try n.nfkd(allocator, "Hello World!");
     defer result.deinit();
@@ -373,10 +323,9 @@ test "nfkd ASCII / no-alloc" {
 
 test "nfkd !ASCII / alloc" {
     const allocator = testing.allocator;
-    var norm_data = try NormData.init(allocator);
-    defer norm_data.deinit();
-    var n = try init(allocator, &norm_data);
-    defer n.deinit();
+    var data = try NormData.init(allocator);
+    defer data.deinit();
+    var n = Self{ .norm_data = &data };
 
     var result = try n.nfkd(allocator, "Héllo World! \u{3d3}");
     defer result.deinit();
@@ -529,10 +478,9 @@ fn nfxc(self: Self, allocator: std.mem.Allocator, str: []const u8, form: Form) !
 
 test "nfc" {
     const allocator = testing.allocator;
-    var norm_data = try NormData.init(allocator);
-    defer norm_data.deinit();
-    var n = try init(allocator, &norm_data);
-    defer n.deinit();
+    var data = try NormData.init(allocator);
+    defer data.deinit();
+    var n = Self{ .norm_data = &data };
 
     var result = try n.nfc(allocator, "Complex char: \u{3D2}\u{301}");
     defer result.deinit();
@@ -542,10 +490,9 @@ test "nfc" {
 
 test "nfkc" {
     const allocator = testing.allocator;
-    var norm_data = try NormData.init(allocator);
-    defer norm_data.deinit();
-    var n = try init(allocator, &norm_data);
-    defer n.deinit();
+    var data = try NormData.init(allocator);
+    defer data.deinit();
+    var n = Self{ .norm_data = &data };
 
     var result = try n.nfkc(allocator, "Complex char: \u{03A5}\u{0301}");
     defer result.deinit();
@@ -603,10 +550,9 @@ pub fn eql(self: Self, allocator: std.mem.Allocator, a: []const u8, b: []const u
 
 test "eql" {
     const allocator = testing.allocator;
-    var norm_data = try NormData.init(allocator);
-    defer norm_data.deinit();
-    var n = try init(allocator, &norm_data);
-    defer n.deinit();
+    var data = try NormData.init(allocator);
+    defer data.deinit();
+    var n = Self{ .norm_data = &data };
 
     try std.testing.expect(try n.eql(allocator, "foé", "foe\u{0301}"));
     try std.testing.expect(try n.eql(allocator, "foϓ", "fo\u{03D2}\u{0301}"));
@@ -672,10 +618,9 @@ pub fn eqlCaseless(self: Self, allocator: std.mem.Allocator, a: []const u8, b: [
 
 test "eqlCaseless" {
     const allocator = testing.allocator;
-    var norm_data = try NormData.init(allocator);
-    defer norm_data.deinit();
-    var n = try init(allocator, &norm_data);
-    defer n.deinit();
+    var data = try NormData.init(allocator);
+    defer data.deinit();
+    var n = Self{ .norm_data = &data };
 
     try std.testing.expect(try n.eqlCaseless(allocator, "Foϓ", "fo\u{03D2}\u{0301}"));
     try std.testing.expect(try n.eqlCaseless(allocator, "FOÉ", "foe\u{0301}")); // foÉ == foé
@@ -709,10 +654,9 @@ pub fn isFcd(self: Self, str: []const u8) bool {
 
 test "isFcd" {
     const allocator = testing.allocator;
-    var norm_data = try NormData.init(allocator);
-    defer norm_data.deinit();
-    var n = try init(allocator, &norm_data);
-    defer n.deinit();
+    var data = try NormData.init(allocator);
+    defer data.deinit();
+    var n = Self{ .norm_data = &data };
 
     const is_nfc = "José \u{3D3}";
     try std.testing.expect(n.isFcd(is_nfc));
@@ -729,10 +673,9 @@ test "Unicode normalization tests" {
     defer arena.deinit();
     var allocator = arena.allocator();
 
-    var norm_data = try NormData.init(allocator);
-    defer norm_data.deinit();
-    var n = try init(allocator, &norm_data);
-    defer n.deinit();
+    var data = try NormData.init(allocator);
+    defer data.deinit();
+    var n = Self{ .norm_data = &data };
 
     var file = try std.fs.cwd().openFile("data/unicode/NormalizationTest.txt", .{});
     defer file.close();
diff --git a/src/DisplayWidthData.zig b/src/WidthData.zig
index 32f8658..32f8658 100644
--- a/src/DisplayWidthData.zig
+++ b/src/WidthData.zig
diff --git a/src/autogen/compatibility_decompositions.txt.deflate b/src/autogen/compatibility_decompositions.txt.deflate
deleted file mode 100644
index 0370b4c..0000000
--- a/src/autogen/compatibility_decompositions.txt.deflate
+++ /dev/null
diff --git a/src/main.zig b/src/main.zig
index 05c2ea4..2c2cf8c 100644
--- a/src/main.zig
+++ b/src/main.zig
@@ -16,9 +16,9 @@ const std = @import("std");
 // const ascii = @import("ascii");
 // const ascii = std.ascii;
 
-// const norm = @import("ziglyph").Normalizer;
+// const Normalizer = @import("ziglyph").Normalizer;
 const NormData = @import("Normalizer").NormData;
-const norm = @import("Normalizer");
+const Normalizer = @import("Normalizer");
 
 pub fn main() !void {
     var args_iter = std.process.args();
@@ -32,10 +32,9 @@ pub fn main() !void {
     const input = try std.fs.cwd().readFileAlloc(allocator, in_path, std.math.maxInt(u32));
     defer allocator.free(input);
 
-    var norm_data = try NormData.init(allocator);
-    defer norm_data.deinit();
-    var n = try norm.init(allocator, &norm_data);
-    defer n.deinit();
+    var data = try NormData.init(allocator);
+    defer data.deinit();
+    var n = Normalizer{ .norm_data = &data };
     // var n = try norm.init(allocator);
     // defer n.deinit();