From 4ce891a8ce5336da39180964792110e131756cdd Mon Sep 17 00:00:00 2001 From: Jose Colon Rodriguez Date: Wed, 27 Mar 2024 21:52:02 -0400 Subject: ScriptsData and made all Datas const --- src/CanonData.zig | 2 +- src/CaseData.zig | 10 +-- src/CaseFold.zig | 8 +- src/CombiningData.zig | 2 +- src/CompatData.zig | 2 +- src/DisplayWidth.zig | 10 +-- src/FoldData.zig | 2 +- src/GenCatData.zig | 2 +- src/GraphemeData.zig | 2 +- src/HangulData.zig | 2 +- src/NormData.zig | 2 +- src/NormPropsData.zig | 2 +- src/Normalize.zig | 58 ++++++------- src/NumericData.zig | 4 +- src/ScriptsData.zig | 226 ++++++++++++++++++++++++++++++++++++++++++++++++++ src/WidthData.zig | 2 +- src/grapheme.zig | 4 +- 17 files changed, 283 insertions(+), 57 deletions(-) create mode 100644 src/ScriptsData.zig (limited to 'src') diff --git a/src/CanonData.zig b/src/CanonData.zig index 36895ff..9f1deb8 100644 --- a/src/CanonData.zig +++ b/src/CanonData.zig @@ -42,7 +42,7 @@ pub fn init(allocator: mem.Allocator) !Self { return self; } -pub fn deinit(self: *Self) void { +pub fn deinit(self: *const Self) void { self.nfc.deinit(); for (self.nfd) |slice| self.allocator.free(slice); self.allocator.free(self.nfd); diff --git a/src/CaseData.zig b/src/CaseData.zig index 4f06636..c9ccc1e 100644 --- a/src/CaseData.zig +++ b/src/CaseData.zig @@ -77,7 +77,7 @@ pub fn init(allocator: mem.Allocator) !Self { return self; } -pub fn deinit(self: *Self) void { +pub fn deinit(self: *const Self) void { self.allocator.free(self.case_map); self.allocator.free(self.prop_s1); self.allocator.free(self.prop_s2); @@ -103,7 +103,7 @@ pub fn isUpperStr(self: Self, str: []const u8) bool { } test "isUpperStr" { - var cd = try init(testing.allocator); + const cd = try init(testing.allocator); defer cd.deinit(); try testing.expect(cd.isUpperStr("HELLO, WORLD 2112!")); @@ -138,7 +138,7 @@ pub fn toUpperStr( } test "toUpperStr" { - var cd = try init(testing.allocator); + const cd = try init(testing.allocator); defer cd.deinit(); const uppered = try cd.toUpperStr(testing.allocator, "Hello, World 2112!"); @@ -161,7 +161,7 @@ pub fn isLowerStr(self: Self, str: []const u8) bool { } test "isLowerStr" { - var cd = try init(testing.allocator); + const cd = try init(testing.allocator); defer cd.deinit(); try testing.expect(cd.isLowerStr("hello, world 2112!")); @@ -196,7 +196,7 @@ pub fn toLowerStr( } test "toLowerStr" { - var cd = try init(testing.allocator); + const cd = try init(testing.allocator); defer cd.deinit(); const lowered = try cd.toLowerStr(testing.allocator, "Hello, World 2112!"); diff --git a/src/CaseFold.zig b/src/CaseFold.zig index e00d03b..9b10e16 100644 --- a/src/CaseFold.zig +++ b/src/CaseFold.zig @@ -90,11 +90,11 @@ pub fn compatCaselessMatch( test "compatCaselessMatch" { const allocator = testing.allocator; - var norm_data = try Normalize.NormData.init(allocator); + const norm_data = try Normalize.NormData.init(allocator); defer norm_data.deinit(); const n = Normalize{ .norm_data = &norm_data }; - var fold_data = try FoldData.init(allocator); + const fold_data = try FoldData.init(allocator); defer fold_data.deinit(); const caser = Self{ .fold_data = &fold_data }; @@ -163,11 +163,11 @@ pub fn canonCaselessMatch( test "canonCaselessMatch" { const allocator = testing.allocator; - var norm_data = try Normalize.NormData.init(allocator); + const norm_data = try Normalize.NormData.init(allocator); defer norm_data.deinit(); const n = Normalize{ .norm_data = &norm_data }; - var fold_data = try FoldData.init(allocator); + const fold_data = try FoldData.init(allocator); defer fold_data.deinit(); const caser = Self{ .fold_data = &fold_data }; diff --git a/src/CombiningData.zig b/src/CombiningData.zig index 95c947d..c67638c 100644 --- a/src/CombiningData.zig +++ b/src/CombiningData.zig @@ -32,7 +32,7 @@ pub fn init(allocator: mem.Allocator) !Self { return self; } -pub fn deinit(self: *Self) void { +pub fn deinit(self: *const Self) void { self.allocator.free(self.s1); self.allocator.free(self.s2); } diff --git a/src/CompatData.zig b/src/CompatData.zig index fd7f678..67c43e6 100644 --- a/src/CompatData.zig +++ b/src/CompatData.zig @@ -37,7 +37,7 @@ pub fn init(allocator: mem.Allocator) !Self { return self; } -pub fn deinit(self: *Self) void { +pub fn deinit(self: *const Self) void { for (self.nfkd) |slice| { if (slice.len != 0) self.allocator.free(slice); } diff --git a/src/DisplayWidth.zig b/src/DisplayWidth.zig index 8d5eb0f..e547adf 100644 --- a/src/DisplayWidth.zig +++ b/src/DisplayWidth.zig @@ -56,7 +56,7 @@ pub fn strWidth(self: Self, str: []const u8) usize { } test "strWidth" { - var data = try DisplayWidthData.init(testing.allocator); + const data = try DisplayWidthData.init(testing.allocator); defer data.deinit(); const self = Self{ .data = &data }; @@ -157,7 +157,7 @@ pub fn center( test "center" { const allocator = testing.allocator; - var data = try DisplayWidthData.init(allocator); + const data = try DisplayWidthData.init(allocator); defer data.deinit(); const self = Self{ .data = &data }; @@ -236,7 +236,7 @@ pub fn padLeft( test "padLeft" { const allocator = testing.allocator; - var data = try DisplayWidthData.init(allocator); + const data = try DisplayWidthData.init(allocator); defer data.deinit(); const self = Self{ .data = &data }; @@ -286,7 +286,7 @@ pub fn padRight( test "padRight" { const allocator = testing.allocator; - var data = try DisplayWidthData.init(allocator); + const data = try DisplayWidthData.init(allocator); defer data.deinit(); const self = Self{ .data = &data }; @@ -339,7 +339,7 @@ pub fn wrap( test "wrap" { const allocator = testing.allocator; - var data = try DisplayWidthData.init(allocator); + const data = try DisplayWidthData.init(allocator); defer data.deinit(); const self = Self{ .data = &data }; diff --git a/src/FoldData.zig b/src/FoldData.zig index 2a9a1f5..e387447 100644 --- a/src/FoldData.zig +++ b/src/FoldData.zig @@ -41,7 +41,7 @@ pub fn init(allocator: mem.Allocator) !Self { return self; } -pub fn deinit(self: *Self) void { +pub fn deinit(self: *const Self) void { for (self.fold) |slice| self.allocator.free(slice); self.allocator.free(self.fold); self.allocator.free(self.cwcf); diff --git a/src/GenCatData.zig b/src/GenCatData.zig index b45135b..37ae037 100644 --- a/src/GenCatData.zig +++ b/src/GenCatData.zig @@ -71,7 +71,7 @@ pub fn init(allocator: mem.Allocator) !Self { return self; } -pub fn deinit(self: *Self) void { +pub fn deinit(self: *const Self) void { self.allocator.free(self.s1); self.allocator.free(self.s2); self.allocator.free(self.s3); diff --git a/src/GraphemeData.zig b/src/GraphemeData.zig index e418dea..971929a 100644 --- a/src/GraphemeData.zig +++ b/src/GraphemeData.zig @@ -64,7 +64,7 @@ pub fn init(allocator: mem.Allocator) !Self { return self; } -pub fn deinit(self: *Self) void { +pub fn deinit(self: *const Self) void { self.allocator.free(self.s1); self.allocator.free(self.s2); self.allocator.free(self.s3); diff --git a/src/HangulData.zig b/src/HangulData.zig index b97424c..ec360e9 100644 --- a/src/HangulData.zig +++ b/src/HangulData.zig @@ -41,7 +41,7 @@ pub fn init(allocator: mem.Allocator) !Self { return self; } -pub fn deinit(self: *Self) void { +pub fn deinit(self: *const Self) void { self.allocator.free(self.s1); self.allocator.free(self.s2); } diff --git a/src/NormData.zig b/src/NormData.zig index 8a7fa49..413619a 100644 --- a/src/NormData.zig +++ b/src/NormData.zig @@ -26,7 +26,7 @@ pub fn init(allocator: std.mem.Allocator) !Self { }; } -pub fn deinit(self: *Self) void { +pub fn deinit(self: *const Self) void { self.canon_data.deinit(); self.ccc_data.deinit(); self.compat_data.deinit(); diff --git a/src/NormPropsData.zig b/src/NormPropsData.zig index 3c49712..893a8d0 100644 --- a/src/NormPropsData.zig +++ b/src/NormPropsData.zig @@ -32,7 +32,7 @@ pub fn init(allocator: mem.Allocator) !Self { return self; } -pub fn deinit(self: *Self) void { +pub fn deinit(self: *const Self) void { self.allocator.free(self.s1); self.allocator.free(self.s2); } diff --git a/src/Normalize.zig b/src/Normalize.zig index 6ef7c90..daf774d 100644 --- a/src/Normalize.zig +++ b/src/Normalize.zig @@ -177,7 +177,7 @@ fn decompose( test "decompose" { const allocator = testing.allocator; - var data = try NormData.init(allocator); + const data = try NormData.init(allocator); defer data.deinit(); var n = Self{ .norm_data = &data }; @@ -225,7 +225,7 @@ pub const Result = struct { allocator: ?mem.Allocator = null, slice: []const u8, - pub fn deinit(self: *Result) void { + pub fn deinit(self: *const Result) void { if (self.allocator) |allocator| allocator.free(self.slice); } }; @@ -297,11 +297,11 @@ fn nfxd(self: Self, allocator: mem.Allocator, str: []const u8, form: Form) !Resu test "nfd ASCII / no-alloc" { const allocator = testing.allocator; - var data = try NormData.init(allocator); + const data = try NormData.init(allocator); defer data.deinit(); - var n = Self{ .norm_data = &data }; + const n = Self{ .norm_data = &data }; - var result = try n.nfd(allocator, "Hello World!"); + const result = try n.nfd(allocator, "Hello World!"); defer result.deinit(); try testing.expectEqualStrings("Hello World!", result.slice); @@ -309,11 +309,11 @@ test "nfd ASCII / no-alloc" { test "nfd !ASCII / alloc" { const allocator = testing.allocator; - var data = try NormData.init(allocator); + const data = try NormData.init(allocator); defer data.deinit(); - var n = Self{ .norm_data = &data }; + const n = Self{ .norm_data = &data }; - var result = try n.nfd(allocator, "Héllo World! \u{3d3}"); + const result = try n.nfd(allocator, "Héllo World! \u{3d3}"); defer result.deinit(); try testing.expectEqualStrings("He\u{301}llo World! \u{3d2}\u{301}", result.slice); @@ -321,11 +321,11 @@ test "nfd !ASCII / alloc" { test "nfkd ASCII / no-alloc" { const allocator = testing.allocator; - var data = try NormData.init(allocator); + const data = try NormData.init(allocator); defer data.deinit(); - var n = Self{ .norm_data = &data }; + const n = Self{ .norm_data = &data }; - var result = try n.nfkd(allocator, "Hello World!"); + const result = try n.nfkd(allocator, "Hello World!"); defer result.deinit(); try testing.expectEqualStrings("Hello World!", result.slice); @@ -333,11 +333,11 @@ test "nfkd ASCII / no-alloc" { test "nfkd !ASCII / alloc" { const allocator = testing.allocator; - var data = try NormData.init(allocator); + const data = try NormData.init(allocator); defer data.deinit(); - var n = Self{ .norm_data = &data }; + const n = Self{ .norm_data = &data }; - var result = try n.nfkd(allocator, "Héllo World! \u{3d3}"); + const result = try n.nfkd(allocator, "Héllo World! \u{3d3}"); defer result.deinit(); try testing.expectEqualStrings("He\u{301}llo World! \u{3a5}\u{301}", result.slice); @@ -532,11 +532,11 @@ fn nfxc(self: Self, allocator: mem.Allocator, str: []const u8, form: Form) !Resu test "nfc" { const allocator = testing.allocator; - var data = try NormData.init(allocator); + const data = try NormData.init(allocator); defer data.deinit(); - var n = Self{ .norm_data = &data }; + const n = Self{ .norm_data = &data }; - var result = try n.nfc(allocator, "Complex char: \u{3D2}\u{301}"); + const result = try n.nfc(allocator, "Complex char: \u{3D2}\u{301}"); defer result.deinit(); try testing.expectEqualStrings("Complex char: \u{3D3}", result.slice); @@ -544,11 +544,11 @@ test "nfc" { test "nfkc" { const allocator = testing.allocator; - var data = try NormData.init(allocator); + const data = try NormData.init(allocator); defer data.deinit(); - var n = Self{ .norm_data = &data }; + const n = Self{ .norm_data = &data }; - var result = try n.nfkc(allocator, "Complex char: \u{03A5}\u{0301}"); + const result = try n.nfkc(allocator, "Complex char: \u{03A5}\u{0301}"); defer result.deinit(); try testing.expectEqualStrings("Complex char: \u{038E}", result.slice); @@ -556,9 +556,9 @@ test "nfkc" { /// Tests for equality of `a` and `b` after normalizing to NFC. pub fn eql(self: Self, allocator: mem.Allocator, a: []const u8, b: []const u8) !bool { - var norm_result_a = try self.nfc(allocator, a); + const norm_result_a = try self.nfc(allocator, a); defer norm_result_a.deinit(); - var norm_result_b = try self.nfc(allocator, b); + const norm_result_b = try self.nfc(allocator, b); defer norm_result_b.deinit(); return mem.eql(u8, norm_result_a.slice, norm_result_b.slice); @@ -566,9 +566,9 @@ pub fn eql(self: Self, allocator: mem.Allocator, a: []const u8, b: []const u8) ! test "eql" { const allocator = testing.allocator; - var data = try NormData.init(allocator); + const data = try NormData.init(allocator); defer data.deinit(); - var n = Self{ .norm_data = &data }; + const n = Self{ .norm_data = &data }; try testing.expect(try n.eql(allocator, "foé", "foe\u{0301}")); try testing.expect(try n.eql(allocator, "foϓ", "fo\u{03D2}\u{0301}")); @@ -601,9 +601,9 @@ fn isFcd(self: Self, str: []const u8) bool { test "isFcd" { const allocator = testing.allocator; - var data = try NormData.init(allocator); + const data = try NormData.init(allocator); defer data.deinit(); - var n = Self{ .norm_data = &data }; + const n = Self{ .norm_data = &data }; const is_nfc = "José \u{3D3}"; try testing.expect(n.isFcd(is_nfc)); @@ -620,9 +620,9 @@ test "Unicode normalization tests" { defer arena.deinit(); var allocator = arena.allocator(); - var data = try NormData.init(allocator); + const data = try NormData.init(allocator); defer data.deinit(); - var n = Self{ .norm_data = &data }; + const n = Self{ .norm_data = &data }; var file = try fs.cwd().openFile("data/unicode/NormalizationTest.txt", .{}); defer file.close(); @@ -721,7 +721,7 @@ test "Unicode normalization tests" { } const want = w_buf.items; - var got = try n.nfkd(allocator, input); + const got = try n.nfkd(allocator, input); defer got.deinit(); try testing.expectEqualStrings(want, got.slice); diff --git a/src/NumericData.zig b/src/NumericData.zig index baf8f11..210d623 100644 --- a/src/NumericData.zig +++ b/src/NumericData.zig @@ -33,7 +33,7 @@ pub fn init(allocator: mem.Allocator) !Self { return self; } -pub fn deinit(self: *Self) void { +pub fn deinit(self: *const Self) void { self.allocator.free(self.s1); self.allocator.free(self.s2); } @@ -59,7 +59,7 @@ pub inline fn isDecimal(self: Self, cp: u21) bool { } test "isDecimal" { - var self = try init(testing.allocator); + const self = try init(testing.allocator); defer self.deinit(); try testing.expect(self.isNumber('\u{277f}')); diff --git a/src/ScriptsData.zig b/src/ScriptsData.zig new file mode 100644 index 0000000..ac1c46a --- /dev/null +++ b/src/ScriptsData.zig @@ -0,0 +1,226 @@ +const std = @import("std"); +const builtin = @import("builtin"); +const compress = std.compress; +const mem = std.mem; +const testing = std.testing; + +/// Script +pub const Script = enum { + none, + Adlam, + Ahom, + Anatolian_Hieroglyphs, + Arabic, + Armenian, + Avestan, + Balinese, + Bamum, + Bassa_Vah, + Batak, + Bengali, + Bhaiksuki, + Bopomofo, + Brahmi, + Braille, + Buginese, + Buhid, + Canadian_Aboriginal, + Carian, + Caucasian_Albanian, + Chakma, + Cham, + Cherokee, + Chorasmian, + Common, + Coptic, + Cuneiform, + Cypriot, + Cypro_Minoan, + Cyrillic, + Deseret, + Devanagari, + Dives_Akuru, + Dogra, + Duployan, + Egyptian_Hieroglyphs, + Elbasan, + Elymaic, + Ethiopic, + Georgian, + Glagolitic, + Gothic, + Grantha, + Greek, + Gujarati, + Gunjala_Gondi, + Gurmukhi, + Han, + Hangul, + Hanifi_Rohingya, + Hanunoo, + Hatran, + Hebrew, + Hiragana, + Imperial_Aramaic, + Inherited, + Inscriptional_Pahlavi, + Inscriptional_Parthian, + Javanese, + Kaithi, + Kannada, + Katakana, + Kawi, + Kayah_Li, + Kharoshthi, + Khitan_Small_Script, + Khmer, + Khojki, + Khudawadi, + Lao, + Latin, + Lepcha, + Limbu, + Linear_A, + Linear_B, + Lisu, + Lycian, + Lydian, + Mahajani, + Makasar, + Malayalam, + Mandaic, + Manichaean, + Marchen, + Masaram_Gondi, + Medefaidrin, + Meetei_Mayek, + Mende_Kikakui, + Meroitic_Cursive, + Meroitic_Hieroglyphs, + Miao, + Modi, + Mongolian, + Mro, + Multani, + Myanmar, + Nabataean, + Nag_Mundari, + Nandinagari, + New_Tai_Lue, + Newa, + Nko, + Nushu, + Nyiakeng_Puachue_Hmong, + Ogham, + Ol_Chiki, + Old_Hungarian, + Old_Italic, + Old_North_Arabian, + Old_Permic, + Old_Persian, + Old_Sogdian, + Old_South_Arabian, + Old_Turkic, + Old_Uyghur, + Oriya, + Osage, + Osmanya, + Pahawh_Hmong, + Palmyrene, + Pau_Cin_Hau, + Phags_Pa, + Phoenician, + Psalter_Pahlavi, + Rejang, + Runic, + Samaritan, + Saurashtra, + Sharada, + Shavian, + Siddham, + SignWriting, + Sinhala, + Sogdian, + Sora_Sompeng, + Soyombo, + Sundanese, + Syloti_Nagri, + Syriac, + Tagalog, + Tagbanwa, + Tai_Le, + Tai_Tham, + Tai_Viet, + Takri, + Tamil, + Tangsa, + Tangut, + Telugu, + Thaana, + Thai, + Tibetan, + Tifinagh, + Tirhuta, + Toto, + Ugaritic, + Vai, + Vithkuqi, + Wancho, + Warang_Citi, + Yezidi, + Yi, + Zanabazar_Square, +}; + +allocator: mem.Allocator, +s1: []u16 = undefined, +s2: []u8 = undefined, +s3: []u8 = undefined, + +const Self = @This(); + +pub fn init(allocator: mem.Allocator) !Self { + const decompressor = compress.deflate.decompressor; + const in_bytes = @embedFile("scripts"); + var in_fbs = std.io.fixedBufferStream(in_bytes); + var in_decomp = try decompressor(allocator, in_fbs.reader(), null); + defer in_decomp.deinit(); + var reader = in_decomp.reader(); + + const endian = builtin.cpu.arch.endian(); + + var self = Self{ .allocator = allocator }; + + const s1_len: u16 = try reader.readInt(u16, endian); + self.s1 = try allocator.alloc(u16, s1_len); + for (0..s1_len) |i| self.s1[i] = try reader.readInt(u16, endian); + + const s2_len: u16 = try reader.readInt(u16, endian); + self.s2 = try allocator.alloc(u8, s2_len); + _ = try reader.readAll(self.s2); + + const s3_len: u16 = try reader.readInt(u8, endian); + self.s3 = try allocator.alloc(u8, s3_len); + _ = try reader.readAll(self.s3); + + return self; +} + +pub fn deinit(self: *const Self) void { + self.allocator.free(self.s1); + self.allocator.free(self.s2); + self.allocator.free(self.s3); +} + +/// Lookup the Script type for `cp`. +pub fn script(self: Self, cp: u21) ?Script { + const byte = self.s3[self.s2[self.s1[cp >> 8] + (cp & 0xff)]]; + if (byte == 0) return null; + return @enumFromInt(byte); +} + +test "script" { + const self = try init(std.testing.allocator); + defer self.deinit(); + try testing.expectEqual(Script.Latin, self.script('A').?); +} diff --git a/src/WidthData.zig b/src/WidthData.zig index 32f8658..d17f0cd 100644 --- a/src/WidthData.zig +++ b/src/WidthData.zig @@ -39,7 +39,7 @@ pub fn init(allocator: mem.Allocator) !Self { return self; } -pub fn deinit(self: *Self) void { +pub fn deinit(self: *const Self) void { self.allocator.free(self.s1); self.allocator.free(self.s2); self.g_data.deinit(); diff --git a/src/grapheme.zig b/src/grapheme.zig index e55a6a4..ad43cfd 100644 --- a/src/grapheme.zig +++ b/src/grapheme.zig @@ -237,7 +237,7 @@ test "Segmentation GraphemeIterator" { var buf_reader = std.io.bufferedReader(file.reader()); var input_stream = buf_reader.reader(); - var data = try GraphemeData.init(allocator); + const data = try GraphemeData.init(allocator); defer data.deinit(); var buf: [4096]u8 = undefined; @@ -302,7 +302,7 @@ test "Segmentation ZWJ and ZWSP emoji sequences" { const with_zwsp = seq_1 ++ "\u{200B}" ++ seq_2; const no_joiner = seq_1 ++ seq_2; - var data = try GraphemeData.init(std.testing.allocator); + const data = try GraphemeData.init(std.testing.allocator); defer data.deinit(); var iter = Iterator.init(with_zwj, &data); -- cgit v1.2.3