summaryrefslogtreecommitdiff
path: root/src/common/string_util.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'src/common/string_util.cpp')
-rw-r--r--src/common/string_util.cpp134
1 files changed, 6 insertions, 128 deletions
diff --git a/src/common/string_util.cpp b/src/common/string_util.cpp
index c9a5425a7..731d1db34 100644
--- a/src/common/string_util.cpp
+++ b/src/common/string_util.cpp
@@ -5,6 +5,7 @@
5#include <algorithm> 5#include <algorithm>
6#include <cctype> 6#include <cctype>
7#include <cerrno> 7#include <cerrno>
8#include <codecvt>
8#include <cstdio> 9#include <cstdio>
9#include <cstdlib> 10#include <cstdlib>
10#include <cstring> 11#include <cstring>
@@ -13,11 +14,7 @@
13#include "common/string_util.h" 14#include "common/string_util.h"
14 15
15#ifdef _WIN32 16#ifdef _WIN32
16#include <codecvt>
17#include <windows.h> 17#include <windows.h>
18#include "common/common_funcs.h"
19#else
20#include <iconv.h>
21#endif 18#endif
22 19
23namespace Common { 20namespace Common {
@@ -195,11 +192,9 @@ std::string ReplaceAll(std::string result, const std::string& src, const std::st
195 return result; 192 return result;
196} 193}
197 194
198#ifdef _WIN32
199
200std::string UTF16ToUTF8(const std::u16string& input) { 195std::string UTF16ToUTF8(const std::u16string& input) {
201#if _MSC_VER >= 1900 196#ifdef _MSC_VER
202 // Workaround for missing char16_t/char32_t instantiations in MSVC2015 197 // Workaround for missing char16_t/char32_t instantiations in MSVC2017
203 std::wstring_convert<std::codecvt_utf8_utf16<__int16>, __int16> convert; 198 std::wstring_convert<std::codecvt_utf8_utf16<__int16>, __int16> convert;
204 std::basic_string<__int16> tmp_buffer(input.cbegin(), input.cend()); 199 std::basic_string<__int16> tmp_buffer(input.cbegin(), input.cend());
205 return convert.to_bytes(tmp_buffer); 200 return convert.to_bytes(tmp_buffer);
@@ -210,8 +205,8 @@ std::string UTF16ToUTF8(const std::u16string& input) {
210} 205}
211 206
212std::u16string UTF8ToUTF16(const std::string& input) { 207std::u16string UTF8ToUTF16(const std::string& input) {
213#if _MSC_VER >= 1900 208#ifdef _MSC_VER
214 // Workaround for missing char16_t/char32_t instantiations in MSVC2015 209 // Workaround for missing char16_t/char32_t instantiations in MSVC2017
215 std::wstring_convert<std::codecvt_utf8_utf16<__int16>, __int16> convert; 210 std::wstring_convert<std::codecvt_utf8_utf16<__int16>, __int16> convert;
216 auto tmp_buffer = convert.from_bytes(input); 211 auto tmp_buffer = convert.from_bytes(input);
217 return std::u16string(tmp_buffer.cbegin(), tmp_buffer.cend()); 212 return std::u16string(tmp_buffer.cbegin(), tmp_buffer.cend());
@@ -221,6 +216,7 @@ std::u16string UTF8ToUTF16(const std::string& input) {
221#endif 216#endif
222} 217}
223 218
219#ifdef _WIN32
224static std::wstring CPToUTF16(u32 code_page, const std::string& input) { 220static std::wstring CPToUTF16(u32 code_page, const std::string& input) {
225 const auto size = 221 const auto size =
226 MultiByteToWideChar(code_page, 0, input.data(), static_cast<int>(input.size()), nullptr, 0); 222 MultiByteToWideChar(code_page, 0, input.data(), static_cast<int>(input.size()), nullptr, 0);
@@ -261,124 +257,6 @@ std::wstring UTF8ToUTF16W(const std::string& input) {
261 return CPToUTF16(CP_UTF8, input); 257 return CPToUTF16(CP_UTF8, input);
262} 258}
263 259
264std::string SHIFTJISToUTF8(const std::string& input) {
265 return UTF16ToUTF8(CPToUTF16(932, input));
266}
267
268std::string CP1252ToUTF8(const std::string& input) {
269 return UTF16ToUTF8(CPToUTF16(1252, input));
270}
271
272#else
273
274template <typename T>
275static std::string CodeToUTF8(const char* fromcode, const std::basic_string<T>& input) {
276 iconv_t const conv_desc = iconv_open("UTF-8", fromcode);
277 if ((iconv_t)(-1) == conv_desc) {
278 LOG_ERROR(Common, "Iconv initialization failure [{}]: {}", fromcode, strerror(errno));
279 iconv_close(conv_desc);
280 return {};
281 }
282
283 const std::size_t in_bytes = sizeof(T) * input.size();
284 // Multiply by 4, which is the max number of bytes to encode a codepoint
285 const std::size_t out_buffer_size = 4 * in_bytes;
286
287 std::string out_buffer(out_buffer_size, '\0');
288
289 auto src_buffer = &input[0];
290 std::size_t src_bytes = in_bytes;
291 auto dst_buffer = &out_buffer[0];
292 std::size_t dst_bytes = out_buffer.size();
293
294 while (0 != src_bytes) {
295 std::size_t const iconv_result =
296 iconv(conv_desc, (char**)(&src_buffer), &src_bytes, &dst_buffer, &dst_bytes);
297
298 if (static_cast<std::size_t>(-1) == iconv_result) {
299 if (EILSEQ == errno || EINVAL == errno) {
300 // Try to skip the bad character
301 if (0 != src_bytes) {
302 --src_bytes;
303 ++src_buffer;
304 }
305 } else {
306 LOG_ERROR(Common, "iconv failure [{}]: {}", fromcode, strerror(errno));
307 break;
308 }
309 }
310 }
311
312 std::string result;
313 out_buffer.resize(out_buffer_size - dst_bytes);
314 out_buffer.swap(result);
315
316 iconv_close(conv_desc);
317
318 return result;
319}
320
321std::u16string UTF8ToUTF16(const std::string& input) {
322 iconv_t const conv_desc = iconv_open("UTF-16LE", "UTF-8");
323 if ((iconv_t)(-1) == conv_desc) {
324 LOG_ERROR(Common, "Iconv initialization failure [UTF-8]: {}", strerror(errno));
325 iconv_close(conv_desc);
326 return {};
327 }
328
329 const std::size_t in_bytes = sizeof(char) * input.size();
330 // Multiply by 4, which is the max number of bytes to encode a codepoint
331 const std::size_t out_buffer_size = 4 * sizeof(char16_t) * in_bytes;
332
333 std::u16string out_buffer(out_buffer_size, char16_t{});
334
335 char* src_buffer = const_cast<char*>(&input[0]);
336 std::size_t src_bytes = in_bytes;
337 char* dst_buffer = (char*)(&out_buffer[0]);
338 std::size_t dst_bytes = out_buffer.size();
339
340 while (0 != src_bytes) {
341 std::size_t const iconv_result =
342 iconv(conv_desc, &src_buffer, &src_bytes, &dst_buffer, &dst_bytes);
343
344 if (static_cast<std::size_t>(-1) == iconv_result) {
345 if (EILSEQ == errno || EINVAL == errno) {
346 // Try to skip the bad character
347 if (0 != src_bytes) {
348 --src_bytes;
349 ++src_buffer;
350 }
351 } else {
352 LOG_ERROR(Common, "iconv failure [UTF-8]: {}", strerror(errno));
353 break;
354 }
355 }
356 }
357
358 std::u16string result;
359 out_buffer.resize(out_buffer_size - dst_bytes);
360 out_buffer.swap(result);
361
362 iconv_close(conv_desc);
363
364 return result;
365}
366
367std::string UTF16ToUTF8(const std::u16string& input) {
368 return CodeToUTF8("UTF-16LE", input);
369}
370
371std::string CP1252ToUTF8(const std::string& input) {
372 // return CodeToUTF8("CP1252//TRANSLIT", input);
373 // return CodeToUTF8("CP1252//IGNORE", input);
374 return CodeToUTF8("CP1252", input);
375}
376
377std::string SHIFTJISToUTF8(const std::string& input) {
378 // return CodeToUTF8("CP932", input);
379 return CodeToUTF8("SJIS", input);
380}
381
382#endif 260#endif
383 261
384std::string StringFromFixedZeroTerminatedBuffer(const char* buffer, std::size_t max_len) { 262std::string StringFromFixedZeroTerminatedBuffer(const char* buffer, std::size_t max_len) {