incentive/psopeeps-newserv
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

handle JP heart symbol correctly

Browse Source
This commit is contained in:
Martin Michelsen
2024-03-03 20:07:37 -08:00
parent 7f71b87b9b
commit 2432d8b32b
8 changed files with 5432 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/Main.cc
+1 -1
View File
@@ -1445,7 +1445,7 @@ Action a_extract_bml("extract-bml", "\
Action a_decode_sjis(
"decode-sjis", nullptr, +[](Arguments& args) {
string data = read_input_data(args);
string result = tt_sjis_to_utf8(data);
string result = tt_sega_sjis_to_utf8(data);
write_output_data(args, result.data(), result.size(), "txt");
});
src/QuestScript.cc
+4 -4
View File
@@ -55,7 +55,7 @@ static string escape_string(const string& data, TextEncoding encoding = TextEnco
decoded = tt_utf16_to_utf8(data);
break;
case TextEncoding::SJIS:
decoded = tt_sjis_to_utf8(data);
decoded = tt_sega_sjis_to_utf8(data);
break;
case TextEncoding::ISO8859:
decoded = tt_8859_to_utf8(data);
@@ -1234,7 +1234,7 @@ std::string disassemble_quest_script(const void* data, size_t size, Version vers
} else {
string s = cmd_r.get_cstr();
if (def->flags & F_PASS) {
arg_stack_values.emplace_back(language ? tt_8859_to_utf8(s) : tt_sjis_to_utf8(s));
arg_stack_values.emplace_back(language ? tt_8859_to_utf8(s) : tt_sega_sjis_to_utf8(s));
}
dasm_arg = escape_string(s, encoding_for_language(language));
}
@@ -1969,7 +1969,7 @@ std::string assemble_quest_script(const std::string& text) {
auto add_cstr = [&](const string& text) -> void {
switch (quest_version) {
case Version::DC_NTE:
code_w.write(tt_utf8_to_sjis(text));
code_w.write(tt_utf8_to_sega_sjis(text));
code_w.put_u8(0);
break;
case Version::DC_V1_11_2000_PROTOTYPE:
@@ -1980,7 +1980,7 @@ std::string assemble_quest_script(const std::string& text) {
case Version::GC_EP3_NTE:
case Version::GC_EP3:
case Version::XB_V3:
code_w.write(quest_language ? tt_utf8_to_8859(text) : tt_utf8_to_sjis(text));
code_w.write(quest_language ? tt_utf8_to_8859(text) : tt_utf8_to_sega_sjis(text));
code_w.put_u8(0);
break;
case Version::PC_NTE:
src/SendCommands.cc
+2 -1
View File
@@ -2,6 +2,7 @@
#include <event2/buffer.h>
#include <inttypes.h>
#include <stdarg.h>
#include <string.h>
#include <functional>
@@ -853,7 +854,7 @@ string prepare_chat_data(
data.append(text);
return tt_utf8_to_utf16(data);
} else if (version == Version::DC_NTE) {
data.append(tt_utf8_to_sjis(text));
data.append(tt_utf8_to_sega_sjis(text));
return data;
} else {
data.append(tt_encode_marked(text, language, false));
src/Text.cc
+90 -47
View File
@@ -22,52 +22,55 @@ TextTranscoder::TextTranscoder(const char* to, const char* from)
}
}
TextTranscoder::TextTranscoder(TextTranscoder&& other) : ic(other.ic) {
other.ic = this->INVALID_IC;
}
TextTranscoder& TextTranscoder::operator=(TextTranscoder&& other) {
this->ic = other.ic;
other.ic = this->INVALID_IC;
return *this;
}
TextTranscoder::~TextTranscoder() {
iconv_close(this->ic);
}
TextTranscoder::Result TextTranscoder::operator()(
void* dest, size_t dest_size, const void* src, size_t src_bytes, bool truncate_oversize_result) {
void* dest, size_t dest_bytes, const void* src, size_t src_bytes, bool truncate_oversize_result) {
// Clear any conversion state left over from the previous call
iconv(this->ic, nullptr, nullptr, nullptr, nullptr);
void* orig_dest = dest;
const void* orig_src = src;
size_t ret = iconv(
this->ic,
reinterpret_cast<char**>(const_cast<void**>(&src)),
&src_bytes,
reinterpret_cast<char**>(&dest),
&dest_size);
while (src_bytes > 0) {
size_t src_bytes_before = src_bytes;
size_t ret = iconv(this->ic, reinterpret_cast<char**>(const_cast<void**>(&src)), &src_bytes, reinterpret_cast<char**>(&dest), &dest_bytes);
size_t bytes_read = reinterpret_cast<const char*>(src) - reinterpret_cast<const char*>(orig_src);
if (ret == this->FAILURE_RESULT) {
switch (errno) {
case EILSEQ:
throw runtime_error(string_printf("untranslatable character at position 0x%zX", bytes_read));
case EINVAL:
throw runtime_error(string_printf("incomplete multibyte sequence at position 0x%zX", bytes_read));
case E2BIG:
if (!truncate_oversize_result) {
throw runtime_error("string does not fit in buffer");
} else {
size_t bytes_read = reinterpret_cast<const char*>(src) - reinterpret_cast<const char*>(orig_src);
if (ret == this->FAILURE_RESULT) {
switch (errno) {
case EILSEQ: {
string custom_result = this->on_untranslatable(&src, &src_bytes);
if (custom_result.empty()) {
throw runtime_error(string_printf("untranslatable character at position 0x%zX", bytes_read));
} else if (custom_result.size() <= dest_bytes) {
memcpy(dest, custom_result.data(), custom_result.size());
dest = reinterpret_cast<char*>(dest) + custom_result.size();
dest_bytes -= custom_result.size();
} else if (!truncate_oversize_result) {
throw runtime_error("string does not fit in buffer");
}
break;
}
default:
throw runtime_error("transcoding failed: " + string_for_error(errno));
case EINVAL:
throw runtime_error(string_printf("incomplete multibyte sequence at position 0x%zX", bytes_read));
case E2BIG:
if (!truncate_oversize_result) {
throw runtime_error("string does not fit in buffer");
} else {
break;
}
default:
throw runtime_error("transcoding failed: " + string_for_error(errno));
}
} else if (src_bytes_before == src_bytes) {
throw runtime_error("could not transcode any characters");
}
}
size_t bytes_read = reinterpret_cast<const char*>(src) - reinterpret_cast<const char*>(orig_src);
size_t bytes_written = reinterpret_cast<char*>(dest) - reinterpret_cast<char*>(orig_dest);
return Result{
.bytes_read = bytes_read,
@@ -75,21 +78,22 @@ TextTranscoder::Result TextTranscoder::operator()(
};
}
string TextTranscoder::operator()(const void* src, size_t src_size) {
string TextTranscoder::operator()(const void* src, size_t src_bytes) {
// Clear any conversion state left over from the previous call
iconv(this->ic, nullptr, nullptr, nullptr, nullptr);
const void* orig_src = src;
deque<string> blocks;
while (src_size > 0) {
while (src_bytes > 0) {
// Assume 2x input size on average, but always allocate at least 8 bytes
string& block = blocks.emplace_back(max<size_t>((src_size << 1), 8), '\0');
string& block = blocks.emplace_back(max<size_t>((src_bytes << 1), 8), '\0');
char* dest = block.data();
size_t dest_size = block.size();
size_t src_bytes_before = src_bytes;
size_t ret = iconv(
this->ic,
reinterpret_cast<char**>(const_cast<void**>(&src)),
&src_size,
&src_bytes,
reinterpret_cast<char**>(&dest),
&dest_size);
block.resize(block.size() - dest_size);
@@ -97,8 +101,14 @@ string TextTranscoder::operator()(const void* src, size_t src_size) {
size_t bytes_read = reinterpret_cast<const char*>(src) - reinterpret_cast<const char*>(orig_src);
if (ret == this->FAILURE_RESULT) {
switch (errno) {
case EILSEQ:
throw runtime_error(string_printf("untranslatable character at position %zu", bytes_read));
case EILSEQ: {
string custom_result = this->on_untranslatable(&src, &src_bytes);
if (custom_result.empty()) {
throw runtime_error(string_printf("untranslatable character at position %zu", bytes_read));
}
blocks.emplace_back(std::move(custom_result));
break;
}
case EINVAL:
throw runtime_error(string_printf("incomplete multibyte sequence at position %zu", bytes_read));
case E2BIG:
@@ -106,9 +116,8 @@ string TextTranscoder::operator()(const void* src, size_t src_size) {
default:
throw runtime_error("transcoding failed: " + string_for_error(errno));
}
}
if ((bytes_read == 0) && (src_size != 0)) {
throw runtime_error("failed to transcode input data");
} else if (src_bytes_before == src_bytes) {
throw runtime_error("could not transcode any characters");
}
}
@@ -119,10 +128,44 @@ string TextTranscoder::operator()(const string& data) {
return this->operator()(data.data(), data.size());
}
std::string TextTranscoder::on_untranslatable(const void**, size_t*) const {
return "";
}
TextTranscoderCustomSJISToUTF8::TextTranscoderCustomSJISToUTF8() : TextTranscoder("UTF-8", "SHIFT_JIS") {}
std::string TextTranscoderCustomSJISToUTF8::on_untranslatable(const void** src, size_t* size) const {
// Sega implemented a single nonstandard Shift-JIS character on PSO GC (and
// probably XB as well): the heart symbol, encoded as F040. Understandably,
// libiconv doesn't know what to do with it because it's not actually part of
// Shift-JIS, so we have to handle it manually here.
if ((*size >= 2) && !memcmp(*src, "\xF0\x40", 2)) {
*src = reinterpret_cast<const char*>(*src) + 2;
*size -= 2;
return "\xE2\x99\xA5";
} else {
return "";
}
}
TextTranscoderUTF8ToCustomSJIS::TextTranscoderUTF8ToCustomSJIS() : TextTranscoder("SHIFT_JIS", "UTF-8") {}
std::string TextTranscoderUTF8ToCustomSJIS::on_untranslatable(const void** src, size_t* size) const {
if ((*size >= 3) && !memcmp(*src, "\xE2\x99\xA5", 3)) {
*src = reinterpret_cast<const char*>(*src) + 3;
*size -= 3;
return "\xF0\x40";
} else {
return "";
}
}
TextTranscoder tt_8859_to_utf8("UTF-8", "ISO-8859-1");
TextTranscoder tt_utf8_to_8859("ISO-8859-1", "UTF-8");
TextTranscoder tt_sjis_to_utf8("UTF-8", "SHIFT_JIS");
TextTranscoder tt_utf8_to_sjis("SHIFT_JIS", "UTF-8");
TextTranscoder tt_standard_sjis_to_utf8("UTF-8", "SHIFT_JIS");
TextTranscoder tt_utf8_to_standard_sjis("SHIFT_JIS", "UTF-8");
TextTranscoderCustomSJISToUTF8 tt_sega_sjis_to_utf8;
TextTranscoderUTF8ToCustomSJIS tt_utf8_to_sega_sjis;
TextTranscoder tt_utf16_to_utf8("UTF-8", "UTF-16LE");
TextTranscoder tt_utf8_to_utf16("UTF-16LE", "UTF-8");
TextTranscoder tt_ascii_to_utf8("UTF-8", "ASCII");
@@ -136,11 +179,11 @@ string tt_encode_marked_optional(const string& utf8, uint8_t default_language, b
try {
return tt_utf8_to_8859(utf8);
} catch (const exception& e) {
return "\tJ" + tt_utf8_to_sjis(utf8);
return "\tJ" + tt_utf8_to_sega_sjis(utf8);
}
} else {
try {
return tt_utf8_to_sjis(utf8);
return tt_utf8_to_sega_sjis(utf8);
} catch (const exception& e) {
return "\tE" + tt_utf8_to_8859(utf8);
}
@@ -159,11 +202,11 @@ string tt_encode_marked(const string& utf8, uint8_t default_language, bool is_ut
try {
return "\tE" + tt_utf8_to_8859(utf8);
} catch (const exception& e) {
return "\tJ" + tt_utf8_to_sjis(utf8);
return "\tJ" + tt_utf8_to_sega_sjis(utf8);
}
} else {
try {
return "\tJ" + tt_utf8_to_sjis(utf8);
return "\tJ" + tt_utf8_to_sega_sjis(utf8);
} catch (const exception& e) {
return "\tE" + tt_utf8_to_8859(utf8);
}
@@ -181,12 +224,12 @@ string tt_decode_marked(const string& data, uint8_t default_language, bool is_ut
} else {
if (data.size() >= 2 && data[0] == '\t') {
if (data[1] == 'J') {
return tt_sjis_to_utf8(data.substr(2));
return tt_sega_sjis_to_utf8(data.substr(2));
} else if (data[1] == 'E') {
return tt_8859_to_utf8(data.substr(2));
}
}
return default_language ? tt_8859_to_utf8(data) : tt_sjis_to_utf8(data);
return default_language ? tt_8859_to_utf8(data) : tt_sega_sjis_to_utf8(data);
}
}
src/Text.hh
+33 -11
View File
@@ -17,10 +17,10 @@ class TextTranscoder {
public:
TextTranscoder(const char* to, const char* from);
TextTranscoder(const TextTranscoder&) = delete;
TextTranscoder(TextTranscoder&&);
TextTranscoder(TextTranscoder&&) = delete;
TextTranscoder& operator=(const TextTranscoder&) = delete;
TextTranscoder& operator=(TextTranscoder&&);
~TextTranscoder();
TextTranscoder& operator=(TextTranscoder&&) = delete;
virtual ~TextTranscoder();
struct Result {
size_t bytes_read;
@@ -31,16 +31,38 @@ public:
std::string operator()(const void* src, size_t src_bytes);
std::string operator()(const std::string& data);
private:
protected:
virtual std::string on_untranslatable(const void** src, size_t* size) const;
static const iconv_t INVALID_IC;
static const size_t FAILURE_RESULT;
iconv_t ic;
};
class TextTranscoderCustomSJISToUTF8 : public TextTranscoder {
public:
TextTranscoderCustomSJISToUTF8();
virtual ~TextTranscoderCustomSJISToUTF8() = default;
protected:
virtual std::string on_untranslatable(const void** src, size_t* size) const;
};
class TextTranscoderUTF8ToCustomSJIS : public TextTranscoder {
public:
TextTranscoderUTF8ToCustomSJIS();
virtual ~TextTranscoderUTF8ToCustomSJIS() = default;
protected:
virtual std::string on_untranslatable(const void** src, size_t* size) const;
};
extern TextTranscoder tt_8859_to_utf8;
extern TextTranscoder tt_utf8_to_8859;
extern TextTranscoder tt_sjis_to_utf8;
extern TextTranscoder tt_utf8_to_sjis;
extern TextTranscoder tt_standard_sjis_to_utf8;
extern TextTranscoder tt_utf8_to_standard_sjis;
extern TextTranscoderCustomSJISToUTF8 tt_sega_sjis_to_utf8;
extern TextTranscoderUTF8ToCustomSJIS tt_utf8_to_sega_sjis;
extern TextTranscoder tt_utf16_to_utf8;
extern TextTranscoder tt_utf8_to_utf16;
extern TextTranscoder tt_ascii_to_utf8;
@@ -434,7 +456,7 @@ struct pstring {
break;
}
case TextEncoding::SJIS: {
auto ret = tt_utf8_to_sjis(this->data, Bytes, s.data(), s.size(), true);
auto ret = tt_utf8_to_sega_sjis(this->data, Bytes, s.data(), s.size(), true);
this->clear_after_bytes(ret.bytes_written);
break;
}
@@ -469,7 +491,7 @@ struct pstring {
case TextEncoding::MARKED: {
if (client_language == 0) {
try {
auto ret = tt_utf8_to_sjis(this->data, Bytes, s.data(), s.size(), true);
auto ret = tt_utf8_to_sega_sjis(this->data, Bytes, s.data(), s.size(), true);
this->clear_after_bytes(ret.bytes_written);
} catch (const std::runtime_error&) {
this->data[0] = '\t';
@@ -484,7 +506,7 @@ struct pstring {
} catch (const std::runtime_error&) {
this->data[0] = '\t';
this->data[1] = 'J';
auto ret = tt_utf8_to_sjis(this->data + 2, Bytes - 2, s.data(), s.size(), true);
auto ret = tt_utf8_to_sega_sjis(this->data + 2, Bytes - 2, s.data(), s.size(), true);
this->clear_after_bytes(ret.bytes_written + 2);
}
}
@@ -536,7 +558,7 @@ struct pstring {
case TextEncoding::ISO8859:
return tt_8859_to_utf8(this->data, this->used_chars_8());
case TextEncoding::SJIS:
return tt_sjis_to_utf8(this->data, this->used_chars_8());
return tt_sega_sjis_to_utf8(this->data, this->used_chars_8());
case TextEncoding::UTF16:
return tt_utf16_to_utf8(this->data, this->used_chars_16() * 2);
case TextEncoding::UTF16_ALWAYS_MARKED: {
@@ -563,7 +585,7 @@ struct pstring {
}
return client_language
? tt_8859_to_utf8(&this->data[offset], this->used_chars_8() - offset)
: tt_sjis_to_utf8(&this->data[offset], this->used_chars_8() - offset);
: tt_sega_sjis_to_utf8(&this->data[offset], this->used_chars_8() - offset);
}
default:
throw std::logic_error("unknown text encoding");
src/TextIndex.cc
+3 -3
View File
@@ -187,7 +187,7 @@ BinaryTextSet::BinaryTextSet(const std::string& pr2_data, size_t collection_coun
}
used_offsets.emplace(root_offset);
auto& tt = is_sjis ? tt_sjis_to_utf8 : tt_8859_to_utf8;
auto& tt = is_sjis ? tt_sega_sjis_to_utf8 : tt_8859_to_utf8;
collection_offsets_r.go(0);
while (!collection_offsets_r.eof()) {
@@ -286,7 +286,7 @@ void BinaryTextAndKeyboardsSet::parse_t(const string& pr2_data, bool is_sjis) {
using U32T = std::conditional_t<IsBigEndian, be_uint32_t, le_uint32_t>;
using U16T = std::conditional_t<IsBigEndian, be_uint16_t, le_uint16_t>;
auto& tt = is_sjis ? tt_sjis_to_utf8 : tt_8859_to_utf8;
auto& tt = is_sjis ? tt_sega_sjis_to_utf8 : tt_8859_to_utf8;
// The structure is as follows:
// Footer:
@@ -355,7 +355,7 @@ pair<string, string> BinaryTextAndKeyboardsSet::serialize_t(bool is_sjis) const
using U32T = std::conditional_t<IsBigEndian, be_uint32_t, le_uint32_t>;
using U16T = std::conditional_t<IsBigEndian, be_uint16_t, le_uint16_t>;
auto& tt = is_sjis ? tt_utf8_to_sjis : tt_utf8_to_8859;
auto& tt = is_sjis ? tt_utf8_to_sega_sjis : tt_utf8_to_8859;
StringWriter w;
::set<size_t> relocation_offsets;
src/WordSelectTable.cc
+1 -1
View File
@@ -76,7 +76,7 @@ void WordSelectSet::parse_non_windows_t(const std::string& data, bool use_sjis)
auto string_offset_r = r.sub(root.strings_table, sizeof(U32T) * StringTableCount);
while (!string_offset_r.eof()) {
string raw_s = r.pget_cstr(string_offset_r.template get<U32T>());
this->strings.emplace_back(use_sjis ? tt_sjis_to_utf8(raw_s) : tt_8859_to_utf8(raw_s));
this->strings.emplace_back(use_sjis ? tt_sega_sjis_to_utf8(raw_s) : tt_8859_to_utf8(raw_s));
}
}
tests/GC-HeartSymbol.txt
+5298
View File
File diff suppressed because it is too large Load Diff