handle JP heart symbol correctly

src/Main.cc

@@ -1445,7 +1445,7 @@ Action a_extract_bml("extract-bml", "\
 Action a_decode_sjis(
     "decode-sjis", nullptr, +[](Arguments& args) {
       string data = read_input_data(args);
-      string result = tt_sjis_to_utf8(data);
+      string result = tt_sega_sjis_to_utf8(data);
       write_output_data(args, result.data(), result.size(), "txt");
     });
 

src/QuestScript.cc

@@ -55,7 +55,7 @@ static string escape_string(const string& data, TextEncoding encoding = TextEnco
         decoded = tt_utf16_to_utf8(data);
         break;
       case TextEncoding::SJIS:
-        decoded = tt_sjis_to_utf8(data);
+        decoded = tt_sega_sjis_to_utf8(data);
         break;
       case TextEncoding::ISO8859:
         decoded = tt_8859_to_utf8(data);
@@ -1234,7 +1234,7 @@ std::string disassemble_quest_script(const void* data, size_t size, Version vers
                   } else {
                     string s = cmd_r.get_cstr();
                     if (def->flags & F_PASS) {
-                      arg_stack_values.emplace_back(language ? tt_8859_to_utf8(s) : tt_sjis_to_utf8(s));
+                      arg_stack_values.emplace_back(language ? tt_8859_to_utf8(s) : tt_sega_sjis_to_utf8(s));
                     }
                     dasm_arg = escape_string(s, encoding_for_language(language));
                   }
@@ -1969,7 +1969,7 @@ std::string assemble_quest_script(const std::string& text) {
             auto add_cstr = [&](const string& text) -> void {
               switch (quest_version) {
                 case Version::DC_NTE:
-                  code_w.write(tt_utf8_to_sjis(text));
+                  code_w.write(tt_utf8_to_sega_sjis(text));
                   code_w.put_u8(0);
                   break;
                 case Version::DC_V1_11_2000_PROTOTYPE:
@@ -1980,7 +1980,7 @@ std::string assemble_quest_script(const std::string& text) {
                 case Version::GC_EP3_NTE:
                 case Version::GC_EP3:
                 case Version::XB_V3:
-                  code_w.write(quest_language ? tt_utf8_to_8859(text) : tt_utf8_to_sjis(text));
+                  code_w.write(quest_language ? tt_utf8_to_8859(text) : tt_utf8_to_sega_sjis(text));
                   code_w.put_u8(0);
                   break;
                 case Version::PC_NTE:

src/SendCommands.cc

@@ -2,6 +2,7 @@
 
 #include <event2/buffer.h>
 #include <inttypes.h>
+#include <stdarg.h>
 #include <string.h>
 
 #include <functional>
@@ -853,7 +854,7 @@ string prepare_chat_data(
     data.append(text);
     return tt_utf8_to_utf16(data);
   } else if (version == Version::DC_NTE) {
-    data.append(tt_utf8_to_sjis(text));
+    data.append(tt_utf8_to_sega_sjis(text));
     return data;
   } else {
     data.append(tt_encode_marked(text, language, false));

src/Text.cc

@@ -22,52 +22,55 @@ TextTranscoder::TextTranscoder(const char* to, const char* from)
   }
 }
 
-TextTranscoder::TextTranscoder(TextTranscoder&& other) : ic(other.ic) {
-  other.ic = this->INVALID_IC;
-}
-
-TextTranscoder& TextTranscoder::operator=(TextTranscoder&& other) {
-  this->ic = other.ic;
-  other.ic = this->INVALID_IC;
-  return *this;
-}
-
 TextTranscoder::~TextTranscoder() {
   iconv_close(this->ic);
 }
 
 TextTranscoder::Result TextTranscoder::operator()(
-    void* dest, size_t dest_size, const void* src, size_t src_bytes, bool truncate_oversize_result) {
+    void* dest, size_t dest_bytes, const void* src, size_t src_bytes, bool truncate_oversize_result) {
   // Clear any conversion state left over from the previous call
   iconv(this->ic, nullptr, nullptr, nullptr, nullptr);
 
   void* orig_dest = dest;
   const void* orig_src = src;
-  size_t ret = iconv(
+  while (src_bytes > 0) {
-      this->ic,
+    size_t src_bytes_before = src_bytes;
-      reinterpret_cast<char**>(const_cast<void**>(&src)),
+    size_t ret = iconv(this->ic, reinterpret_cast<char**>(const_cast<void**>(&src)), &src_bytes, reinterpret_cast<char**>(&dest), &dest_bytes);
-      &src_bytes,
-      reinterpret_cast<char**>(&dest),
-      &dest_size);
 
-  size_t bytes_read = reinterpret_cast<const char*>(src) - reinterpret_cast<const char*>(orig_src);
+    size_t bytes_read = reinterpret_cast<const char*>(src) - reinterpret_cast<const char*>(orig_src);
-  if (ret == this->FAILURE_RESULT) {
+    if (ret == this->FAILURE_RESULT) {
-    switch (errno) {
+      switch (errno) {
-      case EILSEQ:
+        case EILSEQ: {
-        throw runtime_error(string_printf("untranslatable character at position 0x%zX", bytes_read));
+          string custom_result = this->on_untranslatable(&src, &src_bytes);
-      case EINVAL:
+          if (custom_result.empty()) {
-        throw runtime_error(string_printf("incomplete multibyte sequence at position 0x%zX", bytes_read));
+            throw runtime_error(string_printf("untranslatable character at position 0x%zX", bytes_read));
-      case E2BIG:
+          } else if (custom_result.size() <= dest_bytes) {
-        if (!truncate_oversize_result) {
+            memcpy(dest, custom_result.data(), custom_result.size());
-          throw runtime_error("string does not fit in buffer");
+            dest = reinterpret_cast<char*>(dest) + custom_result.size();
-        } else {
+            dest_bytes -= custom_result.size();
+          } else if (!truncate_oversize_result) {
+            throw runtime_error("string does not fit in buffer");
+          }
           break;
         }
-      default:
+        case EINVAL:
-        throw runtime_error("transcoding failed: " + string_for_error(errno));
+          throw runtime_error(string_printf("incomplete multibyte sequence at position 0x%zX", bytes_read));
+        case E2BIG:
+          if (!truncate_oversize_result) {
+            throw runtime_error("string does not fit in buffer");
+          } else {
+            break;
+          }
+        default:
+          throw runtime_error("transcoding failed: " + string_for_error(errno));
+      }
+
+    } else if (src_bytes_before == src_bytes) {
+      throw runtime_error("could not transcode any characters");
     }
   }
 
+  size_t bytes_read = reinterpret_cast<const char*>(src) - reinterpret_cast<const char*>(orig_src);
   size_t bytes_written = reinterpret_cast<char*>(dest) - reinterpret_cast<char*>(orig_dest);
   return Result{
       .bytes_read = bytes_read,
@@ -75,21 +78,22 @@ TextTranscoder::Result TextTranscoder::operator()(
   };
 }
 
-string TextTranscoder::operator()(const void* src, size_t src_size) {
+string TextTranscoder::operator()(const void* src, size_t src_bytes) {
   // Clear any conversion state left over from the previous call
   iconv(this->ic, nullptr, nullptr, nullptr, nullptr);
 
   const void* orig_src = src;
   deque<string> blocks;
-  while (src_size > 0) {
+  while (src_bytes > 0) {
     // Assume 2x input size on average, but always allocate at least 8 bytes
-    string& block = blocks.emplace_back(max<size_t>((src_size << 1), 8), '\0');
+    string& block = blocks.emplace_back(max<size_t>((src_bytes << 1), 8), '\0');
     char* dest = block.data();
     size_t dest_size = block.size();
+    size_t src_bytes_before = src_bytes;
     size_t ret = iconv(
         this->ic,
         reinterpret_cast<char**>(const_cast<void**>(&src)),
-        &src_size,
+        &src_bytes,
         reinterpret_cast<char**>(&dest),
         &dest_size);
     block.resize(block.size() - dest_size);
@@ -97,8 +101,14 @@ string TextTranscoder::operator()(const void* src, size_t src_size) {
     size_t bytes_read = reinterpret_cast<const char*>(src) - reinterpret_cast<const char*>(orig_src);
     if (ret == this->FAILURE_RESULT) {
       switch (errno) {
-        case EILSEQ:
+        case EILSEQ: {
-          throw runtime_error(string_printf("untranslatable character at position %zu", bytes_read));
+          string custom_result = this->on_untranslatable(&src, &src_bytes);
+          if (custom_result.empty()) {
+            throw runtime_error(string_printf("untranslatable character at position %zu", bytes_read));
+          }
+          blocks.emplace_back(std::move(custom_result));
+          break;
+        }
         case EINVAL:
           throw runtime_error(string_printf("incomplete multibyte sequence at position %zu", bytes_read));
         case E2BIG:
@@ -106,9 +116,8 @@ string TextTranscoder::operator()(const void* src, size_t src_size) {
         default:
           throw runtime_error("transcoding failed: " + string_for_error(errno));
       }
-    }
+    } else if (src_bytes_before == src_bytes) {
-    if ((bytes_read == 0) && (src_size != 0)) {
+      throw runtime_error("could not transcode any characters");
-      throw runtime_error("failed to transcode input data");
     }
   }
 
@@ -119,10 +128,44 @@ string TextTranscoder::operator()(const string& data) {
   return this->operator()(data.data(), data.size());
 }
 
+std::string TextTranscoder::on_untranslatable(const void**, size_t*) const {
+  return "";
+}
+
+TextTranscoderCustomSJISToUTF8::TextTranscoderCustomSJISToUTF8() : TextTranscoder("UTF-8", "SHIFT_JIS") {}
+
+std::string TextTranscoderCustomSJISToUTF8::on_untranslatable(const void** src, size_t* size) const {
+  // Sega implemented a single nonstandard Shift-JIS character on PSO GC (and
+  // probably XB as well): the heart symbol, encoded as F040. Understandably,
+  // libiconv doesn't know what to do with it because it's not actually part of
+  // Shift-JIS, so we have to handle it manually here.
+  if ((*size >= 2) && !memcmp(*src, "\xF0\x40", 2)) {
+    *src = reinterpret_cast<const char*>(*src) + 2;
+    *size -= 2;
+    return "\xE2\x99\xA5";
+  } else {
+    return "";
+  }
+}
+
+TextTranscoderUTF8ToCustomSJIS::TextTranscoderUTF8ToCustomSJIS() : TextTranscoder("SHIFT_JIS", "UTF-8") {}
+
+std::string TextTranscoderUTF8ToCustomSJIS::on_untranslatable(const void** src, size_t* size) const {
+  if ((*size >= 3) && !memcmp(*src, "\xE2\x99\xA5", 3)) {
+    *src = reinterpret_cast<const char*>(*src) + 3;
+    *size -= 3;
+    return "\xF0\x40";
+  } else {
+    return "";
+  }
+}
+
 TextTranscoder tt_8859_to_utf8("UTF-8", "ISO-8859-1");
 TextTranscoder tt_utf8_to_8859("ISO-8859-1", "UTF-8");
-TextTranscoder tt_sjis_to_utf8("UTF-8", "SHIFT_JIS");
+TextTranscoder tt_standard_sjis_to_utf8("UTF-8", "SHIFT_JIS");
-TextTranscoder tt_utf8_to_sjis("SHIFT_JIS", "UTF-8");
+TextTranscoder tt_utf8_to_standard_sjis("SHIFT_JIS", "UTF-8");
+TextTranscoderCustomSJISToUTF8 tt_sega_sjis_to_utf8;
+TextTranscoderUTF8ToCustomSJIS tt_utf8_to_sega_sjis;
 TextTranscoder tt_utf16_to_utf8("UTF-8", "UTF-16LE");
 TextTranscoder tt_utf8_to_utf16("UTF-16LE", "UTF-8");
 TextTranscoder tt_ascii_to_utf8("UTF-8", "ASCII");
@@ -136,11 +179,11 @@ string tt_encode_marked_optional(const string& utf8, uint8_t default_language, b
       try {
         return tt_utf8_to_8859(utf8);
       } catch (const exception& e) {
-        return "\tJ" + tt_utf8_to_sjis(utf8);
+        return "\tJ" + tt_utf8_to_sega_sjis(utf8);
       }
     } else {
       try {
-        return tt_utf8_to_sjis(utf8);
+        return tt_utf8_to_sega_sjis(utf8);
       } catch (const exception& e) {
         return "\tE" + tt_utf8_to_8859(utf8);
       }
@@ -159,11 +202,11 @@ string tt_encode_marked(const string& utf8, uint8_t default_language, bool is_ut
       try {
         return "\tE" + tt_utf8_to_8859(utf8);
       } catch (const exception& e) {
-        return "\tJ" + tt_utf8_to_sjis(utf8);
+        return "\tJ" + tt_utf8_to_sega_sjis(utf8);
       }
     } else {
       try {
-        return "\tJ" + tt_utf8_to_sjis(utf8);
+        return "\tJ" + tt_utf8_to_sega_sjis(utf8);
       } catch (const exception& e) {
         return "\tE" + tt_utf8_to_8859(utf8);
       }
@@ -181,12 +224,12 @@ string tt_decode_marked(const string& data, uint8_t default_language, bool is_ut
   } else {
     if (data.size() >= 2 && data[0] == '\t') {
       if (data[1] == 'J') {
-        return tt_sjis_to_utf8(data.substr(2));
+        return tt_sega_sjis_to_utf8(data.substr(2));
       } else if (data[1] == 'E') {
         return tt_8859_to_utf8(data.substr(2));
       }
     }
-    return default_language ? tt_8859_to_utf8(data) : tt_sjis_to_utf8(data);
+    return default_language ? tt_8859_to_utf8(data) : tt_sega_sjis_to_utf8(data);
   }
 }
 

src/Text.hh

@@ -17,10 +17,10 @@ class TextTranscoder {
 public:
   TextTranscoder(const char* to, const char* from);
   TextTranscoder(const TextTranscoder&) = delete;
-  TextTranscoder(TextTranscoder&&);
+  TextTranscoder(TextTranscoder&&) = delete;
   TextTranscoder& operator=(const TextTranscoder&) = delete;
-  TextTranscoder& operator=(TextTranscoder&&);
+  TextTranscoder& operator=(TextTranscoder&&) = delete;
-  ~TextTranscoder();
+  virtual ~TextTranscoder();
 
   struct Result {
     size_t bytes_read;
@@ -31,16 +31,38 @@ public:
   std::string operator()(const void* src, size_t src_bytes);
   std::string operator()(const std::string& data);
 
-private:
+protected:
+  virtual std::string on_untranslatable(const void** src, size_t* size) const;
+
   static const iconv_t INVALID_IC;
   static const size_t FAILURE_RESULT;
   iconv_t ic;
 };
 
+class TextTranscoderCustomSJISToUTF8 : public TextTranscoder {
+public:
+  TextTranscoderCustomSJISToUTF8();
+  virtual ~TextTranscoderCustomSJISToUTF8() = default;
+
+protected:
+  virtual std::string on_untranslatable(const void** src, size_t* size) const;
+};
+
+class TextTranscoderUTF8ToCustomSJIS : public TextTranscoder {
+public:
+  TextTranscoderUTF8ToCustomSJIS();
+  virtual ~TextTranscoderUTF8ToCustomSJIS() = default;
+
+protected:
+  virtual std::string on_untranslatable(const void** src, size_t* size) const;
+};
+
 extern TextTranscoder tt_8859_to_utf8;
 extern TextTranscoder tt_utf8_to_8859;
-extern TextTranscoder tt_sjis_to_utf8;
+extern TextTranscoder tt_standard_sjis_to_utf8;
-extern TextTranscoder tt_utf8_to_sjis;
+extern TextTranscoder tt_utf8_to_standard_sjis;
+extern TextTranscoderCustomSJISToUTF8 tt_sega_sjis_to_utf8;
+extern TextTranscoderUTF8ToCustomSJIS tt_utf8_to_sega_sjis;
 extern TextTranscoder tt_utf16_to_utf8;
 extern TextTranscoder tt_utf8_to_utf16;
 extern TextTranscoder tt_ascii_to_utf8;
@@ -434,7 +456,7 @@ struct pstring {
           break;
         }
         case TextEncoding::SJIS: {
-          auto ret = tt_utf8_to_sjis(this->data, Bytes, s.data(), s.size(), true);
+          auto ret = tt_utf8_to_sega_sjis(this->data, Bytes, s.data(), s.size(), true);
           this->clear_after_bytes(ret.bytes_written);
           break;
         }
@@ -469,7 +491,7 @@ struct pstring {
         case TextEncoding::MARKED: {
           if (client_language == 0) {
             try {
-              auto ret = tt_utf8_to_sjis(this->data, Bytes, s.data(), s.size(), true);
+              auto ret = tt_utf8_to_sega_sjis(this->data, Bytes, s.data(), s.size(), true);
               this->clear_after_bytes(ret.bytes_written);
             } catch (const std::runtime_error&) {
               this->data[0] = '\t';
@@ -484,7 +506,7 @@ struct pstring {
             } catch (const std::runtime_error&) {
               this->data[0] = '\t';
               this->data[1] = 'J';
-              auto ret = tt_utf8_to_sjis(this->data + 2, Bytes - 2, s.data(), s.size(), true);
+              auto ret = tt_utf8_to_sega_sjis(this->data + 2, Bytes - 2, s.data(), s.size(), true);
               this->clear_after_bytes(ret.bytes_written + 2);
             }
           }
@@ -536,7 +558,7 @@ struct pstring {
         case TextEncoding::ISO8859:
           return tt_8859_to_utf8(this->data, this->used_chars_8());
         case TextEncoding::SJIS:
-          return tt_sjis_to_utf8(this->data, this->used_chars_8());
+          return tt_sega_sjis_to_utf8(this->data, this->used_chars_8());
         case TextEncoding::UTF16:
           return tt_utf16_to_utf8(this->data, this->used_chars_16() * 2);
         case TextEncoding::UTF16_ALWAYS_MARKED: {
@@ -563,7 +585,7 @@ struct pstring {
           }
           return client_language
               ? tt_8859_to_utf8(&this->data[offset], this->used_chars_8() - offset)
-              : tt_sjis_to_utf8(&this->data[offset], this->used_chars_8() - offset);
+              : tt_sega_sjis_to_utf8(&this->data[offset], this->used_chars_8() - offset);
         }
         default:
           throw std::logic_error("unknown text encoding");

src/TextIndex.cc

@@ -187,7 +187,7 @@ BinaryTextSet::BinaryTextSet(const std::string& pr2_data, size_t collection_coun
   }
   used_offsets.emplace(root_offset);
 
-  auto& tt = is_sjis ? tt_sjis_to_utf8 : tt_8859_to_utf8;
+  auto& tt = is_sjis ? tt_sega_sjis_to_utf8 : tt_8859_to_utf8;
 
   collection_offsets_r.go(0);
   while (!collection_offsets_r.eof()) {
@@ -286,7 +286,7 @@ void BinaryTextAndKeyboardsSet::parse_t(const string& pr2_data, bool is_sjis) {
   using U32T = std::conditional_t<IsBigEndian, be_uint32_t, le_uint32_t>;
   using U16T = std::conditional_t<IsBigEndian, be_uint16_t, le_uint16_t>;
 
-  auto& tt = is_sjis ? tt_sjis_to_utf8 : tt_8859_to_utf8;
+  auto& tt = is_sjis ? tt_sega_sjis_to_utf8 : tt_8859_to_utf8;
 
   // The structure is as follows:
   // Footer:
@@ -355,7 +355,7 @@ pair<string, string> BinaryTextAndKeyboardsSet::serialize_t(bool is_sjis) const
   using U32T = std::conditional_t<IsBigEndian, be_uint32_t, le_uint32_t>;
   using U16T = std::conditional_t<IsBigEndian, be_uint16_t, le_uint16_t>;
 
-  auto& tt = is_sjis ? tt_utf8_to_sjis : tt_utf8_to_8859;
+  auto& tt = is_sjis ? tt_utf8_to_sega_sjis : tt_utf8_to_8859;
 
   StringWriter w;
   ::set<size_t> relocation_offsets;

src/WordSelectTable.cc

@@ -76,7 +76,7 @@ void WordSelectSet::parse_non_windows_t(const std::string& data, bool use_sjis)
     auto string_offset_r = r.sub(root.strings_table, sizeof(U32T) * StringTableCount);
     while (!string_offset_r.eof()) {
       string raw_s = r.pget_cstr(string_offset_r.template get<U32T>());
-      this->strings.emplace_back(use_sjis ? tt_sjis_to_utf8(raw_s) : tt_8859_to_utf8(raw_s));
+      this->strings.emplace_back(use_sjis ? tt_sega_sjis_to_utf8(raw_s) : tt_8859_to_utf8(raw_s));
     }
   }