rewrite ItemNameIndex and index all game text

src/ChatCommands.cc

@@ -295,7 +295,7 @@ static void server_command_quest(shared_ptr<Client> c, const std::string& args)
 
   auto s = c->require_server_state();
   auto l = c->require_lobby();
-  auto q = s->quest_index_for_version(effective_version)->get(stoul(args));
+  auto q = s->quest_index(effective_version)->get(stoul(args));
   if (!q) {
     send_text_message(c, "$C6Quest not found");
   } else {
@@ -1611,7 +1611,7 @@ static void server_command_item(shared_ptr<Client> c, const std::string& args) {
   check_is_game(l, true);
   check_cheats_enabled(l, c);
 
-  ItemData item = s->item_name_index->parse_item_description(c->version(), args);
+  ItemData item = s->parse_item_description(c->version(), args);
   item.id = l->generate_item_id(c->lobby_client_id);
 
   if ((l->drop_mode == Lobby::DropMode::SERVER_PRIVATE) || (l->drop_mode == Lobby::DropMode::SERVER_DUPLICATE)) {
@@ -1644,7 +1644,7 @@ static void proxy_command_item(shared_ptr<ProxyServer::LinkedSession> ses, const
 
   bool set_drop = (!args.empty() && (args[0] == '!'));
 
-  ItemData item = s->item_name_index->parse_item_description(ses->version(), (set_drop ? args.substr(1) : args));
+  ItemData item = s->parse_item_description(ses->version(), (set_drop ? args.substr(1) : args));
   item.id = random_object<uint32_t>() | 0x80000000;
 
   if (set_drop) {

src/Client.cc

@@ -1021,3 +1021,46 @@ void Client::use_character_bank(int8_t index) {
     }
   }
 }
+
+void Client::print_inventory(FILE* stream) const {
+  auto p = this->character();
+  shared_ptr<const ItemNameIndex> name_index;
+  try {
+    name_index = this->require_server_state()->item_name_index(this->version());
+  } catch (const runtime_error&) {
+  }
+  fprintf(stream, "[PlayerInventory] Meseta: %" PRIu32 "\n", p->disp.stats.meseta.load());
+  fprintf(stream, "[PlayerInventory] %hhu items\n", p->inventory.num_items);
+  for (size_t x = 0; x < p->inventory.num_items; x++) {
+    const auto& item = p->inventory.items[x];
+    auto hex = item.data.hex();
+    if (name_index) {
+      auto name = name_index->describe_item(item.data);
+      fprintf(stream, "[PlayerInventory]   %2zu: [+%08" PRIX32 "] %s (%s)\n", x, item.flags.load(), hex.c_str(), name.c_str());
+    } else {
+      fprintf(stream, "[PlayerInventory]   %2zu: [+%08" PRIX32 "] %s\n", x, item.flags.load(), hex.c_str());
+    }
+  }
+}
+
+void Client::print_bank(FILE* stream) const {
+  auto p = this->character();
+  shared_ptr<const ItemNameIndex> name_index;
+  try {
+    name_index = this->require_server_state()->item_name_index(this->version());
+  } catch (const runtime_error&) {
+  }
+  fprintf(stream, "[PlayerBank] Meseta: %" PRIu32 "\n", p->bank.meseta.load());
+  fprintf(stream, "[PlayerBank] %" PRIu32 " items\n", p->bank.num_items.load());
+  for (size_t x = 0; x < p->bank.num_items; x++) {
+    const auto& item = p->bank.items[x];
+    const char* present_token = item.present ? "" : " (missing present flag)";
+    auto hex = item.data.hex();
+    if (name_index) {
+      auto name = name_index->describe_item(item.data);
+      fprintf(stream, "[PlayerBank]   %3zu: %s (%s) (x%hu)%s\n", x, hex.c_str(), name.c_str(), item.amount.load(), present_token);
+    } else {
+      fprintf(stream, "[PlayerBank]   %3zu: %s (x%hu)%s\n", x, hex.c_str(), item.amount.load(), present_token);
+    }
+  }
+}

src/Client.hh

@@ -349,6 +349,9 @@ public:
   void use_character_bank(int8_t bb_character_index);
   void use_default_bank();
 
+  void print_inventory(FILE* stream) const;
+  void print_bank(FILE* stream) const;
+
 private:
   // The overlay character data is used in battle and challenge modes, when
   // character data is temporarily replaced in-game. In other play modes and in

src/Episode3/DataIndexes.cc

@@ -764,7 +764,7 @@ string CardDefinition::Effect::str_for_arg(const string& arg) {
   }
 }
 
-string CardDefinition::Effect::str(const char* separator, const TextArchive* text_archive) const {
+string CardDefinition::Effect::str(const char* separator, const TextSet* text_archive) const {
   vector<string> tokens;
   tokens.emplace_back(string_printf("%hhu:", this->effect_num));
   {
@@ -802,7 +802,7 @@ string CardDefinition::Effect::str(const char* separator, const TextArchive* tex
   const char* name = nullptr;
   if (this->name_index && text_archive) {
     try {
-      name = text_archive->get_string(45, this->name_index).c_str();
+      name = text_archive->get(45, this->name_index).c_str();
     } catch (const exception&) {
     }
   }
@@ -1061,7 +1061,7 @@ static const char* name_for_assist_ai_param_target(uint8_t target) {
   }
 }
 
-string CardDefinition::str(bool single_line, const TextArchive* text_archive) const {
+string CardDefinition::str(bool single_line, const TextSet* text_archive) const {
   string type_str;
   try {
     type_str = name_for_card_type(this->type);

src/Episode3/DataIndexes.hh

@@ -15,7 +15,7 @@
 
 #include "../PlayerSubordinates.hh"
 #include "../Text.hh"
-#include "../TextArchive.hh"
+#include "../TextIndex.hh"
 
 namespace Episode3 {
 
@@ -514,7 +514,7 @@ struct CardDefinition {
 
     bool is_empty() const;
     static std::string str_for_arg(const std::string& arg);
-    std::string str(const char* separator = ", ", const TextArchive* text_archive = nullptr) const;
+    std::string str(const char* separator = ", ", const TextSet* text_archive = nullptr) const;
   } __attribute__((packed));
 
   /* 0000 */ be_uint32_t card_id;
@@ -780,7 +780,7 @@ struct CardDefinition {
   CardClass card_class() const;
 
   void decode_range();
-  std::string str(bool single_line = true, const TextArchive* text_archive = nullptr) const;
+  std::string str(bool single_line = true, const TextSet* text_archive = nullptr) const;
 } __attribute__((packed)); // 0x128 bytes in total
 
 struct CardDefinitionsFooter {

src/ItemNameIndex.cc

@@ -4,37 +4,67 @@
 
 using namespace std;
 
-ItemNameIndex::ItemNameIndex(JSON&& v2_names, JSON&& v3_names, JSON&& v4_names) {
+// class ItemNameIndex {
-  auto get_or_create_meta = [&](uint32_t primary_identifier) {
+// public:
-    shared_ptr<ItemMetadata> meta;
+//   ItemNameIndex(std::shared_ptr<const ItemParameterTable> pmt, const std::vector<std::string>& name_coll);
-    try {
+//   std::string describe_item(const ItemData& item, bool include_color_escapes = false) const;
-      return this->primary_identifier_index.at(primary_identifier);
+//   ItemData parse_item_description(const std::string& description) const;
-    } catch (const out_of_range&) {
+// private:
-      auto meta = make_shared<ItemMetadata>();
+//   ItemData parse_item_description_phase(const std::string& description, bool skip_special) const;
-      meta->primary_identifier = primary_identifier;
+//   std::shared_ptr<const ItemParameterTable> item_parameter_table;
-      this->primary_identifier_index.emplace(primary_identifier, meta);
+//   struct ItemMetadata {
-      return meta;
+//     uint32_t primary_identifier;
+//     std::string name;
+//   };
+//   std::unordered_map<uint32_t, std::shared_ptr<ItemMetadata>> primary_identifier_indexes;
+//   std::map<std::string, std::shared_ptr<ItemMetadata>> name_indexes;
+// };
+
+ItemNameIndex::ItemNameIndex(
+    std::shared_ptr<const ItemParameterTable> item_parameter_table,
+    const std::vector<std::string>& name_coll)
+    : item_parameter_table(item_parameter_table) {
+
+  auto find_items_1d = [&](uint64_t data1, size_t position) -> size_t {
+    ItemData item(data1, 0);
+    for (size_t x = 0; x < 0x100; x++) {
+      item.data1[position] = x;
+      uint32_t id;
+      try {
+        id = this->item_parameter_table->get_item_id(item);
+      } catch (const out_of_range&) {
+        return x;
+      }
+      const string* name = nullptr;
+      try {
+        name = &name_coll.at(id);
+      } catch (const out_of_range&) {
+      }
+
+      if (name) {
+        auto meta = make_shared<ItemMetadata>();
+        meta->primary_identifier = item.primary_identifier();
+        meta->name = *name;
+        this->primary_identifier_index.emplace(meta->primary_identifier, meta);
+        this->name_index.emplace(tolower(meta->name), meta);
+      }
+    }
+    return 0x100;
+  };
+  auto find_items_2d = [&](uint64_t data1) {
+    for (size_t x = 0; x < 0x100; x++) {
+      if (find_items_1d(data1 | (static_cast<uint64_t>(x) << 48), 2) == 0) {
+        break;
+      }
     }
   };
 
-  for (const auto& it : v2_names.as_dict()) {
+  find_items_2d(0x0000000000000000);
-    uint32_t primary_identifier = stoul(it.first, nullptr, 16);
+  find_items_1d(0x0101000000000000, 2);
-    auto meta = get_or_create_meta(primary_identifier);
+  find_items_1d(0x0102000000000000, 2);
-    meta->v2_name = std::move(it.second->as_string());
+  find_items_1d(0x0103000000000000, 2);
-    this->v2_name_index.emplace(tolower(meta->v2_name), meta);
+  find_items_1d(0x0200000000000000, 1);
-  }
+  find_items_2d(0x0300000000000000);
-  for (const auto& it : v3_names.as_dict()) {
-    uint32_t primary_identifier = stoul(it.first, nullptr, 16);
-    auto meta = get_or_create_meta(primary_identifier);
-    meta->v3_name = std::move(it.second->as_string());
-    this->v3_name_index.emplace(tolower(meta->v3_name), meta);
-  }
-  for (const auto& it : v4_names.as_dict()) {
-    uint32_t primary_identifier = stoul(it.first, nullptr, 16);
-    auto meta = get_or_create_meta(primary_identifier);
-    meta->v4_name = std::move(it.second->as_string());
-    this->v4_name_index.emplace(tolower(meta->v4_name), meta);
-  }
 }
 
 static const char* s_rank_name_characters = "\0ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_";
@@ -106,11 +136,10 @@ const array<const char*, 0x11> name_for_s_rank_special = {
 };
 
 std::string ItemNameIndex::describe_item(
-    Version version,
     const ItemData& item,
-    std::shared_ptr<const ItemParameterTable> item_parameter_table) const {
+    bool include_color_escapes) const {
   if (item.data1[0] == 0x04) {
-    return string_printf("%s%" PRIu32 " Meseta", item_parameter_table ? "$C7" : "", item.data2d.load());
+    return string_printf("%s%" PRIu32 " Meseta", include_color_escapes ? "$C7" : "", item.data2d.load());
   }
 
   vector<string> ret_tokens;
@@ -172,18 +201,7 @@ std::string ItemNameIndex::describe_item(
   } else {
     try {
       auto meta = this->primary_identifier_index.at(primary_identifier);
-      const string* name;
+      ret_tokens.emplace_back(meta->name);
-      if (is_v4(version)) {
-        name = &meta->v4_name;
-      } else if (is_v3(version)) {
-        name = &meta->v3_name;
-      } else {
-        name = &meta->v2_name;
-      }
-      if (name->empty()) {
-        throw out_of_range("item does not exist");
-      }
-      ret_tokens.emplace_back(*name);
 
     } catch (const out_of_range&) {
       ret_tokens.emplace_back(string_printf("!ID:%06" PRIX32, primary_identifier));
@@ -347,10 +365,10 @@ std::string ItemNameIndex::describe_item(
   }
 
   string ret = join(ret_tokens, " ");
-  if (item_parameter_table) {
+  if (include_color_escapes) {
     if (item.is_s_rank_weapon()) {
       return "$C4" + ret;
-    } else if (item_parameter_table->is_item_rare(item)) {
+    } else if (this->item_parameter_table->is_item_rare(item)) {
       return "$C6" + ret;
     } else if (item.has_bonuses()) {
       return "$C2" + ret;
@@ -362,23 +380,23 @@ std::string ItemNameIndex::describe_item(
   }
 }
 
-ItemData ItemNameIndex::parse_item_description(Version version, const std::string& desc) const {
+ItemData ItemNameIndex::parse_item_description(const std::string& desc) const {
   ItemData ret;
   try {
-    ret = this->parse_item_description_phase(version, desc, false);
+    ret = this->parse_item_description_phase(desc, false);
   } catch (const exception& e1) {
     try {
-      ret = this->parse_item_description_phase(version, desc, true);
+      ret = this->parse_item_description_phase(desc, true);
     } catch (const exception& e2) {
       try {
         ret = ItemData::from_data(parse_data_string(desc));
       } catch (const exception& ed) {
         if (strcmp(e1.what(), e2.what())) {
-          throw runtime_error(string_printf("cannot parse item description \"%s\" in %s (as text 1: %s) (as text 2: %s) (as data: %s)",
+          throw runtime_error(string_printf("cannot parse item description \"%s\" (as text 1: %s) (as text 2: %s) (as data: %s)",
-              desc.c_str(), name_for_enum(version), e1.what(), e2.what(), ed.what()));
+              desc.c_str(), e1.what(), e2.what(), ed.what()));
         } else {
-          throw runtime_error(string_printf("cannot parse item description \"%s\" in %s (as text: %s) (as data: %s)",
+          throw runtime_error(string_printf("cannot parse item description \"%s\" (as text: %s) (as data: %s)",
-              desc.c_str(), name_for_enum(version), e1.what(), ed.what()));
+              desc.c_str(), e1.what(), ed.what()));
         }
       }
     }
@@ -387,7 +405,7 @@ ItemData ItemNameIndex::parse_item_description(Version version, const std::strin
   return ret;
 }
 
-ItemData ItemNameIndex::parse_item_description_phase(Version version, const std::string& description, bool skip_special) const {
+ItemData ItemNameIndex::parse_item_description_phase(const std::string& description, bool skip_special) const {
   ItemData ret;
   ret.data1d.clear(0);
   ret.id = 0xFFFFFFFF;
@@ -448,24 +466,15 @@ ItemData ItemNameIndex::parse_item_description_phase(Version version, const std:
     }
   }
 
-  const map<string, shared_ptr<ItemMetadata>>* name_index;
+  auto name_it = this->name_index.lower_bound(desc);
-  if (is_v4(version)) {
-    name_index = &this->v4_name_index;
-  } else if (is_v3(version)) {
-    name_index = &this->v3_name_index;
-  } else {
-    name_index = &this->v2_name_index;
-  }
-
-  auto name_it = name_index->lower_bound(desc);
   // Look up to 3 places before the lower bound. We have to do this to catch
   // cases like Sange vs. Sange & Yasha - if the input is like "Sange 0/...",
   // then we'll see Sange & Yasha first, which we should skip.
   size_t lookback = 0;
   while (lookback < 4) {
-    if (name_it != name_index->end() && desc.starts_with(name_it->first)) {
+    if (name_it != this->name_index.end() && desc.starts_with(name_it->first)) {
       break;
-    } else if (name_it == name_index->begin()) {
+    } else if (name_it == this->name_index.begin()) {
       throw runtime_error("no such item");
     } else {
       name_it--;

src/ItemNameIndex.hh

@@ -14,26 +14,32 @@
 
 class ItemNameIndex {
 public:
-  ItemNameIndex(JSON&& v2_names, JSON&& v3_names, JSON&& v4_names);
-
-  std::string describe_item(
-      Version version,
-      const ItemData& item,
-      std::shared_ptr<const ItemParameterTable> item_parameter_table = nullptr) const;
-  ItemData parse_item_description(Version version, const std::string& description) const;
-
-private:
-  ItemData parse_item_description_phase(Version version, const std::string& description, bool skip_special) const;
-
   struct ItemMetadata {
     uint32_t primary_identifier;
-    std::string v2_name;
+    std::string name;
-    std::string v3_name;
-    std::string v4_name;
   };
 
-  std::unordered_map<uint32_t, std::shared_ptr<ItemMetadata>> primary_identifier_index;
+  ItemNameIndex(std::shared_ptr<const ItemParameterTable> pmt, const std::vector<std::string>& name_coll);
-  std::map<std::string, std::shared_ptr<ItemMetadata>> v2_name_index;
+
-  std::map<std::string, std::shared_ptr<ItemMetadata>> v3_name_index;
+  inline size_t entry_count() const {
-  std::map<std::string, std::shared_ptr<ItemMetadata>> v4_name_index;
+    return this->primary_identifier_index.size();
+  }
+
+  inline const std::unordered_map<uint32_t, std::shared_ptr<const ItemMetadata>>& all_by_primary_identifier() const {
+    return this->primary_identifier_index;
+  }
+  inline const std::map<std::string, std::shared_ptr<const ItemMetadata>>& all_by_name() const {
+    return this->name_index;
+  }
+
+  std::string describe_item(const ItemData& item, bool include_color_escapes = false) const;
+  ItemData parse_item_description(const std::string& description) const;
+
+private:
+  ItemData parse_item_description_phase(const std::string& description, bool skip_special) const;
+
+  std::shared_ptr<const ItemParameterTable> item_parameter_table;
+
+  std::unordered_map<uint32_t, std::shared_ptr<const ItemMetadata>> primary_identifier_index;
+  std::map<std::string, std::shared_ptr<const ItemMetadata>> name_index;
 };

src/ItemParameterTable.cc

@@ -52,13 +52,13 @@ ItemParameterTable::ItemParameterTable(shared_ptr<const string> data, Version ve
 
 const ItemParameterTable::WeaponV4& ItemParameterTable::get_weapon(uint8_t data1_1, uint8_t data1_2) const {
   if (data1_1 >= this->num_weapon_classes) {
-    throw runtime_error("weapon ID out of range");
+    throw out_of_range("weapon ID out of range");
   }
 
   if (this->offsets_v4) {
     const auto& co = this->r.pget<ArrayRefLE>(this->offsets_v4->weapon_table + sizeof(ArrayRefLE) * data1_1);
     if (data1_2 >= co.count) {
-      throw runtime_error("weapon ID out of range");
+      throw out_of_range("weapon ID out of range");
     }
     return this->r.pget<WeaponV4>(co.offset + sizeof(WeaponV4) * data1_2);
   }
@@ -67,12 +67,12 @@ const ItemParameterTable::WeaponV4& ItemParameterTable::get_weapon(uint8_t data1
   try {
     return this->parsed_weapons.at(key);
   } catch (const std::out_of_range&) {
-    auto& def_v4 = this->parsed_weapons.emplace(key, WeaponV4{}).first->second;
+    WeaponV4 def_v4;
 
     if (this->offsets_v2) {
       const auto& co = this->r.pget<ArrayRefLE>(this->offsets_v2->weapon_table + sizeof(ArrayRefLE) * data1_1);
       if (data1_2 >= co.count) {
-        throw runtime_error("weapon ID out of range");
+        throw out_of_range("weapon ID out of range");
       }
       const auto& def_v2 = this->r.pget<WeaponV2>(co.offset + sizeof(WeaponV2) * data1_2);
       def_v4.base.id = def_v2.base.id;
@@ -90,7 +90,7 @@ const ItemParameterTable::WeaponV4& ItemParameterTable::get_weapon(uint8_t data1
     } else if (this->offsets_v3) {
       const auto& co = this->r.pget<ArrayRefBE>(this->offsets_v3->weapon_table + sizeof(ArrayRefBE) * data1_1);
       if (data1_2 >= co.count) {
-        throw runtime_error("weapon ID out of range");
+        throw out_of_range("weapon ID out of range");
       }
       const auto& def_v3 = this->r.pget<WeaponV3<true>>(co.offset + sizeof(WeaponV3<true>) * data1_2);
       def_v4.base.id = def_v3.base.id.load();
@@ -126,19 +126,19 @@ const ItemParameterTable::WeaponV4& ItemParameterTable::get_weapon(uint8_t data1
       throw logic_error("table is not v2, v3, or v4");
     }
 
-    return def_v4;
+    return this->parsed_weapons.emplace(key, def_v4).first->second;
   }
 }
 
 const ItemParameterTable::ArmorOrShieldV4& ItemParameterTable::get_armor_or_shield(uint8_t data1_1, uint8_t data1_2) const {
   if ((data1_1 < 1) || (data1_1 > 2)) {
-    throw runtime_error("armor/shield class ID out of range");
+    throw out_of_range("armor/shield class ID out of range");
   }
 
   if (this->offsets_v4) {
     const auto& co = this->r.pget<ArrayRefLE>(this->offsets_v4->armor_table + sizeof(ArrayRefLE) * (data1_1 - 1));
     if (data1_2 >= co.count) {
-      throw runtime_error("armor/shield ID out of range");
+      throw out_of_range("armor/shield ID out of range");
     }
     return this->r.pget<ArmorOrShieldV4>(co.offset + sizeof(ArmorOrShieldV4) * data1_2);
   }
@@ -151,15 +151,12 @@ const ItemParameterTable::ArmorOrShieldV4& ItemParameterTable::get_armor_or_shie
     }
     return ret;
   } catch (const std::out_of_range&) {
-    if (data1_2 >= parsed_vec.size()) {
+    ArmorOrShieldV4 def_v4;
-      parsed_vec.resize(data1_2 + 1);
-    }
-    auto& def_v4 = parsed_vec[data1_2];
 
     if (this->offsets_v2) {
       const auto& co = this->r.pget<ArrayRefLE>(this->offsets_v2->armor_table + sizeof(ArrayRefLE) * (data1_1 - 1));
       if (data1_2 >= co.count) {
-        throw runtime_error("armor/shield ID out of range");
+        throw out_of_range("armor/shield ID out of range");
       }
       const auto& def_v2 = this->r.pget<ArmorOrShieldV2>(co.offset + sizeof(ArmorOrShieldV2) * data1_2);
       def_v4.base.id = def_v2.base.id;
@@ -183,7 +180,7 @@ const ItemParameterTable::ArmorOrShieldV4& ItemParameterTable::get_armor_or_shie
     } else if (this->offsets_v3) {
       const auto& co = this->r.pget<ArrayRefBE>(this->offsets_v3->armor_table + sizeof(ArrayRefBE) * (data1_1 - 1));
       if (data1_2 >= co.count) {
-        throw runtime_error("armor/shield ID out of range");
+        throw out_of_range("armor/shield ID out of range");
       }
       const auto& def_v3 = this->r.pget<ArmorOrShieldV3>(co.offset + sizeof(ArmorOrShieldV3) * data1_2);
       def_v4.base.id = def_v3.base.id.load();
@@ -210,7 +207,11 @@ const ItemParameterTable::ArmorOrShieldV4& ItemParameterTable::get_armor_or_shie
       throw logic_error("table is not v2, v3, or v4");
     }
 
-    return def_v4;
+    if (data1_2 >= parsed_vec.size()) {
+      parsed_vec.resize(data1_2 + 1);
+    }
+    parsed_vec[data1_2] = def_v4;
+    return parsed_vec[data1_2];
   }
 }
 
@@ -218,7 +219,7 @@ const ItemParameterTable::UnitV4& ItemParameterTable::get_unit(uint8_t data1_2)
   if (this->offsets_v4) {
     const auto& co = this->r.pget<ArrayRefLE>(this->offsets_v4->unit_table);
     if (data1_2 >= co.count) {
-      throw runtime_error("unit ID out of range");
+      throw out_of_range("unit ID out of range");
     }
     return this->r.pget<UnitV4>(co.offset + sizeof(UnitV4) * data1_2);
   }
@@ -230,15 +231,12 @@ const ItemParameterTable::UnitV4& ItemParameterTable::get_unit(uint8_t data1_2)
     }
     return ret;
   } catch (const std::out_of_range&) {
-    if (data1_2 >= this->parsed_units.size()) {
+    UnitV4 def_v4;
-      this->parsed_units.resize(data1_2 + 1);
-    }
-    auto& def_v4 = this->parsed_units[data1_2];
 
     if (this->offsets_v2) {
       const auto& co = this->r.pget<ArrayRefLE>(this->offsets_v2->unit_table);
       if (data1_2 >= co.count) {
-        throw runtime_error("unit ID out of range");
+        throw out_of_range("unit ID out of range");
       }
       const auto& def_v2 = this->r.pget<UnitV2>(co.offset + sizeof(UnitV2) * data1_2);
       def_v4.base.id = def_v2.base.id;
@@ -249,7 +247,7 @@ const ItemParameterTable::UnitV4& ItemParameterTable::get_unit(uint8_t data1_2)
     } else if (this->offsets_v3) {
       const auto& co = this->r.pget<ArrayRefBE>(this->offsets_v3->unit_table);
       if (data1_2 >= co.count) {
-        throw runtime_error("unit ID out of range");
+        throw out_of_range("unit ID out of range");
       }
       const auto& def_v3 = this->r.pget<UnitV3>(co.offset + sizeof(UnitV3) * data1_2);
       def_v4.base.id = def_v3.base.id.load();
@@ -263,7 +261,11 @@ const ItemParameterTable::UnitV4& ItemParameterTable::get_unit(uint8_t data1_2)
       throw logic_error("table is not v2, v3, or v4");
     }
 
-    return def_v4;
+    if (data1_2 >= this->parsed_units.size()) {
+      this->parsed_units.resize(data1_2 + 1);
+    }
+    this->parsed_units[data1_2] = def_v4;
+    return this->parsed_units[data1_2];
   }
 }
 
@@ -271,7 +273,7 @@ const ItemParameterTable::MagV4& ItemParameterTable::get_mag(uint8_t data1_1) co
   if (this->offsets_v4) {
     const auto& co = this->r.pget<ArrayRefLE>(this->offsets_v4->mag_table);
     if (data1_1 >= co.count) {
-      throw runtime_error("mag ID out of range");
+      throw out_of_range("mag ID out of range");
     }
     return this->r.pget<MagV4>(co.offset + sizeof(MagV4) * data1_1);
   }
@@ -283,15 +285,12 @@ const ItemParameterTable::MagV4& ItemParameterTable::get_mag(uint8_t data1_1) co
     }
     return ret;
   } catch (const std::out_of_range&) {
-    if (data1_1 >= this->parsed_mags.size()) {
+    MagV4 def_v4;
-      this->parsed_mags.resize(data1_1 + 1);
-    }
-    auto& def_v4 = this->parsed_mags[data1_1];
 
     if (this->offsets_v2) {
       const auto& co = this->r.pget<ArrayRefLE>(this->offsets_v2->mag_table);
       if (data1_1 >= co.count) {
-        throw runtime_error("mag ID out of range");
+        throw out_of_range("mag ID out of range");
       }
       const auto& def_v2 = this->r.pget<MagV2>(co.offset + sizeof(MagV2) * data1_1);
       def_v4.base.id = def_v2.base.id;
@@ -311,7 +310,7 @@ const ItemParameterTable::MagV4& ItemParameterTable::get_mag(uint8_t data1_1) co
     } else if (this->offsets_v3) {
       const auto& co = this->r.pget<ArrayRefBE>(this->offsets_v3->mag_table);
       if (data1_1 >= co.count) {
-        throw runtime_error("mag ID out of range");
+        throw out_of_range("mag ID out of range");
       }
       const auto& def_v3 = this->r.pget<MagV3>(co.offset + sizeof(MagV3) * data1_1);
       def_v4.base.id = def_v3.base.id.load();
@@ -334,19 +333,23 @@ const ItemParameterTable::MagV4& ItemParameterTable::get_mag(uint8_t data1_1) co
       throw logic_error("table is not v2, v3, or v4");
     }
 
-    return def_v4;
+    if (data1_1 >= this->parsed_mags.size()) {
+      this->parsed_mags.resize(data1_1 + 1);
+    }
+    this->parsed_mags[data1_1] = def_v4;
+    return this->parsed_mags[data1_1];
   }
 }
 
 const ItemParameterTable::ToolV4& ItemParameterTable::get_tool(uint8_t data1_1, uint8_t data1_2) const {
   if (data1_1 >= this->num_tool_classes) {
-    throw runtime_error("tool class ID out of range");
+    throw out_of_range("tool class ID out of range");
   }
 
   if (this->offsets_v4) {
     const auto& co = this->r.pget<ArrayRefLE>(this->offsets_v4->tool_table + sizeof(ArrayRefLE) * data1_1);
     if (data1_2 >= co.count) {
-      throw runtime_error("tool ID out of range");
+      throw out_of_range("tool ID out of range");
     }
     return this->r.pget<ToolV4>(co.offset + sizeof(ToolV4) * data1_2);
   }
@@ -355,12 +358,12 @@ const ItemParameterTable::ToolV4& ItemParameterTable::get_tool(uint8_t data1_1,
   try {
     return this->parsed_tools.at(key);
   } catch (const std::out_of_range&) {
-    auto& def_v4 = this->parsed_tools.emplace(key, ToolV4{}).first->second;
+    ToolV4 def_v4;
 
     if (this->offsets_v2) {
       const auto& co = this->r.pget<ArrayRefLE>(this->offsets_v2->tool_table + sizeof(ArrayRefLE) * data1_1);
       if (data1_2 >= co.count) {
-        throw runtime_error("tool ID out of range");
+        throw out_of_range("tool ID out of range");
       }
       const auto& def_v2 = this->r.pget<ToolV2>(co.offset + sizeof(ToolV2) * data1_2);
       def_v4.base.id = def_v2.base.id;
@@ -372,7 +375,7 @@ const ItemParameterTable::ToolV4& ItemParameterTable::get_tool(uint8_t data1_1,
     } else if (this->offsets_v3) {
       const auto& co = this->r.pget<ArrayRefBE>(this->offsets_v3->tool_table + sizeof(ArrayRefBE) * data1_1);
       if (data1_2 >= co.count) {
-        throw runtime_error("tool ID out of range");
+        throw out_of_range("tool ID out of range");
       }
       const auto& def_v3 = this->r.pget<ToolV3>(co.offset + sizeof(ToolV3) * data1_2);
       def_v4.base.id = def_v3.base.id.load();
@@ -387,7 +390,7 @@ const ItemParameterTable::ToolV4& ItemParameterTable::get_tool(uint8_t data1_1,
       throw logic_error("table is not v2, v3, or v4");
     }
 
-    return def_v4;
+    return this->parsed_tools.emplace(key, def_v4).first->second;
   }
 }
 
@@ -472,10 +475,10 @@ float ItemParameterTable::get_sale_divisor(uint8_t data1_0, uint8_t data1_1) con
 const ItemParameterTable::MagFeedResult& ItemParameterTable::get_mag_feed_result(
     uint8_t table_index, uint8_t item_index) const {
   if (table_index >= 8) {
-    throw runtime_error("invalid mag feed table index");
+    throw out_of_range("invalid mag feed table index");
   }
   if (item_index >= 11) {
-    throw runtime_error("invalid mag feed item index");
+    throw out_of_range("invalid mag feed item index");
   }
 
   uint32_t offset;
@@ -521,7 +524,7 @@ uint8_t ItemParameterTable::get_special_stars(uint8_t det) const {
 const ItemParameterTable::Special<false>& ItemParameterTable::get_special(uint8_t special) const {
   special &= 0x3F;
   if (special >= this->num_specials) {
-    throw runtime_error("invalid special index");
+    throw out_of_range("invalid special index");
   }
 
   if (this->offsets_v2) {
@@ -545,10 +548,10 @@ const ItemParameterTable::Special<false>& ItemParameterTable::get_special(uint8_
 
 uint8_t ItemParameterTable::get_max_tech_level(uint8_t char_class, uint8_t tech_num) const {
   if (char_class >= 12) {
-    throw runtime_error("invalid character class");
+    throw out_of_range("invalid character class");
   }
   if (tech_num >= 19) {
-    throw runtime_error("invalid technique number");
+    throw out_of_range("invalid technique number");
   }
 
   if (this->offsets_v2) {
@@ -762,7 +765,7 @@ std::pair<const ItemParameterTable::EventItem*, size_t> ItemParameterTable::get_
     uint32_t base_offset, uint8_t event_number) const {
   const auto& co = this->r.pget<ArrayRef<IsBigEndian>>(base_offset);
   if (event_number >= co.count) {
-    throw runtime_error("invalid event number");
+    throw out_of_range("invalid event number");
   }
   const auto& event_co = this->r.pget<ArrayRef<IsBigEndian>>(co.offset + sizeof(ArrayRef<IsBigEndian>) * event_number);
   const auto* defs = &this->r.pget<EventItem>(event_co.offset, event_co.count * sizeof(EventItem));

src/ItemParameterTable.hh

@@ -369,6 +369,9 @@ private:
     /* 40 / 5704 */ le_uint32_t shield_effect_table; // -> [8-byte structs]
   } __attribute__((packed));
 
+  // TODO: The GC NTE ItemPMT format is intermediate between V2 and V3 - the
+  // Offsets struct is 0x50 bytes. Figure it out and add support here.
+
   template <bool IsBigEndian>
   struct TableOffsetsV3V4 {
     using U32T = typename std::conditional<IsBigEndian, be_uint32_t, le_uint32_t>::type;

src/Items.cc

@@ -23,7 +23,7 @@ void player_use_item(shared_ptr<Client> c, size_t item_index, shared_ptr<PSOLFGE
     // Nothing to do (it should be deleted)
 
   } else if (item_identifier == 0x030200) { // Technique disk
-    auto item_parameter_table = s->item_parameter_table_for_version(c->version());
+    auto item_parameter_table = s->item_parameter_table(c->version());
     uint8_t max_level = item_parameter_table->get_max_tech_level(player->disp.visual.char_class, item.data.data1[4]);
     if (item.data.data1[2] > max_level) {
       throw runtime_error("technique level too high");
@@ -43,7 +43,7 @@ void player_use_item(shared_ptr<Client> c, size_t item_index, shared_ptr<PSOLFGE
     // get an accurate picture of what's actually in the player's inventory, so
     // there's no way to know if we would be enforcing the correct grind limit.
     if (is_v3_or_later) {
-      auto item_parameter_table = s->item_parameter_table_for_version(c->version());
+      auto item_parameter_table = s->item_parameter_table(c->version());
       auto weapon_def = item_parameter_table->get_weapon(weapon.data.data1[1], weapon.data.data1[2]);
       if (weapon.data.data1[3] >= weapon_def.max_grind) {
         throw runtime_error("weapon already at maximum grind");
@@ -168,7 +168,7 @@ void player_use_item(shared_ptr<Client> c, size_t item_index, shared_ptr<PSOLFGE
 
   } else if ((item_identifier & 0xFFFF00) == 0x031500) {
     // Christmas Present, etc. - use unwrap_table + probabilities therein
-    auto item_parameter_table = s->item_parameter_table_for_version(c->version());
+    auto item_parameter_table = s->item_parameter_table(c->version());
     auto table = item_parameter_table->get_event_items(item.data.data1[2]);
     size_t sum = 0;
     for (size_t z = 0; z < table.second; z++) {
@@ -207,7 +207,7 @@ void player_use_item(shared_ptr<Client> c, size_t item_index, shared_ptr<PSOLFGE
         continue;
       }
       try {
-        auto item_parameter_table = s->item_parameter_table_for_version(c->version());
+        auto item_parameter_table = s->item_parameter_table(c->version());
         const auto& combo = item_parameter_table->get_item_combination(item.data, inv_item.data);
         if (combo.char_class != 0xFF && combo.char_class != player->disp.visual.char_class) {
           throw runtime_error("item combination requires specific char_class");
@@ -497,7 +497,7 @@ void player_feed_mag(std::shared_ptr<Client> c, size_t mag_item_index, size_t fe
   apply_mag_feed_result(
       player->inventory.items[mag_item_index].data,
       player->inventory.items[fed_item_index].data,
-      s->item_parameter_table_for_version(c->version()),
+      s->item_parameter_table(c->version()),
       s->mag_evolution_table,
       player->disp.visual.char_class,
       player->disp.visual.section_id,

src/Lobby.cc

@@ -251,7 +251,7 @@ void Lobby::create_item_creator() {
       s->tool_random_set,
       s->weapon_random_sets.at(this->difficulty),
       s->tekker_adjustment_set,
-      s->item_parameter_table_for_version(this->base_version),
+      s->item_parameter_table(this->base_version),
       this->base_version,
       this->episode,
       (this->mode == GameMode::SOLO) ? GameMode::NORMAL : this->mode,
@@ -351,7 +351,7 @@ void Lobby::load_maps() {
 
   this->log.info("Generated objects list (%zu entries):", this->map->objects.size());
   for (size_t z = 0; z < this->map->objects.size(); z++) {
-    string o_str = this->map->objects[z].str(s->item_name_index);
+    string o_str = this->map->objects[z].str();
     this->log.info("(K-%zX) %s", z, o_str.c_str());
   }
   this->log.info("Generated enemies list (%zu entries):", this->map->enemies.size());

src/Main.cc

@@ -43,8 +43,7 @@
 #include "ServerState.hh"
 #include "StaticGameData.hh"
 #include "Text.hh"
-#include "TextArchive.hh"
+#include "TextIndex.hh"
-#include "UnicodeTextSet.hh"
 
 using namespace std;
 
@@ -955,7 +954,7 @@ Action a_decode_gci(
       auto decoded = decode_gci_data(read_input_data(args), num_threads, dec_seed, skip_checksum);
       save_file(input_filename + ".dec", decoded);
     });
-Action a_decode_vmg(
+Action a_decode_vms(
     "decode-vms", nullptr, +[](Arguments& args) {
       string input_filename = args.get<string>(1, false);
       if (input_filename.empty() || (input_filename == "-")) {
@@ -1171,25 +1170,35 @@ Action a_decode_sjis(
     });
 
 Action a_decode_text_archive(
-    "decode-text-archive", nullptr, +[](Arguments& args) {
+    "decode-text-archive", "\
+  decode-text-archive [INPUT-FILENAME [OUTPUT-FILENAME]]\n\
+    Decode a text archive to JSON. --collections=NUM_COLLECTIONS is given,\n\
+    expects a fixed number of collections in the input. If --has-pr3 is given,\n\
+    expects the input not to have a REL footer.\n",
+    +[](Arguments& args) {
       string data = read_input_data(args);
-      TextArchive a(data, args.get<bool>("big-endian"));
+
-      JSON j = a.json();
+      unique_ptr<TextSet> ts;
+      size_t collection_count = args.get<size_t>("collections", 0);
+      if (collection_count) {
+        ts = make_unique<BinaryTextSet>(data, collection_count, !args.get<bool>("has-pr3"));
+      } else {
+        ts = make_unique<BinaryTextAndKeyboardsSet>(data, args.get<bool>("big-endian"));
+      }
+      JSON j = ts->json();
       string out_data = j.serialize(JSON::SerializeOption::FORMAT);
       write_output_data(args, out_data.data(), out_data.size(), "json");
     });
 Action a_encode_text_archive(
     "encode-text-archive", "\
   decode-text-archive [INPUT-FILENAME [OUTPUT-FILENAME]]\n\
-  encode-text-archive [INPUT-FILENAME [OUTPUT-FILENAME]]\n\
+    Encode a text archive. Currently only supports GC and Xbox format.\n",
-    Decode a text archive (e.g. TextEnglish.pr2) to JSON for easy editing, or\n\
-    encode a JSON file to a text archive.\n",
     +[](Arguments& args) {
       const string& input_filename = args.get<string>(1, false);
       const string& output_filename = args.get<string>(2, false);
 
       auto json = JSON::parse(read_input_data(args));
-      TextArchive a(json);
+      BinaryTextAndKeyboardsSet a(json);
       auto result = a.serialize(args.get<bool>("big-endian"));
       if (output_filename.empty()) {
         if (input_filename.empty() || (input_filename == "-")) {
@@ -1214,14 +1223,8 @@ Action a_encode_text_archive(
 
 Action a_decode_unicode_text_set(
     "decode-unicode-text-set", nullptr, +[](Arguments& args) {
-      auto collections = parse_unicode_text_set(read_input_data(args));
+      UnicodeTextSet uts(read_input_data(args));
-      JSON j = JSON::list();
+      JSON j = uts.json();
-      for (const auto& collection : collections) {
-        JSON& coll_j = j.emplace_back(JSON::list());
-        for (const auto& s : collection) {
-          coll_j.emplace_back(s);
-        }
-      }
       string out_data = j.serialize(JSON::SerializeOption::FORMAT);
       write_output_data(args, out_data.data(), out_data.size(), "json");
     });
@@ -1232,15 +1235,8 @@ Action a_encode_unicode_text_set(
     Decode a Unicode text set (e.g. unitxt_e.prs) to JSON for easy editing, or\n\
     encode a JSON file to a Unicode text set.\n",
     +[](Arguments& args) {
-      auto json = JSON::parse(read_input_data(args));
+      UnicodeTextSet uts(JSON::parse(read_input_data(args)));
-      vector<vector<string>> collections;
+      string encoded = uts.serialize();
-      for (const auto& coll_json : json.as_list()) {
-        auto& collection = collections.emplace_back();
-        for (const auto& s_json : coll_json->as_list()) {
-          collection.emplace_back(std::move(s_json->as_string()));
-        }
-      }
-      string encoded = serialize_unicode_text_set(collections);
       write_output_data(args, encoded.data(), encoded.size(), "prs");
     });
 
@@ -1256,30 +1252,22 @@ Action a_decode_word_select_set(
       auto version = get_cli_version(args);
 
       string unitxt_filename = args.get<string>("unitxt");
-      vector<string> unitxt_collection;
+      const vector<string>* unitxt_collection;
       if (!unitxt_filename.empty()) {
-        vector<vector<string>> unitxt_data;
+        unique_ptr<UnicodeTextSet> uts;
         if (ends_with(unitxt_filename, ".prs")) {
-          unitxt_data = parse_unicode_text_set(load_file(unitxt_filename));
+          uts = make_unique<UnicodeTextSet>(load_file(unitxt_filename));
         } else if (ends_with(unitxt_filename, ".json")) {
-          auto json = JSON::parse(load_file(unitxt_filename));
+          uts = make_unique<UnicodeTextSet>(JSON::parse(load_file(unitxt_filename)));
-          for (const auto& coll_it : json.as_list()) {
-            auto& coll = unitxt_data.emplace_back();
-            for (const auto& str_it : coll_it->as_list()) {
-              coll.emplace_back(str_it->as_string());
-            }
-          }
         } else {
           throw runtime_error("unitxt filename must end in .prs or .json");
         }
-        if (version == Version::BB_V4) {
+        unitxt_collection = &uts->get((version == Version::BB_V4) ? 0 : 35);
-          unitxt_collection = std::move(unitxt_data.at(0));
+      } else {
-        } else {
+        unitxt_collection = nullptr;
-          unitxt_collection = std::move(unitxt_data.at(35));
-        }
       }
 
-      WordSelectSet ws(read_input_data(args), version, &unitxt_collection, args.get<bool>("japanese"));
+      WordSelectSet ws(read_input_data(args), version, unitxt_collection, args.get<bool>("japanese"));
       ws.print(stdout);
     });
 Action a_print_word_select_table(
@@ -1289,17 +1277,18 @@ Action a_print_word_select_table(
     given, prints the table sorted by token ID for that version. If no version\n\
     option is given, prints the token table sorted by canonical name.\n",
     +[](Arguments& args) {
-      auto table = ServerState::load_word_select_table_from_system();
+      ServerState s;
-      Version v = Version::UNKNOWN;
+      s.load_objects("word_select_table");
+      Version v;
       try {
         v = get_cli_version(args);
       } catch (const runtime_error&) {
+        v = Version::UNKNOWN;
       }
-
       if (v != Version::UNKNOWN) {
-        table->print_index(stdout, v);
+        s.word_select_table->print_index(stdout, v);
       } else {
-        table->print(stdout);
+        s.word_select_table->print(stdout);
       }
     });
 
@@ -1343,21 +1332,20 @@ Action a_convert_rare_item_set(
       .afs (PSO V2 little-endian AFS archive)\n\
       .rel (Schtserv rare table; cannot be used in output filename)\n",
     +[](Arguments& args) {
-      auto name_index = make_shared<ItemNameIndex>(
+      auto version = get_cli_version(args);
-          JSON::parse(load_file("system/item-tables/names-v2.json")),
+
-          JSON::parse(load_file("system/item-tables/names-v3.json")),
+      ServerState s;
-          JSON::parse(load_file("system/item-tables/names-v4.json")));
+      s.load_objects("item_name_indexes");
 
       string input_filename = args.get<string>(1, false);
       if (input_filename.empty() || (input_filename == "-")) {
         throw runtime_error("input filename must be given");
       }
-      auto version = get_cli_version(args);
 
       auto data = make_shared<string>(read_input_data(args));
       shared_ptr<RareItemSet> rs;
       if (ends_with(input_filename, ".json")) {
-        rs = make_shared<RareItemSet>(JSON::parse(*data), version, name_index);
+        rs = make_shared<RareItemSet>(JSON::parse(*data), s.item_name_index(version));
       } else if (ends_with(input_filename, ".gsl")) {
         rs = make_shared<RareItemSet>(GSLArchive(data, false), false);
       } else if (ends_with(input_filename, ".gslb")) {
@@ -1372,9 +1360,9 @@ Action a_convert_rare_item_set(
 
       string output_filename = args.get<string>(2, false);
       if (output_filename.empty() || (output_filename == "-")) {
-        rs->print_all_collections(stdout, version, name_index);
+        rs->print_all_collections(stdout, s.item_name_index(version));
       } else if (ends_with(output_filename, ".json")) {
-        string data = rs->serialize_json(version, name_index);
+        string data = rs->serialize_json(s.item_name_index(version));
         write_output_data(args, data.data(), data.size(), nullptr);
       } else if (ends_with(output_filename, ".gsl")) {
         string data = rs->serialize_gsl(args.get<bool>("big-endian"));
@@ -1401,20 +1389,13 @@ Action a_describe_item(
       string description = args.get<string>(1);
       auto version = get_cli_version(args);
 
-      auto name_index = make_shared<ItemNameIndex>(
+      ServerState s;
-          JSON::parse(load_file("system/item-tables/names-v2.json")),
+      s.load_objects("item_name_indexes");
-          JSON::parse(load_file("system/item-tables/names-v3.json")),
+      auto name_index = s.item_name_index(version);
-          JSON::parse(load_file("system/item-tables/names-v4.json")));
-      auto pmt_data_v2 = make_shared<string>(prs_decompress(load_file("system/item-tables/ItemPMT-v2.prs")));
-      auto pmt_v2 = make_shared<ItemParameterTable>(pmt_data_v2, ItemParameterTable::Version::V2);
-      auto pmt_data_v3 = make_shared<string>(prs_decompress(load_file("system/item-tables/ItemPMT-gc.prs")));
-      auto pmt_v3 = make_shared<ItemParameterTable>(pmt_data_v3, ItemParameterTable::Version::V3);
-      auto pmt_data_v4 = make_shared<string>(prs_decompress(load_file("system/item-tables/ItemPMT-bb.prs")));
-      auto pmt_v4 = make_shared<ItemParameterTable>(pmt_data_v4, ItemParameterTable::Version::V4);
 
-      ItemData item = name_index->parse_item_description(version, description);
+      ItemData item = name_index->parse_item_description(description);
 
-      string desc = name_index->describe_item(version, item);
+      string desc = name_index->describe_item(item);
       log_info("Data (decoded):    %02hhX%02hhX%02hhX%02hhX %02hhX%02hhX%02hhX%02hhX %02hhX%02hhX%02hhX%02hhX -------- %02hhX%02hhX%02hhX%02hhX",
           item.data1[0], item.data1[1], item.data1[2], item.data1[3],
           item.data1[4], item.data1[5], item.data1[6], item.data1[7],
@@ -1422,7 +1403,7 @@ Action a_describe_item(
           item.data2[0], item.data2[1], item.data2[2], item.data2[3]);
 
       ItemData item_v1 = item;
-      item_v1.encode_for_version(Version::PC_V2, pmt_v2);
+      item_v1.encode_for_version(Version::PC_V2, s.item_parameter_table(Version::PC_V2));
       ItemData item_v1_decoded = item_v1;
       item_v1_decoded.decode_for_version(Version::PC_V2);
 
@@ -1441,7 +1422,7 @@ Action a_describe_item(
       }
 
       ItemData item_gc = item;
-      item_gc.encode_for_version(Version::GC_V3, pmt_v3);
+      item_gc.encode_for_version(Version::GC_V3, s.item_parameter_table(Version::GC_V3));
       ItemData item_gc_decoded = item_gc;
       item_gc_decoded.decode_for_version(Version::GC_V3);
 
@@ -1461,11 +1442,49 @@ Action a_describe_item(
 
       log_info("Description: %s", desc.c_str());
 
-      size_t purchase_price = pmt_v4->price_for_item(item);
+      size_t purchase_price = s.item_parameter_table(Version::BB_V4)->price_for_item(item);
       size_t sale_price = purchase_price >> 3;
       log_info("Purchase price: %zu; sale price: %zu", purchase_price, sale_price);
     });
 
+Action a_name_all_items(
+    "name-all-items", nullptr, +[](Arguments&) {
+      ServerState s;
+      s.load_objects("item_name_indexes");
+
+      set<uint32_t> all_primary_identifiers;
+      for (const auto& index : s.item_name_indexes) {
+        if (index) {
+          for (const auto& it : index->all_by_primary_identifier()) {
+            all_primary_identifiers.emplace(it.first);
+          }
+        }
+      }
+
+      fprintf(stderr, "IDENT :");
+      for (size_t v_s = 0; v_s < NUM_VERSIONS; v_s++) {
+        Version version = static_cast<Version>(v_s);
+        const auto& index = s.item_name_indexes.at(v_s);
+        if (index) {
+          fprintf(stderr, " %30s", name_for_enum(version));
+        }
+      }
+      fputc('\n', stderr);
+
+      for (uint32_t primary_identifier : all_primary_identifiers) {
+        fprintf(stderr, "%06" PRIX32 ":", primary_identifier);
+        for (size_t v_s = 0; v_s < NUM_VERSIONS; v_s++) {
+          const auto& index = s.item_name_indexes.at(v_s);
+          if (index) {
+            ItemData item(static_cast<uint64_t>(primary_identifier) << 40);
+            string name = index->describe_item(item);
+            fprintf(stderr, " %30s", name.c_str());
+          }
+        }
+        fputc('\n', stderr);
+      }
+    });
+
 Action a_show_ep3_cards(
     "show-ep3-cards", "\
   show-ep3-cards\n\

src/Map.cc

@@ -113,17 +113,10 @@ string Map::Enemy::str() const {
       this->source_index, name_for_enum(this->type), this->floor, this->state_flags, this->last_hit_by_client_id);
 }
 
-string Map::Object::str(shared_ptr<const ItemNameIndex> name_index) const {
+string Map::Object::str() const {
   if (this->param1 <= 0.0f) {
-    string item_name;
+    return string_printf("[Map::Object source %zX %04hX @%04hX p1=%g (specialized: %08" PRIX32 " %08" PRIX32 " %08" PRIX32 ") floor=%02hhX item_drop_checked=%s]",
-    try {
+        this->source_index, this->base_type, this->section, this->param1, this->param4, this->param5, this->param6, this->floor, this->item_drop_checked ? "true" : "false");
-      auto item = ItemCreator::base_item_for_specialized_box(this->param4, this->param5, this->param6);
-      item_name = name_index ? name_index->describe_item(Version::BB_V4, item) : item.hex();
-    } catch (const exception& e) {
-      item_name = string_printf("(failed: %s)", e.what());
-    }
-    return string_printf("[Map::Object source %zX %04hX @%04hX p1=%g (specialized: %s) floor=%02hhX item_drop_checked=%s]",
-        this->source_index, this->base_type, this->section, this->param1, item_name.c_str(), this->floor, this->item_drop_checked ? "true" : "false");
   } else {
     return string_printf("[Map::Object source %zX %04hX @%04hX p1=%g (generic) p456=[%08" PRIX32 " %08" PRIX32 " %08" PRIX32 "] floor=%02hhX item_drop_checked=%s]",
         this->source_index, this->base_type, this->section, this->param1, this->param4, this->param5, this->param6,

src/Map.hh

@@ -10,7 +10,6 @@
 #include <vector>
 
 #include "BattleParamsIndex.hh"
-#include "ItemNameIndex.hh"
 #include "PSOEncryption.hh"
 #include "StaticGameData.hh"
 #include "Text.hh"
@@ -217,7 +216,7 @@ struct Map {
     uint8_t floor;
     bool item_drop_checked;
 
-    std::string str(std::shared_ptr<const ItemNameIndex> name_index) const;
+    std::string str() const;
   };
 
   struct Enemy {

src/PlayerSubordinates.cc

@@ -678,7 +678,7 @@ void PlayerInventory::encode_for_client(shared_ptr<Client> c) {
     }
   }
 
-  auto item_parameter_table = c->require_server_state()->item_parameter_table_for_version(c->version());
+  auto item_parameter_table = c->require_server_state()->item_parameter_table(c->version());
   for (size_t z = 0; z < this->items.size(); z++) {
     this->items[z].data.encode_for_version(c->version(), item_parameter_table);
   }

src/RareItemSet.cc

@@ -17,7 +17,7 @@ string RareItemSet::ExpandedDrop::str() const {
       this->probability, frac.first, frac.second, this->item_code[0], this->item_code[1], this->item_code[2]);
 }
 
-string RareItemSet::ExpandedDrop::str(Version version, shared_ptr<const ItemNameIndex> name_index) const {
+string RareItemSet::ExpandedDrop::str(shared_ptr<const ItemNameIndex> name_index) const {
   ItemData item;
   item.data1[0] = this->item_code[0];
   item.data1[1] = this->item_code[1];
@@ -25,7 +25,7 @@ string RareItemSet::ExpandedDrop::str(Version version, shared_ptr<const ItemName
 
   string ret = this->str();
   ret += " (";
-  ret += name_index->describe_item(version, item);
+  ret += name_index->describe_item(item);
   ret += ")";
   return ret;
 }
@@ -303,7 +303,7 @@ RareItemSet::RareItemSet(const string& rel_data, bool is_big_endian) {
   }
 }
 
-RareItemSet::RareItemSet(const JSON& json, Version version, shared_ptr<const ItemNameIndex> name_index) {
+RareItemSet::RareItemSet(const JSON& json, shared_ptr<const ItemNameIndex> name_index) {
   for (const auto& mode_it : json.as_dict()) {
     static const unordered_map<string, GameMode> mode_keys(
         {{"Normal", GameMode::NORMAL}, {"Battle", GameMode::BATTLE}, {"Challenge", GameMode::CHALLENGE}, {"Solo", GameMode::SOLO}});
@@ -369,7 +369,7 @@ RareItemSet::RareItemSet(const JSON& json, Version version, shared_ptr<const Ite
                 if (!name_index) {
                   throw runtime_error("item name index is not available");
                 }
-                ItemData data = name_index->parse_item_description(version, item_desc.as_string());
+                ItemData data = name_index->parse_item_description(item_desc.as_string());
                 d.item_code[0] = data.data1[0];
                 d.item_code[1] = data.data1[1];
                 d.item_code[2] = data.data1[2];
@@ -427,7 +427,7 @@ std::string RareItemSet::serialize_gsl(bool big_endian) const {
   return GSLArchive::generate(files, big_endian);
 }
 
-std::string RareItemSet::serialize_json(Version version, shared_ptr<const ItemNameIndex> name_index) const {
+std::string RareItemSet::serialize_json(shared_ptr<const ItemNameIndex> name_index) const {
   auto modes_dict = JSON::dict();
   static const array<GameMode, 4> modes = {GameMode::NORMAL, GameMode::BATTLE, GameMode::CHALLENGE, GameMode::SOLO};
   for (const auto& mode : modes) {
@@ -458,7 +458,7 @@ std::string RareItemSet::serialize_json(Version version, shared_ptr<const ItemNa
                 data.data1[0] = spec.item_code[0];
                 data.data1[1] = spec.item_code[1];
                 data.data1[2] = spec.item_code[2];
-                spec_json.emplace_back(name_index->describe_item(version, data));
+                spec_json.emplace_back(name_index->describe_item(data));
               }
               for (const auto& enemy_type : enemy_types) {
                 if (enemy_type_valid_for_episode(episode, enemy_type)) {
@@ -485,7 +485,7 @@ std::string RareItemSet::serialize_json(Version version, shared_ptr<const ItemNa
                 data.data1[0] = spec.item_code[0];
                 data.data1[1] = spec.item_code[1];
                 data.data1[2] = spec.item_code[2];
-                area_list.back().emplace_back(name_index->describe_item(version, data));
+                area_list.back().emplace_back(name_index->describe_item(data));
               }
             }
 
@@ -512,7 +512,6 @@ std::string RareItemSet::serialize_json(Version version, shared_ptr<const ItemNa
 
 void RareItemSet::print_collection(
     FILE* stream,
-    Version version,
     GameMode mode,
     Episode episode,
     uint8_t difficulty,
@@ -544,7 +543,7 @@ void RareItemSet::print_collection(
     }
 
     for (const auto& spec : collection->rt_index_to_specs[z]) {
-      string s = name_index ? spec.str(version, name_index) : spec.str();
+      string s = name_index ? spec.str(name_index) : spec.str();
       fprintf(stream, "    %02zX: %s (%s)\n", z, s.c_str(), enemy_types_str.c_str());
     }
   }
@@ -552,14 +551,13 @@ void RareItemSet::print_collection(
   fprintf(stream, "  Box rares:\n");
   for (size_t area = 0; area < collection->box_area_to_specs.size(); area++) {
     for (const auto& spec : collection->box_area_to_specs[area]) {
-      string s = name_index ? spec.str(version, name_index) : spec.str();
+      string s = name_index ? spec.str(name_index) : spec.str();
       fprintf(stream, "    (area %02zX) %s\n", area, s.c_str());
     }
   }
 }
 
-void RareItemSet::print_all_collections(
+void RareItemSet::print_all_collections(FILE* stream, std::shared_ptr<const ItemNameIndex> name_index) const {
-    FILE* stream, Version version, std::shared_ptr<const ItemNameIndex> name_index) const {
   static const array<GameMode, 4> modes = {GameMode::NORMAL, GameMode::BATTLE, GameMode::CHALLENGE, GameMode::SOLO};
   static const array<Episode, 3> episodes = {Episode::EP1, Episode::EP2, Episode::EP4};
   for (GameMode mode : modes) {
@@ -567,7 +565,7 @@ void RareItemSet::print_all_collections(
       for (uint8_t difficulty = 0; difficulty < 4; difficulty++) {
         for (uint8_t section_id = 0; section_id < 10; section_id++) {
           try {
-            this->print_collection(stream, version, mode, episode, difficulty, section_id, name_index);
+            this->print_collection(stream, mode, episode, difficulty, section_id, name_index);
           } catch (const out_of_range& e) {
           }
         }

src/RareItemSet.hh

@@ -22,14 +22,14 @@ public:
     parray<uint8_t, 3> item_code;
 
     std::string str() const;
-    std::string str(Version version, std::shared_ptr<const ItemNameIndex> name_index) const;
+    std::string str(std::shared_ptr<const ItemNameIndex> name_index) const;
   };
 
   RareItemSet();
   RareItemSet(const AFSArchive& afs, bool is_v1);
   RareItemSet(const GSLArchive& gsl, bool is_big_endian);
   RareItemSet(const std::string& rel, bool is_big_endian);
-  RareItemSet(const JSON& json, Version version, std::shared_ptr<const ItemNameIndex> name_index = nullptr);
+  RareItemSet(const JSON& json, std::shared_ptr<const ItemNameIndex> name_index = nullptr);
   ~RareItemSet() = default;
 
   std::vector<ExpandedDrop> get_enemy_specs(GameMode mode, Episode episode, uint8_t difficulty, uint8_t secid, uint8_t rt_index) const;
@@ -37,17 +37,16 @@ public:
 
   std::string serialize_afs(bool is_v1) const;
   std::string serialize_gsl(bool big_endian) const;
-  std::string serialize_json(Version version, std::shared_ptr<const ItemNameIndex> name_index = nullptr) const;
+  std::string serialize_json(std::shared_ptr<const ItemNameIndex> name_index = nullptr) const;
 
   void print_collection(
       FILE* stream,
-      Version version,
       GameMode mode,
       Episode episode,
       uint8_t difficulty,
       uint8_t section_id,
       std::shared_ptr<const ItemNameIndex> name_index = nullptr) const;
-  void print_all_collections(FILE* stream, Version version, std::shared_ptr<const ItemNameIndex> name_index = nullptr) const;
+  void print_all_collections(FILE* stream, std::shared_ptr<const ItemNameIndex> name_index = nullptr) const;
 
 protected:
   struct SpecCollection {

src/ReceiveCommands.cc

@@ -132,7 +132,7 @@ void send_client_to_proxy_server(shared_ptr<Client> c) {
 
 static void send_proxy_destinations_menu(shared_ptr<Client> c) {
   auto s = c->require_server_state();
-  send_menu(c, s->proxy_destinations_menu_for_version(c->version()));
+  send_menu(c, s->proxy_destinations_menu(c->version()));
 }
 
 static bool send_enable_send_function_call_if_applicable(shared_ptr<Client> c) {
@@ -1762,7 +1762,7 @@ static void on_D6_V3(shared_ptr<Client> c, uint16_t, uint32_t, string& data) {
   check_size_v(data.size(), 0);
   if (c->config.check_flag(Client::Flag::IN_INFORMATION_MENU)) {
     auto s = c->require_server_state();
-    send_menu(c, s->information_menu_for_version(c->version()));
+    send_menu(c, s->information_menu(c->version()));
   } else if (c->config.check_flag(Client::Flag::AT_WELCOME_MESSAGE)) {
     c->config.clear_flag(Client::Flag::AT_WELCOME_MESSAGE);
     send_enable_send_function_call_if_applicable(c);
@@ -1784,7 +1784,7 @@ static void on_09(shared_ptr<Client> c, uint16_t, uint32_t, string& data) {
     case MenuID::QUEST_EP1:
     case MenuID::QUEST_EP2: {
       bool is_download_quest = !c->lobby.lock();
-      auto quest_index = s->quest_index_for_version(c->version());
+      auto quest_index = s->quest_index(c->version());
       if (!quest_index) {
         send_quest_info(c, "$C6Quests are not available.", is_download_quest);
       } else {
@@ -2178,7 +2178,7 @@ static void on_10(shared_ptr<Client> c, uint16_t, uint32_t, string& data) {
 
         case MainMenuItemID::INFORMATION: {
           auto s = c->require_server_state();
-          send_menu(c, s->information_menu_for_version(c->version()));
+          send_menu(c, s->information_menu(c->version()));
           c->config.set_flag(Client::Flag::IN_INFORMATION_MENU);
           break;
         }
@@ -2199,7 +2199,7 @@ static void on_10(shared_ptr<Client> c, uint16_t, uint32_t, string& data) {
             // always the download quest menu. (Episode 3 does actually have
             // online quests, but they're served via a server data request
             // instead of the file download paradigm that other versions use.)
-            auto quest_index = s->quest_index_for_version(c->version());
+            auto quest_index = s->quest_index(c->version());
             const auto& categories = quest_index->categories(menu_type, Episode::EP3, c->version());
             if (categories.size() == 1) {
               auto quests = quest_index->filter(menu_type, Episode::EP3, c->version(), categories[0]->category_id);
@@ -2208,7 +2208,7 @@ static void on_10(shared_ptr<Client> c, uint16_t, uint32_t, string& data) {
             }
           }
 
-          send_quest_categories_menu(c, s->quest_index_for_version(c->version()), menu_type, Episode::NONE);
+          send_quest_categories_menu(c, s->quest_index(c->version()), menu_type, Episode::NONE);
           break;
         }
 
@@ -2346,7 +2346,7 @@ static void on_10(shared_ptr<Client> c, uint16_t, uint32_t, string& data) {
         auto s = c->require_server_state();
         const pair<string, uint16_t>* dest = nullptr;
         try {
-          dest = &s->proxy_destinations_for_version(c->version()).at(item_id);
+          dest = &s->proxy_destinations(c->version()).at(item_id);
         } catch (const out_of_range&) {
         }
 
@@ -2453,7 +2453,7 @@ static void on_10(shared_ptr<Client> c, uint16_t, uint32_t, string& data) {
 
     case MenuID::QUEST_CATEGORIES: {
       auto s = c->require_server_state();
-      auto quest_index = s->quest_index_for_version(c->version());
+      auto quest_index = s->quest_index(c->version());
       if (!quest_index) {
         send_lobby_message_box(c, "$C6Quests are not available.");
         break;
@@ -2494,7 +2494,7 @@ static void on_10(shared_ptr<Client> c, uint16_t, uint32_t, string& data) {
     case MenuID::QUEST_EP1:
     case MenuID::QUEST_EP2: {
       auto s = c->require_server_state();
-      auto quest_index = s->quest_index_for_version(c->version());
+      auto quest_index = s->quest_index(c->version());
       if (!quest_index) {
         send_lobby_message_box(c, "$C6Quests are not\navailable.");
         break;
@@ -2707,7 +2707,7 @@ static void on_08_E6(shared_ptr<Client> c, uint16_t command, uint32_t, string& d
 static void on_1F(shared_ptr<Client> c, uint16_t, uint32_t, string& data) {
   check_size_v(data.size(), 0);
   auto s = c->require_server_state();
-  send_menu(c, s->information_menu_for_version(c->version()), true);
+  send_menu(c, s->information_menu(c->version()), true);
 }
 
 static void on_A0(shared_ptr<Client> c, uint16_t, uint32_t, string&) {
@@ -2842,7 +2842,7 @@ static void on_A2(shared_ptr<Client> c, uint16_t, uint32_t flag, string& data) {
           throw logic_error("invalid game mode");
       }
     }
-    send_quest_categories_menu(c, s->quest_index_for_version(c->version()), menu_type, l->episode);
+    send_quest_categories_menu(c, s->quest_index(c->version()), menu_type, l->episode);
   }
 }
 

src/ReceiveSubcommands.cc

@@ -832,7 +832,7 @@ static void on_sync_joining_player_disp_and_inventory(
       throw logic_error("6x70 command from unknown game version");
   }
 
-  parsed->transcode_inventory_items(c->version(), target->version(), s->item_parameter_table_for_version(target->version()));
+  parsed->transcode_inventory_items(c->version(), target->version(), s->item_parameter_table(target->version()));
   parsed->visual.enforce_lobby_join_limits_for_version(target->version());
 
   switch (target->version()) {
@@ -1349,7 +1349,7 @@ static void on_player_drop_item(shared_ptr<Client> c, uint8_t command, uint8_t f
     auto name = s->describe_item(c->version(), item, false);
     l->log.info("Player %hu dropped item %08" PRIX32 " (%s) at %hu:(%g, %g)",
         cmd.header.client_id.load(), cmd.item_id.load(), name.c_str(), cmd.floor.load(), cmd.x.load(), cmd.z.load());
-    p->print_inventory(stderr, c->version(), s->item_name_index);
+    c->print_inventory(stderr);
   }
 
   forward_subcommand(c, command, flag, data, size);
@@ -1374,7 +1374,7 @@ void forward_subcommand_with_item_transcode_t(shared_ptr<Client> c, uint8_t comm
       out_cmd.header.subcommand = translate_subcommand_number(lc->version(), c->version(), out_cmd.header.subcommand);
       if (out_cmd.header.subcommand) {
         out_cmd.item_data.decode_for_version(c->version());
-        out_cmd.item_data.encode_for_version(lc->version(), s->item_parameter_table_for_version(lc->version()));
+        out_cmd.item_data.encode_for_version(lc->version(), s->item_parameter_table(lc->version()));
         send_command_t(lc, command, flag, out_cmd);
       } else {
         lc->log.info("Subcommand cannot be translated to client\'s version");
@@ -1409,7 +1409,7 @@ static void on_create_inventory_item_t(shared_ptr<Client> c, uint8_t command, ui
     auto s = c->require_server_state();
     auto name = s->describe_item(c->version(), item, false);
     l->log.info("Player %hu created inventory item %08" PRIX32 " (%s)", c->lobby_client_id, item.id.load(), name.c_str());
-    p->print_inventory(stderr, c->version(), s->item_name_index);
+    c->print_inventory(stderr);
   }
 
   forward_subcommand_with_item_transcode_t(c, command, flag, cmd);
@@ -1450,7 +1450,7 @@ static void on_drop_partial_stack_t(shared_ptr<Client> c, uint8_t command, uint8
     auto name = s->describe_item(c->version(), item, false);
     l->log.info("Player %hu split stack to create floor item %08" PRIX32 " (%s) at %hu:(%g, %g)",
         cmd.header.client_id.load(), item.id.load(), name.c_str(), cmd.floor.load(), cmd.x.load(), cmd.z.load());
-    c->character()->print_inventory(stderr, c->version(), s->item_name_index);
+    c->print_inventory(stderr);
   }
 
   forward_subcommand_with_item_transcode_t(c, command, flag, cmd);
@@ -1498,7 +1498,7 @@ static void on_drop_partial_stack_bb(shared_ptr<Client> c, uint8_t command, uint
       auto name = s->describe_item(c->version(), item, false);
       l->log.info("Player %hu split stack %08" PRIX32 " (removed: %s) at %hu:(%g, %g)",
           cmd.header.client_id.load(), cmd.item_id.load(), name.c_str(), cmd.floor.load(), cmd.x.load(), cmd.z.load());
-      p->print_inventory(stderr, c->version(), s->item_name_index);
+      c->print_inventory(stderr);
     }
 
   } else {
@@ -1525,14 +1525,14 @@ static void on_buy_shop_item(shared_ptr<Client> c, uint8_t command, uint8_t flag
   l->on_item_id_generated_externally(item.id);
   p->add_item(item);
 
-  size_t price = s->item_parameter_table_for_version(c->version())->price_for_item(item);
+  size_t price = s->item_parameter_table(c->version())->price_for_item(item);
   p->remove_meseta(price, c->version() != Version::BB_V4);
 
   if (l->log.should_log(LogLevel::INFO)) {
     auto name = s->describe_item(c->version(), item, false);
     l->log.info("Player %hu bought item %08" PRIX32 " (%s) from shop (%zu Meseta)",
         cmd.header.client_id.load(), item.id.load(), name.c_str(), price);
-    p->print_inventory(stderr, c->version(), s->item_name_index);
+    c->print_inventory(stderr);
   }
 
   forward_subcommand_with_item_transcode_t(c, command, flag, cmd);
@@ -1575,7 +1575,7 @@ static void on_box_or_enemy_item_drop_t(shared_ptr<Client> c, uint8_t command, u
       out_cmd.header.subcommand = translate_subcommand_number(lc->version(), c->version(), out_cmd.header.subcommand);
       if (out_cmd.header.subcommand) {
         out_cmd.item.item.decode_for_version(c->version());
-        out_cmd.item.item.encode_for_version(lc->version(), s->item_parameter_table_for_version(lc->version()));
+        out_cmd.item.item.encode_for_version(lc->version(), s->item_parameter_table(lc->version()));
         send_command_t(lc, command, flag, out_cmd);
       } else {
         lc->log.info("Subcommand cannot be translated to client\'s version");
@@ -1639,7 +1639,7 @@ static void on_pick_up_item_generic(
       auto s = c->require_server_state();
       auto name = s->describe_item(c->version(), fi->data, false);
       l->log.info("Player %hu picked up %08" PRIX32 " (%s)", client_id, item_id, name.c_str());
-      p->print_inventory(stderr, c->version(), s->item_name_index);
+      c->print_inventory(stderr);
     }
 
     auto s = c->require_server_state();
@@ -1725,7 +1725,7 @@ static void on_use_item(
   if (l->log.should_log(LogLevel::INFO)) {
     l->log.info("Player %hhu used item %hu:%08" PRIX32 " (%s)",
         c->lobby_client_id, cmd.header.client_id.load(), cmd.item_id.load(), name.c_str());
-    p->print_inventory(stderr, c->version(), s->item_name_index);
+    c->print_inventory(stderr);
   }
 
   forward_subcommand(c, command, flag, data, size);
@@ -1771,7 +1771,7 @@ static void on_feed_mag(
     l->log.info("Player %hhu fed item %hu:%08" PRIX32 " (%s) to mag %hu:%08" PRIX32 " (%s)",
         c->lobby_client_id, cmd.header.client_id.load(), cmd.fed_item_id.load(), fed_name.c_str(),
         cmd.header.client_id.load(), cmd.mag_item_id.load(), mag_name.c_str());
-    p->print_inventory(stderr, c->version(), s->item_name_index);
+    c->print_inventory(stderr);
   }
 
   forward_subcommand(c, command, flag, data, size);
@@ -1806,7 +1806,7 @@ static void on_open_shop_bb_or_ep3_battle_subs(shared_ptr<Client> c, uint8_t com
     }
     for (auto& item : c->bb_shop_contents[cmd.shop_type]) {
       item.id = 0xFFFFFFFF;
-      item.data2d = s->item_parameter_table_for_version(c->version())->price_for_item(item);
+      item.data2d = s->item_parameter_table(c->version())->price_for_item(item);
     }
 
     send_shop(c, cmd.shop_type);
@@ -1863,10 +1863,10 @@ static void on_ep3_private_word_select_bb_bank_action(shared_ptr<Client> c, uint
         send_destroy_item_to_lobby(c, cmd.item_id, cmd.item_amount, true);
 
         if (l->log.should_log(LogLevel::INFO)) {
-          string name = s->item_name_index->describe_item(Version::BB_V4, item);
+          string name = s->describe_item(Version::BB_V4, item, false);
           l->log.info("Player %hu deposited item %08" PRIX32 " (x%hhu) (%s) in the bank",
               c->lobby_client_id, cmd.item_id.load(), cmd.item_amount, name.c_str());
-          c->character()->print_inventory(stderr, c->version(), s->item_name_index);
+          c->print_inventory(stderr);
         }
       }
 
@@ -1892,10 +1892,10 @@ static void on_ep3_private_word_select_bb_bank_action(shared_ptr<Client> c, uint
         send_create_inventory_item_to_lobby(c, c->lobby_client_id, item);
 
         if (l->log.should_log(LogLevel::INFO)) {
-          string name = s->item_name_index->describe_item(Version::BB_V4, item);
+          string name = s->describe_item(Version::BB_V4, item, false);
           l->log.info("Player %hu withdrew item %08" PRIX32 " (x%hhu) (%s) from the bank",
               c->lobby_client_id, item.id.load(), cmd.item_amount, name.c_str());
-          c->character()->print_inventory(stderr, c->version(), s->item_name_index);
+          c->print_inventory(stderr);
         }
       }
 
@@ -2133,7 +2133,7 @@ static void on_entity_drop_item_request(shared_ptr<Client> c, uint8_t command, u
       if (item.empty()) {
         l->log.info("No item was created");
       } else {
-        string name = s->item_name_index->describe_item(l->base_version, item);
+        string name = s->describe_item(l->base_version, item, false);
         l->log.info("Entity %04hX (area %02hX) created item %s", cmd.entity_id.load(), cmd.effective_area, name.c_str());
         if (l->drop_mode == Lobby::DropMode::SERVER_DUPLICATE) {
           for (const auto& lc : l->clients) {
@@ -2163,7 +2163,7 @@ static void on_entity_drop_item_request(shared_ptr<Client> c, uint8_t command, u
           if (item.empty()) {
             l->log.info("No item was created for %s", lc->channel.name.c_str());
           } else {
-            string name = s->item_name_index->describe_item(l->base_version, item);
+            string name = s->describe_item(l->base_version, item, false);
             l->log.info("Entity %04hX (area %02hX) created item %s", cmd.entity_id.load(), cmd.effective_area, name.c_str());
             item.id = l->generate_item_id(0xFF);
             l->log.info("Creating item %08" PRIX32 " at %02hhX:%g,%g for %s",
@@ -2455,7 +2455,7 @@ static void on_steal_exp_bb(shared_ptr<Client> c, uint8_t, uint8_t, void* data,
   const auto& inventory = p->inventory;
   const auto& weapon = inventory.items[inventory.find_equipped_item(EquipSlot::WEAPON)];
 
-  auto item_parameter_table = s->item_parameter_table_for_version(c->version());
+  auto item_parameter_table = s->item_parameter_table(c->version());
 
   uint8_t special_id = 0;
   if (((weapon.data.data1[1] < 0x0A) && (weapon.data.data1[2] < 0x05)) ||
@@ -2574,7 +2574,7 @@ static void on_enemy_exp_request_bb(shared_ptr<Client> c, uint8_t, uint8_t, void
     for (size_t z = 0; z < inventory.num_items; z++) {
       auto& item = inventory.items[z];
       if ((item.flags & 0x08) &&
-          s->item_parameter_table_for_version(c->version())->is_unsealable_item(item.data)) {
+          s->item_parameter_table(c->version())->is_unsealable_item(item.data)) {
         item.data.set_sealed_item_kill_count(item.data.get_sealed_item_kill_count() + 1);
       }
     }
@@ -2644,7 +2644,7 @@ void on_transfer_item_via_mail_message_bb(shared_ptr<Client> c, uint8_t command,
     auto name = s->describe_item(c->version(), item, false);
     l->log.info("Player %hhu sent inventory item %hu:%08" PRIX32 " (%s) x%" PRIu32 " to player %08" PRIX32,
         c->lobby_client_id, cmd.header.client_id.load(), cmd.item_id.load(), name.c_str(), cmd.amount.load(), cmd.target_guild_card_number.load());
-    p->print_inventory(stderr, c->version(), s->item_name_index);
+    c->print_inventory(stderr);
   }
 
   // To receive an item, the player must be online, using BB, have a character
@@ -2704,7 +2704,7 @@ void on_exchange_item_for_team_points_bb(shared_ptr<Client> c, uint8_t command,
     auto name = s->describe_item(c->version(), item, false);
     l->log.info("Player %hhu exchanged inventory item %hu:%08" PRIX32 " (%s) for %zu team points",
         c->lobby_client_id, cmd.header.client_id.load(), cmd.item_id.load(), name.c_str(), points);
-    p->print_inventory(stderr, c->version(), s->item_name_index);
+    c->print_inventory(stderr);
   }
 
   forward_subcommand(c, command, flag, data, size);
@@ -2729,7 +2729,7 @@ static void on_destroy_inventory_item(shared_ptr<Client> c, uint8_t command, uin
     auto name = s->describe_item(c->version(), item, false);
     l->log.info("Player %hhu destroyed inventory item %hu:%08" PRIX32 " (%s)",
         c->lobby_client_id, cmd.header.client_id.load(), cmd.item_id.load(), name.c_str());
-    p->print_inventory(stderr, c->version(), s->item_name_index);
+    c->print_inventory(stderr);
   }
   forward_subcommand(c, command, flag, data, size);
 }
@@ -2833,14 +2833,14 @@ static void on_sell_item_at_shop_bb(shared_ptr<Client> c, uint8_t command, uint8
     auto s = c->require_server_state();
     auto p = c->character();
     auto item = p->remove_item(cmd.item_id, cmd.amount, c->version() != Version::BB_V4);
-    size_t price = (s->item_parameter_table_for_version(c->version())->price_for_item(item) >> 3) * cmd.amount;
+    size_t price = (s->item_parameter_table(c->version())->price_for_item(item) >> 3) * cmd.amount;
     p->add_meseta(price);
 
     if (l->log.should_log(LogLevel::INFO)) {
       auto name = s->describe_item(c->version(), item, false);
       l->log.info("Player %hhu sold inventory item %08" PRIX32 " (%s) for %zu Meseta",
           c->lobby_client_id, cmd.item_id.load(), name.c_str(), price);
-      p->print_inventory(stderr, c->version(), s->item_name_index);
+      c->print_inventory(stderr);
     }
 
     forward_subcommand(c, command, flag, data, size);
@@ -2875,7 +2875,7 @@ static void on_buy_shop_item_bb(shared_ptr<Client> c, uint8_t, uint8_t, void* da
       auto name = s->describe_item(c->version(), item, false);
       l->log.info("Player %hhu purchased item %08" PRIX32 " (%s) for %zu meseta",
           c->lobby_client_id, item.id.load(), name.c_str(), price);
-      p->print_inventory(stderr, c->version(), s->item_name_index);
+      c->print_inventory(stderr);
     }
   }
 }
@@ -3246,7 +3246,7 @@ static void on_quest_F960_result_bb(shared_ptr<Client> c, uint8_t, uint8_t, void
 
     item.id = l->generate_item_id(c->lobby_client_id);
     // If it's a weapon, make it unidentified
-    auto item_parameter_table = s->item_parameter_table_for_version(c->version());
+    auto item_parameter_table = s->item_parameter_table(c->version());
     if ((item.data1[0] == 0x00) && (item_parameter_table->is_item_rare(item) || (item.data1[4] != 0))) {
       item.data1[4] |= 0x80;
     }
@@ -3260,12 +3260,12 @@ static void on_quest_F960_result_bb(shared_ptr<Client> c, uint8_t, uint8_t, void
       p->add_item(item);
       send_create_inventory_item_to_lobby(c, c->lobby_client_id, item);
       if (c->log.should_log(LogLevel::INFO)) {
-        string name = s->item_name_index->describe_item(c->version(), item);
+        string name = s->describe_item(c->version(), item, false);
         c->log.info("Awarded item %s", name.c_str());
       }
     } catch (const out_of_range&) {
       if (c->log.should_log(LogLevel::INFO)) {
-        string name = s->item_name_index->describe_item(c->version(), item);
+        string name = s->describe_item(c->version(), item, false);
         c->log.info("Attempted to award item %s, but inventory was full", name.c_str());
       }
     }

src/SaveFileFormats.cc

@@ -578,29 +578,6 @@ void PSOBBCharacterFile::clear_all_material_usage() {
   }
 }
 
-void PSOBBCharacterFile::print_inventory(FILE* stream, Version version, shared_ptr<const ItemNameIndex> name_index) const {
-  fprintf(stream, "[PlayerInventory] Meseta: %" PRIu32 "\n", this->disp.stats.meseta.load());
-  fprintf(stream, "[PlayerInventory] %hhu items\n", this->inventory.num_items);
-  for (size_t x = 0; x < this->inventory.num_items; x++) {
-    const auto& item = this->inventory.items[x];
-    auto name = name_index->describe_item(version, item.data);
-    auto hex = item.data.hex();
-    fprintf(stream, "[PlayerInventory]   %2zu: [+%08" PRIX32 "] %s (%s)\n", x, item.flags.load(), hex.c_str(), name.c_str());
-  }
-}
-
-void PSOBBCharacterFile::print_bank(FILE* stream, Version version, shared_ptr<const ItemNameIndex> name_index) const {
-  fprintf(stream, "[PlayerBank] Meseta: %" PRIu32 "\n", this->bank.meseta.load());
-  fprintf(stream, "[PlayerBank] %" PRIu32 " items\n", this->bank.num_items.load());
-  for (size_t x = 0; x < this->bank.num_items; x++) {
-    const auto& item = this->bank.items[x];
-    const char* present_token = item.present ? "" : " (missing present flag)";
-    auto name = name_index->describe_item(version, item.data);
-    auto hex = item.data.hex();
-    fprintf(stream, "[PlayerBank]   %3zu: %s (%s) (x%hu) %s\n", x, hex.c_str(), name.c_str(), item.amount.load(), present_token);
-  }
-}
-
 const array<PSOBBCharacterFile::DefaultSymbolChatEntry, 6> PSOBBCharacterFile::DEFAULT_SYMBOL_CHATS = {
     DefaultSymbolChatEntry{"\tEHello", 0x28, {0xFFFF, 0x000D, 0xFFFF, 0xFFFF}, {SymbolChat::FacePart{0x05, 0x18, 0x1D, 0x00}, {0x05, 0x28, 0x1D, 0x01}, {0x36, 0x20, 0x2A, 0x00}, {0x3C, 0x00, 0x32, 0x00}, {0xFF, 0x00, 0x00, 0x00}, {0xFF, 0x00, 0x00, 0x00}, {0xFF, 0x00, 0x00, 0x00}, {0xFF, 0x00, 0x00, 0x02}, {0xFF, 0x00, 0x00, 0x02}, {0xFF, 0x00, 0x00, 0x02}, {0xFF, 0x00, 0x00, 0x02}, {0xFF, 0x00, 0x00, 0x02}}},
     DefaultSymbolChatEntry{"\tEGood-bye", 0x74, {0x0476, 0x000C, 0xFFFF, 0xFFFF}, {SymbolChat::FacePart{0x06, 0x15, 0x14, 0x00}, {0x06, 0x2B, 0x14, 0x01}, {0x05, 0x18, 0x1F, 0x00}, {0x05, 0x28, 0x1F, 0x01}, {0x36, 0x20, 0x2A, 0x00}, {0x3C, 0x00, 0x32, 0x00}, {0xFF, 0x00, 0x00, 0x00}, {0xFF, 0x00, 0x00, 0x02}, {0xFF, 0x00, 0x00, 0x02}, {0xFF, 0x00, 0x00, 0x02}, {0xFF, 0x00, 0x00, 0x02}, {0xFF, 0x00, 0x00, 0x02}}},

src/SaveFileFormats.hh

@@ -252,9 +252,6 @@ struct PSOBBCharacterFile {
   uint8_t get_material_usage(MaterialType which) const;
   void set_material_usage(MaterialType which, uint8_t usage);
   void clear_all_material_usage();
-
-  void print_inventory(FILE* stream, Version version, std::shared_ptr<const ItemNameIndex> name_index) const;
-  void print_bank(FILE* stream, Version version, std::shared_ptr<const ItemNameIndex> name_index) const;
 } __attribute__((packed));
 
 struct PSOBBGuildCardFile {

src/SendCommands.cc

@@ -2239,7 +2239,7 @@ void send_execute_item_trade(shared_ptr<Client> c, const vector<ItemData>& items
   cmd.item_count = items.size();
   for (size_t x = 0; x < items.size(); x++) {
     cmd.item_datas[x] = items[x];
-    cmd.item_datas[x].encode_for_version(c->version(), s->item_parameter_table_for_version(c->version()));
+    cmd.item_datas[x].encode_for_version(c->version(), s->item_parameter_table(c->version()));
   }
   send_command_t(c, 0xD3, 0x00, cmd);
 }
@@ -2431,7 +2431,7 @@ void send_game_item_state(shared_ptr<Client> c) {
       fi.unknown_a2 = 0;
       fi.drop_number = (floor == 0) ? 0xFFFF : (decompressed_header.next_drop_number_per_floor.at(floor - 1)++);
       fi.item = item->data;
-      fi.item.encode_for_version(c->version(), s->item_parameter_table_for_version(c->version()));
+      fi.item.encode_for_version(c->version(), s->item_parameter_table(c->version()));
       floor_items_w.put(fi);
 
       decompressed_header.floor_item_count_per_floor.at(floor)++;
@@ -2509,7 +2509,7 @@ void send_drop_item_to_channel(shared_ptr<ServerState> s, Channel& ch, const Ite
   uint8_t subcommand = get_pre_v1_subcommand(ch.version, 0x51, 0x58, 0x5F);
   G_DropItem_PC_V3_BB_6x5F cmd = {
       {{subcommand, 0x0B, 0x0000}, {floor, from_enemy, entity_id, x, z, 0, 0, item}}, 0};
-  cmd.item.item.encode_for_version(ch.version, s->item_parameter_table_for_version(ch.version));
+  cmd.item.item.encode_for_version(ch.version, s->item_parameter_table(ch.version));
   ch.send(0x60, 0x00, &cmd, sizeof(cmd));
 }
 
@@ -2528,7 +2528,7 @@ void send_drop_stacked_item_to_channel(
     shared_ptr<ServerState> s, Channel& ch, const ItemData& item, uint8_t floor, float x, float z) {
   uint8_t subcommand = get_pre_v1_subcommand(ch.version, 0x4F, 0x56, 0x5D);
   G_DropStackedItem_PC_V3_BB_6x5D cmd = {{{subcommand, 0x0A, 0x0000}, floor, 0, x, z, item}, 0};
-  cmd.item_data.encode_for_version(ch.version, s->item_parameter_table_for_version(ch.version));
+  cmd.item_data.encode_for_version(ch.version, s->item_parameter_table(ch.version));
   ch.send(0x60, 0x00, &cmd, sizeof(cmd));
 }
 

src/ServerShell.cc

@@ -806,7 +806,7 @@ Proxy session commands:\n\
     }
 
     auto s = ses->require_server_state();
-    ItemData item = s->item_name_index->parse_item_description(ses->version(), command_args);
+    ItemData item = s->parse_item_description(ses->version(), command_args);
     item.id = random_object<uint32_t>() | 0x80000000;
 
     if (command_name == "set-next-item") {

src/ServerState.cc

@@ -14,7 +14,7 @@
 #include "NetworkAddresses.hh"
 #include "SendCommands.hh"
 #include "Text.hh"
-#include "UnicodeTextSet.hh"
+#include "TextIndex.hh"
 
 using namespace std;
 
@@ -379,7 +379,7 @@ uint32_t ServerState::connect_address_for_client(shared_ptr<Client> c) const {
   }
 }
 
-shared_ptr<const Menu> ServerState::information_menu_for_version(Version version) const {
+shared_ptr<const Menu> ServerState::information_menu(Version version) const {
   if (is_v1_or_v2(version)) {
     return this->information_menu_v2;
   } else if (is_v3(version)) {
@@ -388,7 +388,7 @@ shared_ptr<const Menu> ServerState::information_menu_for_version(Version version
   throw out_of_range("no information menu exists for this version");
 }
 
-shared_ptr<const Menu> ServerState::proxy_destinations_menu_for_version(Version version) const {
+shared_ptr<const Menu> ServerState::proxy_destinations_menu(Version version) const {
   switch (version) {
     case Version::DC_NTE:
     case Version::DC_V1_11_2000_PROTOTYPE:
@@ -410,7 +410,7 @@ shared_ptr<const Menu> ServerState::proxy_destinations_menu_for_version(Version
   }
 }
 
-const vector<pair<string, uint16_t>>& ServerState::proxy_destinations_for_version(Version version) const {
+const vector<pair<string, uint16_t>>& ServerState::proxy_destinations(Version version) const {
   switch (version) {
     case Version::DC_NTE:
     case Version::DC_V1_11_2000_PROTOTYPE:
@@ -432,7 +432,19 @@ const vector<pair<string, uint16_t>>& ServerState::proxy_destinations_for_versio
   }
 }
 
-shared_ptr<const ItemParameterTable> ServerState::item_parameter_table_for_version(Version version) const {
+shared_ptr<const vector<string>> ServerState::information_contents_for_client(shared_ptr<const Client> c) const {
+  return is_v1_or_v2(c->version()) ? this->information_contents_v2 : this->information_contents_v3;
+}
+
+shared_ptr<const QuestIndex> ServerState::quest_index(Version version) const {
+  return is_ep3(version) ? this->ep3_download_quest_index : this->default_quest_index;
+}
+
+void ServerState::dispatch_destroy_lobbies(evutil_socket_t, short, void* ctx) {
+  reinterpret_cast<ServerState*>(ctx)->lobbies_to_destroy.clear();
+}
+
+shared_ptr<const ItemParameterTable> ServerState::item_parameter_table(Version version) const {
   switch (version) {
     case Version::DC_NTE:
     case Version::DC_V1_11_2000_PROTOTYPE:
@@ -454,11 +466,24 @@ shared_ptr<const ItemParameterTable> ServerState::item_parameter_table_for_versi
   }
 }
 
+shared_ptr<const ItemNameIndex> ServerState::item_name_index(Version version) const {
+  auto ret = this->item_name_indexes.at(static_cast<size_t>(version));
+  if (ret == nullptr) {
+    throw runtime_error("no item name index exists for this version");
+  }
+  return ret;
+}
+
+void ServerState::set_item_name_index(Version version, shared_ptr<const ItemNameIndex> new_index) {
+  this->item_name_indexes.at(static_cast<size_t>(version)) = new_index;
+}
+
 string ServerState::describe_item(Version version, const ItemData& item, bool include_color_codes) const {
-  return this->item_name_index->describe_item(
+  return this->item_name_index(version)->describe_item(item, include_color_codes);
-      version,
+}
-      item,
+
-      include_color_codes ? this->item_parameter_table_for_version(version) : nullptr);
+ItemData ServerState::parse_item_description(Version version, const string& description) const {
+  return this->item_name_index(version)->parse_item_description(description);
 }
 
 void ServerState::set_port_configuration(
@@ -829,7 +854,7 @@ void ServerState::parse_config(const JSON& json, bool is_reload) {
       for (const auto& difficulty_it : type_it->as_list()) {
         auto& difficulty_res = type_res.emplace_back();
         for (const auto& item_it : difficulty_it->as_list()) {
-          difficulty_res.emplace_back(this->item_name_index->parse_item_description(Version::BB_V4, item_it->as_string()));
+          difficulty_res.emplace_back(this->parse_item_description(Version::BB_V4, item_it->as_string()));
         }
       }
     }
@@ -840,22 +865,22 @@ void ServerState::parse_config(const JSON& json, bool is_reload) {
     for (const auto& it : json.get_list("QuestF95FResultItems")) {
       auto& list = it->as_list();
       size_t price = list.at(0)->as_int();
-      this->quest_F95F_results.emplace_back(make_pair(price, this->item_name_index->parse_item_description(Version::BB_V4, list.at(1)->as_string())));
+      this->quest_F95F_results.emplace_back(make_pair(price, this->parse_item_description(Version::BB_V4, list.at(1)->as_string())));
     }
   } catch (const out_of_range&) {
   }
   try {
     this->quest_F960_success_results.clear();
-    this->quest_F960_failure_results = QuestF960Result(json.at("QuestF960FailureResultItems"), this->item_name_index);
+    this->quest_F960_failure_results = QuestF960Result(json.at("QuestF960FailureResultItems"), this->item_name_index(Version::BB_V4));
     for (const auto& it : json.get_list("QuestF960SuccessResultItems")) {
-      this->quest_F960_success_results.emplace_back(*it, this->item_name_index);
+      this->quest_F960_success_results.emplace_back(*it, this->item_name_index(Version::BB_V4));
     }
   } catch (const out_of_range&) {
   }
   try {
     this->secret_lottery_results.clear();
     for (const auto& it : json.get_list("SecretLotteryResultItems")) {
-      this->secret_lottery_results.emplace_back(this->item_name_index->parse_item_description(Version::BB_V4, it->as_string()));
+      this->secret_lottery_results.emplace_back(this->parse_item_description(Version::BB_V4, it->as_string()));
     }
   } catch (const out_of_range&) {
   }
@@ -1109,7 +1134,25 @@ void ServerState::load_level_table() {
   this->level_table = make_shared<LevelTableV4>(*this->load_bb_file("PlyLevelTbl.prs"), true);
 }
 
-shared_ptr<WordSelectTable> ServerState::load_word_select_table_from_system() {
+void ServerState::load_text_index() {
+  this->text_index = make_shared<TextIndex>("system/text-sets", [&](Version version, const string& filename) -> shared_ptr<const string> {
+    try {
+      if (version == Version::BB_V4) {
+        return this->load_bb_file(filename);
+      } else {
+        return this->pc_patch_file_index->get("Media/PSO/" + filename)->load_data();
+      }
+    } catch (const out_of_range&) {
+      return nullptr;
+    } catch (const cannot_open_file&) {
+      return nullptr;
+    }
+  });
+}
+
+void ServerState::load_word_select_table() {
+  config_log.info("Loading Word Select table");
+
   vector<vector<string>> name_alias_lists;
   auto json = JSON::parse(load_file("system/word-select/name-alias-lists.json"));
   for (const auto& coll_it : json.as_list()) {
@@ -1119,12 +1162,20 @@ shared_ptr<WordSelectTable> ServerState::load_word_select_table_from_system() {
     }
   }
 
-  config_log.info("(Word select) Loading pc_unitxt.prs");
+  const vector<string>* pc_unitxt_collection = nullptr;
-  vector<vector<string>> pc_unitxt_data = parse_unicode_text_set(load_file("system/word-select/pc_unitxt.prs"));
+  const vector<string>* bb_unitxt_collection = nullptr;
+  unique_ptr<UnicodeTextSet> pc_unitxt_data;
+  if (this->text_index) {
+    config_log.info("(Word select) Using PC_V2 unitxt_e.prs from text index");
+    pc_unitxt_collection = &this->text_index->get(Version::PC_V2, 1, 35);
+  } else {
+    config_log.info("(Word select) Loading PC_V2 unitxt_e.prs");
+    pc_unitxt_data = make_unique<UnicodeTextSet>(load_file("system/word-select/pc_unitxt.prs"));
+    pc_unitxt_collection = &pc_unitxt_data->get(35);
+  }
   config_log.info("(Word select) Loading bb_unitxt_ws.prs");
-  vector<vector<string>> bb_unitxt_data = parse_unicode_text_set(load_file("system/word-select/bb_unitxt_ws.prs"));
+  auto bb_unitxt_data = make_unique<UnicodeTextSet>(load_file("system/word-select/bb_unitxt_ws.prs"));
-  vector<string> pc_unitxt_collection = std::move(pc_unitxt_data.at(35));
+  bb_unitxt_collection = &bb_unitxt_data->get(0);
-  vector<string> bb_unitxt_collection = std::move(bb_unitxt_data.at(0));
 
   config_log.info("(Word select) Loading DC_NTE data");
   WordSelectSet dc_nte_ws(load_file("system/word-select/dc_nte_ws_data.bin"), Version::DC_NTE, nullptr, true);
@@ -1135,9 +1186,9 @@ shared_ptr<WordSelectTable> ServerState::load_word_select_table_from_system() {
   config_log.info("(Word select) Loading DC_V2 data");
   WordSelectSet dc_v2_ws(load_file("system/word-select/dcv2_ws_data.bin"), Version::DC_V2, nullptr, false);
   config_log.info("(Word select) Loading PC_NTE data");
-  WordSelectSet pc_nte_ws(load_file("system/word-select/pc_nte_ws_data.bin"), Version::PC_NTE, &pc_unitxt_collection, false);
+  WordSelectSet pc_nte_ws(load_file("system/word-select/pc_nte_ws_data.bin"), Version::PC_NTE, pc_unitxt_collection, false);
   config_log.info("(Word select) Loading PC_V2 data");
-  WordSelectSet pc_v2_ws(load_file("system/word-select/pc_ws_data.bin"), Version::PC_V2, &pc_unitxt_collection, false);
+  WordSelectSet pc_v2_ws(load_file("system/word-select/pc_ws_data.bin"), Version::PC_V2, pc_unitxt_collection, false);
   config_log.info("(Word select) Loading GC_NTE data");
   WordSelectSet gc_nte_ws(load_file("system/word-select/gc_nte_ws_data.bin"), Version::GC_NTE, nullptr, false);
   config_log.info("(Word select) Loading GC_V3 data");
@@ -1149,30 +1200,72 @@ shared_ptr<WordSelectTable> ServerState::load_word_select_table_from_system() {
   config_log.info("(Word select) Loading XB_V3 data");
   WordSelectSet xb_v3_ws(load_file("system/word-select/xb_ws_data.bin"), Version::XB_V3, nullptr, false);
   config_log.info("(Word select) Loading BB_V4 data");
-  WordSelectSet bb_v4_ws(load_file("system/word-select/bb_ws_data.bin"), Version::BB_V4, &bb_unitxt_collection, false);
+  WordSelectSet bb_v4_ws(load_file("system/word-select/bb_ws_data.bin"), Version::BB_V4, bb_unitxt_collection, false);
 
   config_log.info("(Word select) Generating table");
-  return make_shared<WordSelectTable>(
+  this->word_select_table = make_shared<WordSelectTable>(
       dc_nte_ws, dc_112000_ws, dc_v1_ws, dc_v2_ws,
       pc_nte_ws, pc_v2_ws, gc_nte_ws, gc_v3_ws,
       gc_ep3_nte_ws, gc_ep3_ws, xb_v3_ws, bb_v4_ws,
       name_alias_lists);
 }
 
-void ServerState::load_word_select_table() {
+shared_ptr<ItemNameIndex> ServerState::create_item_name_index_for_version(
-  config_log.info("Loading Word Select table");
+    Version version, shared_ptr<const ItemParameterTable> pmt, shared_ptr<const TextIndex> text_index) {
-  this->word_select_table = this->load_word_select_table_from_system();
+  switch (version) {
+    case Version::DC_NTE:
+      return make_shared<ItemNameIndex>(pmt, text_index->get(Version::DC_NTE, 0, 2));
+    case Version::DC_V1_11_2000_PROTOTYPE:
+      return make_shared<ItemNameIndex>(pmt, text_index->get(Version::DC_V1_11_2000_PROTOTYPE, 1, 2));
+    case Version::DC_V1:
+      return make_shared<ItemNameIndex>(pmt, text_index->get(Version::DC_V1, 1, 2));
+    case Version::DC_V2:
+      return make_shared<ItemNameIndex>(pmt, text_index->get(Version::DC_V2, 1, 3));
+    case Version::PC_NTE:
+      return make_shared<ItemNameIndex>(pmt, text_index->get(Version::PC_NTE, 1, 3));
+    case Version::PC_V2:
+      return make_shared<ItemNameIndex>(pmt, text_index->get(Version::PC_V2, 1, 3));
+    case Version::GC_NTE:
+      return make_shared<ItemNameIndex>(pmt, text_index->get(Version::GC_NTE, 1, 0));
+    case Version::GC_V3:
+      return make_shared<ItemNameIndex>(pmt, text_index->get(Version::GC_V3, 1, 0));
+    case Version::XB_V3:
+      return make_shared<ItemNameIndex>(pmt, text_index->get(Version::XB_V3, 1, 0));
+    case Version::BB_V4:
+      return make_shared<ItemNameIndex>(pmt, text_index->get(Version::BB_V4, 1, 1));
+    default:
+      return nullptr;
+  }
 }
 
-void ServerState::load_item_name_index() {
+void ServerState::load_item_name_indexes() {
-  config_log.info("Loading item name index");
+  config_log.info("Generating item name indexes");
-  this->item_name_index = make_shared<ItemNameIndex>(
+  // TODO: Get ItemPMT files for the versions for which we don't have them
-      JSON::parse(load_file("system/item-tables/names-v2.json")),
+  // (especially DC_V1) and add support for them. Currently we only have three
-      JSON::parse(load_file("system/item-tables/names-v3.json")),
+  // ItemPMTs (PC, GC, and BB), so we can't use them to generate all the name
-      JSON::parse(load_file("system/item-tables/names-v4.json")));
+  // indexes.
+
+  auto pc_v2_index = create_item_name_index_for_version(
+      Version::PC_V2, this->item_parameter_table(Version::PC_V2), this->text_index);
+  this->set_item_name_index(Version::DC_NTE, pc_v2_index);
+  this->set_item_name_index(Version::DC_V1_11_2000_PROTOTYPE, pc_v2_index);
+  this->set_item_name_index(Version::DC_V1, pc_v2_index);
+  this->set_item_name_index(Version::DC_V2, pc_v2_index);
+  this->set_item_name_index(Version::PC_NTE, pc_v2_index);
+  this->set_item_name_index(Version::PC_V2, pc_v2_index);
+
+  auto gc_v3_index = create_item_name_index_for_version(
+      Version::GC_V3, this->item_parameter_table(Version::GC_V3), this->text_index);
+  this->set_item_name_index(Version::GC_NTE, gc_v3_index);
+  this->set_item_name_index(Version::GC_V3, gc_v3_index);
+  this->set_item_name_index(Version::XB_V3, gc_v3_index);
+
+  auto bb_v4_index = create_item_name_index_for_version(
+      Version::BB_V4, this->item_parameter_table(Version::BB_V4), this->text_index);
+  this->set_item_name_index(Version::BB_V4, bb_v4_index);
 }
 
-void ServerState::load_item_tables() {
+void ServerState::load_drop_tables() {
   config_log.info("Loading rare item sets");
   unordered_map<string, shared_ptr<const RareItemSet>> new_rare_item_sets;
   for (const auto& filename : list_directory_sorted("system/item-tables")) {
@@ -1186,16 +1279,16 @@ void ServerState::load_item_tables() {
 
     if (ends_with(filename, "-v1.json")) {
       config_log.info("Loading v1 JSON rare item table %s", filename.c_str());
-      new_rare_item_sets.emplace(basename, make_shared<RareItemSet>(JSON::parse(load_file(path)), Version::DC_V1, this->item_name_index));
+      new_rare_item_sets.emplace(basename, make_shared<RareItemSet>(JSON::parse(load_file(path)), this->item_name_index(Version::DC_V1)));
     } else if (ends_with(filename, "-v2.json")) {
       config_log.info("Loading v2 JSON rare item table %s", filename.c_str());
-      new_rare_item_sets.emplace(basename, make_shared<RareItemSet>(JSON::parse(load_file(path)), Version::PC_V2, this->item_name_index));
+      new_rare_item_sets.emplace(basename, make_shared<RareItemSet>(JSON::parse(load_file(path)), this->item_name_index(Version::PC_V2)));
     } else if (ends_with(filename, "-v3.json")) {
       config_log.info("Loading v3 JSON rare item table %s", filename.c_str());
-      new_rare_item_sets.emplace(basename, make_shared<RareItemSet>(JSON::parse(load_file(path)), Version::GC_V3, this->item_name_index));
+      new_rare_item_sets.emplace(basename, make_shared<RareItemSet>(JSON::parse(load_file(path)), this->item_name_index(Version::GC_V3)));
     } else if (ends_with(filename, "-v4.json")) {
       config_log.info("Loading v4 JSON rare item table %s", filename.c_str());
-      new_rare_item_sets.emplace(basename, make_shared<RareItemSet>(JSON::parse(load_file(path)), Version::BB_V4, this->item_name_index));
+      new_rare_item_sets.emplace(basename, make_shared<RareItemSet>(JSON::parse(load_file(path)), this->item_name_index(Version::BB_V4)));
 
     } else if (ends_with(filename, ".afs")) {
       config_log.info("Loading AFS rare item table %s", filename.c_str());
@@ -1255,7 +1348,9 @@ void ServerState::load_item_tables() {
   config_log.info("Loading tekker adjustment table");
   auto tekker_data = make_shared<string>(load_file("system/item-tables/JudgeItem-gc.rel"));
   this->tekker_adjustment_set = make_shared<TekkerAdjustmentSet>(tekker_data);
+}
 
+void ServerState::load_item_definitions() {
   config_log.info("Loading item definition tables");
   auto pmt_data_v2 = make_shared<string>(prs_decompress(load_file("system/item-tables/ItemPMT-v2.prs")));
   this->item_parameter_table_v2 = make_shared<ItemParameterTable>(pmt_data_v2, ItemParameterTable::Version::V2);

src/ServerState.hh

@@ -138,7 +138,8 @@ struct ServerState : public std::enable_shared_from_this<ServerState> {
   std::shared_ptr<const ItemParameterTable> item_parameter_table_v3;
   std::shared_ptr<const ItemParameterTable> item_parameter_table_v4;
   std::shared_ptr<const MagEvolutionTable> mag_evolution_table;
-  std::shared_ptr<const ItemNameIndex> item_name_index;
+  std::shared_ptr<const TextIndex> text_index;
+  std::array<std::shared_ptr<const ItemNameIndex>, NUM_VERSIONS> item_name_indexes;
   std::shared_ptr<const WordSelectTable> word_select_table;
   std::array<std::shared_ptr<const Map::RareEnemyRates>, 4> rare_enemy_rates_by_difficulty;
   std::shared_ptr<const Map::RareEnemyRates> rare_enemy_rates_challenge;
@@ -257,15 +258,18 @@ struct ServerState : public std::enable_shared_from_this<ServerState> {
 
   uint32_t connect_address_for_client(std::shared_ptr<Client> c) const;
 
-  std::shared_ptr<const Menu> information_menu_for_version(Version version) const;
+  std::shared_ptr<const Menu> information_menu(Version version) const;
-  std::shared_ptr<const Menu> proxy_destinations_menu_for_version(Version version) const;
+  std::shared_ptr<const Menu> proxy_destinations_menu(Version version) const;
-  const std::vector<std::pair<std::string, uint16_t>>& proxy_destinations_for_version(Version version) const;
+  const std::vector<std::pair<std::string, uint16_t>>& proxy_destinations(Version version) const;
 
-  std::shared_ptr<const ItemParameterTable> item_parameter_table_for_version(Version version) const;
+  std::shared_ptr<const ItemParameterTable> item_parameter_table(Version version) const;
+  std::shared_ptr<const ItemNameIndex> item_name_index(Version version) const;
+  void set_item_name_index(Version version, std::shared_ptr<const ItemNameIndex>);
   std::string describe_item(Version version, const ItemData& item, bool include_color_codes) const;
+  ItemData parse_item_description(Version version, const std::string& description) const;
 
   std::shared_ptr<const std::vector<std::string>> information_contents_for_client(std::shared_ptr<const Client> c) const;
-  std::shared_ptr<const QuestIndex> quest_index_for_version(Version version) const;
+  std::shared_ptr<const QuestIndex> quest_index(Version version) const;
 
   void set_port_configuration(const std::vector<PortConfiguration>& port_configs);
 

src/TextArchive.cc

@@ -1,302 +0,0 @@
-#include "TextArchive.hh"
-
-#include <phosg/Encoding.hh>
-#include <phosg/Filesystem.hh>
-#include <phosg/Random.hh>
-#include <set>
-#include <stdexcept>
-
-#include "Compression.hh"
-#include "PSOEncryption.hh"
-#include "Text.hh"
-
-using namespace std;
-
-TextArchive::TextArchive(const string& pr2_data, bool big_endian) {
-  if (big_endian) {
-    this->load_t<true>(pr2_data);
-  } else {
-    this->load_t<false>(pr2_data);
-  }
-}
-
-TextArchive::TextArchive(const JSON& json) {
-  for (const auto& collection_json : json.at("collections").as_list()) {
-    auto& collection = this->collections.emplace_back();
-    for (const auto& string_json : collection_json->as_list()) {
-      collection.emplace_back(string_json->as_string());
-    }
-  }
-
-  for (const auto& keyboard_json : json.at("keyboards").as_list()) {
-    auto& keyboard = this->keyboards.emplace_back(make_unique<Keyboard>());
-    for (size_t y = 0; y < keyboard->size(); y++) {
-      auto& row = keyboard->at(y);
-      const auto& row_json = keyboard_json->at(y);
-      for (size_t x = 0; x < row.size(); x++) {
-        row[x] = row_json.at(x).as_int();
-      }
-    }
-  }
-
-  this->keyboard_selector_width = json.at("keyboard_selector_width").as_int();
-}
-
-JSON TextArchive::json() const {
-  auto collections_json = JSON::list();
-  for (const auto& collection : this->collections) {
-    auto collection_json = JSON::list();
-    for (const auto& s : collection) {
-      collection_json.emplace_back(s);
-    }
-    collections_json.emplace_back(std::move(collection_json));
-  }
-  auto keyboards_json = JSON::list();
-  for (const auto& kb : this->keyboards) {
-    JSON keyboard_json = JSON::list();
-    for (size_t y = 0; y < kb->size(); y++) {
-      const auto& row = kb->at(y);
-      JSON row_json = JSON::list();
-      for (size_t x = 0; x < row.size(); x++) {
-        row_json.emplace_back(row[x]);
-      }
-      keyboard_json.emplace_back(std::move(row_json));
-    }
-    keyboards_json.emplace_back(std::move(keyboard_json));
-  }
-  return JSON::dict({
-      {"collections", std::move(collections_json)},
-      {"keyboards", std::move(keyboards_json)},
-      {"keyboard_selector_width", this->keyboard_selector_width},
-  });
-}
-
-const string& TextArchive::get_string(size_t collection_index, size_t index) const {
-  return this->collections.at(collection_index).at(index);
-}
-
-void TextArchive::set_string(size_t collection_index, size_t index, const string& data) {
-  if (collection_index >= this->collections.size()) {
-    this->collections.resize(collection_index + 1);
-  }
-  auto& coll = this->collections[collection_index];
-  if (index >= coll.size()) {
-    coll.resize(index + 1);
-  }
-  coll[index] = data;
-}
-
-void TextArchive::set_string(size_t collection_index, size_t index, string&& data) {
-  if (collection_index >= this->collections.size()) {
-    this->collections.resize(collection_index + 1);
-  }
-  auto& coll = this->collections[collection_index];
-  if (index >= coll.size()) {
-    coll.resize(index + 1);
-  }
-  coll[index] = std::move(data);
-}
-
-void TextArchive::resize_collection(size_t collection_index, size_t size) {
-  if (collection_index >= this->collections.size()) {
-    this->collections.resize(collection_index + 1);
-  }
-  this->collections[collection_index].resize(size);
-}
-
-void TextArchive::resize_collection(size_t num_collections) {
-  this->collections.resize(num_collections);
-}
-
-TextArchive::Keyboard TextArchive::get_keyboard(size_t kb_index) const {
-  return *this->keyboards.at(kb_index);
-}
-
-void TextArchive::set_keyboard(size_t kb_index, const Keyboard& kb) {
-  if (kb_index >= this->keyboards.size()) {
-    this->keyboards.resize(kb_index + 1);
-  }
-  this->keyboards[kb_index] = make_unique<Keyboard>(kb);
-}
-
-void TextArchive::resize_keyboards(size_t num_keyboards) {
-  this->keyboards.resize(num_keyboards);
-}
-
-pair<string, string> TextArchive::serialize(bool big_endian) const {
-  if (big_endian) {
-    return this->serialize_t<true>();
-  } else {
-    return this->serialize_t<false>();
-  }
-}
-
-template <bool IsBigEndian>
-void TextArchive::load_t(const string& pr2_data) {
-  using U32T = std::conditional_t<IsBigEndian, be_uint32_t, le_uint32_t>;
-  using U16T = std::conditional_t<IsBigEndian, be_uint16_t, le_uint16_t>;
-
-  // The structure is as follows:
-  // Footer:
-  //   U32T keyboard_index_offset ->:
-  //     U8 num_keyboards
-  //     U8 keyboard_selector_width
-  //     U8 unused[2]
-  //     U32T keyboards_offset ->:
-  //       U32T keyboard_offset[num_keyboards] ->:
-  //         U16T key_defs[7][16]
-  //   U32T collections_offset ->:
-  //     U32T[...] strings_offset ->:
-  //       U32T[...] string_offset ->:
-  //         char string[...\0]
-  //   <EOF>
-
-  auto pr2_decrypted = decrypt_pr2_data<IsBigEndian>(pr2_data);
-  auto decompressed = prs_decompress(pr2_decrypted.compressed_data);
-  StringReader r(decompressed);
-
-  // Annoyingly, there doesn't appear to be any bounds-checking on the language
-  // functions, so there are no counts of strings in each collection. We have to
-  // figure out where each collection ends by collecting all the relevant
-  // offsets in the file instead.
-  set<uint32_t> used_offsets;
-  used_offsets.emplace(r.size() - 8);
-
-  uint32_t keyboard_index_offset = r.pget<U32T>(r.size() - 8);
-  used_offsets.emplace(keyboard_index_offset);
-  size_t num_keyboards = r.pget_u8(keyboard_index_offset);
-  this->keyboard_selector_width = r.pget_u8(keyboard_index_offset + 1);
-  uint32_t keyboards_offset = r.pget<U32T>(keyboard_index_offset + 4);
-  used_offsets.emplace(keyboards_offset);
-  while (this->keyboards.size() < num_keyboards) {
-    uint32_t keyboard_offset = r.pget<U32T>(keyboards_offset + 4 * this->keyboards.size());
-    used_offsets.emplace(keyboard_offset);
-    auto& kb = this->keyboards.emplace_back(make_unique<Keyboard>());
-    auto key_r = r.sub(keyboard_offset, sizeof(Keyboard));
-    for (size_t y = 0; y < kb->size(); y++) {
-      auto& row = kb->at(y);
-      for (size_t x = 0; x < row.size(); x++) {
-        row[x] = key_r.get<U16T>();
-      }
-    }
-  }
-
-  uint32_t collections_offset = r.pget<U32T>(r.size() - 4);
-  for (uint32_t offset = collections_offset; !used_offsets.count(offset); offset += 4) {
-    used_offsets.emplace(r.pget<U32T>(offset));
-  }
-  used_offsets.emplace(collections_offset);
-
-  for (uint32_t offset = collections_offset; (offset == collections_offset) || !used_offsets.count(offset); offset += 4) {
-    auto& collection = this->collections.emplace_back();
-    uint32_t first_string_offset_offset = r.pget<U32T>(offset);
-    for (uint32_t string_offset_offset = first_string_offset_offset;
-         (string_offset_offset == first_string_offset_offset) || !used_offsets.count(string_offset_offset);
-         string_offset_offset += 4) {
-      collection.emplace_back(r.pget_cstr(r.pget<U32T>(string_offset_offset)));
-    }
-  }
-}
-
-template <bool IsBigEndian>
-pair<string, string> TextArchive::serialize_t() const {
-  using U32T = std::conditional_t<IsBigEndian, be_uint32_t, le_uint32_t>;
-  using U16T = std::conditional_t<IsBigEndian, be_uint16_t, le_uint16_t>;
-
-  StringWriter w;
-  set<size_t> relocation_offsets;
-  auto put_offset_u32 = [&](uint32_t v) {
-    relocation_offsets.emplace(w.size());
-    w.put<U32T>(v);
-  };
-
-  uint32_t collections_offset;
-  {
-    unordered_map<string, uint32_t> string_to_offset;
-    for (const auto& collection : this->collections) {
-      for (const auto& s : collection) {
-        if (string_to_offset.emplace(s, w.size()).second) {
-          w.write(s);
-          w.put_u8(0);
-          while (w.size() & 3) {
-            w.put_u8(0);
-          }
-        }
-      }
-    }
-
-    vector<uint32_t> collection_offsets;
-    for (const auto& collection : this->collections) {
-      collection_offsets.emplace_back(w.size());
-      for (const auto& s : collection) {
-        put_offset_u32(string_to_offset.at(s));
-      }
-    }
-
-    collections_offset = w.size();
-    for (uint32_t collection_offset : collection_offsets) {
-      put_offset_u32(collection_offset);
-    }
-  }
-
-  uint32_t keyboard_index_offset;
-  {
-    vector<uint32_t> keyboard_offsets;
-    for (const auto& keyboard : this->keyboards) {
-      keyboard_offsets.emplace_back(w.size());
-      for (size_t y = 0; y < keyboard->size(); y++) {
-        const auto& row = keyboard->at(y);
-        for (size_t x = 0; x < row.size(); x++) {
-          w.put<U16T>(row[x]);
-        }
-      }
-    }
-
-    uint32_t keyboards_offset = w.size();
-    for (uint32_t keyboard_offset : keyboard_offsets) {
-      put_offset_u32(keyboard_offset);
-    }
-
-    keyboard_index_offset = w.size();
-    w.put_u8(keyboard_offsets.size());
-    w.put_u8(this->keyboard_selector_width);
-    w.put_u16(0);
-    put_offset_u32(keyboards_offset);
-  }
-
-  put_offset_u32(keyboard_index_offset);
-  put_offset_u32(collections_offset);
-
-  StringWriter reloc_w;
-  reloc_w.put_u32(0);
-  reloc_w.put<U32T>(relocation_offsets.size());
-  reloc_w.put_u64(0);
-  reloc_w.put<U32T>(w.size() - 8);
-  reloc_w.put_u32(0);
-  reloc_w.put_u64(0);
-  {
-    size_t offset = 0;
-    for (size_t reloc_offset : relocation_offsets) {
-      if (reloc_offset & 3) {
-        throw logic_error("misaligned relocation");
-      }
-      size_t num_words = (reloc_offset - offset) >> 2;
-      if (num_words > 0xFFFF) {
-        throw runtime_error("relocation offset too far away");
-      }
-      reloc_w.put<U16T>(num_words);
-      offset = reloc_offset;
-    }
-  }
-
-  const string& pr2_data = w.str();
-  const string& pr3_data = reloc_w.str();
-  print_data(stderr, pr2_data);
-  string pr2_compressed = prs_compress_optimal(pr2_data.data(), pr2_data.size());
-  string pr3_compressed = prs_compress_optimal(pr3_data.data(), pr3_data.size());
-  print_data(stderr, pr2_compressed);
-  string pr2_ret = encrypt_pr2_data<IsBigEndian>(pr2_compressed, pr2_data.size(), random_object<uint32_t>());
-  string pr3_ret = encrypt_pr2_data<IsBigEndian>(pr3_compressed, pr3_data.size(), random_object<uint32_t>());
-  print_data(stderr, pr2_ret);
-  return make_pair(std::move(pr2_ret), std::move(pr3_ret));
-}

src/TextArchive.hh

@@ -1,51 +0,0 @@
-#pragma once
-
-#include <stdint.h>
-
-#include <phosg/JSON.hh>
-#include <string>
-#include <utility>
-#include <vector>
-
-#include "Text.hh"
-
-// This class implements loading and saving of text archives, commonly found in
-// PSO games with filenames like TextEnglish.pr2 and TextEnglish.pr3. The game
-// requires both files, but newserv needs only the pr2 file to load a text
-// archive. When saving (serializing), both pr2 and pr3 files are generated.
-class TextArchive {
-public:
-  using Keyboard = parray<parray<uint16_t, 0x10>, 7>;
-
-  explicit TextArchive(const JSON& json);
-  TextArchive(const std::string& pr2_data, bool big_endian);
-  ~TextArchive() = default;
-
-  JSON json() const;
-
-  const std::string& get_string(size_t collection_index, size_t index) const;
-  void set_string(size_t collection_index, size_t index, const std::string& data);
-  void set_string(size_t collection_index, size_t index, std::string&& data);
-  void resize_collection(size_t collection_index, size_t size);
-  void resize_collection(size_t num_collections);
-
-  Keyboard get_keyboard(size_t kb_index) const;
-  void set_keyboard(size_t kb_index, const Keyboard& kb);
-  void resize_keyboards(size_t num_keyboards);
-
-  uint8_t get_keyboard_selector_width() const;
-  void set_keyboard_selector_width(uint8_t width);
-
-  // Returns (pr2_data, pr3_data)
-  std::pair<std::string, std::string> serialize(bool big_endian) const;
-
-private:
-  template <bool IsBigEndian>
-  void load_t(const std::string& pr2_data);
-  template <bool IsBigEndian>
-  std::pair<std::string, std::string> serialize_t() const;
-
-  std::vector<std::vector<std::string>> collections;
-  std::vector<std::unique_ptr<Keyboard>> keyboards;
-  uint8_t keyboard_selector_width;
-};

src/TextIndex.cc

@@ -0,0 +1,557 @@
+#include "TextIndex.hh"
+
+#include <phosg/Encoding.hh>
+#include <phosg/Filesystem.hh>
+#include <phosg/Random.hh>
+#include <set>
+#include <stdexcept>
+
+#include "Compression.hh"
+#include "Loggers.hh"
+#include "PSOEncryption.hh"
+#include "StaticGameData.hh"
+#include "Text.hh"
+
+using namespace std;
+
+TextSet::TextSet(const JSON& json) {
+  for (const auto& coll_json : json.as_list()) {
+    auto& collection = this->collections.emplace_back();
+    for (const auto& s_json : coll_json->as_list()) {
+      collection.emplace_back(s_json->as_string());
+    }
+  }
+}
+
+TextSet::TextSet(JSON&& json) {
+  for (const auto& coll_json : json.as_list()) {
+    auto& collection = this->collections.emplace_back();
+    for (const auto& s_json : coll_json->as_list()) {
+      collection.emplace_back(std::move(s_json->as_string()));
+    }
+  }
+}
+
+JSON TextSet::json() const {
+  JSON j = JSON::list();
+  for (const auto& collection : this->collections) {
+    JSON& coll_j = j.emplace_back(JSON::list());
+    for (const auto& s : collection) {
+      coll_j.emplace_back(s);
+    }
+  }
+  return j;
+}
+
+size_t TextSet::count(size_t collection_index) const {
+  return this->collections.at(collection_index).size();
+}
+
+size_t TextSet::count() const {
+  return this->collections.size();
+}
+
+const std::string& TextSet::get(size_t collection_index, size_t string_index) const {
+  return this->get(collection_index).at(string_index);
+}
+
+const vector<string>& TextSet::get(size_t collection_index) const {
+  return this->collections.at(collection_index);
+}
+
+void TextSet::set(size_t collection_index, size_t string_index, const std::string& data) {
+  this->ensure_slot_exists(collection_index, string_index);
+  this->collections[collection_index][string_index] = data;
+}
+
+void TextSet::set(size_t collection_index, size_t string_index, std::string&& data) {
+  this->ensure_slot_exists(collection_index, string_index);
+  this->collections[collection_index][string_index] = std::move(data);
+}
+
+void TextSet::set(size_t collection_index, const std::vector<std::string>& coll) {
+  this->ensure_collection_exists(collection_index);
+  this->collections[collection_index] = coll;
+}
+
+void TextSet::set(size_t collection_index, std::vector<std::string>&& coll) {
+  this->ensure_collection_exists(collection_index);
+  this->collections[collection_index] = std::move(coll);
+}
+
+void TextSet::truncate_collection(size_t collection_index, size_t new_entry_count) {
+  if (collection_index >= this->collections.size()) {
+    this->collections.resize(collection_index + 1);
+  }
+  this->collections[collection_index].resize(new_entry_count);
+}
+
+void TextSet::truncate(size_t new_collection_count) {
+  this->collections.resize(new_collection_count);
+}
+
+void TextSet::ensure_slot_exists(size_t collection_index, size_t string_index) {
+  this->ensure_collection_exists(collection_index);
+  auto& coll = this->collections[collection_index];
+  if (string_index >= coll.size()) {
+    coll.resize(string_index + 1);
+  }
+}
+
+void TextSet::ensure_collection_exists(size_t collection_index) {
+  if (collection_index >= this->collections.size()) {
+    this->collections.resize(collection_index + 1);
+  }
+}
+
+UnicodeTextSet::UnicodeTextSet(const string& prs_data) {
+  string data = prs_decompress(prs_data);
+  StringReader r(data);
+
+  uint32_t num_collections = r.get_u32l();
+  deque<uint32_t> collection_sizes;
+  while (collection_sizes.size() < num_collections) {
+    collection_sizes.emplace_back(r.get_u32l());
+  }
+
+  this->collections.reserve(collection_sizes.size());
+  while (!collection_sizes.empty()) {
+    uint32_t num_strings = collection_sizes.front();
+    collection_sizes.pop_front();
+
+    auto& strings = this->collections.emplace_back();
+    strings.reserve(num_strings);
+    while (strings.size() < num_strings) {
+      StringReader sub_r = r.sub(r.get_u32l());
+      StringWriter w;
+      for (uint16_t ch = sub_r.get_u16l(); ch != 0; ch = sub_r.get_u16l()) {
+        w.put_u16l(ch);
+      }
+      strings.emplace_back(tt_utf16_to_utf8(w.str()));
+    }
+  }
+}
+
+string UnicodeTextSet::serialize() const {
+  StringWriter header_w;
+  StringWriter data_w;
+
+  size_t total_num_strings = 0;
+  header_w.put_u32l(this->collections.size());
+  for (const auto& collection : this->collections) {
+    header_w.put_u32l(collection.size());
+    total_num_strings += collection.size();
+  }
+
+  unordered_map<string, uint32_t> encoded;
+
+  size_t data_base_offset = (total_num_strings * 4) + header_w.size();
+  for (const auto& collection : this->collections) {
+    for (const auto& s : collection) {
+      auto encoded_it = encoded.find(s);
+      if (encoded_it == encoded.end()) {
+        uint32_t offset = data_base_offset + data_w.size();
+        encoded_it = encoded.emplace(s, offset).first;
+        string s_utf16 = tt_utf8_to_utf16(s);
+        data_w.write(s_utf16.data(), s_utf16.size());
+        data_w.put_u16(0);
+        while (data_w.size() & 3) {
+          data_w.put_u8(0);
+        }
+      }
+      header_w.put_u32l(encoded_it->second);
+    }
+  }
+
+  header_w.write(data_w.str());
+  return prs_compress_optimal(header_w.str());
+}
+
+BinaryTextSet::BinaryTextSet(const std::string& pr2_data, size_t collection_count, bool has_rel_footer) {
+  auto pr2_decrypted = decrypt_pr2_data<false>(pr2_data);
+  auto decompressed = prs_decompress(pr2_decrypted.compressed_data);
+  StringReader r(decompressed);
+
+  // Annoyingly, there doesn't appear to be any bounds-checking on the language
+  // functions, so there are no counts of strings in each collection. We have to
+  // figure out where each collection ends by collecting all the relevant
+  // offsets in the file instead.
+  ::set<uint32_t> used_offsets;
+  size_t root_offset = has_rel_footer
+      ? r.pget_u32l(r.size() - 0x10)
+      : (r.size() - collection_count * sizeof(le_uint32_t));
+
+  StringReader collection_offsets_r = r.sub(root_offset, collection_count * sizeof(le_uint32_t));
+  while (!collection_offsets_r.eof()) {
+    used_offsets.emplace(collection_offsets_r.get_u32l());
+  }
+  used_offsets.emplace(root_offset);
+
+  collection_offsets_r.go(0);
+  while (!collection_offsets_r.eof()) {
+    auto& collection = this->collections.emplace_back();
+    uint32_t first_string_offset_offset = collection_offsets_r.get_u32l();
+    // TODO: Apparently the early formats do actually include keyboards, but
+    // they're just in the middle of the collections list. Sigh...
+    try {
+      for (uint32_t string_offset_offset = first_string_offset_offset;
+           (string_offset_offset == first_string_offset_offset) || !used_offsets.count(string_offset_offset);
+           string_offset_offset += 4) {
+        collection.emplace_back(r.pget_cstr(r.pget_u32l(string_offset_offset)));
+      }
+    } catch (const out_of_range&) {
+    }
+  }
+}
+
+BinaryTextAndKeyboardsSet::BinaryTextAndKeyboardsSet(const string& pr2_data, bool big_endian) {
+  if (big_endian) {
+    this->parse_t<true>(pr2_data);
+  } else {
+    this->parse_t<false>(pr2_data);
+  }
+}
+
+BinaryTextAndKeyboardsSet::BinaryTextAndKeyboardsSet(const JSON& json) {
+  for (const auto& collection_json : json.at("collections").as_list()) {
+    auto& collection = this->collections.emplace_back();
+    for (const auto& string_json : collection_json->as_list()) {
+      collection.emplace_back(string_json->as_string());
+    }
+  }
+
+  for (const auto& keyboard_json : json.at("keyboards").as_list()) {
+    auto& keyboard = this->keyboards.emplace_back(make_unique<Keyboard>());
+    for (size_t y = 0; y < keyboard->size(); y++) {
+      auto& row = keyboard->at(y);
+      const auto& row_json = keyboard_json->at(y);
+      for (size_t x = 0; x < row.size(); x++) {
+        row[x] = row_json.at(x).as_int();
+      }
+    }
+  }
+
+  this->keyboard_selector_width = json.at("keyboard_selector_width").as_int();
+}
+
+JSON BinaryTextAndKeyboardsSet::json() const {
+  auto collections_json = this->TextSet::json();
+  auto keyboards_json = JSON::list();
+  for (const auto& kb : this->keyboards) {
+    JSON keyboard_json = JSON::list();
+    for (size_t y = 0; y < kb->size(); y++) {
+      const auto& row = kb->at(y);
+      JSON row_json = JSON::list();
+      for (size_t x = 0; x < row.size(); x++) {
+        row_json.emplace_back(row[x]);
+      }
+      keyboard_json.emplace_back(std::move(row_json));
+    }
+    keyboards_json.emplace_back(std::move(keyboard_json));
+  }
+  return JSON::dict({
+      {"collections", std::move(collections_json)},
+      {"keyboards", std::move(keyboards_json)},
+      {"keyboard_selector_width", this->keyboard_selector_width},
+  });
+}
+
+const BinaryTextAndKeyboardsSet::Keyboard& BinaryTextAndKeyboardsSet::get_keyboard(size_t kb_index) const {
+  return *this->keyboards.at(kb_index);
+}
+
+void BinaryTextAndKeyboardsSet::set_keyboard(size_t kb_index, const Keyboard& kb) {
+  if (kb_index >= this->keyboards.size()) {
+    this->keyboards.resize(kb_index + 1);
+  }
+  this->keyboards[kb_index] = make_unique<Keyboard>(kb);
+}
+
+void BinaryTextAndKeyboardsSet::resize_keyboards(size_t num_keyboards) {
+  this->keyboards.resize(num_keyboards);
+}
+
+pair<string, string> BinaryTextAndKeyboardsSet::serialize(bool big_endian) const {
+  if (big_endian) {
+    return this->serialize_t<true>();
+  } else {
+    return this->serialize_t<false>();
+  }
+}
+
+template <bool IsBigEndian>
+void BinaryTextAndKeyboardsSet::parse_t(const string& pr2_data) {
+  using U32T = std::conditional_t<IsBigEndian, be_uint32_t, le_uint32_t>;
+  using U16T = std::conditional_t<IsBigEndian, be_uint16_t, le_uint16_t>;
+
+  // The structure is as follows:
+  // Footer:
+  //   U32T keyboard_index_offset ->:
+  //     U8 num_keyboards
+  //     U8 keyboard_selector_width
+  //     U8 unused[2]
+  //     U32T keyboards_offset ->:
+  //       U32T keyboard_offset[num_keyboards] ->:
+  //         U16T key_defs[7][16]
+  //   U32T collections_offset ->:
+  //     U32T[...] strings_offset ->:
+  //       U32T[...] string_offset ->:
+  //         char string[...\0]
+  //   <EOF>
+
+  auto pr2_decrypted = decrypt_pr2_data<IsBigEndian>(pr2_data);
+  auto decompressed = prs_decompress(pr2_decrypted.compressed_data);
+  StringReader r(decompressed);
+
+  // Annoyingly, there doesn't appear to be any bounds-checking on the language
+  // functions, so there are no counts of strings in each collection. We have to
+  // figure out where each collection ends by collecting all the relevant
+  // offsets in the file instead.
+  ::set<uint32_t> used_offsets;
+  used_offsets.emplace(r.size() - 8);
+
+  uint32_t keyboard_index_offset = r.pget<U32T>(r.size() - 8);
+  used_offsets.emplace(keyboard_index_offset);
+  size_t num_keyboards = r.pget_u8(keyboard_index_offset);
+  this->keyboard_selector_width = r.pget_u8(keyboard_index_offset + 1);
+  uint32_t keyboards_offset = r.pget<U32T>(keyboard_index_offset + 4);
+  used_offsets.emplace(keyboards_offset);
+  while (this->keyboards.size() < num_keyboards) {
+    uint32_t keyboard_offset = r.pget<U32T>(keyboards_offset + 4 * this->keyboards.size());
+    used_offsets.emplace(keyboard_offset);
+    auto& kb = this->keyboards.emplace_back(make_unique<Keyboard>());
+    auto key_r = r.sub(keyboard_offset, sizeof(Keyboard));
+    for (size_t y = 0; y < kb->size(); y++) {
+      auto& row = kb->at(y);
+      for (size_t x = 0; x < row.size(); x++) {
+        row[x] = key_r.get<U16T>();
+      }
+    }
+  }
+
+  uint32_t collections_offset = r.pget<U32T>(r.size() - 4);
+  for (uint32_t offset = collections_offset; !used_offsets.count(offset); offset += 4) {
+    used_offsets.emplace(r.pget<U32T>(offset));
+  }
+  used_offsets.emplace(collections_offset);
+
+  for (uint32_t offset = collections_offset; (offset == collections_offset) || !used_offsets.count(offset); offset += 4) {
+    auto& collection = this->collections.emplace_back();
+    uint32_t first_string_offset_offset = r.pget<U32T>(offset);
+    for (uint32_t string_offset_offset = first_string_offset_offset;
+         (string_offset_offset == first_string_offset_offset) || !used_offsets.count(string_offset_offset);
+         string_offset_offset += 4) {
+      collection.emplace_back(r.pget_cstr(r.pget<U32T>(string_offset_offset)));
+    }
+  }
+}
+
+template <bool IsBigEndian>
+pair<string, string> BinaryTextAndKeyboardsSet::serialize_t() const {
+  using U32T = std::conditional_t<IsBigEndian, be_uint32_t, le_uint32_t>;
+  using U16T = std::conditional_t<IsBigEndian, be_uint16_t, le_uint16_t>;
+
+  StringWriter w;
+  ::set<size_t> relocation_offsets;
+  auto put_offset_u32 = [&](uint32_t v) {
+    relocation_offsets.emplace(w.size());
+    w.put<U32T>(v);
+  };
+
+  uint32_t collections_offset;
+  {
+    unordered_map<string, uint32_t> string_to_offset;
+    for (const auto& collection : this->collections) {
+      for (const auto& s : collection) {
+        if (string_to_offset.emplace(s, w.size()).second) {
+          w.write(s);
+          w.put_u8(0);
+          while (w.size() & 3) {
+            w.put_u8(0);
+          }
+        }
+      }
+    }
+
+    vector<uint32_t> collection_offsets;
+    for (const auto& collection : this->collections) {
+      collection_offsets.emplace_back(w.size());
+      for (const auto& s : collection) {
+        put_offset_u32(string_to_offset.at(s));
+      }
+    }
+
+    collections_offset = w.size();
+    for (uint32_t collection_offset : collection_offsets) {
+      put_offset_u32(collection_offset);
+    }
+  }
+
+  uint32_t keyboard_index_offset;
+  {
+    vector<uint32_t> keyboard_offsets;
+    for (const auto& keyboard : this->keyboards) {
+      keyboard_offsets.emplace_back(w.size());
+      for (size_t y = 0; y < keyboard->size(); y++) {
+        const auto& row = keyboard->at(y);
+        for (size_t x = 0; x < row.size(); x++) {
+          w.put<U16T>(row[x]);
+        }
+      }
+    }
+
+    uint32_t keyboards_offset = w.size();
+    for (uint32_t keyboard_offset : keyboard_offsets) {
+      put_offset_u32(keyboard_offset);
+    }
+
+    keyboard_index_offset = w.size();
+    w.put_u8(keyboard_offsets.size());
+    w.put_u8(this->keyboard_selector_width);
+    w.put_u16(0);
+    put_offset_u32(keyboards_offset);
+  }
+
+  put_offset_u32(keyboard_index_offset);
+  put_offset_u32(collections_offset);
+
+  StringWriter reloc_w;
+  reloc_w.put_u32(0);
+  reloc_w.put<U32T>(relocation_offsets.size());
+  reloc_w.put_u64(0);
+  reloc_w.put<U32T>(w.size() - 8);
+  reloc_w.put_u32(0);
+  reloc_w.put_u64(0);
+  {
+    size_t offset = 0;
+    for (size_t reloc_offset : relocation_offsets) {
+      if (reloc_offset & 3) {
+        throw logic_error("misaligned relocation");
+      }
+      size_t num_words = (reloc_offset - offset) >> 2;
+      if (num_words > 0xFFFF) {
+        throw runtime_error("relocation offset too far away");
+      }
+      reloc_w.put<U16T>(num_words);
+      offset = reloc_offset;
+    }
+  }
+
+  const string& pr2_data = w.str();
+  const string& pr3_data = reloc_w.str();
+  print_data(stderr, pr2_data);
+  string pr2_compressed = prs_compress_optimal(pr2_data.data(), pr2_data.size());
+  string pr3_compressed = prs_compress_optimal(pr3_data.data(), pr3_data.size());
+  print_data(stderr, pr2_compressed);
+  string pr2_ret = encrypt_pr2_data<IsBigEndian>(pr2_compressed, pr2_data.size(), random_object<uint32_t>());
+  string pr3_ret = encrypt_pr2_data<IsBigEndian>(pr3_compressed, pr3_data.size(), random_object<uint32_t>());
+  print_data(stderr, pr2_ret);
+  return make_pair(std::move(pr2_ret), std::move(pr3_ret));
+}
+
+TextIndex::TextIndex(
+    const string& directory,
+    function<shared_ptr<const string>(Version, const string&)> get_patch_file)
+    : log("[TextIndex] ", static_game_data_log.min_level) {
+  if (!directory.empty()) {
+    auto add_version = [&](Version version, const string& subdirectory, function<shared_ptr<TextSet>(const string&)> make_set) -> void {
+      static const map<string, uint8_t> bintext_filenames({
+          {"TextJapanese.pr2", 0x00},
+          {"TextEnglish.pr2", 0x01},
+          {"TextGerman.pr2", 0x02},
+          {"TextFrench.pr2", 0x03},
+          {"TextSpanish.pr2", 0x04},
+      });
+      static const map<string, uint8_t> unitext_filenames({
+          {"unitxt_j.prs", 0x00}, // PC/BB Japanese
+          {"unitxt_e.prs", 0x01}, // PC/BB English
+          {"unitxt_g.prs", 0x02}, // PC/BB German
+          {"unitxt_f.prs", 0x03}, // PC/BB French
+          {"unitxt_s.prs", 0x04}, // PC/BB Spanish
+          {"unitxt_b.prs", 0x05}, // PC Simplified Chinese
+          {"unitxt_cs.prs", 0x05}, // BB Simplified Chinese
+          {"unitxt_t.prs", 0x06}, // PC Traditional Chinese
+          {"unitxt_ct.prs", 0x06}, // BB Traditional Chinese
+          {"unitxt_k.prs", 0x07}, // PC Korean
+          {"unitxt_h.prs", 0x07}, // BB Korean
+      });
+      if (!uses_utf16(version)) {
+        for (const auto& it : bintext_filenames) {
+          string file_path = directory + "/" + subdirectory + "/" + it.first;
+          string json_path = file_path + ".json";
+          if (isfile(json_path)) {
+            this->log.info("Loading %s %c JSON text set from %s", name_for_enum(version), char_for_language_code(it.second), json_path.c_str());
+            this->add_set(version, it.second, make_shared<BinaryTextSet>(JSON::parse(load_file(json_path))));
+          } else if (isfile(file_path)) {
+            this->log.info("Loading %s %c binary text set from %s", name_for_enum(version), char_for_language_code(it.second), file_path.c_str());
+            this->add_set(version, it.second, make_set(load_file(file_path)));
+          }
+        }
+      } else {
+        for (const auto& it : unitext_filenames) {
+          string file_path = directory + "/" + subdirectory + "/" + it.first;
+          string json_path = file_path + ".json";
+          if (isfile(json_path)) {
+            this->log.info("Loading %s %c JSON text set from %s", name_for_enum(version), char_for_language_code(it.second), json_path.c_str());
+            this->add_set(version, it.second, make_shared<UnicodeTextSet>(JSON::parse(load_file(json_path))));
+          } else {
+            auto patch_file = get_patch_file ? get_patch_file(version, it.first) : nullptr;
+            if (patch_file) {
+              this->log.info("Loading %s %c Unicode text set from %s in patch tree", name_for_enum(version), char_for_language_code(it.second), it.first.c_str());
+              this->add_set(version, it.second, make_set(*patch_file));
+            } else {
+              if (isfile(file_path)) {
+                this->log.info("Loading %s %c Unicode text set from %s", name_for_enum(version), char_for_language_code(it.second), file_path.c_str());
+                this->add_set(version, it.second, make_set(load_file(file_path)));
+              }
+            }
+          }
+        }
+      }
+    };
+
+    auto make_binary_dc112000 = +[](const string& data) { return make_shared<BinaryTextSet>(data, 21, true); };
+    auto make_binary_dcnte_dcv1 = +[](const string& data) { return make_shared<BinaryTextSet>(data, 26, true); };
+    auto make_binary_dcv2 = +[](const string& data) { return make_shared<BinaryTextSet>(data, 37, false); };
+    auto make_binary_gc = +[](const string& data) { return make_shared<BinaryTextAndKeyboardsSet>(data, true); };
+    auto make_binary_xb = +[](const string& data) { return make_shared<BinaryTextAndKeyboardsSet>(data, false); };
+    auto make_unitxt = +[](const string& data) { return make_shared<UnicodeTextSet>(data); };
+
+    add_version(Version::DC_NTE, "dc-nte", make_binary_dcnte_dcv1);
+    add_version(Version::DC_V1_11_2000_PROTOTYPE, "dc-11-2000", make_binary_dc112000);
+    add_version(Version::DC_V1, "dc-v1", make_binary_dcnte_dcv1);
+    add_version(Version::DC_V2, "dc-v2", make_binary_dcv2);
+    add_version(Version::PC_NTE, "pc-nte", make_unitxt);
+    add_version(Version::PC_V2, "pc-v2", make_unitxt);
+    add_version(Version::GC_NTE, "gc-nte", make_binary_gc);
+    add_version(Version::GC_V3, "gc-v3", make_binary_gc);
+    add_version(Version::GC_EP3_NTE, "gc-ep3-nte", make_binary_gc);
+    add_version(Version::GC_EP3, "gc-ep3", make_binary_gc);
+    add_version(Version::XB_V3, "xb-v3", make_binary_xb);
+    add_version(Version::BB_V4, "bb-v4", make_unitxt);
+  }
+}
+
+void TextIndex::add_set(Version version, uint8_t language, std::shared_ptr<const TextSet> ts) {
+  this->sets[this->key_for_set(version, language)] = ts;
+}
+
+void TextIndex::delete_set(Version version, uint8_t language) {
+  this->sets.erase(this->key_for_set(version, language));
+}
+
+const std::string& TextIndex::get(Version version, uint8_t language, size_t collection_index, size_t string_index) const {
+  return this->get(version, language)->get(collection_index, string_index);
+}
+
+const std::vector<std::string>& TextIndex::get(Version version, uint8_t language, size_t collection_index) const {
+  return this->get(version, language)->get(collection_index);
+}
+
+std::shared_ptr<const TextSet> TextIndex::get(Version version, uint8_t language) const {
+  return this->sets.at(this->key_for_set(version, language));
+}
+
+uint32_t TextIndex::key_for_set(Version version, uint8_t language) {
+  return (static_cast<uint32_t>(version) << 8) | language;
+}

src/TextIndex.hh

@@ -0,0 +1,111 @@
+#pragma once
+
+#include <stdint.h>
+
+#include <phosg/JSON.hh>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "Text.hh"
+#include "Version.hh"
+
+class TextSet {
+public:
+  virtual ~TextSet() = default;
+  virtual JSON json() const;
+
+  size_t count(size_t collection_index) const;
+  size_t count() const;
+
+  const std::string& get(size_t collection, size_t index) const;
+  const std::vector<std::string>& get(size_t collection) const;
+
+  void set(size_t collection_index, size_t string_index, const std::string& data);
+  void set(size_t collection_index, size_t string_index, std::string&& data);
+  void set(size_t collection_index, const std::vector<std::string>& coll);
+  void set(size_t collection_index, std::vector<std::string>&& coll);
+
+  void truncate_collection(size_t collection, size_t new_entry_count);
+  void truncate(size_t new_collection_count);
+
+protected:
+  std::vector<std::vector<std::string>> collections;
+
+  TextSet() = default;
+  TextSet(const JSON& json);
+  TextSet(JSON&& json);
+
+  void ensure_slot_exists(size_t collection_index, size_t string_index);
+  void ensure_collection_exists(size_t collection_index);
+};
+
+class UnicodeTextSet : public TextSet {
+public:
+  explicit UnicodeTextSet(const JSON& json) : TextSet(json) {}
+  explicit UnicodeTextSet(JSON&& json) : TextSet(json) {}
+  explicit UnicodeTextSet(const std::string& unitxt_prs_data);
+  virtual ~UnicodeTextSet() = default;
+  std::string serialize() const;
+};
+
+class BinaryTextSet : public TextSet {
+public:
+  explicit BinaryTextSet(const JSON& json) : TextSet(json) {}
+  explicit BinaryTextSet(JSON&& json) : TextSet(json) {}
+  BinaryTextSet(const std::string& pr2_data, size_t collection_count, bool has_rel_footer);
+  ~BinaryTextSet() = default;
+  // TODO: Implement serialize functions
+};
+
+class BinaryTextAndKeyboardsSet : public TextSet {
+public:
+  using Keyboard = parray<parray<uint16_t, 0x10>, 7>;
+
+  explicit BinaryTextAndKeyboardsSet(const JSON& json);
+  explicit BinaryTextAndKeyboardsSet(JSON&& json);
+  BinaryTextAndKeyboardsSet(const std::string& pr2_data, bool big_endian);
+  ~BinaryTextAndKeyboardsSet() = default;
+
+  virtual JSON json() const;
+
+  const Keyboard& get_keyboard(size_t kb_index) const;
+  void set_keyboard(size_t kb_index, const Keyboard& kb);
+  void resize_keyboards(size_t num_keyboards);
+
+  uint8_t get_keyboard_selector_width() const;
+  void set_keyboard_selector_width(uint8_t width);
+
+  // Returns (pr2_data, pr3_data)
+  std::pair<std::string, std::string> serialize(bool big_endian) const;
+
+protected:
+  template <bool IsBigEndian>
+  void parse_t(const std::string& pr2_data);
+  template <bool IsBigEndian>
+  std::pair<std::string, std::string> serialize_t() const;
+
+  std::vector<std::unique_ptr<Keyboard>> keyboards;
+  uint8_t keyboard_selector_width;
+};
+
+class TextIndex {
+public:
+  explicit TextIndex(
+      const std::string& directory = "",
+      std::function<std::shared_ptr<const std::string>(Version, const std::string&)> get_patch_file = nullptr);
+  ~TextIndex() = default;
+
+  void add_set(Version version, uint8_t language, std::shared_ptr<const TextSet> ts);
+  void delete_set(Version version, uint8_t language);
+
+  const std::string& get(Version version, uint8_t language, size_t collection_index, size_t string_index) const;
+  const std::vector<std::string>& get(Version version, uint8_t language, size_t collection_index) const;
+  std::shared_ptr<const TextSet> get(Version version, uint8_t language) const;
+
+protected:
+  static uint32_t key_for_set(Version version, uint8_t language);
+
+  PrefixedLogger log;
+  std::unordered_map<uint32_t, std::shared_ptr<const TextSet>> sets;
+};

src/UnicodeTextSet.cc

@@ -1,75 +0,0 @@
-#include "UnicodeTextSet.hh"
-
-#include <phosg/Strings.hh>
-#include <string>
-#include <vector>
-
-#include "Compression.hh"
-
-using namespace std;
-
-vector<vector<string>> parse_unicode_text_set(const string& prs_data) {
-  string data = prs_decompress(prs_data);
-  StringReader r(data);
-
-  uint32_t num_collections = r.get_u32l();
-  deque<uint32_t> collection_sizes;
-  while (collection_sizes.size() < num_collections) {
-    collection_sizes.emplace_back(r.get_u32l());
-  }
-
-  vector<vector<string>> ret;
-  ret.reserve(collection_sizes.size());
-  while (!collection_sizes.empty()) {
-    uint32_t num_strings = collection_sizes.front();
-    collection_sizes.pop_front();
-
-    auto& strings = ret.emplace_back();
-    strings.reserve(num_strings);
-    while (strings.size() < num_strings) {
-      StringReader sub_r = r.sub(r.get_u32l());
-      StringWriter w;
-      for (uint16_t ch = sub_r.get_u16l(); ch != 0; ch = sub_r.get_u16l()) {
-        w.put_u16l(ch);
-      }
-      strings.emplace_back(tt_utf16_to_utf8(w.str()));
-    }
-  }
-  return ret;
-}
-
-string serialize_unicode_text_set(const vector<vector<string>>& collections) {
-  StringWriter header_w;
-  StringWriter data_w;
-
-  size_t total_num_strings = 0;
-  header_w.put_u32l(collections.size());
-  for (const auto& collection : collections) {
-    header_w.put_u32l(collection.size());
-    total_num_strings += collection.size();
-  }
-
-  unordered_map<string, uint32_t> encoded;
-
-  size_t data_base_offset = (total_num_strings * 4) + header_w.size();
-  for (const auto& collection : collections) {
-    for (const auto& s : collection) {
-      auto encoded_it = encoded.find(s);
-      if (encoded_it == encoded.end()) {
-        uint32_t offset = data_base_offset + data_w.size();
-        encoded.emplace(s, offset);
-        string s_utf16 = tt_utf8_to_utf16(s);
-        data_w.write(s_utf16.data(), s_utf16.size());
-        data_w.put_u16(0);
-        while (data_w.size() & 3) {
-          data_w.put_u8(0);
-        }
-      } else {
-        header_w.put_u32l(encoded_it->second);
-      }
-    }
-  }
-
-  header_w.write(data_w.str());
-  return std::move(header_w.str());
-}

src/UnicodeTextSet.hh

@@ -1,7 +0,0 @@
-#pragma once
-
-#include <string>
-#include <vector>
-
-std::vector<std::vector<std::string>> parse_unicode_text_set(const std::string& prs_data);
-std::string serialize_unicode_text_set(const std::vector<std::vector<std::string>>& collections);