diff --git a/src/ChatCommands.cc b/src/ChatCommands.cc index 988ee31e..fa7cb618 100644 --- a/src/ChatCommands.cc +++ b/src/ChatCommands.cc @@ -295,7 +295,7 @@ static void server_command_quest(shared_ptr c, const std::string& args) auto s = c->require_server_state(); auto l = c->require_lobby(); - auto q = s->quest_index_for_version(effective_version)->get(stoul(args)); + auto q = s->quest_index(effective_version)->get(stoul(args)); if (!q) { send_text_message(c, "$C6Quest not found"); } else { @@ -1611,7 +1611,7 @@ static void server_command_item(shared_ptr c, const std::string& args) { check_is_game(l, true); check_cheats_enabled(l, c); - ItemData item = s->item_name_index->parse_item_description(c->version(), args); + ItemData item = s->parse_item_description(c->version(), args); item.id = l->generate_item_id(c->lobby_client_id); if ((l->drop_mode == Lobby::DropMode::SERVER_PRIVATE) || (l->drop_mode == Lobby::DropMode::SERVER_DUPLICATE)) { @@ -1644,7 +1644,7 @@ static void proxy_command_item(shared_ptr ses, const bool set_drop = (!args.empty() && (args[0] == '!')); - ItemData item = s->item_name_index->parse_item_description(ses->version(), (set_drop ? args.substr(1) : args)); + ItemData item = s->parse_item_description(ses->version(), (set_drop ? args.substr(1) : args)); item.id = random_object() | 0x80000000; if (set_drop) { diff --git a/src/Client.cc b/src/Client.cc index 58adfe1f..81a1abb4 100644 --- a/src/Client.cc +++ b/src/Client.cc @@ -1021,3 +1021,46 @@ void Client::use_character_bank(int8_t index) { } } } + +void Client::print_inventory(FILE* stream) const { + auto p = this->character(); + shared_ptr name_index; + try { + name_index = this->require_server_state()->item_name_index(this->version()); + } catch (const runtime_error&) { + } + fprintf(stream, "[PlayerInventory] Meseta: %" PRIu32 "\n", p->disp.stats.meseta.load()); + fprintf(stream, "[PlayerInventory] %hhu items\n", p->inventory.num_items); + for (size_t x = 0; x < p->inventory.num_items; x++) { + const auto& item = p->inventory.items[x]; + auto hex = item.data.hex(); + if (name_index) { + auto name = name_index->describe_item(item.data); + fprintf(stream, "[PlayerInventory] %2zu: [+%08" PRIX32 "] %s (%s)\n", x, item.flags.load(), hex.c_str(), name.c_str()); + } else { + fprintf(stream, "[PlayerInventory] %2zu: [+%08" PRIX32 "] %s\n", x, item.flags.load(), hex.c_str()); + } + } +} + +void Client::print_bank(FILE* stream) const { + auto p = this->character(); + shared_ptr name_index; + try { + name_index = this->require_server_state()->item_name_index(this->version()); + } catch (const runtime_error&) { + } + fprintf(stream, "[PlayerBank] Meseta: %" PRIu32 "\n", p->bank.meseta.load()); + fprintf(stream, "[PlayerBank] %" PRIu32 " items\n", p->bank.num_items.load()); + for (size_t x = 0; x < p->bank.num_items; x++) { + const auto& item = p->bank.items[x]; + const char* present_token = item.present ? "" : " (missing present flag)"; + auto hex = item.data.hex(); + if (name_index) { + auto name = name_index->describe_item(item.data); + fprintf(stream, "[PlayerBank] %3zu: %s (%s) (x%hu)%s\n", x, hex.c_str(), name.c_str(), item.amount.load(), present_token); + } else { + fprintf(stream, "[PlayerBank] %3zu: %s (x%hu)%s\n", x, hex.c_str(), item.amount.load(), present_token); + } + } +} diff --git a/src/Client.hh b/src/Client.hh index c52b9e9a..d3819b82 100644 --- a/src/Client.hh +++ b/src/Client.hh @@ -349,6 +349,9 @@ public: void use_character_bank(int8_t bb_character_index); void use_default_bank(); + void print_inventory(FILE* stream) const; + void print_bank(FILE* stream) const; + private: // The overlay character data is used in battle and challenge modes, when // character data is temporarily replaced in-game. In other play modes and in diff --git a/src/Episode3/DataIndexes.cc b/src/Episode3/DataIndexes.cc index d12192be..95644020 100644 --- a/src/Episode3/DataIndexes.cc +++ b/src/Episode3/DataIndexes.cc @@ -764,7 +764,7 @@ string CardDefinition::Effect::str_for_arg(const string& arg) { } } -string CardDefinition::Effect::str(const char* separator, const TextArchive* text_archive) const { +string CardDefinition::Effect::str(const char* separator, const TextSet* text_archive) const { vector tokens; tokens.emplace_back(string_printf("%hhu:", this->effect_num)); { @@ -802,7 +802,7 @@ string CardDefinition::Effect::str(const char* separator, const TextArchive* tex const char* name = nullptr; if (this->name_index && text_archive) { try { - name = text_archive->get_string(45, this->name_index).c_str(); + name = text_archive->get(45, this->name_index).c_str(); } catch (const exception&) { } } @@ -1061,7 +1061,7 @@ static const char* name_for_assist_ai_param_target(uint8_t target) { } } -string CardDefinition::str(bool single_line, const TextArchive* text_archive) const { +string CardDefinition::str(bool single_line, const TextSet* text_archive) const { string type_str; try { type_str = name_for_card_type(this->type); diff --git a/src/Episode3/DataIndexes.hh b/src/Episode3/DataIndexes.hh index 9e44f51c..727adc1d 100644 --- a/src/Episode3/DataIndexes.hh +++ b/src/Episode3/DataIndexes.hh @@ -15,7 +15,7 @@ #include "../PlayerSubordinates.hh" #include "../Text.hh" -#include "../TextArchive.hh" +#include "../TextIndex.hh" namespace Episode3 { @@ -514,7 +514,7 @@ struct CardDefinition { bool is_empty() const; static std::string str_for_arg(const std::string& arg); - std::string str(const char* separator = ", ", const TextArchive* text_archive = nullptr) const; + std::string str(const char* separator = ", ", const TextSet* text_archive = nullptr) const; } __attribute__((packed)); /* 0000 */ be_uint32_t card_id; @@ -780,7 +780,7 @@ struct CardDefinition { CardClass card_class() const; void decode_range(); - std::string str(bool single_line = true, const TextArchive* text_archive = nullptr) const; + std::string str(bool single_line = true, const TextSet* text_archive = nullptr) const; } __attribute__((packed)); // 0x128 bytes in total struct CardDefinitionsFooter { diff --git a/src/ItemNameIndex.cc b/src/ItemNameIndex.cc index cfce7f0e..54a7748e 100644 --- a/src/ItemNameIndex.cc +++ b/src/ItemNameIndex.cc @@ -4,37 +4,67 @@ using namespace std; -ItemNameIndex::ItemNameIndex(JSON&& v2_names, JSON&& v3_names, JSON&& v4_names) { - auto get_or_create_meta = [&](uint32_t primary_identifier) { - shared_ptr meta; - try { - return this->primary_identifier_index.at(primary_identifier); - } catch (const out_of_range&) { - auto meta = make_shared(); - meta->primary_identifier = primary_identifier; - this->primary_identifier_index.emplace(primary_identifier, meta); - return meta; +// class ItemNameIndex { +// public: +// ItemNameIndex(std::shared_ptr pmt, const std::vector& name_coll); +// std::string describe_item(const ItemData& item, bool include_color_escapes = false) const; +// ItemData parse_item_description(const std::string& description) const; +// private: +// ItemData parse_item_description_phase(const std::string& description, bool skip_special) const; +// std::shared_ptr item_parameter_table; +// struct ItemMetadata { +// uint32_t primary_identifier; +// std::string name; +// }; +// std::unordered_map> primary_identifier_indexes; +// std::map> name_indexes; +// }; + +ItemNameIndex::ItemNameIndex( + std::shared_ptr item_parameter_table, + const std::vector& name_coll) + : item_parameter_table(item_parameter_table) { + + auto find_items_1d = [&](uint64_t data1, size_t position) -> size_t { + ItemData item(data1, 0); + for (size_t x = 0; x < 0x100; x++) { + item.data1[position] = x; + uint32_t id; + try { + id = this->item_parameter_table->get_item_id(item); + } catch (const out_of_range&) { + return x; + } + const string* name = nullptr; + try { + name = &name_coll.at(id); + } catch (const out_of_range&) { + } + + if (name) { + auto meta = make_shared(); + meta->primary_identifier = item.primary_identifier(); + meta->name = *name; + this->primary_identifier_index.emplace(meta->primary_identifier, meta); + this->name_index.emplace(tolower(meta->name), meta); + } + } + return 0x100; + }; + auto find_items_2d = [&](uint64_t data1) { + for (size_t x = 0; x < 0x100; x++) { + if (find_items_1d(data1 | (static_cast(x) << 48), 2) == 0) { + break; + } } }; - for (const auto& it : v2_names.as_dict()) { - uint32_t primary_identifier = stoul(it.first, nullptr, 16); - auto meta = get_or_create_meta(primary_identifier); - meta->v2_name = std::move(it.second->as_string()); - this->v2_name_index.emplace(tolower(meta->v2_name), meta); - } - for (const auto& it : v3_names.as_dict()) { - uint32_t primary_identifier = stoul(it.first, nullptr, 16); - auto meta = get_or_create_meta(primary_identifier); - meta->v3_name = std::move(it.second->as_string()); - this->v3_name_index.emplace(tolower(meta->v3_name), meta); - } - for (const auto& it : v4_names.as_dict()) { - uint32_t primary_identifier = stoul(it.first, nullptr, 16); - auto meta = get_or_create_meta(primary_identifier); - meta->v4_name = std::move(it.second->as_string()); - this->v4_name_index.emplace(tolower(meta->v4_name), meta); - } + find_items_2d(0x0000000000000000); + find_items_1d(0x0101000000000000, 2); + find_items_1d(0x0102000000000000, 2); + find_items_1d(0x0103000000000000, 2); + find_items_1d(0x0200000000000000, 1); + find_items_2d(0x0300000000000000); } static const char* s_rank_name_characters = "\0ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_"; @@ -106,11 +136,10 @@ const array name_for_s_rank_special = { }; std::string ItemNameIndex::describe_item( - Version version, const ItemData& item, - std::shared_ptr item_parameter_table) const { + bool include_color_escapes) const { if (item.data1[0] == 0x04) { - return string_printf("%s%" PRIu32 " Meseta", item_parameter_table ? "$C7" : "", item.data2d.load()); + return string_printf("%s%" PRIu32 " Meseta", include_color_escapes ? "$C7" : "", item.data2d.load()); } vector ret_tokens; @@ -172,18 +201,7 @@ std::string ItemNameIndex::describe_item( } else { try { auto meta = this->primary_identifier_index.at(primary_identifier); - const string* name; - if (is_v4(version)) { - name = &meta->v4_name; - } else if (is_v3(version)) { - name = &meta->v3_name; - } else { - name = &meta->v2_name; - } - if (name->empty()) { - throw out_of_range("item does not exist"); - } - ret_tokens.emplace_back(*name); + ret_tokens.emplace_back(meta->name); } catch (const out_of_range&) { ret_tokens.emplace_back(string_printf("!ID:%06" PRIX32, primary_identifier)); @@ -347,10 +365,10 @@ std::string ItemNameIndex::describe_item( } string ret = join(ret_tokens, " "); - if (item_parameter_table) { + if (include_color_escapes) { if (item.is_s_rank_weapon()) { return "$C4" + ret; - } else if (item_parameter_table->is_item_rare(item)) { + } else if (this->item_parameter_table->is_item_rare(item)) { return "$C6" + ret; } else if (item.has_bonuses()) { return "$C2" + ret; @@ -362,23 +380,23 @@ std::string ItemNameIndex::describe_item( } } -ItemData ItemNameIndex::parse_item_description(Version version, const std::string& desc) const { +ItemData ItemNameIndex::parse_item_description(const std::string& desc) const { ItemData ret; try { - ret = this->parse_item_description_phase(version, desc, false); + ret = this->parse_item_description_phase(desc, false); } catch (const exception& e1) { try { - ret = this->parse_item_description_phase(version, desc, true); + ret = this->parse_item_description_phase(desc, true); } catch (const exception& e2) { try { ret = ItemData::from_data(parse_data_string(desc)); } catch (const exception& ed) { if (strcmp(e1.what(), e2.what())) { - throw runtime_error(string_printf("cannot parse item description \"%s\" in %s (as text 1: %s) (as text 2: %s) (as data: %s)", - desc.c_str(), name_for_enum(version), e1.what(), e2.what(), ed.what())); + throw runtime_error(string_printf("cannot parse item description \"%s\" (as text 1: %s) (as text 2: %s) (as data: %s)", + desc.c_str(), e1.what(), e2.what(), ed.what())); } else { - throw runtime_error(string_printf("cannot parse item description \"%s\" in %s (as text: %s) (as data: %s)", - desc.c_str(), name_for_enum(version), e1.what(), ed.what())); + throw runtime_error(string_printf("cannot parse item description \"%s\" (as text: %s) (as data: %s)", + desc.c_str(), e1.what(), ed.what())); } } } @@ -387,7 +405,7 @@ ItemData ItemNameIndex::parse_item_description(Version version, const std::strin return ret; } -ItemData ItemNameIndex::parse_item_description_phase(Version version, const std::string& description, bool skip_special) const { +ItemData ItemNameIndex::parse_item_description_phase(const std::string& description, bool skip_special) const { ItemData ret; ret.data1d.clear(0); ret.id = 0xFFFFFFFF; @@ -448,24 +466,15 @@ ItemData ItemNameIndex::parse_item_description_phase(Version version, const std: } } - const map>* name_index; - if (is_v4(version)) { - name_index = &this->v4_name_index; - } else if (is_v3(version)) { - name_index = &this->v3_name_index; - } else { - name_index = &this->v2_name_index; - } - - auto name_it = name_index->lower_bound(desc); + auto name_it = this->name_index.lower_bound(desc); // Look up to 3 places before the lower bound. We have to do this to catch // cases like Sange vs. Sange & Yasha - if the input is like "Sange 0/...", // then we'll see Sange & Yasha first, which we should skip. size_t lookback = 0; while (lookback < 4) { - if (name_it != name_index->end() && desc.starts_with(name_it->first)) { + if (name_it != this->name_index.end() && desc.starts_with(name_it->first)) { break; - } else if (name_it == name_index->begin()) { + } else if (name_it == this->name_index.begin()) { throw runtime_error("no such item"); } else { name_it--; diff --git a/src/ItemNameIndex.hh b/src/ItemNameIndex.hh index 26cee92f..15cf76a9 100644 --- a/src/ItemNameIndex.hh +++ b/src/ItemNameIndex.hh @@ -14,26 +14,32 @@ class ItemNameIndex { public: - ItemNameIndex(JSON&& v2_names, JSON&& v3_names, JSON&& v4_names); - - std::string describe_item( - Version version, - const ItemData& item, - std::shared_ptr item_parameter_table = nullptr) const; - ItemData parse_item_description(Version version, const std::string& description) const; - -private: - ItemData parse_item_description_phase(Version version, const std::string& description, bool skip_special) const; - struct ItemMetadata { uint32_t primary_identifier; - std::string v2_name; - std::string v3_name; - std::string v4_name; + std::string name; }; - std::unordered_map> primary_identifier_index; - std::map> v2_name_index; - std::map> v3_name_index; - std::map> v4_name_index; + ItemNameIndex(std::shared_ptr pmt, const std::vector& name_coll); + + inline size_t entry_count() const { + return this->primary_identifier_index.size(); + } + + inline const std::unordered_map>& all_by_primary_identifier() const { + return this->primary_identifier_index; + } + inline const std::map>& all_by_name() const { + return this->name_index; + } + + std::string describe_item(const ItemData& item, bool include_color_escapes = false) const; + ItemData parse_item_description(const std::string& description) const; + +private: + ItemData parse_item_description_phase(const std::string& description, bool skip_special) const; + + std::shared_ptr item_parameter_table; + + std::unordered_map> primary_identifier_index; + std::map> name_index; }; diff --git a/src/ItemParameterTable.cc b/src/ItemParameterTable.cc index 5dc747fc..af76cdb4 100644 --- a/src/ItemParameterTable.cc +++ b/src/ItemParameterTable.cc @@ -52,13 +52,13 @@ ItemParameterTable::ItemParameterTable(shared_ptr data, Version ve const ItemParameterTable::WeaponV4& ItemParameterTable::get_weapon(uint8_t data1_1, uint8_t data1_2) const { if (data1_1 >= this->num_weapon_classes) { - throw runtime_error("weapon ID out of range"); + throw out_of_range("weapon ID out of range"); } if (this->offsets_v4) { const auto& co = this->r.pget(this->offsets_v4->weapon_table + sizeof(ArrayRefLE) * data1_1); if (data1_2 >= co.count) { - throw runtime_error("weapon ID out of range"); + throw out_of_range("weapon ID out of range"); } return this->r.pget(co.offset + sizeof(WeaponV4) * data1_2); } @@ -67,12 +67,12 @@ const ItemParameterTable::WeaponV4& ItemParameterTable::get_weapon(uint8_t data1 try { return this->parsed_weapons.at(key); } catch (const std::out_of_range&) { - auto& def_v4 = this->parsed_weapons.emplace(key, WeaponV4{}).first->second; + WeaponV4 def_v4; if (this->offsets_v2) { const auto& co = this->r.pget(this->offsets_v2->weapon_table + sizeof(ArrayRefLE) * data1_1); if (data1_2 >= co.count) { - throw runtime_error("weapon ID out of range"); + throw out_of_range("weapon ID out of range"); } const auto& def_v2 = this->r.pget(co.offset + sizeof(WeaponV2) * data1_2); def_v4.base.id = def_v2.base.id; @@ -90,7 +90,7 @@ const ItemParameterTable::WeaponV4& ItemParameterTable::get_weapon(uint8_t data1 } else if (this->offsets_v3) { const auto& co = this->r.pget(this->offsets_v3->weapon_table + sizeof(ArrayRefBE) * data1_1); if (data1_2 >= co.count) { - throw runtime_error("weapon ID out of range"); + throw out_of_range("weapon ID out of range"); } const auto& def_v3 = this->r.pget>(co.offset + sizeof(WeaponV3) * data1_2); def_v4.base.id = def_v3.base.id.load(); @@ -126,19 +126,19 @@ const ItemParameterTable::WeaponV4& ItemParameterTable::get_weapon(uint8_t data1 throw logic_error("table is not v2, v3, or v4"); } - return def_v4; + return this->parsed_weapons.emplace(key, def_v4).first->second; } } const ItemParameterTable::ArmorOrShieldV4& ItemParameterTable::get_armor_or_shield(uint8_t data1_1, uint8_t data1_2) const { if ((data1_1 < 1) || (data1_1 > 2)) { - throw runtime_error("armor/shield class ID out of range"); + throw out_of_range("armor/shield class ID out of range"); } if (this->offsets_v4) { const auto& co = this->r.pget(this->offsets_v4->armor_table + sizeof(ArrayRefLE) * (data1_1 - 1)); if (data1_2 >= co.count) { - throw runtime_error("armor/shield ID out of range"); + throw out_of_range("armor/shield ID out of range"); } return this->r.pget(co.offset + sizeof(ArmorOrShieldV4) * data1_2); } @@ -151,15 +151,12 @@ const ItemParameterTable::ArmorOrShieldV4& ItemParameterTable::get_armor_or_shie } return ret; } catch (const std::out_of_range&) { - if (data1_2 >= parsed_vec.size()) { - parsed_vec.resize(data1_2 + 1); - } - auto& def_v4 = parsed_vec[data1_2]; + ArmorOrShieldV4 def_v4; if (this->offsets_v2) { const auto& co = this->r.pget(this->offsets_v2->armor_table + sizeof(ArrayRefLE) * (data1_1 - 1)); if (data1_2 >= co.count) { - throw runtime_error("armor/shield ID out of range"); + throw out_of_range("armor/shield ID out of range"); } const auto& def_v2 = this->r.pget(co.offset + sizeof(ArmorOrShieldV2) * data1_2); def_v4.base.id = def_v2.base.id; @@ -183,7 +180,7 @@ const ItemParameterTable::ArmorOrShieldV4& ItemParameterTable::get_armor_or_shie } else if (this->offsets_v3) { const auto& co = this->r.pget(this->offsets_v3->armor_table + sizeof(ArrayRefBE) * (data1_1 - 1)); if (data1_2 >= co.count) { - throw runtime_error("armor/shield ID out of range"); + throw out_of_range("armor/shield ID out of range"); } const auto& def_v3 = this->r.pget(co.offset + sizeof(ArmorOrShieldV3) * data1_2); def_v4.base.id = def_v3.base.id.load(); @@ -210,7 +207,11 @@ const ItemParameterTable::ArmorOrShieldV4& ItemParameterTable::get_armor_or_shie throw logic_error("table is not v2, v3, or v4"); } - return def_v4; + if (data1_2 >= parsed_vec.size()) { + parsed_vec.resize(data1_2 + 1); + } + parsed_vec[data1_2] = def_v4; + return parsed_vec[data1_2]; } } @@ -218,7 +219,7 @@ const ItemParameterTable::UnitV4& ItemParameterTable::get_unit(uint8_t data1_2) if (this->offsets_v4) { const auto& co = this->r.pget(this->offsets_v4->unit_table); if (data1_2 >= co.count) { - throw runtime_error("unit ID out of range"); + throw out_of_range("unit ID out of range"); } return this->r.pget(co.offset + sizeof(UnitV4) * data1_2); } @@ -230,15 +231,12 @@ const ItemParameterTable::UnitV4& ItemParameterTable::get_unit(uint8_t data1_2) } return ret; } catch (const std::out_of_range&) { - if (data1_2 >= this->parsed_units.size()) { - this->parsed_units.resize(data1_2 + 1); - } - auto& def_v4 = this->parsed_units[data1_2]; + UnitV4 def_v4; if (this->offsets_v2) { const auto& co = this->r.pget(this->offsets_v2->unit_table); if (data1_2 >= co.count) { - throw runtime_error("unit ID out of range"); + throw out_of_range("unit ID out of range"); } const auto& def_v2 = this->r.pget(co.offset + sizeof(UnitV2) * data1_2); def_v4.base.id = def_v2.base.id; @@ -249,7 +247,7 @@ const ItemParameterTable::UnitV4& ItemParameterTable::get_unit(uint8_t data1_2) } else if (this->offsets_v3) { const auto& co = this->r.pget(this->offsets_v3->unit_table); if (data1_2 >= co.count) { - throw runtime_error("unit ID out of range"); + throw out_of_range("unit ID out of range"); } const auto& def_v3 = this->r.pget(co.offset + sizeof(UnitV3) * data1_2); def_v4.base.id = def_v3.base.id.load(); @@ -263,7 +261,11 @@ const ItemParameterTable::UnitV4& ItemParameterTable::get_unit(uint8_t data1_2) throw logic_error("table is not v2, v3, or v4"); } - return def_v4; + if (data1_2 >= this->parsed_units.size()) { + this->parsed_units.resize(data1_2 + 1); + } + this->parsed_units[data1_2] = def_v4; + return this->parsed_units[data1_2]; } } @@ -271,7 +273,7 @@ const ItemParameterTable::MagV4& ItemParameterTable::get_mag(uint8_t data1_1) co if (this->offsets_v4) { const auto& co = this->r.pget(this->offsets_v4->mag_table); if (data1_1 >= co.count) { - throw runtime_error("mag ID out of range"); + throw out_of_range("mag ID out of range"); } return this->r.pget(co.offset + sizeof(MagV4) * data1_1); } @@ -283,15 +285,12 @@ const ItemParameterTable::MagV4& ItemParameterTable::get_mag(uint8_t data1_1) co } return ret; } catch (const std::out_of_range&) { - if (data1_1 >= this->parsed_mags.size()) { - this->parsed_mags.resize(data1_1 + 1); - } - auto& def_v4 = this->parsed_mags[data1_1]; + MagV4 def_v4; if (this->offsets_v2) { const auto& co = this->r.pget(this->offsets_v2->mag_table); if (data1_1 >= co.count) { - throw runtime_error("mag ID out of range"); + throw out_of_range("mag ID out of range"); } const auto& def_v2 = this->r.pget(co.offset + sizeof(MagV2) * data1_1); def_v4.base.id = def_v2.base.id; @@ -311,7 +310,7 @@ const ItemParameterTable::MagV4& ItemParameterTable::get_mag(uint8_t data1_1) co } else if (this->offsets_v3) { const auto& co = this->r.pget(this->offsets_v3->mag_table); if (data1_1 >= co.count) { - throw runtime_error("mag ID out of range"); + throw out_of_range("mag ID out of range"); } const auto& def_v3 = this->r.pget(co.offset + sizeof(MagV3) * data1_1); def_v4.base.id = def_v3.base.id.load(); @@ -334,19 +333,23 @@ const ItemParameterTable::MagV4& ItemParameterTable::get_mag(uint8_t data1_1) co throw logic_error("table is not v2, v3, or v4"); } - return def_v4; + if (data1_1 >= this->parsed_mags.size()) { + this->parsed_mags.resize(data1_1 + 1); + } + this->parsed_mags[data1_1] = def_v4; + return this->parsed_mags[data1_1]; } } const ItemParameterTable::ToolV4& ItemParameterTable::get_tool(uint8_t data1_1, uint8_t data1_2) const { if (data1_1 >= this->num_tool_classes) { - throw runtime_error("tool class ID out of range"); + throw out_of_range("tool class ID out of range"); } if (this->offsets_v4) { const auto& co = this->r.pget(this->offsets_v4->tool_table + sizeof(ArrayRefLE) * data1_1); if (data1_2 >= co.count) { - throw runtime_error("tool ID out of range"); + throw out_of_range("tool ID out of range"); } return this->r.pget(co.offset + sizeof(ToolV4) * data1_2); } @@ -355,12 +358,12 @@ const ItemParameterTable::ToolV4& ItemParameterTable::get_tool(uint8_t data1_1, try { return this->parsed_tools.at(key); } catch (const std::out_of_range&) { - auto& def_v4 = this->parsed_tools.emplace(key, ToolV4{}).first->second; + ToolV4 def_v4; if (this->offsets_v2) { const auto& co = this->r.pget(this->offsets_v2->tool_table + sizeof(ArrayRefLE) * data1_1); if (data1_2 >= co.count) { - throw runtime_error("tool ID out of range"); + throw out_of_range("tool ID out of range"); } const auto& def_v2 = this->r.pget(co.offset + sizeof(ToolV2) * data1_2); def_v4.base.id = def_v2.base.id; @@ -372,7 +375,7 @@ const ItemParameterTable::ToolV4& ItemParameterTable::get_tool(uint8_t data1_1, } else if (this->offsets_v3) { const auto& co = this->r.pget(this->offsets_v3->tool_table + sizeof(ArrayRefBE) * data1_1); if (data1_2 >= co.count) { - throw runtime_error("tool ID out of range"); + throw out_of_range("tool ID out of range"); } const auto& def_v3 = this->r.pget(co.offset + sizeof(ToolV3) * data1_2); def_v4.base.id = def_v3.base.id.load(); @@ -387,7 +390,7 @@ const ItemParameterTable::ToolV4& ItemParameterTable::get_tool(uint8_t data1_1, throw logic_error("table is not v2, v3, or v4"); } - return def_v4; + return this->parsed_tools.emplace(key, def_v4).first->second; } } @@ -472,10 +475,10 @@ float ItemParameterTable::get_sale_divisor(uint8_t data1_0, uint8_t data1_1) con const ItemParameterTable::MagFeedResult& ItemParameterTable::get_mag_feed_result( uint8_t table_index, uint8_t item_index) const { if (table_index >= 8) { - throw runtime_error("invalid mag feed table index"); + throw out_of_range("invalid mag feed table index"); } if (item_index >= 11) { - throw runtime_error("invalid mag feed item index"); + throw out_of_range("invalid mag feed item index"); } uint32_t offset; @@ -521,7 +524,7 @@ uint8_t ItemParameterTable::get_special_stars(uint8_t det) const { const ItemParameterTable::Special& ItemParameterTable::get_special(uint8_t special) const { special &= 0x3F; if (special >= this->num_specials) { - throw runtime_error("invalid special index"); + throw out_of_range("invalid special index"); } if (this->offsets_v2) { @@ -545,10 +548,10 @@ const ItemParameterTable::Special& ItemParameterTable::get_special(uint8_ uint8_t ItemParameterTable::get_max_tech_level(uint8_t char_class, uint8_t tech_num) const { if (char_class >= 12) { - throw runtime_error("invalid character class"); + throw out_of_range("invalid character class"); } if (tech_num >= 19) { - throw runtime_error("invalid technique number"); + throw out_of_range("invalid technique number"); } if (this->offsets_v2) { @@ -762,7 +765,7 @@ std::pair ItemParameterTable::get_ uint32_t base_offset, uint8_t event_number) const { const auto& co = this->r.pget>(base_offset); if (event_number >= co.count) { - throw runtime_error("invalid event number"); + throw out_of_range("invalid event number"); } const auto& event_co = this->r.pget>(co.offset + sizeof(ArrayRef) * event_number); const auto* defs = &this->r.pget(event_co.offset, event_co.count * sizeof(EventItem)); diff --git a/src/ItemParameterTable.hh b/src/ItemParameterTable.hh index 840bff20..68ada94d 100644 --- a/src/ItemParameterTable.hh +++ b/src/ItemParameterTable.hh @@ -369,6 +369,9 @@ private: /* 40 / 5704 */ le_uint32_t shield_effect_table; // -> [8-byte structs] } __attribute__((packed)); + // TODO: The GC NTE ItemPMT format is intermediate between V2 and V3 - the + // Offsets struct is 0x50 bytes. Figure it out and add support here. + template struct TableOffsetsV3V4 { using U32T = typename std::conditional::type; diff --git a/src/Items.cc b/src/Items.cc index 99d6421f..f3b09bb5 100644 --- a/src/Items.cc +++ b/src/Items.cc @@ -23,7 +23,7 @@ void player_use_item(shared_ptr c, size_t item_index, shared_ptritem_parameter_table_for_version(c->version()); + auto item_parameter_table = s->item_parameter_table(c->version()); uint8_t max_level = item_parameter_table->get_max_tech_level(player->disp.visual.char_class, item.data.data1[4]); if (item.data.data1[2] > max_level) { throw runtime_error("technique level too high"); @@ -43,7 +43,7 @@ void player_use_item(shared_ptr c, size_t item_index, shared_ptritem_parameter_table_for_version(c->version()); + auto item_parameter_table = s->item_parameter_table(c->version()); auto weapon_def = item_parameter_table->get_weapon(weapon.data.data1[1], weapon.data.data1[2]); if (weapon.data.data1[3] >= weapon_def.max_grind) { throw runtime_error("weapon already at maximum grind"); @@ -168,7 +168,7 @@ void player_use_item(shared_ptr c, size_t item_index, shared_ptritem_parameter_table_for_version(c->version()); + auto item_parameter_table = s->item_parameter_table(c->version()); auto table = item_parameter_table->get_event_items(item.data.data1[2]); size_t sum = 0; for (size_t z = 0; z < table.second; z++) { @@ -207,7 +207,7 @@ void player_use_item(shared_ptr c, size_t item_index, shared_ptritem_parameter_table_for_version(c->version()); + auto item_parameter_table = s->item_parameter_table(c->version()); const auto& combo = item_parameter_table->get_item_combination(item.data, inv_item.data); if (combo.char_class != 0xFF && combo.char_class != player->disp.visual.char_class) { throw runtime_error("item combination requires specific char_class"); @@ -497,7 +497,7 @@ void player_feed_mag(std::shared_ptr c, size_t mag_item_index, size_t fe apply_mag_feed_result( player->inventory.items[mag_item_index].data, player->inventory.items[fed_item_index].data, - s->item_parameter_table_for_version(c->version()), + s->item_parameter_table(c->version()), s->mag_evolution_table, player->disp.visual.char_class, player->disp.visual.section_id, diff --git a/src/Lobby.cc b/src/Lobby.cc index 7e2abc77..c7760386 100644 --- a/src/Lobby.cc +++ b/src/Lobby.cc @@ -251,7 +251,7 @@ void Lobby::create_item_creator() { s->tool_random_set, s->weapon_random_sets.at(this->difficulty), s->tekker_adjustment_set, - s->item_parameter_table_for_version(this->base_version), + s->item_parameter_table(this->base_version), this->base_version, this->episode, (this->mode == GameMode::SOLO) ? GameMode::NORMAL : this->mode, @@ -351,7 +351,7 @@ void Lobby::load_maps() { this->log.info("Generated objects list (%zu entries):", this->map->objects.size()); for (size_t z = 0; z < this->map->objects.size(); z++) { - string o_str = this->map->objects[z].str(s->item_name_index); + string o_str = this->map->objects[z].str(); this->log.info("(K-%zX) %s", z, o_str.c_str()); } this->log.info("Generated enemies list (%zu entries):", this->map->enemies.size()); diff --git a/src/Main.cc b/src/Main.cc index d9e4b539..7a519477 100644 --- a/src/Main.cc +++ b/src/Main.cc @@ -43,8 +43,7 @@ #include "ServerState.hh" #include "StaticGameData.hh" #include "Text.hh" -#include "TextArchive.hh" -#include "UnicodeTextSet.hh" +#include "TextIndex.hh" using namespace std; @@ -955,7 +954,7 @@ Action a_decode_gci( auto decoded = decode_gci_data(read_input_data(args), num_threads, dec_seed, skip_checksum); save_file(input_filename + ".dec", decoded); }); -Action a_decode_vmg( +Action a_decode_vms( "decode-vms", nullptr, +[](Arguments& args) { string input_filename = args.get(1, false); if (input_filename.empty() || (input_filename == "-")) { @@ -1171,25 +1170,35 @@ Action a_decode_sjis( }); Action a_decode_text_archive( - "decode-text-archive", nullptr, +[](Arguments& args) { + "decode-text-archive", "\ + decode-text-archive [INPUT-FILENAME [OUTPUT-FILENAME]]\n\ + Decode a text archive to JSON. --collections=NUM_COLLECTIONS is given,\n\ + expects a fixed number of collections in the input. If --has-pr3 is given,\n\ + expects the input not to have a REL footer.\n", + +[](Arguments& args) { string data = read_input_data(args); - TextArchive a(data, args.get("big-endian")); - JSON j = a.json(); + + unique_ptr ts; + size_t collection_count = args.get("collections", 0); + if (collection_count) { + ts = make_unique(data, collection_count, !args.get("has-pr3")); + } else { + ts = make_unique(data, args.get("big-endian")); + } + JSON j = ts->json(); string out_data = j.serialize(JSON::SerializeOption::FORMAT); write_output_data(args, out_data.data(), out_data.size(), "json"); }); Action a_encode_text_archive( "encode-text-archive", "\ decode-text-archive [INPUT-FILENAME [OUTPUT-FILENAME]]\n\ - encode-text-archive [INPUT-FILENAME [OUTPUT-FILENAME]]\n\ - Decode a text archive (e.g. TextEnglish.pr2) to JSON for easy editing, or\n\ - encode a JSON file to a text archive.\n", + Encode a text archive. Currently only supports GC and Xbox format.\n", +[](Arguments& args) { const string& input_filename = args.get(1, false); const string& output_filename = args.get(2, false); auto json = JSON::parse(read_input_data(args)); - TextArchive a(json); + BinaryTextAndKeyboardsSet a(json); auto result = a.serialize(args.get("big-endian")); if (output_filename.empty()) { if (input_filename.empty() || (input_filename == "-")) { @@ -1214,14 +1223,8 @@ Action a_encode_text_archive( Action a_decode_unicode_text_set( "decode-unicode-text-set", nullptr, +[](Arguments& args) { - auto collections = parse_unicode_text_set(read_input_data(args)); - JSON j = JSON::list(); - for (const auto& collection : collections) { - JSON& coll_j = j.emplace_back(JSON::list()); - for (const auto& s : collection) { - coll_j.emplace_back(s); - } - } + UnicodeTextSet uts(read_input_data(args)); + JSON j = uts.json(); string out_data = j.serialize(JSON::SerializeOption::FORMAT); write_output_data(args, out_data.data(), out_data.size(), "json"); }); @@ -1232,15 +1235,8 @@ Action a_encode_unicode_text_set( Decode a Unicode text set (e.g. unitxt_e.prs) to JSON for easy editing, or\n\ encode a JSON file to a Unicode text set.\n", +[](Arguments& args) { - auto json = JSON::parse(read_input_data(args)); - vector> collections; - for (const auto& coll_json : json.as_list()) { - auto& collection = collections.emplace_back(); - for (const auto& s_json : coll_json->as_list()) { - collection.emplace_back(std::move(s_json->as_string())); - } - } - string encoded = serialize_unicode_text_set(collections); + UnicodeTextSet uts(JSON::parse(read_input_data(args))); + string encoded = uts.serialize(); write_output_data(args, encoded.data(), encoded.size(), "prs"); }); @@ -1256,30 +1252,22 @@ Action a_decode_word_select_set( auto version = get_cli_version(args); string unitxt_filename = args.get("unitxt"); - vector unitxt_collection; + const vector* unitxt_collection; if (!unitxt_filename.empty()) { - vector> unitxt_data; + unique_ptr uts; if (ends_with(unitxt_filename, ".prs")) { - unitxt_data = parse_unicode_text_set(load_file(unitxt_filename)); + uts = make_unique(load_file(unitxt_filename)); } else if (ends_with(unitxt_filename, ".json")) { - auto json = JSON::parse(load_file(unitxt_filename)); - for (const auto& coll_it : json.as_list()) { - auto& coll = unitxt_data.emplace_back(); - for (const auto& str_it : coll_it->as_list()) { - coll.emplace_back(str_it->as_string()); - } - } + uts = make_unique(JSON::parse(load_file(unitxt_filename))); } else { throw runtime_error("unitxt filename must end in .prs or .json"); } - if (version == Version::BB_V4) { - unitxt_collection = std::move(unitxt_data.at(0)); - } else { - unitxt_collection = std::move(unitxt_data.at(35)); - } + unitxt_collection = &uts->get((version == Version::BB_V4) ? 0 : 35); + } else { + unitxt_collection = nullptr; } - WordSelectSet ws(read_input_data(args), version, &unitxt_collection, args.get("japanese")); + WordSelectSet ws(read_input_data(args), version, unitxt_collection, args.get("japanese")); ws.print(stdout); }); Action a_print_word_select_table( @@ -1289,17 +1277,18 @@ Action a_print_word_select_table( given, prints the table sorted by token ID for that version. If no version\n\ option is given, prints the token table sorted by canonical name.\n", +[](Arguments& args) { - auto table = ServerState::load_word_select_table_from_system(); - Version v = Version::UNKNOWN; + ServerState s; + s.load_objects("word_select_table"); + Version v; try { v = get_cli_version(args); } catch (const runtime_error&) { + v = Version::UNKNOWN; } - if (v != Version::UNKNOWN) { - table->print_index(stdout, v); + s.word_select_table->print_index(stdout, v); } else { - table->print(stdout); + s.word_select_table->print(stdout); } }); @@ -1343,21 +1332,20 @@ Action a_convert_rare_item_set( .afs (PSO V2 little-endian AFS archive)\n\ .rel (Schtserv rare table; cannot be used in output filename)\n", +[](Arguments& args) { - auto name_index = make_shared( - JSON::parse(load_file("system/item-tables/names-v2.json")), - JSON::parse(load_file("system/item-tables/names-v3.json")), - JSON::parse(load_file("system/item-tables/names-v4.json"))); + auto version = get_cli_version(args); + + ServerState s; + s.load_objects("item_name_indexes"); string input_filename = args.get(1, false); if (input_filename.empty() || (input_filename == "-")) { throw runtime_error("input filename must be given"); } - auto version = get_cli_version(args); auto data = make_shared(read_input_data(args)); shared_ptr rs; if (ends_with(input_filename, ".json")) { - rs = make_shared(JSON::parse(*data), version, name_index); + rs = make_shared(JSON::parse(*data), s.item_name_index(version)); } else if (ends_with(input_filename, ".gsl")) { rs = make_shared(GSLArchive(data, false), false); } else if (ends_with(input_filename, ".gslb")) { @@ -1372,9 +1360,9 @@ Action a_convert_rare_item_set( string output_filename = args.get(2, false); if (output_filename.empty() || (output_filename == "-")) { - rs->print_all_collections(stdout, version, name_index); + rs->print_all_collections(stdout, s.item_name_index(version)); } else if (ends_with(output_filename, ".json")) { - string data = rs->serialize_json(version, name_index); + string data = rs->serialize_json(s.item_name_index(version)); write_output_data(args, data.data(), data.size(), nullptr); } else if (ends_with(output_filename, ".gsl")) { string data = rs->serialize_gsl(args.get("big-endian")); @@ -1401,20 +1389,13 @@ Action a_describe_item( string description = args.get(1); auto version = get_cli_version(args); - auto name_index = make_shared( - JSON::parse(load_file("system/item-tables/names-v2.json")), - JSON::parse(load_file("system/item-tables/names-v3.json")), - JSON::parse(load_file("system/item-tables/names-v4.json"))); - auto pmt_data_v2 = make_shared(prs_decompress(load_file("system/item-tables/ItemPMT-v2.prs"))); - auto pmt_v2 = make_shared(pmt_data_v2, ItemParameterTable::Version::V2); - auto pmt_data_v3 = make_shared(prs_decompress(load_file("system/item-tables/ItemPMT-gc.prs"))); - auto pmt_v3 = make_shared(pmt_data_v3, ItemParameterTable::Version::V3); - auto pmt_data_v4 = make_shared(prs_decompress(load_file("system/item-tables/ItemPMT-bb.prs"))); - auto pmt_v4 = make_shared(pmt_data_v4, ItemParameterTable::Version::V4); + ServerState s; + s.load_objects("item_name_indexes"); + auto name_index = s.item_name_index(version); - ItemData item = name_index->parse_item_description(version, description); + ItemData item = name_index->parse_item_description(description); - string desc = name_index->describe_item(version, item); + string desc = name_index->describe_item(item); log_info("Data (decoded): %02hhX%02hhX%02hhX%02hhX %02hhX%02hhX%02hhX%02hhX %02hhX%02hhX%02hhX%02hhX -------- %02hhX%02hhX%02hhX%02hhX", item.data1[0], item.data1[1], item.data1[2], item.data1[3], item.data1[4], item.data1[5], item.data1[6], item.data1[7], @@ -1422,7 +1403,7 @@ Action a_describe_item( item.data2[0], item.data2[1], item.data2[2], item.data2[3]); ItemData item_v1 = item; - item_v1.encode_for_version(Version::PC_V2, pmt_v2); + item_v1.encode_for_version(Version::PC_V2, s.item_parameter_table(Version::PC_V2)); ItemData item_v1_decoded = item_v1; item_v1_decoded.decode_for_version(Version::PC_V2); @@ -1441,7 +1422,7 @@ Action a_describe_item( } ItemData item_gc = item; - item_gc.encode_for_version(Version::GC_V3, pmt_v3); + item_gc.encode_for_version(Version::GC_V3, s.item_parameter_table(Version::GC_V3)); ItemData item_gc_decoded = item_gc; item_gc_decoded.decode_for_version(Version::GC_V3); @@ -1461,11 +1442,49 @@ Action a_describe_item( log_info("Description: %s", desc.c_str()); - size_t purchase_price = pmt_v4->price_for_item(item); + size_t purchase_price = s.item_parameter_table(Version::BB_V4)->price_for_item(item); size_t sale_price = purchase_price >> 3; log_info("Purchase price: %zu; sale price: %zu", purchase_price, sale_price); }); +Action a_name_all_items( + "name-all-items", nullptr, +[](Arguments&) { + ServerState s; + s.load_objects("item_name_indexes"); + + set all_primary_identifiers; + for (const auto& index : s.item_name_indexes) { + if (index) { + for (const auto& it : index->all_by_primary_identifier()) { + all_primary_identifiers.emplace(it.first); + } + } + } + + fprintf(stderr, "IDENT :"); + for (size_t v_s = 0; v_s < NUM_VERSIONS; v_s++) { + Version version = static_cast(v_s); + const auto& index = s.item_name_indexes.at(v_s); + if (index) { + fprintf(stderr, " %30s", name_for_enum(version)); + } + } + fputc('\n', stderr); + + for (uint32_t primary_identifier : all_primary_identifiers) { + fprintf(stderr, "%06" PRIX32 ":", primary_identifier); + for (size_t v_s = 0; v_s < NUM_VERSIONS; v_s++) { + const auto& index = s.item_name_indexes.at(v_s); + if (index) { + ItemData item(static_cast(primary_identifier) << 40); + string name = index->describe_item(item); + fprintf(stderr, " %30s", name.c_str()); + } + } + fputc('\n', stderr); + } + }); + Action a_show_ep3_cards( "show-ep3-cards", "\ show-ep3-cards\n\ diff --git a/src/Map.cc b/src/Map.cc index 941323aa..3aaefbae 100644 --- a/src/Map.cc +++ b/src/Map.cc @@ -113,17 +113,10 @@ string Map::Enemy::str() const { this->source_index, name_for_enum(this->type), this->floor, this->state_flags, this->last_hit_by_client_id); } -string Map::Object::str(shared_ptr name_index) const { +string Map::Object::str() const { if (this->param1 <= 0.0f) { - string item_name; - try { - auto item = ItemCreator::base_item_for_specialized_box(this->param4, this->param5, this->param6); - item_name = name_index ? name_index->describe_item(Version::BB_V4, item) : item.hex(); - } catch (const exception& e) { - item_name = string_printf("(failed: %s)", e.what()); - } - return string_printf("[Map::Object source %zX %04hX @%04hX p1=%g (specialized: %s) floor=%02hhX item_drop_checked=%s]", - this->source_index, this->base_type, this->section, this->param1, item_name.c_str(), this->floor, this->item_drop_checked ? "true" : "false"); + return string_printf("[Map::Object source %zX %04hX @%04hX p1=%g (specialized: %08" PRIX32 " %08" PRIX32 " %08" PRIX32 ") floor=%02hhX item_drop_checked=%s]", + this->source_index, this->base_type, this->section, this->param1, this->param4, this->param5, this->param6, this->floor, this->item_drop_checked ? "true" : "false"); } else { return string_printf("[Map::Object source %zX %04hX @%04hX p1=%g (generic) p456=[%08" PRIX32 " %08" PRIX32 " %08" PRIX32 "] floor=%02hhX item_drop_checked=%s]", this->source_index, this->base_type, this->section, this->param1, this->param4, this->param5, this->param6, diff --git a/src/Map.hh b/src/Map.hh index 9ac77338..a3946eab 100644 --- a/src/Map.hh +++ b/src/Map.hh @@ -10,7 +10,6 @@ #include #include "BattleParamsIndex.hh" -#include "ItemNameIndex.hh" #include "PSOEncryption.hh" #include "StaticGameData.hh" #include "Text.hh" @@ -217,7 +216,7 @@ struct Map { uint8_t floor; bool item_drop_checked; - std::string str(std::shared_ptr name_index) const; + std::string str() const; }; struct Enemy { diff --git a/src/PlayerSubordinates.cc b/src/PlayerSubordinates.cc index 9d8a6917..d57cd4a7 100644 --- a/src/PlayerSubordinates.cc +++ b/src/PlayerSubordinates.cc @@ -678,7 +678,7 @@ void PlayerInventory::encode_for_client(shared_ptr c) { } } - auto item_parameter_table = c->require_server_state()->item_parameter_table_for_version(c->version()); + auto item_parameter_table = c->require_server_state()->item_parameter_table(c->version()); for (size_t z = 0; z < this->items.size(); z++) { this->items[z].data.encode_for_version(c->version(), item_parameter_table); } diff --git a/src/RareItemSet.cc b/src/RareItemSet.cc index eef1ce5d..fbfd28e7 100644 --- a/src/RareItemSet.cc +++ b/src/RareItemSet.cc @@ -17,7 +17,7 @@ string RareItemSet::ExpandedDrop::str() const { this->probability, frac.first, frac.second, this->item_code[0], this->item_code[1], this->item_code[2]); } -string RareItemSet::ExpandedDrop::str(Version version, shared_ptr name_index) const { +string RareItemSet::ExpandedDrop::str(shared_ptr name_index) const { ItemData item; item.data1[0] = this->item_code[0]; item.data1[1] = this->item_code[1]; @@ -25,7 +25,7 @@ string RareItemSet::ExpandedDrop::str(Version version, shared_ptrstr(); ret += " ("; - ret += name_index->describe_item(version, item); + ret += name_index->describe_item(item); ret += ")"; return ret; } @@ -303,7 +303,7 @@ RareItemSet::RareItemSet(const string& rel_data, bool is_big_endian) { } } -RareItemSet::RareItemSet(const JSON& json, Version version, shared_ptr name_index) { +RareItemSet::RareItemSet(const JSON& json, shared_ptr name_index) { for (const auto& mode_it : json.as_dict()) { static const unordered_map mode_keys( {{"Normal", GameMode::NORMAL}, {"Battle", GameMode::BATTLE}, {"Challenge", GameMode::CHALLENGE}, {"Solo", GameMode::SOLO}}); @@ -369,7 +369,7 @@ RareItemSet::RareItemSet(const JSON& json, Version version, shared_ptrparse_item_description(version, item_desc.as_string()); + ItemData data = name_index->parse_item_description(item_desc.as_string()); d.item_code[0] = data.data1[0]; d.item_code[1] = data.data1[1]; d.item_code[2] = data.data1[2]; @@ -427,7 +427,7 @@ std::string RareItemSet::serialize_gsl(bool big_endian) const { return GSLArchive::generate(files, big_endian); } -std::string RareItemSet::serialize_json(Version version, shared_ptr name_index) const { +std::string RareItemSet::serialize_json(shared_ptr name_index) const { auto modes_dict = JSON::dict(); static const array modes = {GameMode::NORMAL, GameMode::BATTLE, GameMode::CHALLENGE, GameMode::SOLO}; for (const auto& mode : modes) { @@ -458,7 +458,7 @@ std::string RareItemSet::serialize_json(Version version, shared_ptrdescribe_item(version, data)); + spec_json.emplace_back(name_index->describe_item(data)); } for (const auto& enemy_type : enemy_types) { if (enemy_type_valid_for_episode(episode, enemy_type)) { @@ -485,7 +485,7 @@ std::string RareItemSet::serialize_json(Version version, shared_ptrdescribe_item(version, data)); + area_list.back().emplace_back(name_index->describe_item(data)); } } @@ -512,7 +512,6 @@ std::string RareItemSet::serialize_json(Version version, shared_ptrrt_index_to_specs[z]) { - string s = name_index ? spec.str(version, name_index) : spec.str(); + string s = name_index ? spec.str(name_index) : spec.str(); fprintf(stream, " %02zX: %s (%s)\n", z, s.c_str(), enemy_types_str.c_str()); } } @@ -552,14 +551,13 @@ void RareItemSet::print_collection( fprintf(stream, " Box rares:\n"); for (size_t area = 0; area < collection->box_area_to_specs.size(); area++) { for (const auto& spec : collection->box_area_to_specs[area]) { - string s = name_index ? spec.str(version, name_index) : spec.str(); + string s = name_index ? spec.str(name_index) : spec.str(); fprintf(stream, " (area %02zX) %s\n", area, s.c_str()); } } } -void RareItemSet::print_all_collections( - FILE* stream, Version version, std::shared_ptr name_index) const { +void RareItemSet::print_all_collections(FILE* stream, std::shared_ptr name_index) const { static const array modes = {GameMode::NORMAL, GameMode::BATTLE, GameMode::CHALLENGE, GameMode::SOLO}; static const array episodes = {Episode::EP1, Episode::EP2, Episode::EP4}; for (GameMode mode : modes) { @@ -567,7 +565,7 @@ void RareItemSet::print_all_collections( for (uint8_t difficulty = 0; difficulty < 4; difficulty++) { for (uint8_t section_id = 0; section_id < 10; section_id++) { try { - this->print_collection(stream, version, mode, episode, difficulty, section_id, name_index); + this->print_collection(stream, mode, episode, difficulty, section_id, name_index); } catch (const out_of_range& e) { } } diff --git a/src/RareItemSet.hh b/src/RareItemSet.hh index ac3d6ab5..b2ec40e0 100644 --- a/src/RareItemSet.hh +++ b/src/RareItemSet.hh @@ -22,14 +22,14 @@ public: parray item_code; std::string str() const; - std::string str(Version version, std::shared_ptr name_index) const; + std::string str(std::shared_ptr name_index) const; }; RareItemSet(); RareItemSet(const AFSArchive& afs, bool is_v1); RareItemSet(const GSLArchive& gsl, bool is_big_endian); RareItemSet(const std::string& rel, bool is_big_endian); - RareItemSet(const JSON& json, Version version, std::shared_ptr name_index = nullptr); + RareItemSet(const JSON& json, std::shared_ptr name_index = nullptr); ~RareItemSet() = default; std::vector get_enemy_specs(GameMode mode, Episode episode, uint8_t difficulty, uint8_t secid, uint8_t rt_index) const; @@ -37,17 +37,16 @@ public: std::string serialize_afs(bool is_v1) const; std::string serialize_gsl(bool big_endian) const; - std::string serialize_json(Version version, std::shared_ptr name_index = nullptr) const; + std::string serialize_json(std::shared_ptr name_index = nullptr) const; void print_collection( FILE* stream, - Version version, GameMode mode, Episode episode, uint8_t difficulty, uint8_t section_id, std::shared_ptr name_index = nullptr) const; - void print_all_collections(FILE* stream, Version version, std::shared_ptr name_index = nullptr) const; + void print_all_collections(FILE* stream, std::shared_ptr name_index = nullptr) const; protected: struct SpecCollection { diff --git a/src/ReceiveCommands.cc b/src/ReceiveCommands.cc index 538f02a0..5451387f 100644 --- a/src/ReceiveCommands.cc +++ b/src/ReceiveCommands.cc @@ -132,7 +132,7 @@ void send_client_to_proxy_server(shared_ptr c) { static void send_proxy_destinations_menu(shared_ptr c) { auto s = c->require_server_state(); - send_menu(c, s->proxy_destinations_menu_for_version(c->version())); + send_menu(c, s->proxy_destinations_menu(c->version())); } static bool send_enable_send_function_call_if_applicable(shared_ptr c) { @@ -1762,7 +1762,7 @@ static void on_D6_V3(shared_ptr c, uint16_t, uint32_t, string& data) { check_size_v(data.size(), 0); if (c->config.check_flag(Client::Flag::IN_INFORMATION_MENU)) { auto s = c->require_server_state(); - send_menu(c, s->information_menu_for_version(c->version())); + send_menu(c, s->information_menu(c->version())); } else if (c->config.check_flag(Client::Flag::AT_WELCOME_MESSAGE)) { c->config.clear_flag(Client::Flag::AT_WELCOME_MESSAGE); send_enable_send_function_call_if_applicable(c); @@ -1784,7 +1784,7 @@ static void on_09(shared_ptr c, uint16_t, uint32_t, string& data) { case MenuID::QUEST_EP1: case MenuID::QUEST_EP2: { bool is_download_quest = !c->lobby.lock(); - auto quest_index = s->quest_index_for_version(c->version()); + auto quest_index = s->quest_index(c->version()); if (!quest_index) { send_quest_info(c, "$C6Quests are not available.", is_download_quest); } else { @@ -2178,7 +2178,7 @@ static void on_10(shared_ptr c, uint16_t, uint32_t, string& data) { case MainMenuItemID::INFORMATION: { auto s = c->require_server_state(); - send_menu(c, s->information_menu_for_version(c->version())); + send_menu(c, s->information_menu(c->version())); c->config.set_flag(Client::Flag::IN_INFORMATION_MENU); break; } @@ -2199,7 +2199,7 @@ static void on_10(shared_ptr c, uint16_t, uint32_t, string& data) { // always the download quest menu. (Episode 3 does actually have // online quests, but they're served via a server data request // instead of the file download paradigm that other versions use.) - auto quest_index = s->quest_index_for_version(c->version()); + auto quest_index = s->quest_index(c->version()); const auto& categories = quest_index->categories(menu_type, Episode::EP3, c->version()); if (categories.size() == 1) { auto quests = quest_index->filter(menu_type, Episode::EP3, c->version(), categories[0]->category_id); @@ -2208,7 +2208,7 @@ static void on_10(shared_ptr c, uint16_t, uint32_t, string& data) { } } - send_quest_categories_menu(c, s->quest_index_for_version(c->version()), menu_type, Episode::NONE); + send_quest_categories_menu(c, s->quest_index(c->version()), menu_type, Episode::NONE); break; } @@ -2346,7 +2346,7 @@ static void on_10(shared_ptr c, uint16_t, uint32_t, string& data) { auto s = c->require_server_state(); const pair* dest = nullptr; try { - dest = &s->proxy_destinations_for_version(c->version()).at(item_id); + dest = &s->proxy_destinations(c->version()).at(item_id); } catch (const out_of_range&) { } @@ -2453,7 +2453,7 @@ static void on_10(shared_ptr c, uint16_t, uint32_t, string& data) { case MenuID::QUEST_CATEGORIES: { auto s = c->require_server_state(); - auto quest_index = s->quest_index_for_version(c->version()); + auto quest_index = s->quest_index(c->version()); if (!quest_index) { send_lobby_message_box(c, "$C6Quests are not available."); break; @@ -2494,7 +2494,7 @@ static void on_10(shared_ptr c, uint16_t, uint32_t, string& data) { case MenuID::QUEST_EP1: case MenuID::QUEST_EP2: { auto s = c->require_server_state(); - auto quest_index = s->quest_index_for_version(c->version()); + auto quest_index = s->quest_index(c->version()); if (!quest_index) { send_lobby_message_box(c, "$C6Quests are not\navailable."); break; @@ -2707,7 +2707,7 @@ static void on_08_E6(shared_ptr c, uint16_t command, uint32_t, string& d static void on_1F(shared_ptr c, uint16_t, uint32_t, string& data) { check_size_v(data.size(), 0); auto s = c->require_server_state(); - send_menu(c, s->information_menu_for_version(c->version()), true); + send_menu(c, s->information_menu(c->version()), true); } static void on_A0(shared_ptr c, uint16_t, uint32_t, string&) { @@ -2842,7 +2842,7 @@ static void on_A2(shared_ptr c, uint16_t, uint32_t flag, string& data) { throw logic_error("invalid game mode"); } } - send_quest_categories_menu(c, s->quest_index_for_version(c->version()), menu_type, l->episode); + send_quest_categories_menu(c, s->quest_index(c->version()), menu_type, l->episode); } } diff --git a/src/ReceiveSubcommands.cc b/src/ReceiveSubcommands.cc index b321b2e3..cd4a58cc 100644 --- a/src/ReceiveSubcommands.cc +++ b/src/ReceiveSubcommands.cc @@ -832,7 +832,7 @@ static void on_sync_joining_player_disp_and_inventory( throw logic_error("6x70 command from unknown game version"); } - parsed->transcode_inventory_items(c->version(), target->version(), s->item_parameter_table_for_version(target->version())); + parsed->transcode_inventory_items(c->version(), target->version(), s->item_parameter_table(target->version())); parsed->visual.enforce_lobby_join_limits_for_version(target->version()); switch (target->version()) { @@ -1349,7 +1349,7 @@ static void on_player_drop_item(shared_ptr c, uint8_t command, uint8_t f auto name = s->describe_item(c->version(), item, false); l->log.info("Player %hu dropped item %08" PRIX32 " (%s) at %hu:(%g, %g)", cmd.header.client_id.load(), cmd.item_id.load(), name.c_str(), cmd.floor.load(), cmd.x.load(), cmd.z.load()); - p->print_inventory(stderr, c->version(), s->item_name_index); + c->print_inventory(stderr); } forward_subcommand(c, command, flag, data, size); @@ -1374,7 +1374,7 @@ void forward_subcommand_with_item_transcode_t(shared_ptr c, uint8_t comm out_cmd.header.subcommand = translate_subcommand_number(lc->version(), c->version(), out_cmd.header.subcommand); if (out_cmd.header.subcommand) { out_cmd.item_data.decode_for_version(c->version()); - out_cmd.item_data.encode_for_version(lc->version(), s->item_parameter_table_for_version(lc->version())); + out_cmd.item_data.encode_for_version(lc->version(), s->item_parameter_table(lc->version())); send_command_t(lc, command, flag, out_cmd); } else { lc->log.info("Subcommand cannot be translated to client\'s version"); @@ -1409,7 +1409,7 @@ static void on_create_inventory_item_t(shared_ptr c, uint8_t command, ui auto s = c->require_server_state(); auto name = s->describe_item(c->version(), item, false); l->log.info("Player %hu created inventory item %08" PRIX32 " (%s)", c->lobby_client_id, item.id.load(), name.c_str()); - p->print_inventory(stderr, c->version(), s->item_name_index); + c->print_inventory(stderr); } forward_subcommand_with_item_transcode_t(c, command, flag, cmd); @@ -1450,7 +1450,7 @@ static void on_drop_partial_stack_t(shared_ptr c, uint8_t command, uint8 auto name = s->describe_item(c->version(), item, false); l->log.info("Player %hu split stack to create floor item %08" PRIX32 " (%s) at %hu:(%g, %g)", cmd.header.client_id.load(), item.id.load(), name.c_str(), cmd.floor.load(), cmd.x.load(), cmd.z.load()); - c->character()->print_inventory(stderr, c->version(), s->item_name_index); + c->print_inventory(stderr); } forward_subcommand_with_item_transcode_t(c, command, flag, cmd); @@ -1498,7 +1498,7 @@ static void on_drop_partial_stack_bb(shared_ptr c, uint8_t command, uint auto name = s->describe_item(c->version(), item, false); l->log.info("Player %hu split stack %08" PRIX32 " (removed: %s) at %hu:(%g, %g)", cmd.header.client_id.load(), cmd.item_id.load(), name.c_str(), cmd.floor.load(), cmd.x.load(), cmd.z.load()); - p->print_inventory(stderr, c->version(), s->item_name_index); + c->print_inventory(stderr); } } else { @@ -1525,14 +1525,14 @@ static void on_buy_shop_item(shared_ptr c, uint8_t command, uint8_t flag l->on_item_id_generated_externally(item.id); p->add_item(item); - size_t price = s->item_parameter_table_for_version(c->version())->price_for_item(item); + size_t price = s->item_parameter_table(c->version())->price_for_item(item); p->remove_meseta(price, c->version() != Version::BB_V4); if (l->log.should_log(LogLevel::INFO)) { auto name = s->describe_item(c->version(), item, false); l->log.info("Player %hu bought item %08" PRIX32 " (%s) from shop (%zu Meseta)", cmd.header.client_id.load(), item.id.load(), name.c_str(), price); - p->print_inventory(stderr, c->version(), s->item_name_index); + c->print_inventory(stderr); } forward_subcommand_with_item_transcode_t(c, command, flag, cmd); @@ -1575,7 +1575,7 @@ static void on_box_or_enemy_item_drop_t(shared_ptr c, uint8_t command, u out_cmd.header.subcommand = translate_subcommand_number(lc->version(), c->version(), out_cmd.header.subcommand); if (out_cmd.header.subcommand) { out_cmd.item.item.decode_for_version(c->version()); - out_cmd.item.item.encode_for_version(lc->version(), s->item_parameter_table_for_version(lc->version())); + out_cmd.item.item.encode_for_version(lc->version(), s->item_parameter_table(lc->version())); send_command_t(lc, command, flag, out_cmd); } else { lc->log.info("Subcommand cannot be translated to client\'s version"); @@ -1639,7 +1639,7 @@ static void on_pick_up_item_generic( auto s = c->require_server_state(); auto name = s->describe_item(c->version(), fi->data, false); l->log.info("Player %hu picked up %08" PRIX32 " (%s)", client_id, item_id, name.c_str()); - p->print_inventory(stderr, c->version(), s->item_name_index); + c->print_inventory(stderr); } auto s = c->require_server_state(); @@ -1725,7 +1725,7 @@ static void on_use_item( if (l->log.should_log(LogLevel::INFO)) { l->log.info("Player %hhu used item %hu:%08" PRIX32 " (%s)", c->lobby_client_id, cmd.header.client_id.load(), cmd.item_id.load(), name.c_str()); - p->print_inventory(stderr, c->version(), s->item_name_index); + c->print_inventory(stderr); } forward_subcommand(c, command, flag, data, size); @@ -1771,7 +1771,7 @@ static void on_feed_mag( l->log.info("Player %hhu fed item %hu:%08" PRIX32 " (%s) to mag %hu:%08" PRIX32 " (%s)", c->lobby_client_id, cmd.header.client_id.load(), cmd.fed_item_id.load(), fed_name.c_str(), cmd.header.client_id.load(), cmd.mag_item_id.load(), mag_name.c_str()); - p->print_inventory(stderr, c->version(), s->item_name_index); + c->print_inventory(stderr); } forward_subcommand(c, command, flag, data, size); @@ -1806,7 +1806,7 @@ static void on_open_shop_bb_or_ep3_battle_subs(shared_ptr c, uint8_t com } for (auto& item : c->bb_shop_contents[cmd.shop_type]) { item.id = 0xFFFFFFFF; - item.data2d = s->item_parameter_table_for_version(c->version())->price_for_item(item); + item.data2d = s->item_parameter_table(c->version())->price_for_item(item); } send_shop(c, cmd.shop_type); @@ -1863,10 +1863,10 @@ static void on_ep3_private_word_select_bb_bank_action(shared_ptr c, uint send_destroy_item_to_lobby(c, cmd.item_id, cmd.item_amount, true); if (l->log.should_log(LogLevel::INFO)) { - string name = s->item_name_index->describe_item(Version::BB_V4, item); + string name = s->describe_item(Version::BB_V4, item, false); l->log.info("Player %hu deposited item %08" PRIX32 " (x%hhu) (%s) in the bank", c->lobby_client_id, cmd.item_id.load(), cmd.item_amount, name.c_str()); - c->character()->print_inventory(stderr, c->version(), s->item_name_index); + c->print_inventory(stderr); } } @@ -1892,10 +1892,10 @@ static void on_ep3_private_word_select_bb_bank_action(shared_ptr c, uint send_create_inventory_item_to_lobby(c, c->lobby_client_id, item); if (l->log.should_log(LogLevel::INFO)) { - string name = s->item_name_index->describe_item(Version::BB_V4, item); + string name = s->describe_item(Version::BB_V4, item, false); l->log.info("Player %hu withdrew item %08" PRIX32 " (x%hhu) (%s) from the bank", c->lobby_client_id, item.id.load(), cmd.item_amount, name.c_str()); - c->character()->print_inventory(stderr, c->version(), s->item_name_index); + c->print_inventory(stderr); } } @@ -2133,7 +2133,7 @@ static void on_entity_drop_item_request(shared_ptr c, uint8_t command, u if (item.empty()) { l->log.info("No item was created"); } else { - string name = s->item_name_index->describe_item(l->base_version, item); + string name = s->describe_item(l->base_version, item, false); l->log.info("Entity %04hX (area %02hX) created item %s", cmd.entity_id.load(), cmd.effective_area, name.c_str()); if (l->drop_mode == Lobby::DropMode::SERVER_DUPLICATE) { for (const auto& lc : l->clients) { @@ -2163,7 +2163,7 @@ static void on_entity_drop_item_request(shared_ptr c, uint8_t command, u if (item.empty()) { l->log.info("No item was created for %s", lc->channel.name.c_str()); } else { - string name = s->item_name_index->describe_item(l->base_version, item); + string name = s->describe_item(l->base_version, item, false); l->log.info("Entity %04hX (area %02hX) created item %s", cmd.entity_id.load(), cmd.effective_area, name.c_str()); item.id = l->generate_item_id(0xFF); l->log.info("Creating item %08" PRIX32 " at %02hhX:%g,%g for %s", @@ -2455,7 +2455,7 @@ static void on_steal_exp_bb(shared_ptr c, uint8_t, uint8_t, void* data, const auto& inventory = p->inventory; const auto& weapon = inventory.items[inventory.find_equipped_item(EquipSlot::WEAPON)]; - auto item_parameter_table = s->item_parameter_table_for_version(c->version()); + auto item_parameter_table = s->item_parameter_table(c->version()); uint8_t special_id = 0; if (((weapon.data.data1[1] < 0x0A) && (weapon.data.data1[2] < 0x05)) || @@ -2574,7 +2574,7 @@ static void on_enemy_exp_request_bb(shared_ptr c, uint8_t, uint8_t, void for (size_t z = 0; z < inventory.num_items; z++) { auto& item = inventory.items[z]; if ((item.flags & 0x08) && - s->item_parameter_table_for_version(c->version())->is_unsealable_item(item.data)) { + s->item_parameter_table(c->version())->is_unsealable_item(item.data)) { item.data.set_sealed_item_kill_count(item.data.get_sealed_item_kill_count() + 1); } } @@ -2644,7 +2644,7 @@ void on_transfer_item_via_mail_message_bb(shared_ptr c, uint8_t command, auto name = s->describe_item(c->version(), item, false); l->log.info("Player %hhu sent inventory item %hu:%08" PRIX32 " (%s) x%" PRIu32 " to player %08" PRIX32, c->lobby_client_id, cmd.header.client_id.load(), cmd.item_id.load(), name.c_str(), cmd.amount.load(), cmd.target_guild_card_number.load()); - p->print_inventory(stderr, c->version(), s->item_name_index); + c->print_inventory(stderr); } // To receive an item, the player must be online, using BB, have a character @@ -2704,7 +2704,7 @@ void on_exchange_item_for_team_points_bb(shared_ptr c, uint8_t command, auto name = s->describe_item(c->version(), item, false); l->log.info("Player %hhu exchanged inventory item %hu:%08" PRIX32 " (%s) for %zu team points", c->lobby_client_id, cmd.header.client_id.load(), cmd.item_id.load(), name.c_str(), points); - p->print_inventory(stderr, c->version(), s->item_name_index); + c->print_inventory(stderr); } forward_subcommand(c, command, flag, data, size); @@ -2729,7 +2729,7 @@ static void on_destroy_inventory_item(shared_ptr c, uint8_t command, uin auto name = s->describe_item(c->version(), item, false); l->log.info("Player %hhu destroyed inventory item %hu:%08" PRIX32 " (%s)", c->lobby_client_id, cmd.header.client_id.load(), cmd.item_id.load(), name.c_str()); - p->print_inventory(stderr, c->version(), s->item_name_index); + c->print_inventory(stderr); } forward_subcommand(c, command, flag, data, size); } @@ -2833,14 +2833,14 @@ static void on_sell_item_at_shop_bb(shared_ptr c, uint8_t command, uint8 auto s = c->require_server_state(); auto p = c->character(); auto item = p->remove_item(cmd.item_id, cmd.amount, c->version() != Version::BB_V4); - size_t price = (s->item_parameter_table_for_version(c->version())->price_for_item(item) >> 3) * cmd.amount; + size_t price = (s->item_parameter_table(c->version())->price_for_item(item) >> 3) * cmd.amount; p->add_meseta(price); if (l->log.should_log(LogLevel::INFO)) { auto name = s->describe_item(c->version(), item, false); l->log.info("Player %hhu sold inventory item %08" PRIX32 " (%s) for %zu Meseta", c->lobby_client_id, cmd.item_id.load(), name.c_str(), price); - p->print_inventory(stderr, c->version(), s->item_name_index); + c->print_inventory(stderr); } forward_subcommand(c, command, flag, data, size); @@ -2875,7 +2875,7 @@ static void on_buy_shop_item_bb(shared_ptr c, uint8_t, uint8_t, void* da auto name = s->describe_item(c->version(), item, false); l->log.info("Player %hhu purchased item %08" PRIX32 " (%s) for %zu meseta", c->lobby_client_id, item.id.load(), name.c_str(), price); - p->print_inventory(stderr, c->version(), s->item_name_index); + c->print_inventory(stderr); } } } @@ -3246,7 +3246,7 @@ static void on_quest_F960_result_bb(shared_ptr c, uint8_t, uint8_t, void item.id = l->generate_item_id(c->lobby_client_id); // If it's a weapon, make it unidentified - auto item_parameter_table = s->item_parameter_table_for_version(c->version()); + auto item_parameter_table = s->item_parameter_table(c->version()); if ((item.data1[0] == 0x00) && (item_parameter_table->is_item_rare(item) || (item.data1[4] != 0))) { item.data1[4] |= 0x80; } @@ -3260,12 +3260,12 @@ static void on_quest_F960_result_bb(shared_ptr c, uint8_t, uint8_t, void p->add_item(item); send_create_inventory_item_to_lobby(c, c->lobby_client_id, item); if (c->log.should_log(LogLevel::INFO)) { - string name = s->item_name_index->describe_item(c->version(), item); + string name = s->describe_item(c->version(), item, false); c->log.info("Awarded item %s", name.c_str()); } } catch (const out_of_range&) { if (c->log.should_log(LogLevel::INFO)) { - string name = s->item_name_index->describe_item(c->version(), item); + string name = s->describe_item(c->version(), item, false); c->log.info("Attempted to award item %s, but inventory was full", name.c_str()); } } diff --git a/src/SaveFileFormats.cc b/src/SaveFileFormats.cc index 77c1858d..f48abc40 100644 --- a/src/SaveFileFormats.cc +++ b/src/SaveFileFormats.cc @@ -578,29 +578,6 @@ void PSOBBCharacterFile::clear_all_material_usage() { } } -void PSOBBCharacterFile::print_inventory(FILE* stream, Version version, shared_ptr name_index) const { - fprintf(stream, "[PlayerInventory] Meseta: %" PRIu32 "\n", this->disp.stats.meseta.load()); - fprintf(stream, "[PlayerInventory] %hhu items\n", this->inventory.num_items); - for (size_t x = 0; x < this->inventory.num_items; x++) { - const auto& item = this->inventory.items[x]; - auto name = name_index->describe_item(version, item.data); - auto hex = item.data.hex(); - fprintf(stream, "[PlayerInventory] %2zu: [+%08" PRIX32 "] %s (%s)\n", x, item.flags.load(), hex.c_str(), name.c_str()); - } -} - -void PSOBBCharacterFile::print_bank(FILE* stream, Version version, shared_ptr name_index) const { - fprintf(stream, "[PlayerBank] Meseta: %" PRIu32 "\n", this->bank.meseta.load()); - fprintf(stream, "[PlayerBank] %" PRIu32 " items\n", this->bank.num_items.load()); - for (size_t x = 0; x < this->bank.num_items; x++) { - const auto& item = this->bank.items[x]; - const char* present_token = item.present ? "" : " (missing present flag)"; - auto name = name_index->describe_item(version, item.data); - auto hex = item.data.hex(); - fprintf(stream, "[PlayerBank] %3zu: %s (%s) (x%hu) %s\n", x, hex.c_str(), name.c_str(), item.amount.load(), present_token); - } -} - const array PSOBBCharacterFile::DEFAULT_SYMBOL_CHATS = { DefaultSymbolChatEntry{"\tEHello", 0x28, {0xFFFF, 0x000D, 0xFFFF, 0xFFFF}, {SymbolChat::FacePart{0x05, 0x18, 0x1D, 0x00}, {0x05, 0x28, 0x1D, 0x01}, {0x36, 0x20, 0x2A, 0x00}, {0x3C, 0x00, 0x32, 0x00}, {0xFF, 0x00, 0x00, 0x00}, {0xFF, 0x00, 0x00, 0x00}, {0xFF, 0x00, 0x00, 0x00}, {0xFF, 0x00, 0x00, 0x02}, {0xFF, 0x00, 0x00, 0x02}, {0xFF, 0x00, 0x00, 0x02}, {0xFF, 0x00, 0x00, 0x02}, {0xFF, 0x00, 0x00, 0x02}}}, DefaultSymbolChatEntry{"\tEGood-bye", 0x74, {0x0476, 0x000C, 0xFFFF, 0xFFFF}, {SymbolChat::FacePart{0x06, 0x15, 0x14, 0x00}, {0x06, 0x2B, 0x14, 0x01}, {0x05, 0x18, 0x1F, 0x00}, {0x05, 0x28, 0x1F, 0x01}, {0x36, 0x20, 0x2A, 0x00}, {0x3C, 0x00, 0x32, 0x00}, {0xFF, 0x00, 0x00, 0x00}, {0xFF, 0x00, 0x00, 0x02}, {0xFF, 0x00, 0x00, 0x02}, {0xFF, 0x00, 0x00, 0x02}, {0xFF, 0x00, 0x00, 0x02}, {0xFF, 0x00, 0x00, 0x02}}}, diff --git a/src/SaveFileFormats.hh b/src/SaveFileFormats.hh index 5da10b92..17eaea2a 100644 --- a/src/SaveFileFormats.hh +++ b/src/SaveFileFormats.hh @@ -252,9 +252,6 @@ struct PSOBBCharacterFile { uint8_t get_material_usage(MaterialType which) const; void set_material_usage(MaterialType which, uint8_t usage); void clear_all_material_usage(); - - void print_inventory(FILE* stream, Version version, std::shared_ptr name_index) const; - void print_bank(FILE* stream, Version version, std::shared_ptr name_index) const; } __attribute__((packed)); struct PSOBBGuildCardFile { diff --git a/src/SendCommands.cc b/src/SendCommands.cc index 8d770b60..65da6fc4 100644 --- a/src/SendCommands.cc +++ b/src/SendCommands.cc @@ -2239,7 +2239,7 @@ void send_execute_item_trade(shared_ptr c, const vector& items cmd.item_count = items.size(); for (size_t x = 0; x < items.size(); x++) { cmd.item_datas[x] = items[x]; - cmd.item_datas[x].encode_for_version(c->version(), s->item_parameter_table_for_version(c->version())); + cmd.item_datas[x].encode_for_version(c->version(), s->item_parameter_table(c->version())); } send_command_t(c, 0xD3, 0x00, cmd); } @@ -2431,7 +2431,7 @@ void send_game_item_state(shared_ptr c) { fi.unknown_a2 = 0; fi.drop_number = (floor == 0) ? 0xFFFF : (decompressed_header.next_drop_number_per_floor.at(floor - 1)++); fi.item = item->data; - fi.item.encode_for_version(c->version(), s->item_parameter_table_for_version(c->version())); + fi.item.encode_for_version(c->version(), s->item_parameter_table(c->version())); floor_items_w.put(fi); decompressed_header.floor_item_count_per_floor.at(floor)++; @@ -2509,7 +2509,7 @@ void send_drop_item_to_channel(shared_ptr s, Channel& ch, const Ite uint8_t subcommand = get_pre_v1_subcommand(ch.version, 0x51, 0x58, 0x5F); G_DropItem_PC_V3_BB_6x5F cmd = { {{subcommand, 0x0B, 0x0000}, {floor, from_enemy, entity_id, x, z, 0, 0, item}}, 0}; - cmd.item.item.encode_for_version(ch.version, s->item_parameter_table_for_version(ch.version)); + cmd.item.item.encode_for_version(ch.version, s->item_parameter_table(ch.version)); ch.send(0x60, 0x00, &cmd, sizeof(cmd)); } @@ -2528,7 +2528,7 @@ void send_drop_stacked_item_to_channel( shared_ptr s, Channel& ch, const ItemData& item, uint8_t floor, float x, float z) { uint8_t subcommand = get_pre_v1_subcommand(ch.version, 0x4F, 0x56, 0x5D); G_DropStackedItem_PC_V3_BB_6x5D cmd = {{{subcommand, 0x0A, 0x0000}, floor, 0, x, z, item}, 0}; - cmd.item_data.encode_for_version(ch.version, s->item_parameter_table_for_version(ch.version)); + cmd.item_data.encode_for_version(ch.version, s->item_parameter_table(ch.version)); ch.send(0x60, 0x00, &cmd, sizeof(cmd)); } diff --git a/src/ServerShell.cc b/src/ServerShell.cc index 91c396d5..56536750 100644 --- a/src/ServerShell.cc +++ b/src/ServerShell.cc @@ -806,7 +806,7 @@ Proxy session commands:\n\ } auto s = ses->require_server_state(); - ItemData item = s->item_name_index->parse_item_description(ses->version(), command_args); + ItemData item = s->parse_item_description(ses->version(), command_args); item.id = random_object() | 0x80000000; if (command_name == "set-next-item") { diff --git a/src/ServerState.cc b/src/ServerState.cc index ce4b7703..c5b9d200 100644 --- a/src/ServerState.cc +++ b/src/ServerState.cc @@ -14,7 +14,7 @@ #include "NetworkAddresses.hh" #include "SendCommands.hh" #include "Text.hh" -#include "UnicodeTextSet.hh" +#include "TextIndex.hh" using namespace std; @@ -379,7 +379,7 @@ uint32_t ServerState::connect_address_for_client(shared_ptr c) const { } } -shared_ptr ServerState::information_menu_for_version(Version version) const { +shared_ptr ServerState::information_menu(Version version) const { if (is_v1_or_v2(version)) { return this->information_menu_v2; } else if (is_v3(version)) { @@ -388,7 +388,7 @@ shared_ptr ServerState::information_menu_for_version(Version version throw out_of_range("no information menu exists for this version"); } -shared_ptr ServerState::proxy_destinations_menu_for_version(Version version) const { +shared_ptr ServerState::proxy_destinations_menu(Version version) const { switch (version) { case Version::DC_NTE: case Version::DC_V1_11_2000_PROTOTYPE: @@ -410,7 +410,7 @@ shared_ptr ServerState::proxy_destinations_menu_for_version(Version } } -const vector>& ServerState::proxy_destinations_for_version(Version version) const { +const vector>& ServerState::proxy_destinations(Version version) const { switch (version) { case Version::DC_NTE: case Version::DC_V1_11_2000_PROTOTYPE: @@ -432,7 +432,19 @@ const vector>& ServerState::proxy_destinations_for_versio } } -shared_ptr ServerState::item_parameter_table_for_version(Version version) const { +shared_ptr> ServerState::information_contents_for_client(shared_ptr c) const { + return is_v1_or_v2(c->version()) ? this->information_contents_v2 : this->information_contents_v3; +} + +shared_ptr ServerState::quest_index(Version version) const { + return is_ep3(version) ? this->ep3_download_quest_index : this->default_quest_index; +} + +void ServerState::dispatch_destroy_lobbies(evutil_socket_t, short, void* ctx) { + reinterpret_cast(ctx)->lobbies_to_destroy.clear(); +} + +shared_ptr ServerState::item_parameter_table(Version version) const { switch (version) { case Version::DC_NTE: case Version::DC_V1_11_2000_PROTOTYPE: @@ -454,11 +466,24 @@ shared_ptr ServerState::item_parameter_table_for_versi } } +shared_ptr ServerState::item_name_index(Version version) const { + auto ret = this->item_name_indexes.at(static_cast(version)); + if (ret == nullptr) { + throw runtime_error("no item name index exists for this version"); + } + return ret; +} + +void ServerState::set_item_name_index(Version version, shared_ptr new_index) { + this->item_name_indexes.at(static_cast(version)) = new_index; +} + string ServerState::describe_item(Version version, const ItemData& item, bool include_color_codes) const { - return this->item_name_index->describe_item( - version, - item, - include_color_codes ? this->item_parameter_table_for_version(version) : nullptr); + return this->item_name_index(version)->describe_item(item, include_color_codes); +} + +ItemData ServerState::parse_item_description(Version version, const string& description) const { + return this->item_name_index(version)->parse_item_description(description); } void ServerState::set_port_configuration( @@ -829,7 +854,7 @@ void ServerState::parse_config(const JSON& json, bool is_reload) { for (const auto& difficulty_it : type_it->as_list()) { auto& difficulty_res = type_res.emplace_back(); for (const auto& item_it : difficulty_it->as_list()) { - difficulty_res.emplace_back(this->item_name_index->parse_item_description(Version::BB_V4, item_it->as_string())); + difficulty_res.emplace_back(this->parse_item_description(Version::BB_V4, item_it->as_string())); } } } @@ -840,22 +865,22 @@ void ServerState::parse_config(const JSON& json, bool is_reload) { for (const auto& it : json.get_list("QuestF95FResultItems")) { auto& list = it->as_list(); size_t price = list.at(0)->as_int(); - this->quest_F95F_results.emplace_back(make_pair(price, this->item_name_index->parse_item_description(Version::BB_V4, list.at(1)->as_string()))); + this->quest_F95F_results.emplace_back(make_pair(price, this->parse_item_description(Version::BB_V4, list.at(1)->as_string()))); } } catch (const out_of_range&) { } try { this->quest_F960_success_results.clear(); - this->quest_F960_failure_results = QuestF960Result(json.at("QuestF960FailureResultItems"), this->item_name_index); + this->quest_F960_failure_results = QuestF960Result(json.at("QuestF960FailureResultItems"), this->item_name_index(Version::BB_V4)); for (const auto& it : json.get_list("QuestF960SuccessResultItems")) { - this->quest_F960_success_results.emplace_back(*it, this->item_name_index); + this->quest_F960_success_results.emplace_back(*it, this->item_name_index(Version::BB_V4)); } } catch (const out_of_range&) { } try { this->secret_lottery_results.clear(); for (const auto& it : json.get_list("SecretLotteryResultItems")) { - this->secret_lottery_results.emplace_back(this->item_name_index->parse_item_description(Version::BB_V4, it->as_string())); + this->secret_lottery_results.emplace_back(this->parse_item_description(Version::BB_V4, it->as_string())); } } catch (const out_of_range&) { } @@ -1109,7 +1134,25 @@ void ServerState::load_level_table() { this->level_table = make_shared(*this->load_bb_file("PlyLevelTbl.prs"), true); } -shared_ptr ServerState::load_word_select_table_from_system() { +void ServerState::load_text_index() { + this->text_index = make_shared("system/text-sets", [&](Version version, const string& filename) -> shared_ptr { + try { + if (version == Version::BB_V4) { + return this->load_bb_file(filename); + } else { + return this->pc_patch_file_index->get("Media/PSO/" + filename)->load_data(); + } + } catch (const out_of_range&) { + return nullptr; + } catch (const cannot_open_file&) { + return nullptr; + } + }); +} + +void ServerState::load_word_select_table() { + config_log.info("Loading Word Select table"); + vector> name_alias_lists; auto json = JSON::parse(load_file("system/word-select/name-alias-lists.json")); for (const auto& coll_it : json.as_list()) { @@ -1119,12 +1162,20 @@ shared_ptr ServerState::load_word_select_table_from_system() { } } - config_log.info("(Word select) Loading pc_unitxt.prs"); - vector> pc_unitxt_data = parse_unicode_text_set(load_file("system/word-select/pc_unitxt.prs")); + const vector* pc_unitxt_collection = nullptr; + const vector* bb_unitxt_collection = nullptr; + unique_ptr pc_unitxt_data; + if (this->text_index) { + config_log.info("(Word select) Using PC_V2 unitxt_e.prs from text index"); + pc_unitxt_collection = &this->text_index->get(Version::PC_V2, 1, 35); + } else { + config_log.info("(Word select) Loading PC_V2 unitxt_e.prs"); + pc_unitxt_data = make_unique(load_file("system/word-select/pc_unitxt.prs")); + pc_unitxt_collection = &pc_unitxt_data->get(35); + } config_log.info("(Word select) Loading bb_unitxt_ws.prs"); - vector> bb_unitxt_data = parse_unicode_text_set(load_file("system/word-select/bb_unitxt_ws.prs")); - vector pc_unitxt_collection = std::move(pc_unitxt_data.at(35)); - vector bb_unitxt_collection = std::move(bb_unitxt_data.at(0)); + auto bb_unitxt_data = make_unique(load_file("system/word-select/bb_unitxt_ws.prs")); + bb_unitxt_collection = &bb_unitxt_data->get(0); config_log.info("(Word select) Loading DC_NTE data"); WordSelectSet dc_nte_ws(load_file("system/word-select/dc_nte_ws_data.bin"), Version::DC_NTE, nullptr, true); @@ -1135,9 +1186,9 @@ shared_ptr ServerState::load_word_select_table_from_system() { config_log.info("(Word select) Loading DC_V2 data"); WordSelectSet dc_v2_ws(load_file("system/word-select/dcv2_ws_data.bin"), Version::DC_V2, nullptr, false); config_log.info("(Word select) Loading PC_NTE data"); - WordSelectSet pc_nte_ws(load_file("system/word-select/pc_nte_ws_data.bin"), Version::PC_NTE, &pc_unitxt_collection, false); + WordSelectSet pc_nte_ws(load_file("system/word-select/pc_nte_ws_data.bin"), Version::PC_NTE, pc_unitxt_collection, false); config_log.info("(Word select) Loading PC_V2 data"); - WordSelectSet pc_v2_ws(load_file("system/word-select/pc_ws_data.bin"), Version::PC_V2, &pc_unitxt_collection, false); + WordSelectSet pc_v2_ws(load_file("system/word-select/pc_ws_data.bin"), Version::PC_V2, pc_unitxt_collection, false); config_log.info("(Word select) Loading GC_NTE data"); WordSelectSet gc_nte_ws(load_file("system/word-select/gc_nte_ws_data.bin"), Version::GC_NTE, nullptr, false); config_log.info("(Word select) Loading GC_V3 data"); @@ -1149,30 +1200,72 @@ shared_ptr ServerState::load_word_select_table_from_system() { config_log.info("(Word select) Loading XB_V3 data"); WordSelectSet xb_v3_ws(load_file("system/word-select/xb_ws_data.bin"), Version::XB_V3, nullptr, false); config_log.info("(Word select) Loading BB_V4 data"); - WordSelectSet bb_v4_ws(load_file("system/word-select/bb_ws_data.bin"), Version::BB_V4, &bb_unitxt_collection, false); + WordSelectSet bb_v4_ws(load_file("system/word-select/bb_ws_data.bin"), Version::BB_V4, bb_unitxt_collection, false); config_log.info("(Word select) Generating table"); - return make_shared( + this->word_select_table = make_shared( dc_nte_ws, dc_112000_ws, dc_v1_ws, dc_v2_ws, pc_nte_ws, pc_v2_ws, gc_nte_ws, gc_v3_ws, gc_ep3_nte_ws, gc_ep3_ws, xb_v3_ws, bb_v4_ws, name_alias_lists); } -void ServerState::load_word_select_table() { - config_log.info("Loading Word Select table"); - this->word_select_table = this->load_word_select_table_from_system(); +shared_ptr ServerState::create_item_name_index_for_version( + Version version, shared_ptr pmt, shared_ptr text_index) { + switch (version) { + case Version::DC_NTE: + return make_shared(pmt, text_index->get(Version::DC_NTE, 0, 2)); + case Version::DC_V1_11_2000_PROTOTYPE: + return make_shared(pmt, text_index->get(Version::DC_V1_11_2000_PROTOTYPE, 1, 2)); + case Version::DC_V1: + return make_shared(pmt, text_index->get(Version::DC_V1, 1, 2)); + case Version::DC_V2: + return make_shared(pmt, text_index->get(Version::DC_V2, 1, 3)); + case Version::PC_NTE: + return make_shared(pmt, text_index->get(Version::PC_NTE, 1, 3)); + case Version::PC_V2: + return make_shared(pmt, text_index->get(Version::PC_V2, 1, 3)); + case Version::GC_NTE: + return make_shared(pmt, text_index->get(Version::GC_NTE, 1, 0)); + case Version::GC_V3: + return make_shared(pmt, text_index->get(Version::GC_V3, 1, 0)); + case Version::XB_V3: + return make_shared(pmt, text_index->get(Version::XB_V3, 1, 0)); + case Version::BB_V4: + return make_shared(pmt, text_index->get(Version::BB_V4, 1, 1)); + default: + return nullptr; + } } -void ServerState::load_item_name_index() { - config_log.info("Loading item name index"); - this->item_name_index = make_shared( - JSON::parse(load_file("system/item-tables/names-v2.json")), - JSON::parse(load_file("system/item-tables/names-v3.json")), - JSON::parse(load_file("system/item-tables/names-v4.json"))); +void ServerState::load_item_name_indexes() { + config_log.info("Generating item name indexes"); + // TODO: Get ItemPMT files for the versions for which we don't have them + // (especially DC_V1) and add support for them. Currently we only have three + // ItemPMTs (PC, GC, and BB), so we can't use them to generate all the name + // indexes. + + auto pc_v2_index = create_item_name_index_for_version( + Version::PC_V2, this->item_parameter_table(Version::PC_V2), this->text_index); + this->set_item_name_index(Version::DC_NTE, pc_v2_index); + this->set_item_name_index(Version::DC_V1_11_2000_PROTOTYPE, pc_v2_index); + this->set_item_name_index(Version::DC_V1, pc_v2_index); + this->set_item_name_index(Version::DC_V2, pc_v2_index); + this->set_item_name_index(Version::PC_NTE, pc_v2_index); + this->set_item_name_index(Version::PC_V2, pc_v2_index); + + auto gc_v3_index = create_item_name_index_for_version( + Version::GC_V3, this->item_parameter_table(Version::GC_V3), this->text_index); + this->set_item_name_index(Version::GC_NTE, gc_v3_index); + this->set_item_name_index(Version::GC_V3, gc_v3_index); + this->set_item_name_index(Version::XB_V3, gc_v3_index); + + auto bb_v4_index = create_item_name_index_for_version( + Version::BB_V4, this->item_parameter_table(Version::BB_V4), this->text_index); + this->set_item_name_index(Version::BB_V4, bb_v4_index); } -void ServerState::load_item_tables() { +void ServerState::load_drop_tables() { config_log.info("Loading rare item sets"); unordered_map> new_rare_item_sets; for (const auto& filename : list_directory_sorted("system/item-tables")) { @@ -1186,16 +1279,16 @@ void ServerState::load_item_tables() { if (ends_with(filename, "-v1.json")) { config_log.info("Loading v1 JSON rare item table %s", filename.c_str()); - new_rare_item_sets.emplace(basename, make_shared(JSON::parse(load_file(path)), Version::DC_V1, this->item_name_index)); + new_rare_item_sets.emplace(basename, make_shared(JSON::parse(load_file(path)), this->item_name_index(Version::DC_V1))); } else if (ends_with(filename, "-v2.json")) { config_log.info("Loading v2 JSON rare item table %s", filename.c_str()); - new_rare_item_sets.emplace(basename, make_shared(JSON::parse(load_file(path)), Version::PC_V2, this->item_name_index)); + new_rare_item_sets.emplace(basename, make_shared(JSON::parse(load_file(path)), this->item_name_index(Version::PC_V2))); } else if (ends_with(filename, "-v3.json")) { config_log.info("Loading v3 JSON rare item table %s", filename.c_str()); - new_rare_item_sets.emplace(basename, make_shared(JSON::parse(load_file(path)), Version::GC_V3, this->item_name_index)); + new_rare_item_sets.emplace(basename, make_shared(JSON::parse(load_file(path)), this->item_name_index(Version::GC_V3))); } else if (ends_with(filename, "-v4.json")) { config_log.info("Loading v4 JSON rare item table %s", filename.c_str()); - new_rare_item_sets.emplace(basename, make_shared(JSON::parse(load_file(path)), Version::BB_V4, this->item_name_index)); + new_rare_item_sets.emplace(basename, make_shared(JSON::parse(load_file(path)), this->item_name_index(Version::BB_V4))); } else if (ends_with(filename, ".afs")) { config_log.info("Loading AFS rare item table %s", filename.c_str()); @@ -1255,7 +1348,9 @@ void ServerState::load_item_tables() { config_log.info("Loading tekker adjustment table"); auto tekker_data = make_shared(load_file("system/item-tables/JudgeItem-gc.rel")); this->tekker_adjustment_set = make_shared(tekker_data); +} +void ServerState::load_item_definitions() { config_log.info("Loading item definition tables"); auto pmt_data_v2 = make_shared(prs_decompress(load_file("system/item-tables/ItemPMT-v2.prs"))); this->item_parameter_table_v2 = make_shared(pmt_data_v2, ItemParameterTable::Version::V2); diff --git a/src/ServerState.hh b/src/ServerState.hh index b7b581cb..32a8b7b4 100644 --- a/src/ServerState.hh +++ b/src/ServerState.hh @@ -138,7 +138,8 @@ struct ServerState : public std::enable_shared_from_this { std::shared_ptr item_parameter_table_v3; std::shared_ptr item_parameter_table_v4; std::shared_ptr mag_evolution_table; - std::shared_ptr item_name_index; + std::shared_ptr text_index; + std::array, NUM_VERSIONS> item_name_indexes; std::shared_ptr word_select_table; std::array, 4> rare_enemy_rates_by_difficulty; std::shared_ptr rare_enemy_rates_challenge; @@ -257,15 +258,18 @@ struct ServerState : public std::enable_shared_from_this { uint32_t connect_address_for_client(std::shared_ptr c) const; - std::shared_ptr information_menu_for_version(Version version) const; - std::shared_ptr proxy_destinations_menu_for_version(Version version) const; - const std::vector>& proxy_destinations_for_version(Version version) const; + std::shared_ptr information_menu(Version version) const; + std::shared_ptr proxy_destinations_menu(Version version) const; + const std::vector>& proxy_destinations(Version version) const; - std::shared_ptr item_parameter_table_for_version(Version version) const; + std::shared_ptr item_parameter_table(Version version) const; + std::shared_ptr item_name_index(Version version) const; + void set_item_name_index(Version version, std::shared_ptr); std::string describe_item(Version version, const ItemData& item, bool include_color_codes) const; + ItemData parse_item_description(Version version, const std::string& description) const; std::shared_ptr> information_contents_for_client(std::shared_ptr c) const; - std::shared_ptr quest_index_for_version(Version version) const; + std::shared_ptr quest_index(Version version) const; void set_port_configuration(const std::vector& port_configs); diff --git a/src/TextArchive.cc b/src/TextArchive.cc deleted file mode 100644 index a7a0f5a9..00000000 --- a/src/TextArchive.cc +++ /dev/null @@ -1,302 +0,0 @@ -#include "TextArchive.hh" - -#include -#include -#include -#include -#include - -#include "Compression.hh" -#include "PSOEncryption.hh" -#include "Text.hh" - -using namespace std; - -TextArchive::TextArchive(const string& pr2_data, bool big_endian) { - if (big_endian) { - this->load_t(pr2_data); - } else { - this->load_t(pr2_data); - } -} - -TextArchive::TextArchive(const JSON& json) { - for (const auto& collection_json : json.at("collections").as_list()) { - auto& collection = this->collections.emplace_back(); - for (const auto& string_json : collection_json->as_list()) { - collection.emplace_back(string_json->as_string()); - } - } - - for (const auto& keyboard_json : json.at("keyboards").as_list()) { - auto& keyboard = this->keyboards.emplace_back(make_unique()); - for (size_t y = 0; y < keyboard->size(); y++) { - auto& row = keyboard->at(y); - const auto& row_json = keyboard_json->at(y); - for (size_t x = 0; x < row.size(); x++) { - row[x] = row_json.at(x).as_int(); - } - } - } - - this->keyboard_selector_width = json.at("keyboard_selector_width").as_int(); -} - -JSON TextArchive::json() const { - auto collections_json = JSON::list(); - for (const auto& collection : this->collections) { - auto collection_json = JSON::list(); - for (const auto& s : collection) { - collection_json.emplace_back(s); - } - collections_json.emplace_back(std::move(collection_json)); - } - auto keyboards_json = JSON::list(); - for (const auto& kb : this->keyboards) { - JSON keyboard_json = JSON::list(); - for (size_t y = 0; y < kb->size(); y++) { - const auto& row = kb->at(y); - JSON row_json = JSON::list(); - for (size_t x = 0; x < row.size(); x++) { - row_json.emplace_back(row[x]); - } - keyboard_json.emplace_back(std::move(row_json)); - } - keyboards_json.emplace_back(std::move(keyboard_json)); - } - return JSON::dict({ - {"collections", std::move(collections_json)}, - {"keyboards", std::move(keyboards_json)}, - {"keyboard_selector_width", this->keyboard_selector_width}, - }); -} - -const string& TextArchive::get_string(size_t collection_index, size_t index) const { - return this->collections.at(collection_index).at(index); -} - -void TextArchive::set_string(size_t collection_index, size_t index, const string& data) { - if (collection_index >= this->collections.size()) { - this->collections.resize(collection_index + 1); - } - auto& coll = this->collections[collection_index]; - if (index >= coll.size()) { - coll.resize(index + 1); - } - coll[index] = data; -} - -void TextArchive::set_string(size_t collection_index, size_t index, string&& data) { - if (collection_index >= this->collections.size()) { - this->collections.resize(collection_index + 1); - } - auto& coll = this->collections[collection_index]; - if (index >= coll.size()) { - coll.resize(index + 1); - } - coll[index] = std::move(data); -} - -void TextArchive::resize_collection(size_t collection_index, size_t size) { - if (collection_index >= this->collections.size()) { - this->collections.resize(collection_index + 1); - } - this->collections[collection_index].resize(size); -} - -void TextArchive::resize_collection(size_t num_collections) { - this->collections.resize(num_collections); -} - -TextArchive::Keyboard TextArchive::get_keyboard(size_t kb_index) const { - return *this->keyboards.at(kb_index); -} - -void TextArchive::set_keyboard(size_t kb_index, const Keyboard& kb) { - if (kb_index >= this->keyboards.size()) { - this->keyboards.resize(kb_index + 1); - } - this->keyboards[kb_index] = make_unique(kb); -} - -void TextArchive::resize_keyboards(size_t num_keyboards) { - this->keyboards.resize(num_keyboards); -} - -pair TextArchive::serialize(bool big_endian) const { - if (big_endian) { - return this->serialize_t(); - } else { - return this->serialize_t(); - } -} - -template -void TextArchive::load_t(const string& pr2_data) { - using U32T = std::conditional_t; - using U16T = std::conditional_t; - - // The structure is as follows: - // Footer: - // U32T keyboard_index_offset ->: - // U8 num_keyboards - // U8 keyboard_selector_width - // U8 unused[2] - // U32T keyboards_offset ->: - // U32T keyboard_offset[num_keyboards] ->: - // U16T key_defs[7][16] - // U32T collections_offset ->: - // U32T[...] strings_offset ->: - // U32T[...] string_offset ->: - // char string[...\0] - // - - auto pr2_decrypted = decrypt_pr2_data(pr2_data); - auto decompressed = prs_decompress(pr2_decrypted.compressed_data); - StringReader r(decompressed); - - // Annoyingly, there doesn't appear to be any bounds-checking on the language - // functions, so there are no counts of strings in each collection. We have to - // figure out where each collection ends by collecting all the relevant - // offsets in the file instead. - set used_offsets; - used_offsets.emplace(r.size() - 8); - - uint32_t keyboard_index_offset = r.pget(r.size() - 8); - used_offsets.emplace(keyboard_index_offset); - size_t num_keyboards = r.pget_u8(keyboard_index_offset); - this->keyboard_selector_width = r.pget_u8(keyboard_index_offset + 1); - uint32_t keyboards_offset = r.pget(keyboard_index_offset + 4); - used_offsets.emplace(keyboards_offset); - while (this->keyboards.size() < num_keyboards) { - uint32_t keyboard_offset = r.pget(keyboards_offset + 4 * this->keyboards.size()); - used_offsets.emplace(keyboard_offset); - auto& kb = this->keyboards.emplace_back(make_unique()); - auto key_r = r.sub(keyboard_offset, sizeof(Keyboard)); - for (size_t y = 0; y < kb->size(); y++) { - auto& row = kb->at(y); - for (size_t x = 0; x < row.size(); x++) { - row[x] = key_r.get(); - } - } - } - - uint32_t collections_offset = r.pget(r.size() - 4); - for (uint32_t offset = collections_offset; !used_offsets.count(offset); offset += 4) { - used_offsets.emplace(r.pget(offset)); - } - used_offsets.emplace(collections_offset); - - for (uint32_t offset = collections_offset; (offset == collections_offset) || !used_offsets.count(offset); offset += 4) { - auto& collection = this->collections.emplace_back(); - uint32_t first_string_offset_offset = r.pget(offset); - for (uint32_t string_offset_offset = first_string_offset_offset; - (string_offset_offset == first_string_offset_offset) || !used_offsets.count(string_offset_offset); - string_offset_offset += 4) { - collection.emplace_back(r.pget_cstr(r.pget(string_offset_offset))); - } - } -} - -template -pair TextArchive::serialize_t() const { - using U32T = std::conditional_t; - using U16T = std::conditional_t; - - StringWriter w; - set relocation_offsets; - auto put_offset_u32 = [&](uint32_t v) { - relocation_offsets.emplace(w.size()); - w.put(v); - }; - - uint32_t collections_offset; - { - unordered_map string_to_offset; - for (const auto& collection : this->collections) { - for (const auto& s : collection) { - if (string_to_offset.emplace(s, w.size()).second) { - w.write(s); - w.put_u8(0); - while (w.size() & 3) { - w.put_u8(0); - } - } - } - } - - vector collection_offsets; - for (const auto& collection : this->collections) { - collection_offsets.emplace_back(w.size()); - for (const auto& s : collection) { - put_offset_u32(string_to_offset.at(s)); - } - } - - collections_offset = w.size(); - for (uint32_t collection_offset : collection_offsets) { - put_offset_u32(collection_offset); - } - } - - uint32_t keyboard_index_offset; - { - vector keyboard_offsets; - for (const auto& keyboard : this->keyboards) { - keyboard_offsets.emplace_back(w.size()); - for (size_t y = 0; y < keyboard->size(); y++) { - const auto& row = keyboard->at(y); - for (size_t x = 0; x < row.size(); x++) { - w.put(row[x]); - } - } - } - - uint32_t keyboards_offset = w.size(); - for (uint32_t keyboard_offset : keyboard_offsets) { - put_offset_u32(keyboard_offset); - } - - keyboard_index_offset = w.size(); - w.put_u8(keyboard_offsets.size()); - w.put_u8(this->keyboard_selector_width); - w.put_u16(0); - put_offset_u32(keyboards_offset); - } - - put_offset_u32(keyboard_index_offset); - put_offset_u32(collections_offset); - - StringWriter reloc_w; - reloc_w.put_u32(0); - reloc_w.put(relocation_offsets.size()); - reloc_w.put_u64(0); - reloc_w.put(w.size() - 8); - reloc_w.put_u32(0); - reloc_w.put_u64(0); - { - size_t offset = 0; - for (size_t reloc_offset : relocation_offsets) { - if (reloc_offset & 3) { - throw logic_error("misaligned relocation"); - } - size_t num_words = (reloc_offset - offset) >> 2; - if (num_words > 0xFFFF) { - throw runtime_error("relocation offset too far away"); - } - reloc_w.put(num_words); - offset = reloc_offset; - } - } - - const string& pr2_data = w.str(); - const string& pr3_data = reloc_w.str(); - print_data(stderr, pr2_data); - string pr2_compressed = prs_compress_optimal(pr2_data.data(), pr2_data.size()); - string pr3_compressed = prs_compress_optimal(pr3_data.data(), pr3_data.size()); - print_data(stderr, pr2_compressed); - string pr2_ret = encrypt_pr2_data(pr2_compressed, pr2_data.size(), random_object()); - string pr3_ret = encrypt_pr2_data(pr3_compressed, pr3_data.size(), random_object()); - print_data(stderr, pr2_ret); - return make_pair(std::move(pr2_ret), std::move(pr3_ret)); -} diff --git a/src/TextArchive.hh b/src/TextArchive.hh deleted file mode 100644 index f99f7f2b..00000000 --- a/src/TextArchive.hh +++ /dev/null @@ -1,51 +0,0 @@ -#pragma once - -#include - -#include -#include -#include -#include - -#include "Text.hh" - -// This class implements loading and saving of text archives, commonly found in -// PSO games with filenames like TextEnglish.pr2 and TextEnglish.pr3. The game -// requires both files, but newserv needs only the pr2 file to load a text -// archive. When saving (serializing), both pr2 and pr3 files are generated. -class TextArchive { -public: - using Keyboard = parray, 7>; - - explicit TextArchive(const JSON& json); - TextArchive(const std::string& pr2_data, bool big_endian); - ~TextArchive() = default; - - JSON json() const; - - const std::string& get_string(size_t collection_index, size_t index) const; - void set_string(size_t collection_index, size_t index, const std::string& data); - void set_string(size_t collection_index, size_t index, std::string&& data); - void resize_collection(size_t collection_index, size_t size); - void resize_collection(size_t num_collections); - - Keyboard get_keyboard(size_t kb_index) const; - void set_keyboard(size_t kb_index, const Keyboard& kb); - void resize_keyboards(size_t num_keyboards); - - uint8_t get_keyboard_selector_width() const; - void set_keyboard_selector_width(uint8_t width); - - // Returns (pr2_data, pr3_data) - std::pair serialize(bool big_endian) const; - -private: - template - void load_t(const std::string& pr2_data); - template - std::pair serialize_t() const; - - std::vector> collections; - std::vector> keyboards; - uint8_t keyboard_selector_width; -}; diff --git a/src/TextIndex.cc b/src/TextIndex.cc new file mode 100644 index 00000000..c42c0320 --- /dev/null +++ b/src/TextIndex.cc @@ -0,0 +1,557 @@ +#include "TextIndex.hh" + +#include +#include +#include +#include +#include + +#include "Compression.hh" +#include "Loggers.hh" +#include "PSOEncryption.hh" +#include "StaticGameData.hh" +#include "Text.hh" + +using namespace std; + +TextSet::TextSet(const JSON& json) { + for (const auto& coll_json : json.as_list()) { + auto& collection = this->collections.emplace_back(); + for (const auto& s_json : coll_json->as_list()) { + collection.emplace_back(s_json->as_string()); + } + } +} + +TextSet::TextSet(JSON&& json) { + for (const auto& coll_json : json.as_list()) { + auto& collection = this->collections.emplace_back(); + for (const auto& s_json : coll_json->as_list()) { + collection.emplace_back(std::move(s_json->as_string())); + } + } +} + +JSON TextSet::json() const { + JSON j = JSON::list(); + for (const auto& collection : this->collections) { + JSON& coll_j = j.emplace_back(JSON::list()); + for (const auto& s : collection) { + coll_j.emplace_back(s); + } + } + return j; +} + +size_t TextSet::count(size_t collection_index) const { + return this->collections.at(collection_index).size(); +} + +size_t TextSet::count() const { + return this->collections.size(); +} + +const std::string& TextSet::get(size_t collection_index, size_t string_index) const { + return this->get(collection_index).at(string_index); +} + +const vector& TextSet::get(size_t collection_index) const { + return this->collections.at(collection_index); +} + +void TextSet::set(size_t collection_index, size_t string_index, const std::string& data) { + this->ensure_slot_exists(collection_index, string_index); + this->collections[collection_index][string_index] = data; +} + +void TextSet::set(size_t collection_index, size_t string_index, std::string&& data) { + this->ensure_slot_exists(collection_index, string_index); + this->collections[collection_index][string_index] = std::move(data); +} + +void TextSet::set(size_t collection_index, const std::vector& coll) { + this->ensure_collection_exists(collection_index); + this->collections[collection_index] = coll; +} + +void TextSet::set(size_t collection_index, std::vector&& coll) { + this->ensure_collection_exists(collection_index); + this->collections[collection_index] = std::move(coll); +} + +void TextSet::truncate_collection(size_t collection_index, size_t new_entry_count) { + if (collection_index >= this->collections.size()) { + this->collections.resize(collection_index + 1); + } + this->collections[collection_index].resize(new_entry_count); +} + +void TextSet::truncate(size_t new_collection_count) { + this->collections.resize(new_collection_count); +} + +void TextSet::ensure_slot_exists(size_t collection_index, size_t string_index) { + this->ensure_collection_exists(collection_index); + auto& coll = this->collections[collection_index]; + if (string_index >= coll.size()) { + coll.resize(string_index + 1); + } +} + +void TextSet::ensure_collection_exists(size_t collection_index) { + if (collection_index >= this->collections.size()) { + this->collections.resize(collection_index + 1); + } +} + +UnicodeTextSet::UnicodeTextSet(const string& prs_data) { + string data = prs_decompress(prs_data); + StringReader r(data); + + uint32_t num_collections = r.get_u32l(); + deque collection_sizes; + while (collection_sizes.size() < num_collections) { + collection_sizes.emplace_back(r.get_u32l()); + } + + this->collections.reserve(collection_sizes.size()); + while (!collection_sizes.empty()) { + uint32_t num_strings = collection_sizes.front(); + collection_sizes.pop_front(); + + auto& strings = this->collections.emplace_back(); + strings.reserve(num_strings); + while (strings.size() < num_strings) { + StringReader sub_r = r.sub(r.get_u32l()); + StringWriter w; + for (uint16_t ch = sub_r.get_u16l(); ch != 0; ch = sub_r.get_u16l()) { + w.put_u16l(ch); + } + strings.emplace_back(tt_utf16_to_utf8(w.str())); + } + } +} + +string UnicodeTextSet::serialize() const { + StringWriter header_w; + StringWriter data_w; + + size_t total_num_strings = 0; + header_w.put_u32l(this->collections.size()); + for (const auto& collection : this->collections) { + header_w.put_u32l(collection.size()); + total_num_strings += collection.size(); + } + + unordered_map encoded; + + size_t data_base_offset = (total_num_strings * 4) + header_w.size(); + for (const auto& collection : this->collections) { + for (const auto& s : collection) { + auto encoded_it = encoded.find(s); + if (encoded_it == encoded.end()) { + uint32_t offset = data_base_offset + data_w.size(); + encoded_it = encoded.emplace(s, offset).first; + string s_utf16 = tt_utf8_to_utf16(s); + data_w.write(s_utf16.data(), s_utf16.size()); + data_w.put_u16(0); + while (data_w.size() & 3) { + data_w.put_u8(0); + } + } + header_w.put_u32l(encoded_it->second); + } + } + + header_w.write(data_w.str()); + return prs_compress_optimal(header_w.str()); +} + +BinaryTextSet::BinaryTextSet(const std::string& pr2_data, size_t collection_count, bool has_rel_footer) { + auto pr2_decrypted = decrypt_pr2_data(pr2_data); + auto decompressed = prs_decompress(pr2_decrypted.compressed_data); + StringReader r(decompressed); + + // Annoyingly, there doesn't appear to be any bounds-checking on the language + // functions, so there are no counts of strings in each collection. We have to + // figure out where each collection ends by collecting all the relevant + // offsets in the file instead. + ::set used_offsets; + size_t root_offset = has_rel_footer + ? r.pget_u32l(r.size() - 0x10) + : (r.size() - collection_count * sizeof(le_uint32_t)); + + StringReader collection_offsets_r = r.sub(root_offset, collection_count * sizeof(le_uint32_t)); + while (!collection_offsets_r.eof()) { + used_offsets.emplace(collection_offsets_r.get_u32l()); + } + used_offsets.emplace(root_offset); + + collection_offsets_r.go(0); + while (!collection_offsets_r.eof()) { + auto& collection = this->collections.emplace_back(); + uint32_t first_string_offset_offset = collection_offsets_r.get_u32l(); + // TODO: Apparently the early formats do actually include keyboards, but + // they're just in the middle of the collections list. Sigh... + try { + for (uint32_t string_offset_offset = first_string_offset_offset; + (string_offset_offset == first_string_offset_offset) || !used_offsets.count(string_offset_offset); + string_offset_offset += 4) { + collection.emplace_back(r.pget_cstr(r.pget_u32l(string_offset_offset))); + } + } catch (const out_of_range&) { + } + } +} + +BinaryTextAndKeyboardsSet::BinaryTextAndKeyboardsSet(const string& pr2_data, bool big_endian) { + if (big_endian) { + this->parse_t(pr2_data); + } else { + this->parse_t(pr2_data); + } +} + +BinaryTextAndKeyboardsSet::BinaryTextAndKeyboardsSet(const JSON& json) { + for (const auto& collection_json : json.at("collections").as_list()) { + auto& collection = this->collections.emplace_back(); + for (const auto& string_json : collection_json->as_list()) { + collection.emplace_back(string_json->as_string()); + } + } + + for (const auto& keyboard_json : json.at("keyboards").as_list()) { + auto& keyboard = this->keyboards.emplace_back(make_unique()); + for (size_t y = 0; y < keyboard->size(); y++) { + auto& row = keyboard->at(y); + const auto& row_json = keyboard_json->at(y); + for (size_t x = 0; x < row.size(); x++) { + row[x] = row_json.at(x).as_int(); + } + } + } + + this->keyboard_selector_width = json.at("keyboard_selector_width").as_int(); +} + +JSON BinaryTextAndKeyboardsSet::json() const { + auto collections_json = this->TextSet::json(); + auto keyboards_json = JSON::list(); + for (const auto& kb : this->keyboards) { + JSON keyboard_json = JSON::list(); + for (size_t y = 0; y < kb->size(); y++) { + const auto& row = kb->at(y); + JSON row_json = JSON::list(); + for (size_t x = 0; x < row.size(); x++) { + row_json.emplace_back(row[x]); + } + keyboard_json.emplace_back(std::move(row_json)); + } + keyboards_json.emplace_back(std::move(keyboard_json)); + } + return JSON::dict({ + {"collections", std::move(collections_json)}, + {"keyboards", std::move(keyboards_json)}, + {"keyboard_selector_width", this->keyboard_selector_width}, + }); +} + +const BinaryTextAndKeyboardsSet::Keyboard& BinaryTextAndKeyboardsSet::get_keyboard(size_t kb_index) const { + return *this->keyboards.at(kb_index); +} + +void BinaryTextAndKeyboardsSet::set_keyboard(size_t kb_index, const Keyboard& kb) { + if (kb_index >= this->keyboards.size()) { + this->keyboards.resize(kb_index + 1); + } + this->keyboards[kb_index] = make_unique(kb); +} + +void BinaryTextAndKeyboardsSet::resize_keyboards(size_t num_keyboards) { + this->keyboards.resize(num_keyboards); +} + +pair BinaryTextAndKeyboardsSet::serialize(bool big_endian) const { + if (big_endian) { + return this->serialize_t(); + } else { + return this->serialize_t(); + } +} + +template +void BinaryTextAndKeyboardsSet::parse_t(const string& pr2_data) { + using U32T = std::conditional_t; + using U16T = std::conditional_t; + + // The structure is as follows: + // Footer: + // U32T keyboard_index_offset ->: + // U8 num_keyboards + // U8 keyboard_selector_width + // U8 unused[2] + // U32T keyboards_offset ->: + // U32T keyboard_offset[num_keyboards] ->: + // U16T key_defs[7][16] + // U32T collections_offset ->: + // U32T[...] strings_offset ->: + // U32T[...] string_offset ->: + // char string[...\0] + // + + auto pr2_decrypted = decrypt_pr2_data(pr2_data); + auto decompressed = prs_decompress(pr2_decrypted.compressed_data); + StringReader r(decompressed); + + // Annoyingly, there doesn't appear to be any bounds-checking on the language + // functions, so there are no counts of strings in each collection. We have to + // figure out where each collection ends by collecting all the relevant + // offsets in the file instead. + ::set used_offsets; + used_offsets.emplace(r.size() - 8); + + uint32_t keyboard_index_offset = r.pget(r.size() - 8); + used_offsets.emplace(keyboard_index_offset); + size_t num_keyboards = r.pget_u8(keyboard_index_offset); + this->keyboard_selector_width = r.pget_u8(keyboard_index_offset + 1); + uint32_t keyboards_offset = r.pget(keyboard_index_offset + 4); + used_offsets.emplace(keyboards_offset); + while (this->keyboards.size() < num_keyboards) { + uint32_t keyboard_offset = r.pget(keyboards_offset + 4 * this->keyboards.size()); + used_offsets.emplace(keyboard_offset); + auto& kb = this->keyboards.emplace_back(make_unique()); + auto key_r = r.sub(keyboard_offset, sizeof(Keyboard)); + for (size_t y = 0; y < kb->size(); y++) { + auto& row = kb->at(y); + for (size_t x = 0; x < row.size(); x++) { + row[x] = key_r.get(); + } + } + } + + uint32_t collections_offset = r.pget(r.size() - 4); + for (uint32_t offset = collections_offset; !used_offsets.count(offset); offset += 4) { + used_offsets.emplace(r.pget(offset)); + } + used_offsets.emplace(collections_offset); + + for (uint32_t offset = collections_offset; (offset == collections_offset) || !used_offsets.count(offset); offset += 4) { + auto& collection = this->collections.emplace_back(); + uint32_t first_string_offset_offset = r.pget(offset); + for (uint32_t string_offset_offset = first_string_offset_offset; + (string_offset_offset == first_string_offset_offset) || !used_offsets.count(string_offset_offset); + string_offset_offset += 4) { + collection.emplace_back(r.pget_cstr(r.pget(string_offset_offset))); + } + } +} + +template +pair BinaryTextAndKeyboardsSet::serialize_t() const { + using U32T = std::conditional_t; + using U16T = std::conditional_t; + + StringWriter w; + ::set relocation_offsets; + auto put_offset_u32 = [&](uint32_t v) { + relocation_offsets.emplace(w.size()); + w.put(v); + }; + + uint32_t collections_offset; + { + unordered_map string_to_offset; + for (const auto& collection : this->collections) { + for (const auto& s : collection) { + if (string_to_offset.emplace(s, w.size()).second) { + w.write(s); + w.put_u8(0); + while (w.size() & 3) { + w.put_u8(0); + } + } + } + } + + vector collection_offsets; + for (const auto& collection : this->collections) { + collection_offsets.emplace_back(w.size()); + for (const auto& s : collection) { + put_offset_u32(string_to_offset.at(s)); + } + } + + collections_offset = w.size(); + for (uint32_t collection_offset : collection_offsets) { + put_offset_u32(collection_offset); + } + } + + uint32_t keyboard_index_offset; + { + vector keyboard_offsets; + for (const auto& keyboard : this->keyboards) { + keyboard_offsets.emplace_back(w.size()); + for (size_t y = 0; y < keyboard->size(); y++) { + const auto& row = keyboard->at(y); + for (size_t x = 0; x < row.size(); x++) { + w.put(row[x]); + } + } + } + + uint32_t keyboards_offset = w.size(); + for (uint32_t keyboard_offset : keyboard_offsets) { + put_offset_u32(keyboard_offset); + } + + keyboard_index_offset = w.size(); + w.put_u8(keyboard_offsets.size()); + w.put_u8(this->keyboard_selector_width); + w.put_u16(0); + put_offset_u32(keyboards_offset); + } + + put_offset_u32(keyboard_index_offset); + put_offset_u32(collections_offset); + + StringWriter reloc_w; + reloc_w.put_u32(0); + reloc_w.put(relocation_offsets.size()); + reloc_w.put_u64(0); + reloc_w.put(w.size() - 8); + reloc_w.put_u32(0); + reloc_w.put_u64(0); + { + size_t offset = 0; + for (size_t reloc_offset : relocation_offsets) { + if (reloc_offset & 3) { + throw logic_error("misaligned relocation"); + } + size_t num_words = (reloc_offset - offset) >> 2; + if (num_words > 0xFFFF) { + throw runtime_error("relocation offset too far away"); + } + reloc_w.put(num_words); + offset = reloc_offset; + } + } + + const string& pr2_data = w.str(); + const string& pr3_data = reloc_w.str(); + print_data(stderr, pr2_data); + string pr2_compressed = prs_compress_optimal(pr2_data.data(), pr2_data.size()); + string pr3_compressed = prs_compress_optimal(pr3_data.data(), pr3_data.size()); + print_data(stderr, pr2_compressed); + string pr2_ret = encrypt_pr2_data(pr2_compressed, pr2_data.size(), random_object()); + string pr3_ret = encrypt_pr2_data(pr3_compressed, pr3_data.size(), random_object()); + print_data(stderr, pr2_ret); + return make_pair(std::move(pr2_ret), std::move(pr3_ret)); +} + +TextIndex::TextIndex( + const string& directory, + function(Version, const string&)> get_patch_file) + : log("[TextIndex] ", static_game_data_log.min_level) { + if (!directory.empty()) { + auto add_version = [&](Version version, const string& subdirectory, function(const string&)> make_set) -> void { + static const map bintext_filenames({ + {"TextJapanese.pr2", 0x00}, + {"TextEnglish.pr2", 0x01}, + {"TextGerman.pr2", 0x02}, + {"TextFrench.pr2", 0x03}, + {"TextSpanish.pr2", 0x04}, + }); + static const map unitext_filenames({ + {"unitxt_j.prs", 0x00}, // PC/BB Japanese + {"unitxt_e.prs", 0x01}, // PC/BB English + {"unitxt_g.prs", 0x02}, // PC/BB German + {"unitxt_f.prs", 0x03}, // PC/BB French + {"unitxt_s.prs", 0x04}, // PC/BB Spanish + {"unitxt_b.prs", 0x05}, // PC Simplified Chinese + {"unitxt_cs.prs", 0x05}, // BB Simplified Chinese + {"unitxt_t.prs", 0x06}, // PC Traditional Chinese + {"unitxt_ct.prs", 0x06}, // BB Traditional Chinese + {"unitxt_k.prs", 0x07}, // PC Korean + {"unitxt_h.prs", 0x07}, // BB Korean + }); + if (!uses_utf16(version)) { + for (const auto& it : bintext_filenames) { + string file_path = directory + "/" + subdirectory + "/" + it.first; + string json_path = file_path + ".json"; + if (isfile(json_path)) { + this->log.info("Loading %s %c JSON text set from %s", name_for_enum(version), char_for_language_code(it.second), json_path.c_str()); + this->add_set(version, it.second, make_shared(JSON::parse(load_file(json_path)))); + } else if (isfile(file_path)) { + this->log.info("Loading %s %c binary text set from %s", name_for_enum(version), char_for_language_code(it.second), file_path.c_str()); + this->add_set(version, it.second, make_set(load_file(file_path))); + } + } + } else { + for (const auto& it : unitext_filenames) { + string file_path = directory + "/" + subdirectory + "/" + it.first; + string json_path = file_path + ".json"; + if (isfile(json_path)) { + this->log.info("Loading %s %c JSON text set from %s", name_for_enum(version), char_for_language_code(it.second), json_path.c_str()); + this->add_set(version, it.second, make_shared(JSON::parse(load_file(json_path)))); + } else { + auto patch_file = get_patch_file ? get_patch_file(version, it.first) : nullptr; + if (patch_file) { + this->log.info("Loading %s %c Unicode text set from %s in patch tree", name_for_enum(version), char_for_language_code(it.second), it.first.c_str()); + this->add_set(version, it.second, make_set(*patch_file)); + } else { + if (isfile(file_path)) { + this->log.info("Loading %s %c Unicode text set from %s", name_for_enum(version), char_for_language_code(it.second), file_path.c_str()); + this->add_set(version, it.second, make_set(load_file(file_path))); + } + } + } + } + } + }; + + auto make_binary_dc112000 = +[](const string& data) { return make_shared(data, 21, true); }; + auto make_binary_dcnte_dcv1 = +[](const string& data) { return make_shared(data, 26, true); }; + auto make_binary_dcv2 = +[](const string& data) { return make_shared(data, 37, false); }; + auto make_binary_gc = +[](const string& data) { return make_shared(data, true); }; + auto make_binary_xb = +[](const string& data) { return make_shared(data, false); }; + auto make_unitxt = +[](const string& data) { return make_shared(data); }; + + add_version(Version::DC_NTE, "dc-nte", make_binary_dcnte_dcv1); + add_version(Version::DC_V1_11_2000_PROTOTYPE, "dc-11-2000", make_binary_dc112000); + add_version(Version::DC_V1, "dc-v1", make_binary_dcnte_dcv1); + add_version(Version::DC_V2, "dc-v2", make_binary_dcv2); + add_version(Version::PC_NTE, "pc-nte", make_unitxt); + add_version(Version::PC_V2, "pc-v2", make_unitxt); + add_version(Version::GC_NTE, "gc-nte", make_binary_gc); + add_version(Version::GC_V3, "gc-v3", make_binary_gc); + add_version(Version::GC_EP3_NTE, "gc-ep3-nte", make_binary_gc); + add_version(Version::GC_EP3, "gc-ep3", make_binary_gc); + add_version(Version::XB_V3, "xb-v3", make_binary_xb); + add_version(Version::BB_V4, "bb-v4", make_unitxt); + } +} + +void TextIndex::add_set(Version version, uint8_t language, std::shared_ptr ts) { + this->sets[this->key_for_set(version, language)] = ts; +} + +void TextIndex::delete_set(Version version, uint8_t language) { + this->sets.erase(this->key_for_set(version, language)); +} + +const std::string& TextIndex::get(Version version, uint8_t language, size_t collection_index, size_t string_index) const { + return this->get(version, language)->get(collection_index, string_index); +} + +const std::vector& TextIndex::get(Version version, uint8_t language, size_t collection_index) const { + return this->get(version, language)->get(collection_index); +} + +std::shared_ptr TextIndex::get(Version version, uint8_t language) const { + return this->sets.at(this->key_for_set(version, language)); +} + +uint32_t TextIndex::key_for_set(Version version, uint8_t language) { + return (static_cast(version) << 8) | language; +} diff --git a/src/TextIndex.hh b/src/TextIndex.hh new file mode 100644 index 00000000..f6465def --- /dev/null +++ b/src/TextIndex.hh @@ -0,0 +1,111 @@ +#pragma once + +#include + +#include +#include +#include +#include + +#include "Text.hh" +#include "Version.hh" + +class TextSet { +public: + virtual ~TextSet() = default; + virtual JSON json() const; + + size_t count(size_t collection_index) const; + size_t count() const; + + const std::string& get(size_t collection, size_t index) const; + const std::vector& get(size_t collection) const; + + void set(size_t collection_index, size_t string_index, const std::string& data); + void set(size_t collection_index, size_t string_index, std::string&& data); + void set(size_t collection_index, const std::vector& coll); + void set(size_t collection_index, std::vector&& coll); + + void truncate_collection(size_t collection, size_t new_entry_count); + void truncate(size_t new_collection_count); + +protected: + std::vector> collections; + + TextSet() = default; + TextSet(const JSON& json); + TextSet(JSON&& json); + + void ensure_slot_exists(size_t collection_index, size_t string_index); + void ensure_collection_exists(size_t collection_index); +}; + +class UnicodeTextSet : public TextSet { +public: + explicit UnicodeTextSet(const JSON& json) : TextSet(json) {} + explicit UnicodeTextSet(JSON&& json) : TextSet(json) {} + explicit UnicodeTextSet(const std::string& unitxt_prs_data); + virtual ~UnicodeTextSet() = default; + std::string serialize() const; +}; + +class BinaryTextSet : public TextSet { +public: + explicit BinaryTextSet(const JSON& json) : TextSet(json) {} + explicit BinaryTextSet(JSON&& json) : TextSet(json) {} + BinaryTextSet(const std::string& pr2_data, size_t collection_count, bool has_rel_footer); + ~BinaryTextSet() = default; + // TODO: Implement serialize functions +}; + +class BinaryTextAndKeyboardsSet : public TextSet { +public: + using Keyboard = parray, 7>; + + explicit BinaryTextAndKeyboardsSet(const JSON& json); + explicit BinaryTextAndKeyboardsSet(JSON&& json); + BinaryTextAndKeyboardsSet(const std::string& pr2_data, bool big_endian); + ~BinaryTextAndKeyboardsSet() = default; + + virtual JSON json() const; + + const Keyboard& get_keyboard(size_t kb_index) const; + void set_keyboard(size_t kb_index, const Keyboard& kb); + void resize_keyboards(size_t num_keyboards); + + uint8_t get_keyboard_selector_width() const; + void set_keyboard_selector_width(uint8_t width); + + // Returns (pr2_data, pr3_data) + std::pair serialize(bool big_endian) const; + +protected: + template + void parse_t(const std::string& pr2_data); + template + std::pair serialize_t() const; + + std::vector> keyboards; + uint8_t keyboard_selector_width; +}; + +class TextIndex { +public: + explicit TextIndex( + const std::string& directory = "", + std::function(Version, const std::string&)> get_patch_file = nullptr); + ~TextIndex() = default; + + void add_set(Version version, uint8_t language, std::shared_ptr ts); + void delete_set(Version version, uint8_t language); + + const std::string& get(Version version, uint8_t language, size_t collection_index, size_t string_index) const; + const std::vector& get(Version version, uint8_t language, size_t collection_index) const; + std::shared_ptr get(Version version, uint8_t language) const; + +protected: + static uint32_t key_for_set(Version version, uint8_t language); + + PrefixedLogger log; + std::unordered_map> sets; +}; diff --git a/src/UnicodeTextSet.cc b/src/UnicodeTextSet.cc deleted file mode 100644 index 3972dc4f..00000000 --- a/src/UnicodeTextSet.cc +++ /dev/null @@ -1,75 +0,0 @@ -#include "UnicodeTextSet.hh" - -#include -#include -#include - -#include "Compression.hh" - -using namespace std; - -vector> parse_unicode_text_set(const string& prs_data) { - string data = prs_decompress(prs_data); - StringReader r(data); - - uint32_t num_collections = r.get_u32l(); - deque collection_sizes; - while (collection_sizes.size() < num_collections) { - collection_sizes.emplace_back(r.get_u32l()); - } - - vector> ret; - ret.reserve(collection_sizes.size()); - while (!collection_sizes.empty()) { - uint32_t num_strings = collection_sizes.front(); - collection_sizes.pop_front(); - - auto& strings = ret.emplace_back(); - strings.reserve(num_strings); - while (strings.size() < num_strings) { - StringReader sub_r = r.sub(r.get_u32l()); - StringWriter w; - for (uint16_t ch = sub_r.get_u16l(); ch != 0; ch = sub_r.get_u16l()) { - w.put_u16l(ch); - } - strings.emplace_back(tt_utf16_to_utf8(w.str())); - } - } - return ret; -} - -string serialize_unicode_text_set(const vector>& collections) { - StringWriter header_w; - StringWriter data_w; - - size_t total_num_strings = 0; - header_w.put_u32l(collections.size()); - for (const auto& collection : collections) { - header_w.put_u32l(collection.size()); - total_num_strings += collection.size(); - } - - unordered_map encoded; - - size_t data_base_offset = (total_num_strings * 4) + header_w.size(); - for (const auto& collection : collections) { - for (const auto& s : collection) { - auto encoded_it = encoded.find(s); - if (encoded_it == encoded.end()) { - uint32_t offset = data_base_offset + data_w.size(); - encoded.emplace(s, offset); - string s_utf16 = tt_utf8_to_utf16(s); - data_w.write(s_utf16.data(), s_utf16.size()); - data_w.put_u16(0); - while (data_w.size() & 3) { - data_w.put_u8(0); - } - } else { - header_w.put_u32l(encoded_it->second); - } - } - } - - header_w.write(data_w.str()); - return std::move(header_w.str()); -} diff --git a/src/UnicodeTextSet.hh b/src/UnicodeTextSet.hh deleted file mode 100644 index 99d4e0e0..00000000 --- a/src/UnicodeTextSet.hh +++ /dev/null @@ -1,7 +0,0 @@ -#pragma once - -#include -#include - -std::vector> parse_unicode_text_set(const std::string& prs_data); -std::string serialize_unicode_text_set(const std::vector>& collections); diff --git a/system/text-sets/bb-v4/unitxt_cs.prs b/system/text-sets/bb-v4/unitxt_cs.prs new file mode 100755 index 00000000..e5ad9fdd Binary files /dev/null and b/system/text-sets/bb-v4/unitxt_cs.prs differ diff --git a/system/text-sets/bb-v4/unitxt_ct.prs b/system/text-sets/bb-v4/unitxt_ct.prs new file mode 100755 index 00000000..b830666f Binary files /dev/null and b/system/text-sets/bb-v4/unitxt_ct.prs differ diff --git a/system/text-sets/bb-v4/unitxt_e.prs b/system/text-sets/bb-v4/unitxt_e.prs new file mode 100755 index 00000000..80934ac8 Binary files /dev/null and b/system/text-sets/bb-v4/unitxt_e.prs differ diff --git a/system/text-sets/bb-v4/unitxt_f.prs b/system/text-sets/bb-v4/unitxt_f.prs new file mode 100755 index 00000000..e9bed801 Binary files /dev/null and b/system/text-sets/bb-v4/unitxt_f.prs differ diff --git a/system/text-sets/bb-v4/unitxt_g.prs b/system/text-sets/bb-v4/unitxt_g.prs new file mode 100755 index 00000000..4880601b Binary files /dev/null and b/system/text-sets/bb-v4/unitxt_g.prs differ diff --git a/system/text-sets/bb-v4/unitxt_h.prs b/system/text-sets/bb-v4/unitxt_h.prs new file mode 100755 index 00000000..ff68603d Binary files /dev/null and b/system/text-sets/bb-v4/unitxt_h.prs differ diff --git a/system/text-sets/bb-v4/unitxt_j.prs b/system/text-sets/bb-v4/unitxt_j.prs new file mode 100755 index 00000000..6f25fa4d Binary files /dev/null and b/system/text-sets/bb-v4/unitxt_j.prs differ diff --git a/system/text-sets/bb-v4/unitxt_s.prs b/system/text-sets/bb-v4/unitxt_s.prs new file mode 100755 index 00000000..16842b66 Binary files /dev/null and b/system/text-sets/bb-v4/unitxt_s.prs differ diff --git a/system/word-select/bb_unitxt_ws.prs b/system/text-sets/bb-v4/unitxt_ws_e.prs similarity index 100% rename from system/word-select/bb_unitxt_ws.prs rename to system/text-sets/bb-v4/unitxt_ws_e.prs diff --git a/system/word-select/bb_ws_data.bin b/system/text-sets/bb-v4/ws_data.bin similarity index 100% rename from system/word-select/bb_ws_data.bin rename to system/text-sets/bb-v4/ws_data.bin diff --git a/system/text-sets/dc-11-2000/TextEnglish.pr2 b/system/text-sets/dc-11-2000/TextEnglish.pr2 new file mode 100644 index 00000000..2da2cba5 Binary files /dev/null and b/system/text-sets/dc-11-2000/TextEnglish.pr2 differ diff --git a/system/text-sets/dc-11-2000/TextFrench.pr2 b/system/text-sets/dc-11-2000/TextFrench.pr2 new file mode 100644 index 00000000..42133835 Binary files /dev/null and b/system/text-sets/dc-11-2000/TextFrench.pr2 differ diff --git a/system/text-sets/dc-11-2000/TextGerman.pr2 b/system/text-sets/dc-11-2000/TextGerman.pr2 new file mode 100644 index 00000000..e39a4d45 Binary files /dev/null and b/system/text-sets/dc-11-2000/TextGerman.pr2 differ diff --git a/system/text-sets/dc-11-2000/TextJapanese.pr2 b/system/text-sets/dc-11-2000/TextJapanese.pr2 new file mode 100644 index 00000000..0a4da6cb Binary files /dev/null and b/system/text-sets/dc-11-2000/TextJapanese.pr2 differ diff --git a/system/text-sets/dc-11-2000/TextSpanish.pr2 b/system/text-sets/dc-11-2000/TextSpanish.pr2 new file mode 100644 index 00000000..fdc73238 Binary files /dev/null and b/system/text-sets/dc-11-2000/TextSpanish.pr2 differ diff --git a/system/word-select/dc_112000_ws_data.bin b/system/text-sets/dc-11-2000/ws_data.bin similarity index 100% rename from system/word-select/dc_112000_ws_data.bin rename to system/text-sets/dc-11-2000/ws_data.bin diff --git a/system/text-sets/dc-nte/TextJapanese.pr2 b/system/text-sets/dc-nte/TextJapanese.pr2 new file mode 100644 index 00000000..f17849aa Binary files /dev/null and b/system/text-sets/dc-nte/TextJapanese.pr2 differ diff --git a/system/word-select/dc_nte_ws_data.bin b/system/text-sets/dc-nte/ws_data.bin similarity index 100% rename from system/word-select/dc_nte_ws_data.bin rename to system/text-sets/dc-nte/ws_data.bin diff --git a/system/text-sets/dc-v1/TextEnglish.pr2 b/system/text-sets/dc-v1/TextEnglish.pr2 new file mode 100644 index 00000000..f17849aa Binary files /dev/null and b/system/text-sets/dc-v1/TextEnglish.pr2 differ diff --git a/system/text-sets/dc-v1/TextFrench.pr2 b/system/text-sets/dc-v1/TextFrench.pr2 new file mode 100644 index 00000000..7ec911fa Binary files /dev/null and b/system/text-sets/dc-v1/TextFrench.pr2 differ diff --git a/system/text-sets/dc-v1/TextGerman.pr2 b/system/text-sets/dc-v1/TextGerman.pr2 new file mode 100644 index 00000000..dfb1ca0e Binary files /dev/null and b/system/text-sets/dc-v1/TextGerman.pr2 differ diff --git a/system/text-sets/dc-v1/TextJapanese.pr2 b/system/text-sets/dc-v1/TextJapanese.pr2 new file mode 100644 index 00000000..eecab395 Binary files /dev/null and b/system/text-sets/dc-v1/TextJapanese.pr2 differ diff --git a/system/text-sets/dc-v1/TextSpanish.pr2 b/system/text-sets/dc-v1/TextSpanish.pr2 new file mode 100644 index 00000000..709f2c48 Binary files /dev/null and b/system/text-sets/dc-v1/TextSpanish.pr2 differ diff --git a/system/word-select/dcv1_ws_data.bin b/system/text-sets/dc-v1/ws_data.bin similarity index 100% rename from system/word-select/dcv1_ws_data.bin rename to system/text-sets/dc-v1/ws_data.bin diff --git a/system/text-sets/dc-v2/TextEnglish.pr2 b/system/text-sets/dc-v2/TextEnglish.pr2 new file mode 100644 index 00000000..dd4d40f2 Binary files /dev/null and b/system/text-sets/dc-v2/TextEnglish.pr2 differ diff --git a/system/text-sets/dc-v2/TextFrench.pr2 b/system/text-sets/dc-v2/TextFrench.pr2 new file mode 100644 index 00000000..30421ecb Binary files /dev/null and b/system/text-sets/dc-v2/TextFrench.pr2 differ diff --git a/system/text-sets/dc-v2/TextGerman.pr2 b/system/text-sets/dc-v2/TextGerman.pr2 new file mode 100644 index 00000000..80a7dcbe Binary files /dev/null and b/system/text-sets/dc-v2/TextGerman.pr2 differ diff --git a/system/text-sets/dc-v2/TextJapanese.pr2 b/system/text-sets/dc-v2/TextJapanese.pr2 new file mode 100644 index 00000000..f4dd75ed Binary files /dev/null and b/system/text-sets/dc-v2/TextJapanese.pr2 differ diff --git a/system/text-sets/dc-v2/TextSpanish.pr2 b/system/text-sets/dc-v2/TextSpanish.pr2 new file mode 100644 index 00000000..71c853ad Binary files /dev/null and b/system/text-sets/dc-v2/TextSpanish.pr2 differ diff --git a/system/word-select/dcv2_ws_data.bin b/system/text-sets/dc-v2/ws_data.bin similarity index 100% rename from system/word-select/dcv2_ws_data.bin rename to system/text-sets/dc-v2/ws_data.bin diff --git a/system/text-sets/gc-ep3-nte/TextEnglish.pr2 b/system/text-sets/gc-ep3-nte/TextEnglish.pr2 new file mode 100755 index 00000000..173c52d8 Binary files /dev/null and b/system/text-sets/gc-ep3-nte/TextEnglish.pr2 differ diff --git a/system/text-sets/gc-ep3-nte/TextFrench.pr2 b/system/text-sets/gc-ep3-nte/TextFrench.pr2 new file mode 100755 index 00000000..55198e15 Binary files /dev/null and b/system/text-sets/gc-ep3-nte/TextFrench.pr2 differ diff --git a/system/text-sets/gc-ep3-nte/TextGerman.pr2 b/system/text-sets/gc-ep3-nte/TextGerman.pr2 new file mode 100755 index 00000000..49623e88 Binary files /dev/null and b/system/text-sets/gc-ep3-nte/TextGerman.pr2 differ diff --git a/system/text-sets/gc-ep3-nte/TextJapanese.pr2 b/system/text-sets/gc-ep3-nte/TextJapanese.pr2 new file mode 100755 index 00000000..c0d97bed Binary files /dev/null and b/system/text-sets/gc-ep3-nte/TextJapanese.pr2 differ diff --git a/system/text-sets/gc-ep3-nte/TextSpanish.pr2 b/system/text-sets/gc-ep3-nte/TextSpanish.pr2 new file mode 100755 index 00000000..b593b555 Binary files /dev/null and b/system/text-sets/gc-ep3-nte/TextSpanish.pr2 differ diff --git a/system/word-select/gc_ep3_nte_ws_data.bin b/system/text-sets/gc-ep3-nte/ws_data.bin similarity index 100% rename from system/word-select/gc_ep3_nte_ws_data.bin rename to system/text-sets/gc-ep3-nte/ws_data.bin diff --git a/system/text-sets/gc-ep3/TextEnglish.pr2 b/system/text-sets/gc-ep3/TextEnglish.pr2 new file mode 100755 index 00000000..eb722cdd Binary files /dev/null and b/system/text-sets/gc-ep3/TextEnglish.pr2 differ diff --git a/system/text-sets/gc-ep3/TextFrench.pr2 b/system/text-sets/gc-ep3/TextFrench.pr2 new file mode 100755 index 00000000..81a1ca35 Binary files /dev/null and b/system/text-sets/gc-ep3/TextFrench.pr2 differ diff --git a/system/text-sets/gc-ep3/TextGerman.pr2 b/system/text-sets/gc-ep3/TextGerman.pr2 new file mode 100755 index 00000000..253b9f64 Binary files /dev/null and b/system/text-sets/gc-ep3/TextGerman.pr2 differ diff --git a/system/text-sets/gc-ep3/TextJapanese.pr2 b/system/text-sets/gc-ep3/TextJapanese.pr2 new file mode 100755 index 00000000..a2278e80 Binary files /dev/null and b/system/text-sets/gc-ep3/TextJapanese.pr2 differ diff --git a/system/text-sets/gc-ep3/TextSpanish.pr2 b/system/text-sets/gc-ep3/TextSpanish.pr2 new file mode 100755 index 00000000..f5a25d75 Binary files /dev/null and b/system/text-sets/gc-ep3/TextSpanish.pr2 differ diff --git a/system/word-select/gc_ep3_ws_data.bin b/system/text-sets/gc-ep3/ws_data.bin similarity index 100% rename from system/word-select/gc_ep3_ws_data.bin rename to system/text-sets/gc-ep3/ws_data.bin diff --git a/system/text-sets/gc-nte/TextEnglish.pr2 b/system/text-sets/gc-nte/TextEnglish.pr2 new file mode 100755 index 00000000..3bdd6b91 Binary files /dev/null and b/system/text-sets/gc-nte/TextEnglish.pr2 differ diff --git a/system/text-sets/gc-nte/TextFrench.pr2 b/system/text-sets/gc-nte/TextFrench.pr2 new file mode 100755 index 00000000..ad035543 Binary files /dev/null and b/system/text-sets/gc-nte/TextFrench.pr2 differ diff --git a/system/text-sets/gc-nte/TextGerman.pr2 b/system/text-sets/gc-nte/TextGerman.pr2 new file mode 100755 index 00000000..1e811364 Binary files /dev/null and b/system/text-sets/gc-nte/TextGerman.pr2 differ diff --git a/system/text-sets/gc-nte/TextJapanese.pr2 b/system/text-sets/gc-nte/TextJapanese.pr2 new file mode 100755 index 00000000..3c9883fe Binary files /dev/null and b/system/text-sets/gc-nte/TextJapanese.pr2 differ diff --git a/system/text-sets/gc-nte/TextSpanish.pr2 b/system/text-sets/gc-nte/TextSpanish.pr2 new file mode 100755 index 00000000..148094c5 Binary files /dev/null and b/system/text-sets/gc-nte/TextSpanish.pr2 differ diff --git a/system/word-select/gc_nte_ws_data.bin b/system/text-sets/gc-nte/ws_data.bin similarity index 100% rename from system/word-select/gc_nte_ws_data.bin rename to system/text-sets/gc-nte/ws_data.bin diff --git a/system/text-sets/gc-v3/TextEnglish.pr2 b/system/text-sets/gc-v3/TextEnglish.pr2 new file mode 100755 index 00000000..5c4955b5 Binary files /dev/null and b/system/text-sets/gc-v3/TextEnglish.pr2 differ diff --git a/system/text-sets/gc-v3/TextFrench.pr2 b/system/text-sets/gc-v3/TextFrench.pr2 new file mode 100755 index 00000000..3afaeaf1 Binary files /dev/null and b/system/text-sets/gc-v3/TextFrench.pr2 differ diff --git a/system/text-sets/gc-v3/TextGerman.pr2 b/system/text-sets/gc-v3/TextGerman.pr2 new file mode 100755 index 00000000..7f5c74e6 Binary files /dev/null and b/system/text-sets/gc-v3/TextGerman.pr2 differ diff --git a/system/text-sets/gc-v3/TextJapanese.pr2 b/system/text-sets/gc-v3/TextJapanese.pr2 new file mode 100755 index 00000000..7b690939 Binary files /dev/null and b/system/text-sets/gc-v3/TextJapanese.pr2 differ diff --git a/system/text-sets/gc-v3/TextSpanish.pr2 b/system/text-sets/gc-v3/TextSpanish.pr2 new file mode 100755 index 00000000..eab039ab Binary files /dev/null and b/system/text-sets/gc-v3/TextSpanish.pr2 differ diff --git a/system/word-select/gc_ws_data.bin b/system/text-sets/gc-v3/ws_data.bin similarity index 100% rename from system/word-select/gc_ws_data.bin rename to system/text-sets/gc-v3/ws_data.bin diff --git a/system/text-sets/pc-nte/unitxt_b.prs b/system/text-sets/pc-nte/unitxt_b.prs new file mode 100644 index 00000000..a4a4202c Binary files /dev/null and b/system/text-sets/pc-nte/unitxt_b.prs differ diff --git a/system/word-select/pc_nte_unitxt.prs b/system/text-sets/pc-nte/unitxt_e.prs similarity index 100% rename from system/word-select/pc_nte_unitxt.prs rename to system/text-sets/pc-nte/unitxt_e.prs diff --git a/system/text-sets/pc-nte/unitxt_f.prs b/system/text-sets/pc-nte/unitxt_f.prs new file mode 100644 index 00000000..521f026e Binary files /dev/null and b/system/text-sets/pc-nte/unitxt_f.prs differ diff --git a/system/text-sets/pc-nte/unitxt_g.prs b/system/text-sets/pc-nte/unitxt_g.prs new file mode 100644 index 00000000..89704dbc Binary files /dev/null and b/system/text-sets/pc-nte/unitxt_g.prs differ diff --git a/system/text-sets/pc-nte/unitxt_j.prs b/system/text-sets/pc-nte/unitxt_j.prs new file mode 100644 index 00000000..27269e4f Binary files /dev/null and b/system/text-sets/pc-nte/unitxt_j.prs differ diff --git a/system/text-sets/pc-nte/unitxt_k.prs b/system/text-sets/pc-nte/unitxt_k.prs new file mode 100644 index 00000000..41ae2a54 Binary files /dev/null and b/system/text-sets/pc-nte/unitxt_k.prs differ diff --git a/system/text-sets/pc-nte/unitxt_s.prs b/system/text-sets/pc-nte/unitxt_s.prs new file mode 100644 index 00000000..3ae85e28 Binary files /dev/null and b/system/text-sets/pc-nte/unitxt_s.prs differ diff --git a/system/text-sets/pc-nte/unitxt_t.prs b/system/text-sets/pc-nte/unitxt_t.prs new file mode 100644 index 00000000..ac2d19d7 Binary files /dev/null and b/system/text-sets/pc-nte/unitxt_t.prs differ diff --git a/system/word-select/pc_nte_ws_data.bin b/system/text-sets/pc-nte/ws_data.bin similarity index 100% rename from system/word-select/pc_nte_ws_data.bin rename to system/text-sets/pc-nte/ws_data.bin diff --git a/system/text-sets/pc-v2/unitxt_b.prs b/system/text-sets/pc-v2/unitxt_b.prs new file mode 100644 index 00000000..9df4b451 Binary files /dev/null and b/system/text-sets/pc-v2/unitxt_b.prs differ diff --git a/system/word-select/pc_unitxt.prs b/system/text-sets/pc-v2/unitxt_e.prs similarity index 100% rename from system/word-select/pc_unitxt.prs rename to system/text-sets/pc-v2/unitxt_e.prs diff --git a/system/text-sets/pc-v2/unitxt_f.prs b/system/text-sets/pc-v2/unitxt_f.prs new file mode 100644 index 00000000..2206f691 Binary files /dev/null and b/system/text-sets/pc-v2/unitxt_f.prs differ diff --git a/system/text-sets/pc-v2/unitxt_g.prs b/system/text-sets/pc-v2/unitxt_g.prs new file mode 100644 index 00000000..a1c3ffcd Binary files /dev/null and b/system/text-sets/pc-v2/unitxt_g.prs differ diff --git a/system/text-sets/pc-v2/unitxt_j.prs b/system/text-sets/pc-v2/unitxt_j.prs new file mode 100644 index 00000000..4f6b2008 Binary files /dev/null and b/system/text-sets/pc-v2/unitxt_j.prs differ diff --git a/system/text-sets/pc-v2/unitxt_k.prs b/system/text-sets/pc-v2/unitxt_k.prs new file mode 100644 index 00000000..a47c6e9b Binary files /dev/null and b/system/text-sets/pc-v2/unitxt_k.prs differ diff --git a/system/text-sets/pc-v2/unitxt_s.prs b/system/text-sets/pc-v2/unitxt_s.prs new file mode 100644 index 00000000..8649c379 Binary files /dev/null and b/system/text-sets/pc-v2/unitxt_s.prs differ diff --git a/system/text-sets/pc-v2/unitxt_t.prs b/system/text-sets/pc-v2/unitxt_t.prs new file mode 100644 index 00000000..ebaaf996 Binary files /dev/null and b/system/text-sets/pc-v2/unitxt_t.prs differ diff --git a/system/word-select/pc_ws_data.bin b/system/text-sets/pc-v2/ws_data.bin similarity index 100% rename from system/word-select/pc_ws_data.bin rename to system/text-sets/pc-v2/ws_data.bin diff --git a/system/word-select/name-alias-lists.json b/system/text-sets/ws-name-alias-lists.json similarity index 100% rename from system/word-select/name-alias-lists.json rename to system/text-sets/ws-name-alias-lists.json diff --git a/system/text-sets/xb-v3/TextEnglish.pr2 b/system/text-sets/xb-v3/TextEnglish.pr2 new file mode 100644 index 00000000..2c404879 Binary files /dev/null and b/system/text-sets/xb-v3/TextEnglish.pr2 differ diff --git a/system/text-sets/xb-v3/TextFrench.pr2 b/system/text-sets/xb-v3/TextFrench.pr2 new file mode 100644 index 00000000..bf343ee9 Binary files /dev/null and b/system/text-sets/xb-v3/TextFrench.pr2 differ diff --git a/system/text-sets/xb-v3/TextGerman.pr2 b/system/text-sets/xb-v3/TextGerman.pr2 new file mode 100644 index 00000000..bb590030 Binary files /dev/null and b/system/text-sets/xb-v3/TextGerman.pr2 differ diff --git a/system/text-sets/xb-v3/TextJapanese.pr2 b/system/text-sets/xb-v3/TextJapanese.pr2 new file mode 100644 index 00000000..5d1e0afb Binary files /dev/null and b/system/text-sets/xb-v3/TextJapanese.pr2 differ diff --git a/system/text-sets/xb-v3/TextSpanish.pr2 b/system/text-sets/xb-v3/TextSpanish.pr2 new file mode 100644 index 00000000..02c9dd2c Binary files /dev/null and b/system/text-sets/xb-v3/TextSpanish.pr2 differ diff --git a/system/word-select/xb_ws_data.bin b/system/text-sets/xb-v3/xb_ws_data.bin similarity index 100% rename from system/word-select/xb_ws_data.bin rename to system/text-sets/xb-v3/xb_ws_data.bin