add ItemParameterTable binary serialization; make JSON the default format

src/ItemNameIndex.cc

@@ -1086,7 +1086,7 @@ void ItemNameIndex::print_table(FILE* stream) const {
 
   phosg::fwrite_fmt(stream, "ITEM COMBINATIONS\n");
   phosg::fwrite_fmt(stream, "  ---USE + -EQUIP => RESULT MLV GND LVL CLS\n");
-  for (const auto& combo_list_it : pmt->all_item_combinations()) {
+  for (const auto& combo_list_it : pmt->item_combinations_index()) {
     for (const auto& combo : combo_list_it.second) {
       phosg::fwrite_fmt(stream, "  {:02X}{:02X}{:02X} + {:02X}{:02X}{:02X} => {:02X}{:02X}{:02X}",
           combo.used_item[0], combo.used_item[1], combo.used_item[2],
@@ -1222,22 +1222,22 @@ void ItemNameIndex::print_table(FILE* stream) const {
 
   phosg::fwrite_fmt(stream, "SOUND REMAPS\n");
   phosg::fwrite_fmt(stream, "  -SOUND1- => RT:[...] CC:[...]\n");
-  for (const auto& [sound_id, remaps] : pmt->get_all_sound_remaps()) {
+  for (const auto& remap : pmt->get_all_sound_remaps()) {
     std::string rt_str;
-    for (uint32_t rt_sound_id : remaps.by_rt_index) {
+    for (uint32_t rt_sound_id : remap.by_rt_index) {
       if (!rt_str.empty()) {
         rt_str += ",";
       }
       rt_str += std::format("{:08X}", rt_sound_id);
     }
     std::string cc_str;
-    for (uint32_t cc_sound_id : remaps.by_char_class) {
+    for (uint32_t cc_sound_id : remap.by_char_class) {
       if (!cc_str.empty()) {
         cc_str += ",";
       }
       cc_str += std::format("{:08X}", cc_sound_id);
     }
-    phosg::fwrite_fmt(stream, "  {:08X} => RT:[{}] CC:[{}]\n", sound_id, rt_str, cc_str);
+    phosg::fwrite_fmt(stream, "  {:08X} => RT:[{}] CC:[{}]\n", remap.sound_id, rt_str, cc_str);
   }
 
   phosg::fwrite_fmt(stream, "TECH BOOSTS\n");

src/ItemParameterTable.hh

@@ -68,6 +68,7 @@ public:
     uint8_t special = 0;
     uint8_t ata = 0;
     uint8_t stat_boost_entry_index = 0;
+    parray<uint8_t, 3> v2_unknown_a9;
     uint8_t projectile = 0;
     int8_t trail1_x = 0;
     int8_t trail1_y = 0;
@@ -361,7 +362,7 @@ public:
   static std::shared_ptr<ItemParameterTable> from_json(const phosg::JSON& json);
 
   phosg::JSON json() const;
-  // std::string serialize_binary() const; // TODO
+  std::string serialize_binary(Version version) const;
 
   std::set<uint32_t> compute_all_valid_primary_identifiers() const;
 
@@ -389,6 +390,7 @@ public:
   virtual const Mag& get_mag(uint8_t data1_1) const = 0;
 
   // weapon_kind_table accessors (data1_1 in [0, num_weapon_classes()])
+  virtual size_t num_weapon_kinds() const = 0;
   virtual uint8_t get_weapon_kind(uint8_t data1_1) const = 0;
 
   // photon_color_table accessors
@@ -401,6 +403,7 @@ public:
 
   // weapon_sale_divisor_table and non_weapon_sale_divisor_table accessors (data1_0 in [0, 1, 2]; data1_1 in [0,
   // num_weapon_classes()] for weapons or ignored otherwise)
+  virtual size_t num_weapon_sale_divisors() const = 0;
   virtual float get_sale_divisor(uint8_t data1_0, uint8_t data1_1) const = 0;
 
   // mag_feed_table accessors (table_index in [0, 7], item_index in [0, 10])
@@ -451,12 +454,14 @@ public:
   virtual uint8_t get_max_tech_level(uint8_t char_class, uint8_t tech_num) const = 0;
 
   // combination_table accessors
-  virtual const std::map<uint32_t, std::vector<ItemCombination>>& all_item_combinations() const = 0;
+  virtual size_t num_item_combinations() const = 0;
+  virtual const ItemCombination& get_item_combination(size_t index) const = 0;
+  const std::map<uint32_t, std::vector<ItemCombination>>& item_combinations_index() const;
   const std::vector<ItemCombination>& all_combinations_for_used_item(const ItemData& used_item) const;
   const ItemCombination& get_item_combination(const ItemData& used_item, const ItemData& equipped_item) const;
 
   // sound_remap_table accessors
-  virtual const std::unordered_map<uint32_t, SoundRemaps>& get_all_sound_remaps() const = 0;
+  virtual const std::vector<SoundRemaps>& get_all_sound_remaps() const = 0;
 
   // tech_boost_table accessors
   virtual size_t num_tech_boosts() const = 0;
@@ -487,4 +492,6 @@ public:
 
 protected:
   ItemParameterTable() = default;
+
+  mutable std::optional<std::map<uint32_t, std::vector<ItemCombination>>> item_combination_index;
 };

src/ItemTranslationTable.cc

@@ -46,18 +46,18 @@ ItemTranslationTable::ItemTranslationTable(
       uint32_t e_id = this->entries[z].id_for_version[v_s];
       if (is_canonical(e_id)) {
         if (!entry_index.count(e_id)) {
-          throw logic_error(std::format("(row {} version {}) canonical ID {:X} is missing from the index", z, phosg::name_for_enum(v), e_id));
+          throw logic_error(std::format("(row {} version {}) canonical ID {:08X} is missing from the index", z, phosg::name_for_enum(v), e_id));
         }
         try {
           item_parameter_table->definition_for_primary_identifier(e_id);
         } catch (const out_of_range&) {
-          throw runtime_error(std::format("(row {} version {}) ID {:X} not defined in item parameter table", z, phosg::name_for_enum(v), e_id));
+          throw runtime_error(std::format("(row {} version {}) ID {:08X} not defined in item parameter table", z, phosg::name_for_enum(v), e_id));
         }
         if (!remaining_identifiers.erase(e_id)) {
-          throw runtime_error(std::format("(row {} version {}) ID {:X} not in item parameter table's primary identifier list", z, phosg::name_for_enum(v), e_id));
+          throw runtime_error(std::format("(row {} version {}) ID {:08X} not in item parameter table's primary identifier list", z, phosg::name_for_enum(v), e_id));
         }
       } else if (!entry_index.count(make_canonical(e_id))) {
-        throw runtime_error(std::format("(row {} version {}) ID {:X} refers to nonexistent canonical ID", z, phosg::name_for_enum(v), e_id));
+        throw runtime_error(std::format("(row {} version {}) ID {:08X} refers to nonexistent canonical ID", z, phosg::name_for_enum(v), e_id));
       }
     }
 

src/Main.cc

@@ -2387,10 +2387,20 @@ Action a_compare_common_item_set(
       cs1->print_diff(stdout, *cs2);
     });
 
-Action a_convert_item_parameter_table(
-    "decode-item-parameter-table", nullptr,
+Action a_decode_item_parameter_table(
+    "decode-item-parameter-table", "\
+  decode-item-parameter-table [INPUT-FILENAME [OUTPUT-FILENAME]] [OPTIONS...]\n\
+    Converts an ItemPMT file into a JSON item parameter table. A version\n\
+    option is required. Use --hex to make item codes in the output readable;\n\
+    however, this option also uses nonstandard JSON syntax - newserv can parse\n\
+    it, but many other JSON parsers can\'t. Expects compressed input (a .prs\n\
+    file) by default; use --decompressed if the input is not compressed.\n",
     +[](phosg::Arguments& args) {
-      auto data = std::make_shared<string>(read_input_data(args));
+      auto input_data = read_input_data(args);
+      if (!args.get<bool>("decompressed")) {
+        input_data = prs_decompress(input_data);
+      }
+      auto data = std::make_shared<string>(std::move(input_data));
       auto pmt = ItemParameterTable::from_binary(data, get_cli_version(args, Version::BB_V4));
       auto json = pmt->json();
       uint32_t serialize_options = phosg::JSON::SerializeOption::FORMAT | phosg::JSON::SerializeOption::SORT_DICT_KEYS;
@@ -2401,6 +2411,23 @@ Action a_convert_item_parameter_table(
       write_output_data(args, json_data.data(), json_data.size(), nullptr);
     });
 
+Action a_encode_item_parameter_table(
+    "encode-item-parameter-table", "\
+  encode-item-parameter-table [INPUT-FILENAME [OUTPUT-FILENAME]] [OPTIONS...]\n\
+    Converts a JSON item parameter table into an ItemPMT file compatible with\n\
+    the game client. A version option is required. By default the output will\n\
+    be compressed, as the client expects; use --decompressed to get\n\
+    uncompressed output.\n",
+    +[](phosg::Arguments& args) {
+      auto json = phosg::JSON::parse(read_input_data(args));
+      auto pmt = ItemParameterTable::from_json(json);
+      string data = pmt->serialize_binary(get_cli_version(args, Version::BB_V4));
+      if (!args.get<bool>("decompressed")) {
+        data = prs_compress_optimal(data);
+      }
+      write_output_data(args, data.data(), data.size(), nullptr);
+    });
+
 Action a_describe_item(
     "describe-item", "\
   describe-item DATA-OR-DESCRIPTION\n\

src/SendCommands.cc

@@ -690,42 +690,16 @@ void send_guild_card_chunk_bb(shared_ptr<Client> c, size_t chunk_index) {
   send_command(c, 0x02DC, 0x00000000, &cmd, sizeof(cmd) - sizeof(cmd.data) + data_size);
 }
 
-static const vector<string> stream_file_entries = {
-    "ItemMagEdit.prs",
-    "ItemPMT.prs",
-    "BattleParamEntry.dat",
-    "BattleParamEntry_on.dat",
-    "BattleParamEntry_lab.dat",
-    "BattleParamEntry_lab_on.dat",
-    "BattleParamEntry_ep4.dat",
-    "BattleParamEntry_ep4_on.dat",
-    "PlyLevelTbl.prs",
-};
-
 void send_stream_file_index_bb(shared_ptr<Client> c) {
   auto s = c->require_server_state();
 
   vector<S_StreamFileIndexEntry_BB_01EB> entries;
-  size_t offset = 0;
-  for (const string& filename : stream_file_entries) {
-    string key = "system/blueburst/" + filename;
-    auto cache_res = s->bb_stream_files_cache->get_or_load(key);
+  for (const auto& sf_entry : s->bb_stream_file->entries) {
     auto& e = entries.emplace_back();
-    e.size = cache_res.file->data->size();
-    // Computing the checksum can be slow, so we cache it along with the file data. If the cache result was just
-    // populated, then it may be different, so we always recompute the checksum in that case.
-    if (cache_res.generate_called) {
-      e.checksum = crc32(cache_res.file->data->data(), e.size);
-      s->bb_stream_files_cache->replace_obj<uint32_t>(key + ".crc32", e.checksum);
-    } else {
-      auto compute_checksum = [&](const string&) -> uint32_t {
-        return crc32(cache_res.file->data->data(), e.size);
-      };
-      e.checksum = s->bb_stream_files_cache->get_obj<uint32_t>(key + ".crc32", compute_checksum).obj;
-    }
-    e.offset = offset;
-    e.filename.encode(filename);
-    offset += e.size;
+    e.size = sf_entry.size;
+    e.checksum = sf_entry.checksum;
+    e.offset = sf_entry.offset;
+    e.filename.encode(sf_entry.filename);
   }
   send_command_vt(c, 0x01EB, entries.size(), entries);
 }
@@ -733,30 +707,14 @@ void send_stream_file_index_bb(shared_ptr<Client> c) {
 void send_stream_file_chunk_bb(shared_ptr<Client> c, uint32_t chunk_index) {
   auto s = c->require_server_state();
 
-  auto cache_result = s->bb_stream_files_cache->get(
-      "<BB stream file>", [&](const string&) -> string {
-        size_t bytes = 0;
-        for (const auto& name : stream_file_entries) {
-          bytes += s->bb_stream_files_cache->get_or_load("system/blueburst/" + name).file->data->size();
-        }
-
-        string ret;
-        ret.reserve(bytes);
-        for (const auto& name : stream_file_entries) {
-          ret += *s->bb_stream_files_cache->get_or_load("system/blueburst/" + name).file->data;
-        }
-        return ret;
-      });
-  const auto& contents = cache_result.file->data;
-
   S_StreamFileChunk_BB_02EB chunk_cmd;
   chunk_cmd.chunk_index = chunk_index;
   size_t offset = sizeof(chunk_cmd.data) * chunk_index;
-  if (offset > contents->size()) {
+  if (offset > s->bb_stream_file->data.size()) {
     throw runtime_error("client requested chunk beyond end of stream file");
   }
-  size_t bytes = min<size_t>(contents->size() - offset, sizeof(chunk_cmd.data));
-  chunk_cmd.data.assign_range(reinterpret_cast<const uint8_t*>(contents->data() + offset), bytes, 0);
+  size_t bytes = min<size_t>(s->bb_stream_file->data.size() - offset, sizeof(chunk_cmd.data));
+  chunk_cmd.data.assign_range(reinterpret_cast<const uint8_t*>(s->bb_stream_file->data.data() + offset), bytes, 0);
 
   size_t cmd_size = offsetof(S_StreamFileChunk_BB_02EB, data) + bytes;
   cmd_size = (cmd_size + 3) & ~3;

src/ServerState.cc

@@ -79,7 +79,6 @@ ServerState::ServerState(const string& config_filename, bool is_replay)
       config_filename(config_filename),
       is_replay(is_replay),
       thread_pool(make_unique<asio::thread_pool>()),
-      bb_stream_files_cache(new FileContentsCache(3600000000ULL)),
       bb_system_cache(new FileContentsCache(3600000000ULL)),
       gba_files_cache(new FileContentsCache(3600000000ULL)) {}
 
@@ -1831,8 +1830,6 @@ vector<shared_ptr<const SuperMap>> ServerState::supermaps_for_variations(
 }
 
 void ServerState::clear_file_caches() {
-  config_log.info_f("Clearing BB stream file cache");
-  this->bb_stream_files_cache.reset(new FileContentsCache(3600000000ULL));
   config_log.info_f("Clearing BB system cache");
   this->bb_system_cache.reset(new FileContentsCache(3600000000ULL));
   config_log.info_f("Clearing GBA file cache");
@@ -2148,10 +2145,9 @@ void ServerState::load_item_definitions() {
   config_log.info_f("Loading item definition tables");
   for (size_t v_s = NUM_PATCH_VERSIONS; v_s < NUM_VERSIONS; v_s++) {
     Version v = static_cast<Version>(v_s);
-    string path = std::format("system/item-tables/ItemPMT-{}.prs", file_path_token_for_version(v));
+    string path = std::format("system/item-tables/item-parameter-table-{}.json", file_path_token_for_version(v));
     config_log.debug_f("Loading item definition table {}", path);
-    auto data = make_shared<string>(prs_decompress(phosg::load_file(path)));
-    new_item_parameter_tables[v_s] = ItemParameterTable::from_binary(data, v);
+    new_item_parameter_tables[v_s] = ItemParameterTable::from_json(phosg::JSON::parse(phosg::load_file(path)));
   }
 
   auto json = phosg::JSON::parse(phosg::load_file("system/item-tables/translation-table.json"));
@@ -2227,6 +2223,39 @@ void ServerState::load_dol_files() {
   this->dol_file_index = make_shared<DOLFileIndex>("system/dol");
 }
 
+void ServerState::generate_bb_stream_file() {
+  config_log.info_f("Generating BB stream file");
+  auto sf = std::make_shared<BBStreamFile>();
+
+  auto add_file = [&](const std::string& filename, std::string&& file_data = "") -> void {
+    if (file_data.empty()) {
+      file_data = phosg::load_file("system/blueburst/" + filename);
+    }
+    auto& e = sf->entries.emplace_back();
+    e.size = file_data.size();
+    e.checksum = phosg::crc32(file_data.data(), file_data.size());
+    e.offset = sf->data.size();
+    e.filename = filename;
+    sf->data += file_data;
+    config_log.debug_f(
+        "[BBStreamFile] Added file {} at offset {:08X} ({:08X} bytes) with checksum {:08X}; total size is now {:08X}",
+        filename, e.offset, e.size, e.checksum, sf->data.size());
+  };
+
+  add_file("BattleParamEntry.dat");
+  add_file("BattleParamEntry_on.dat");
+  add_file("BattleParamEntry_lab.dat");
+  add_file("BattleParamEntry_lab_on.dat");
+  add_file("BattleParamEntry_ep4.dat");
+  add_file("BattleParamEntry_ep4_on.dat");
+  add_file("PlyLevelTbl.prs");
+  add_file("ItemMagEdit.prs");
+  auto pmt = this->item_parameter_table(Version::BB_V4);
+  add_file("ItemPMT.prs", prs_compress_optimal(pmt->serialize_binary(Version::BB_V4)));
+
+  this->bb_stream_file = sf;
+}
+
 void ServerState::create_default_lobbies() {
   if (this->default_lobbies_created) {
     return;
@@ -2306,6 +2335,7 @@ void ServerState::load_all(bool enable_thread_pool) {
   this->load_config_late();
   this->load_teams();
   this->load_quest_index();
+  this->generate_bb_stream_file();
 }
 
 void ServerState::disconnect_all_banned_clients() {

src/ServerState.hh

@@ -65,6 +65,17 @@ struct CheatFlags {
   explicit CheatFlags(const phosg::JSON& json);
 };
 
+struct BBStreamFile {
+  struct Entry {
+    uint32_t offset;
+    uint32_t size;
+    uint32_t checksum; // crc32
+    std::string filename;
+  };
+  std::vector<Entry> entries;
+  std::string data;
+};
+
 struct ServerState : public std::enable_shared_from_this<ServerState> {
   enum class RunShellBehavior {
     DEFAULT = 0,
@@ -180,7 +191,7 @@ struct ServerState : public std::enable_shared_from_this<ServerState> {
   std::unordered_map<uint64_t, std::shared_ptr<const SuperMap>> supermap_for_source_hash_sum;
   std::unordered_map<uint32_t, std::shared_ptr<const SuperMap>> supermap_for_free_play_key;
   std::shared_ptr<const RoomLayoutIndex> room_layout_index;
-  std::shared_ptr<FileContentsCache> bb_stream_files_cache;
+  std::shared_ptr<const BBStreamFile> bb_stream_file;
   std::shared_ptr<FileContentsCache> bb_system_cache;
   std::shared_ptr<FileContentsCache> gba_files_cache;
   std::shared_ptr<const DOLFileIndex> dol_file_index;
@@ -442,6 +453,7 @@ struct ServerState : public std::enable_shared_from_this<ServerState> {
   void load_quest_index(bool raise_on_any_failure = false);
   void compile_functions(bool raise_on_any_failure = false);
   void load_dol_files();
+  void generate_bb_stream_file();
 
   void load_all(bool enable_thread_pool);
 

src/ShellCommands.cc

@@ -211,14 +211,17 @@ ShellCommand c_reload(
           args.s->load_set_data_tables();
         } else if (type == "battle-params") {
           args.s->load_battle_params();
+          args.s->generate_bb_stream_file();
         } else if (type == "level-tables") {
           args.s->load_level_tables();
+          args.s->generate_bb_stream_file();
         } else if (type == "text-index") {
           args.s->load_text_index();
         } else if (type == "word-select") {
           args.s->load_word_select_table();
         } else if (type == "item-definitions") {
           args.s->load_item_definitions();
+          args.s->generate_bb_stream_file();
         } else if (type == "item-name-index") {
           args.s->load_item_name_indexes();
         } else if (type == "drop-tables") {