add ItemParameterTable binary serialization; make JSON the default format

README.md

@@ -781,7 +781,7 @@ newserv has many CLI options, which can be used to access functionality other th
 The data formats that newserv can convert to/from are:
 
 | Format                              | Encode/compress action        | Decode/extract action         |
-|-------------------------------------|---------------------------|------------------------------|
+|-------------------------------------|-------------------------------|-------------------------------|
 | PRS compression                     | `compress-prs`                | `decompress-prs`              |
 | PR2/PRC compression                 | `compress-pr2`                | `decompress-pr2`              |
 | BC0 compression                     | `compress-bc0`                | `decompress-bc0`              |
@@ -803,13 +803,14 @@ The data formats that newserv can convert to/from are:
 | BML archive (.bml)                  | None                          | `extract-bml`                 |
 | PPK archive (.ppk)                  | None                          | `extract-ppk`                 |
 | GSL archive (.gsl)                  | `generate-gsl`                | `extract-gsl`                 |
-| GVM texture (.gvm)                  | `encode-gvm`              | None                         |
+| GVM texture (.gvm)                  | `encode-gvm`                  | None (use resource_dasm)      |
 | Bitmap font (.fon)                  | `encode-bitmap-font`          | `decode-bitmap-font`          |
 | Text archive                        | `encode-text-archive`         | `decode-text-archive`         |
 | Unicode text set                    | `encode-unicode-text-set`     | `decode-unicode-text-set`     |
 | Word Select data set                | None                          | `decode-word-select-set`      |
 | Set data table                      | None                          | `disassemble-set-data-table`  |
 | Rare item table (AFS/GSL/JSON/HTML) | `convert-rare-item-set`       | `convert-rare-item-set`       |
+| Item definitions (ItemPMT)          | `encode-item-parameter-table` | `decode-item-parameter-table` |
 
 There are several actions that don't fit well into the table above, which let you do other things:
 

TODO.md

@@ -30,8 +30,6 @@
 
 ## PSOBB
 
-- Figure out why Pouilly Slime EXP doesn't work
 - Make server-specified rare enemies work with maps loaded by the proxy
-- Implement serialization for various table types (ItemPMT, ItemPT, etc.)
 - Record some BB tests
 - Add all necessary Guild Card number rewrites in BB commands on the proxy

src/ItemNameIndex.cc

@@ -1086,7 +1086,7 @@ void ItemNameIndex::print_table(FILE* stream) const {
 
   phosg::fwrite_fmt(stream, "ITEM COMBINATIONS\n");
   phosg::fwrite_fmt(stream, "  ---USE + -EQUIP => RESULT MLV GND LVL CLS\n");
-  for (const auto& combo_list_it : pmt->all_item_combinations()) {
+  for (const auto& combo_list_it : pmt->item_combinations_index()) {
     for (const auto& combo : combo_list_it.second) {
       phosg::fwrite_fmt(stream, "  {:02X}{:02X}{:02X} + {:02X}{:02X}{:02X} => {:02X}{:02X}{:02X}",
           combo.used_item[0], combo.used_item[1], combo.used_item[2],
@@ -1222,22 +1222,22 @@ void ItemNameIndex::print_table(FILE* stream) const {
 
   phosg::fwrite_fmt(stream, "SOUND REMAPS\n");
   phosg::fwrite_fmt(stream, "  -SOUND1- => RT:[...] CC:[...]\n");
-  for (const auto& [sound_id, remaps] : pmt->get_all_sound_remaps()) {
+  for (const auto& remap : pmt->get_all_sound_remaps()) {
     std::string rt_str;
-    for (uint32_t rt_sound_id : remaps.by_rt_index) {
+    for (uint32_t rt_sound_id : remap.by_rt_index) {
       if (!rt_str.empty()) {
         rt_str += ",";
       }
       rt_str += std::format("{:08X}", rt_sound_id);
     }
     std::string cc_str;
-    for (uint32_t cc_sound_id : remaps.by_char_class) {
+    for (uint32_t cc_sound_id : remap.by_char_class) {
       if (!cc_str.empty()) {
         cc_str += ",";
       }
       cc_str += std::format("{:08X}", cc_sound_id);
     }
-    phosg::fwrite_fmt(stream, "  {:08X} => RT:[{}] CC:[{}]\n", sound_id, rt_str, cc_str);
+    phosg::fwrite_fmt(stream, "  {:08X} => RT:[{}] CC:[{}]\n", remap.sound_id, rt_str, cc_str);
   }
 
   phosg::fwrite_fmt(stream, "TECH BOOSTS\n");

src/ItemParameterTable.hh

@@ -68,6 +68,7 @@ public:
     uint8_t special = 0;
     uint8_t ata = 0;
     uint8_t stat_boost_entry_index = 0;
+    parray<uint8_t, 3> v2_unknown_a9;
     uint8_t projectile = 0;
     int8_t trail1_x = 0;
     int8_t trail1_y = 0;
@@ -361,7 +362,7 @@ public:
   static std::shared_ptr<ItemParameterTable> from_json(const phosg::JSON& json);
 
   phosg::JSON json() const;
-  // std::string serialize_binary() const; // TODO
+  std::string serialize_binary(Version version) const;
 
   std::set<uint32_t> compute_all_valid_primary_identifiers() const;
 
@@ -389,6 +390,7 @@ public:
   virtual const Mag& get_mag(uint8_t data1_1) const = 0;
 
   // weapon_kind_table accessors (data1_1 in [0, num_weapon_classes()])
+  virtual size_t num_weapon_kinds() const = 0;
   virtual uint8_t get_weapon_kind(uint8_t data1_1) const = 0;
 
   // photon_color_table accessors
@@ -401,6 +403,7 @@ public:
 
   // weapon_sale_divisor_table and non_weapon_sale_divisor_table accessors (data1_0 in [0, 1, 2]; data1_1 in [0,
   // num_weapon_classes()] for weapons or ignored otherwise)
+  virtual size_t num_weapon_sale_divisors() const = 0;
   virtual float get_sale_divisor(uint8_t data1_0, uint8_t data1_1) const = 0;
 
   // mag_feed_table accessors (table_index in [0, 7], item_index in [0, 10])
@@ -451,12 +454,14 @@ public:
   virtual uint8_t get_max_tech_level(uint8_t char_class, uint8_t tech_num) const = 0;
 
   // combination_table accessors
-  virtual const std::map<uint32_t, std::vector<ItemCombination>>& all_item_combinations() const = 0;
+  virtual size_t num_item_combinations() const = 0;
+  virtual const ItemCombination& get_item_combination(size_t index) const = 0;
+  const std::map<uint32_t, std::vector<ItemCombination>>& item_combinations_index() const;
   const std::vector<ItemCombination>& all_combinations_for_used_item(const ItemData& used_item) const;
   const ItemCombination& get_item_combination(const ItemData& used_item, const ItemData& equipped_item) const;
 
   // sound_remap_table accessors
-  virtual const std::unordered_map<uint32_t, SoundRemaps>& get_all_sound_remaps() const = 0;
+  virtual const std::vector<SoundRemaps>& get_all_sound_remaps() const = 0;
 
   // tech_boost_table accessors
   virtual size_t num_tech_boosts() const = 0;
@@ -487,4 +492,6 @@ public:
 
 protected:
   ItemParameterTable() = default;
+
+  mutable std::optional<std::map<uint32_t, std::vector<ItemCombination>>> item_combination_index;
 };

src/ItemTranslationTable.cc

@@ -46,18 +46,18 @@ ItemTranslationTable::ItemTranslationTable(
       uint32_t e_id = this->entries[z].id_for_version[v_s];
       if (is_canonical(e_id)) {
         if (!entry_index.count(e_id)) {
-          throw logic_error(std::format("(row {} version {}) canonical ID {:X} is missing from the index", z, phosg::name_for_enum(v), e_id));
+          throw logic_error(std::format("(row {} version {}) canonical ID {:08X} is missing from the index", z, phosg::name_for_enum(v), e_id));
         }
         try {
           item_parameter_table->definition_for_primary_identifier(e_id);
         } catch (const out_of_range&) {
-          throw runtime_error(std::format("(row {} version {}) ID {:X} not defined in item parameter table", z, phosg::name_for_enum(v), e_id));
+          throw runtime_error(std::format("(row {} version {}) ID {:08X} not defined in item parameter table", z, phosg::name_for_enum(v), e_id));
         }
         if (!remaining_identifiers.erase(e_id)) {
-          throw runtime_error(std::format("(row {} version {}) ID {:X} not in item parameter table's primary identifier list", z, phosg::name_for_enum(v), e_id));
+          throw runtime_error(std::format("(row {} version {}) ID {:08X} not in item parameter table's primary identifier list", z, phosg::name_for_enum(v), e_id));
         }
       } else if (!entry_index.count(make_canonical(e_id))) {
-        throw runtime_error(std::format("(row {} version {}) ID {:X} refers to nonexistent canonical ID", z, phosg::name_for_enum(v), e_id));
+        throw runtime_error(std::format("(row {} version {}) ID {:08X} refers to nonexistent canonical ID", z, phosg::name_for_enum(v), e_id));
       }
     }
 

src/Main.cc

@@ -2387,10 +2387,20 @@ Action a_compare_common_item_set(
       cs1->print_diff(stdout, *cs2);
     });
 
-Action a_convert_item_parameter_table(
+Action a_decode_item_parameter_table(
-    "decode-item-parameter-table", nullptr,
+    "decode-item-parameter-table", "\
+  decode-item-parameter-table [INPUT-FILENAME [OUTPUT-FILENAME]] [OPTIONS...]\n\
+    Converts an ItemPMT file into a JSON item parameter table. A version\n\
+    option is required. Use --hex to make item codes in the output readable;\n\
+    however, this option also uses nonstandard JSON syntax - newserv can parse\n\
+    it, but many other JSON parsers can\'t. Expects compressed input (a .prs\n\
+    file) by default; use --decompressed if the input is not compressed.\n",
     +[](phosg::Arguments& args) {
-      auto data = std::make_shared<string>(read_input_data(args));
+      auto input_data = read_input_data(args);
+      if (!args.get<bool>("decompressed")) {
+        input_data = prs_decompress(input_data);
+      }
+      auto data = std::make_shared<string>(std::move(input_data));
       auto pmt = ItemParameterTable::from_binary(data, get_cli_version(args, Version::BB_V4));
       auto json = pmt->json();
       uint32_t serialize_options = phosg::JSON::SerializeOption::FORMAT | phosg::JSON::SerializeOption::SORT_DICT_KEYS;
@@ -2401,6 +2411,23 @@ Action a_convert_item_parameter_table(
       write_output_data(args, json_data.data(), json_data.size(), nullptr);
     });
 
+Action a_encode_item_parameter_table(
+    "encode-item-parameter-table", "\
+  encode-item-parameter-table [INPUT-FILENAME [OUTPUT-FILENAME]] [OPTIONS...]\n\
+    Converts a JSON item parameter table into an ItemPMT file compatible with\n\
+    the game client. A version option is required. By default the output will\n\
+    be compressed, as the client expects; use --decompressed to get\n\
+    uncompressed output.\n",
+    +[](phosg::Arguments& args) {
+      auto json = phosg::JSON::parse(read_input_data(args));
+      auto pmt = ItemParameterTable::from_json(json);
+      string data = pmt->serialize_binary(get_cli_version(args, Version::BB_V4));
+      if (!args.get<bool>("decompressed")) {
+        data = prs_compress_optimal(data);
+      }
+      write_output_data(args, data.data(), data.size(), nullptr);
+    });
+
 Action a_describe_item(
     "describe-item", "\
   describe-item DATA-OR-DESCRIPTION\n\

src/SendCommands.cc

@@ -690,42 +690,16 @@ void send_guild_card_chunk_bb(shared_ptr<Client> c, size_t chunk_index) {
   send_command(c, 0x02DC, 0x00000000, &cmd, sizeof(cmd) - sizeof(cmd.data) + data_size);
 }
 
-static const vector<string> stream_file_entries = {
-    "ItemMagEdit.prs",
-    "ItemPMT.prs",
-    "BattleParamEntry.dat",
-    "BattleParamEntry_on.dat",
-    "BattleParamEntry_lab.dat",
-    "BattleParamEntry_lab_on.dat",
-    "BattleParamEntry_ep4.dat",
-    "BattleParamEntry_ep4_on.dat",
-    "PlyLevelTbl.prs",
-};
-
 void send_stream_file_index_bb(shared_ptr<Client> c) {
   auto s = c->require_server_state();
 
   vector<S_StreamFileIndexEntry_BB_01EB> entries;
-  size_t offset = 0;
+  for (const auto& sf_entry : s->bb_stream_file->entries) {
-  for (const string& filename : stream_file_entries) {
-    string key = "system/blueburst/" + filename;
-    auto cache_res = s->bb_stream_files_cache->get_or_load(key);
     auto& e = entries.emplace_back();
-    e.size = cache_res.file->data->size();
+    e.size = sf_entry.size;
-    // Computing the checksum can be slow, so we cache it along with the file data. If the cache result was just
+    e.checksum = sf_entry.checksum;
-    // populated, then it may be different, so we always recompute the checksum in that case.
+    e.offset = sf_entry.offset;
-    if (cache_res.generate_called) {
+    e.filename.encode(sf_entry.filename);
-      e.checksum = crc32(cache_res.file->data->data(), e.size);
-      s->bb_stream_files_cache->replace_obj<uint32_t>(key + ".crc32", e.checksum);
-    } else {
-      auto compute_checksum = [&](const string&) -> uint32_t {
-        return crc32(cache_res.file->data->data(), e.size);
-      };
-      e.checksum = s->bb_stream_files_cache->get_obj<uint32_t>(key + ".crc32", compute_checksum).obj;
-    }
-    e.offset = offset;
-    e.filename.encode(filename);
-    offset += e.size;
   }
   send_command_vt(c, 0x01EB, entries.size(), entries);
 }
@@ -733,30 +707,14 @@ void send_stream_file_index_bb(shared_ptr<Client> c) {
 void send_stream_file_chunk_bb(shared_ptr<Client> c, uint32_t chunk_index) {
   auto s = c->require_server_state();
 
-  auto cache_result = s->bb_stream_files_cache->get(
-      "<BB stream file>", [&](const string&) -> string {
-        size_t bytes = 0;
-        for (const auto& name : stream_file_entries) {
-          bytes += s->bb_stream_files_cache->get_or_load("system/blueburst/" + name).file->data->size();
-        }
-
-        string ret;
-        ret.reserve(bytes);
-        for (const auto& name : stream_file_entries) {
-          ret += *s->bb_stream_files_cache->get_or_load("system/blueburst/" + name).file->data;
-        }
-        return ret;
-      });
-  const auto& contents = cache_result.file->data;
-
   S_StreamFileChunk_BB_02EB chunk_cmd;
   chunk_cmd.chunk_index = chunk_index;
   size_t offset = sizeof(chunk_cmd.data) * chunk_index;
-  if (offset > contents->size()) {
+  if (offset > s->bb_stream_file->data.size()) {
     throw runtime_error("client requested chunk beyond end of stream file");
   }
-  size_t bytes = min<size_t>(contents->size() - offset, sizeof(chunk_cmd.data));
+  size_t bytes = min<size_t>(s->bb_stream_file->data.size() - offset, sizeof(chunk_cmd.data));
-  chunk_cmd.data.assign_range(reinterpret_cast<const uint8_t*>(contents->data() + offset), bytes, 0);
+  chunk_cmd.data.assign_range(reinterpret_cast<const uint8_t*>(s->bb_stream_file->data.data() + offset), bytes, 0);
 
   size_t cmd_size = offsetof(S_StreamFileChunk_BB_02EB, data) + bytes;
   cmd_size = (cmd_size + 3) & ~3;

src/ServerState.cc

@@ -79,7 +79,6 @@ ServerState::ServerState(const string& config_filename, bool is_replay)
       config_filename(config_filename),
       is_replay(is_replay),
       thread_pool(make_unique<asio::thread_pool>()),
-      bb_stream_files_cache(new FileContentsCache(3600000000ULL)),
       bb_system_cache(new FileContentsCache(3600000000ULL)),
       gba_files_cache(new FileContentsCache(3600000000ULL)) {}
 
@@ -1831,8 +1830,6 @@ vector<shared_ptr<const SuperMap>> ServerState::supermaps_for_variations(
 }
 
 void ServerState::clear_file_caches() {
-  config_log.info_f("Clearing BB stream file cache");
-  this->bb_stream_files_cache.reset(new FileContentsCache(3600000000ULL));
   config_log.info_f("Clearing BB system cache");
   this->bb_system_cache.reset(new FileContentsCache(3600000000ULL));
   config_log.info_f("Clearing GBA file cache");
@@ -2148,10 +2145,9 @@ void ServerState::load_item_definitions() {
   config_log.info_f("Loading item definition tables");
   for (size_t v_s = NUM_PATCH_VERSIONS; v_s < NUM_VERSIONS; v_s++) {
     Version v = static_cast<Version>(v_s);
-    string path = std::format("system/item-tables/ItemPMT-{}.prs", file_path_token_for_version(v));
+    string path = std::format("system/item-tables/item-parameter-table-{}.json", file_path_token_for_version(v));
     config_log.debug_f("Loading item definition table {}", path);
-    auto data = make_shared<string>(prs_decompress(phosg::load_file(path)));
+    new_item_parameter_tables[v_s] = ItemParameterTable::from_json(phosg::JSON::parse(phosg::load_file(path)));
-    new_item_parameter_tables[v_s] = ItemParameterTable::from_binary(data, v);
   }
 
   auto json = phosg::JSON::parse(phosg::load_file("system/item-tables/translation-table.json"));
@@ -2227,6 +2223,39 @@ void ServerState::load_dol_files() {
   this->dol_file_index = make_shared<DOLFileIndex>("system/dol");
 }
 
+void ServerState::generate_bb_stream_file() {
+  config_log.info_f("Generating BB stream file");
+  auto sf = std::make_shared<BBStreamFile>();
+
+  auto add_file = [&](const std::string& filename, std::string&& file_data = "") -> void {
+    if (file_data.empty()) {
+      file_data = phosg::load_file("system/blueburst/" + filename);
+    }
+    auto& e = sf->entries.emplace_back();
+    e.size = file_data.size();
+    e.checksum = phosg::crc32(file_data.data(), file_data.size());
+    e.offset = sf->data.size();
+    e.filename = filename;
+    sf->data += file_data;
+    config_log.debug_f(
+        "[BBStreamFile] Added file {} at offset {:08X} ({:08X} bytes) with checksum {:08X}; total size is now {:08X}",
+        filename, e.offset, e.size, e.checksum, sf->data.size());
+  };
+
+  add_file("BattleParamEntry.dat");
+  add_file("BattleParamEntry_on.dat");
+  add_file("BattleParamEntry_lab.dat");
+  add_file("BattleParamEntry_lab_on.dat");
+  add_file("BattleParamEntry_ep4.dat");
+  add_file("BattleParamEntry_ep4_on.dat");
+  add_file("PlyLevelTbl.prs");
+  add_file("ItemMagEdit.prs");
+  auto pmt = this->item_parameter_table(Version::BB_V4);
+  add_file("ItemPMT.prs", prs_compress_optimal(pmt->serialize_binary(Version::BB_V4)));
+
+  this->bb_stream_file = sf;
+}
+
 void ServerState::create_default_lobbies() {
   if (this->default_lobbies_created) {
     return;
@@ -2306,6 +2335,7 @@ void ServerState::load_all(bool enable_thread_pool) {
   this->load_config_late();
   this->load_teams();
   this->load_quest_index();
+  this->generate_bb_stream_file();
 }
 
 void ServerState::disconnect_all_banned_clients() {

src/ServerState.hh

@@ -65,6 +65,17 @@ struct CheatFlags {
   explicit CheatFlags(const phosg::JSON& json);
 };
 
+struct BBStreamFile {
+  struct Entry {
+    uint32_t offset;
+    uint32_t size;
+    uint32_t checksum; // crc32
+    std::string filename;
+  };
+  std::vector<Entry> entries;
+  std::string data;
+};
+
 struct ServerState : public std::enable_shared_from_this<ServerState> {
   enum class RunShellBehavior {
     DEFAULT = 0,
@@ -180,7 +191,7 @@ struct ServerState : public std::enable_shared_from_this<ServerState> {
   std::unordered_map<uint64_t, std::shared_ptr<const SuperMap>> supermap_for_source_hash_sum;
   std::unordered_map<uint32_t, std::shared_ptr<const SuperMap>> supermap_for_free_play_key;
   std::shared_ptr<const RoomLayoutIndex> room_layout_index;
-  std::shared_ptr<FileContentsCache> bb_stream_files_cache;
+  std::shared_ptr<const BBStreamFile> bb_stream_file;
   std::shared_ptr<FileContentsCache> bb_system_cache;
   std::shared_ptr<FileContentsCache> gba_files_cache;
   std::shared_ptr<const DOLFileIndex> dol_file_index;
@@ -442,6 +453,7 @@ struct ServerState : public std::enable_shared_from_this<ServerState> {
   void load_quest_index(bool raise_on_any_failure = false);
   void compile_functions(bool raise_on_any_failure = false);
   void load_dol_files();
+  void generate_bb_stream_file();
 
   void load_all(bool enable_thread_pool);
 

src/ShellCommands.cc

@@ -211,14 +211,17 @@ ShellCommand c_reload(
           args.s->load_set_data_tables();
         } else if (type == "battle-params") {
           args.s->load_battle_params();
+          args.s->generate_bb_stream_file();
         } else if (type == "level-tables") {
           args.s->load_level_tables();
+          args.s->generate_bb_stream_file();
         } else if (type == "text-index") {
           args.s->load_text_index();
         } else if (type == "word-select") {
           args.s->load_word_select_table();
         } else if (type == "item-definitions") {
           args.s->load_item_definitions();
+          args.s->generate_bb_stream_file();
         } else if (type == "item-name-index") {
           args.s->load_item_name_indexes();
         } else if (type == "drop-tables") {

system/blueburst/ItemPMT.prs

@@ -1 +0,0 @@
-../item-tables/ItemPMT-bb-v4.prs

system/item-tables/ItemPMT-dc-v2.prs

@@ -1 +0,0 @@
-ItemPMT-pc-v2.prs

system/item-tables/ItemPMT-gc-ep3-nte.prs

@@ -1 +0,0 @@
-ItemPMT-gc-v3.prs

system/item-tables/ItemPMT-gc-ep3.prs

@@ -1 +0,0 @@
-ItemPMT-gc-v3.prs

system/item-tables/ItemPMT-pc-nte.prs

@@ -1 +0,0 @@
-ItemPMT-pc-v2.prs

system/item-tables/translation-table.json

@@ -1,16 +1,12 @@
 [
-  // This table maps in-game primary identifiers across client versions. The
+  // This table maps in-game primary identifiers across client versions. The high bit of the entries in this table
-  // high bit of the entries in this table specifies the entry type; if this
+  // specifies the entry type; if this bit is set, the entry is not "canonical"; that is, it's a one-way mapping and
-  // bit is set, the entry is not "canonical"; that is, it's a one-way mapping
+  // the client does not actually have that item defined. Each primary identifier must have at most one "canonical"
-  // and the client does not actually have that item defined. Each primary
+  // listing for each version (but may have none). To convert an item from client A to client B:
-  // identifier must have at most one "canonical" listing for each version
+  // 1. Scan the column for A's version in the table to find the cell that contains the primary identifier sent by A.
-  // (but may have none). To convert an item from client A to client B:
+  //    (We optimize this with an index.)
-  // 1. Scan the column for A's version in the table to find the cell that
-  //    contains the primary identifier sent by A. (We optimize this with an
-  //    index.)
   // 2. Move left or right to the column for B's version.
-  // 3. Take the value from that cell, clear the high bit if needed, and use
+  // 3. Take the value from that cell, clear the high bit if needed, and use that as the primary identifier for B.
-  //    that as the primary identifier to show to B.
   // This logic is implemented in ItemTranslationTable.cc.
 
   // DC_NTE--  11/2000---  DC_V1-----  DC_V2-----  PC_NTE----  PC_V2-----  GC_NTE----  GC_V3-----  GC_EP3_NTE  GC_EP3----  XB_V3-----  BB_V4-----  NAME
@@ -134,7 +130,6 @@
   [0x000F0100, 0x000F0100, 0x000F0100, 0x000F0100, 0x000F0100, 0x000F0100, 0x800F0000, 0x000F0100, 0x000F0100, 0x000F0100, 0x000F0100, 0x000F0100, "ANGRY FIST"],
   [0x000F0200, 0x000F0200, 0x000F0200, 0x000F0200, 0x000F0200, 0x000F0200, 0x800F0000, 0x000F0200, 0x000F0200, 0x000F0200, 0x000F0200, 0x000F0200, "GOD HAND"],
   [0x000F0300, 0x000F0300, 0x000F0300, 0x000F0300, 0x000F0300, 0x000F0300, 0x800F0000, 0x000F0300, 0x000F0300, 0x000F0300, 0x000F0300, 0x000F0300, "SONIC KNUCKLE"],
-  [0x000F0400, 0x000F0400, 0x000F0400, 0x000F0400, 0x000F0400, 0x000F0400, 0x80020500, 0x80100000, 0x80100000, 0x80100000, 0x80100000, 0x80100000, "OROTIAGITO (v0-v2)"],
   [0x800F0300, 0x800F0300, 0x800F0300, 0x800F0300, 0x800F0300, 0x800F0300, 0x800F0000, 0x800F0300, 0x800F0300, 0x800F0300, 0x800F0300, 0x000F0400, "LOGiN"],
   [0x00100000, 0x00100000, 0x00100000, 0x00100000, 0x00100000, 0x00100000, 0x00100000, 0x00100000, 0x00100000, 0x00100000, 0x00100000, 0x00100000, "OROTIAGITO"],
   [0x00100100, 0x00100100, 0x00100100, 0x00100100, 0x00100100, 0x00100100, 0x80100000, 0x00100100, 0x00100100, 0x00100100, 0x00100100, 0x00100100, "AGITO (1975)"],
@@ -439,7 +434,9 @@
   [0x80080500, 0x80080500, 0x80080500, 0x80770000, 0x80770000, 0x80770000, 0x80080500, 0x80770000, 0x80770000, 0x80770000, 0x80770000, 0x00EA0000, "TypeME/MECHGUN"],
   [0x80090500, 0x80090500, 0x80090500, 0x80780000, 0x80780000, 0x80780000, 0x80090500, 0x80780000, 0x80780000, 0x80780000, 0x80780000, 0x00EB0000, "TypeSH/SHOT"],
   [0x800C0400, 0x800C0400, 0x800C0400, 0x807B0000, 0x807B0000, 0x807B0000, 0x800C0400, 0x807B0000, 0x807B0000, 0x807B0000, 0x807B0000, 0x00EC0000, "TypeWA/WAND"],
+  [0x80010000, 0x80010000, 0x80010000, 0x00890000, 0x00890000, 0x00890000, 0x008D0000, 0x00AA0000, 0x00AA0000, 0x00AA0000, 0x00AA0000, 0x00ED0000, "???? (WEAPON)"],
 
+  // DC_NTE--  11/2000---  DC_V1-----  DC_V2-----  PC_NTE----  PC_V2-----  GC_NTE----  GC_V3-----  GC_EP3_NTE  GC_EP3----  XB_V3-----  BB_V4-----  NAME
   [0x01010000, 0x01010000, 0x01010000, 0x01010000, 0x01010000, 0x01010000, 0x01010000, 0x01010000, 0x01010000, 0x01010000, 0x01010000, 0x01010000, "Frame"],
   [0x01010100, 0x01010100, 0x01010100, 0x01010100, 0x01010100, 0x01010100, 0x01010100, 0x01010100, 0x01010100, 0x01010100, 0x01010100, 0x01010100, "Armor"],
   [0x01010200, 0x01010200, 0x01010200, 0x01010200, 0x01010200, 0x01010200, 0x01010200, 0x01010200, 0x01010200, 0x01010200, 0x01010200, 0x01010200, "Psy Armor"],
@@ -528,8 +525,9 @@
   [0x81012800, 0x81012800, 0x81012800, 0x81013400, 0x81013400, 0x81013400, 0x81010C00, 0x81013400, 0x81013400, 0x81013400, 0x81013400, 0x01015500, "UNION FIELD"],
   [0x81012800, 0x81012800, 0x81012800, 0x81013400, 0x81013400, 0x81013400, 0x81010C00, 0x81013400, 0x81013400, 0x81013400, 0x81013400, 0x01015600, "SAMURAI ARMOR"],
   [0x81012800, 0x81012800, 0x81012800, 0x81013400, 0x81013400, 0x81013400, 0x81010C00, 0x81013400, 0x81013400, 0x81013400, 0x81013400, 0x01015700, "STEALTH SUIT"],
-  [0x81012800, 0x81012800, 0x81012800, 0x81013400, 0x81013400, 0x81013400, 0x01013500, 0x01013500, 0x01013500, 0x01013500, 0x01013500, 0x01015800, "???? (ARMOR)"],
+  [0x81012800, 0x81012800, 0x81012800, 0x01013500, 0x01013500, 0x01013500, 0x01013500, 0x01013500, 0x01013500, 0x01013500, 0x01013500, 0x01015800, "???? (ARMOR)"],
 
+  // DC_NTE--  11/2000---  DC_V1-----  DC_V2-----  PC_NTE----  PC_V2-----  GC_NTE----  GC_V3-----  GC_EP3_NTE  GC_EP3----  XB_V3-----  BB_V4-----  NAME
   [0x01020000, 0x01020000, 0x01020000, 0x01020000, 0x01020000, 0x01020000, 0x01020000, 0x01020000, 0x01020000, 0x01020000, 0x01020000, 0x01020000, "Barrier"],
   [0x01020100, 0x01020100, 0x01020100, 0x01020100, 0x01020100, 0x01020100, 0x01020100, 0x01020100, 0x01020100, 0x01020100, 0x01020100, 0x01020100, "Shield"],
   [0x01020200, 0x01020200, 0x01020200, 0x01020200, 0x01020200, 0x01020200, 0x01020200, 0x01020200, 0x01020200, 0x01020200, 0x01020200, 0x01020200, "Core Shield"],
@@ -695,8 +693,9 @@
   [0x81022600, 0x81022600, 0x81022600, 0x81023500, 0x81023500, 0x81023500, 0x81020C00, 0x81028400, 0x81028400, 0x81028400, 0x81028400, 0x0102A200, "GENPEI (unused)"],
   [0x81022600, 0x81022600, 0x81022600, 0x81023500, 0x81023500, 0x81023500, 0x81020C00, 0x81028400, 0x81028400, 0x81028400, 0x81028400, 0x0102A300, "GENPEI (unused)"],
   [0x81022600, 0x81022600, 0x81022600, 0x81023500, 0x81023500, 0x81023500, 0x81020C00, 0x81028400, 0x81028400, 0x81028400, 0x81028400, 0x0102A400, "GENPEI (unused)"],
-  [0x81022600, 0x81022600, 0x81022600, 0x81023500, 0x81023500, 0x81023500, 0x81020C00, 0x01028500, 0x01028500, 0x01028500, 0x01028500, 0x0102A500, "???? (SHIELD)"],
+  [0x81022600, 0x81022600, 0x81022600, 0x01023A00, 0x01023A00, 0x01023A00, 0x81020C00, 0x01028500, 0x01028500, 0x01028500, 0x01028500, 0x0102A500, "???? (SHIELD)"],
 
+  // DC_NTE--  11/2000---  DC_V1-----  DC_V2-----  PC_NTE----  PC_V2-----  GC_NTE----  GC_V3-----  GC_EP3_NTE  GC_EP3----  XB_V3-----  BB_V4-----  NAME
   [0x01030000, 0x01030000, 0x01030000, 0x01030000, 0x01030000, 0x01030000, 0x01030000, 0x01030000, 0x01030000, 0x01030000, 0x01030000, 0x01030000, "Knight/Power"],
   [0x01030100, 0x01030100, 0x01030100, 0x01030100, 0x01030100, 0x01030100, 0x01030100, 0x01030100, 0x01030100, 0x01030100, 0x01030100, 0x01030100, "General/Power"],
   [0x01030200, 0x01030200, 0x01030200, 0x01030200, 0x01030200, 0x01030200, 0x01030200, 0x01030200, 0x01030200, 0x01030200, 0x01030200, 0x01030200, "Ogre/Power"],
@@ -799,8 +798,9 @@
   [0x81033500, 0x81033500, 0x81033500, 0x81033500, 0x81033500, 0x81033500, 0x81033400, 0x81033500, 0x81033500, 0x81033500, 0x81033500, 0x01036100, "HP/Resurrection"],
   [0x81033800, 0x81033800, 0x81033800, 0x81033800, 0x81033800, 0x81033800, 0x81033700, 0x81033800, 0x81033800, 0x81033800, 0x81033800, 0x01036200, "TP/Resurrection"],
   [0x81033B00, 0x81033B00, 0x81033B00, 0x81033B00, 0x81033B00, 0x81033B00, 0x81033A00, 0x81033B00, 0x81033B00, 0x81033B00, 0x81033B00, 0x01036300, "PB/Increase"],
-  [0x81033B00, 0x81033B00, 0x81033B00, 0x81033B00, 0x81033B00, 0x81033B00, 0x01034700, 0x01034800, 0x01034800, 0x01034800, 0x01034800, 0x01036400, "???? (UNIT)"],
+  [0x81033B00, 0x81033B00, 0x81033B00, 0x01034400, 0x01034400, 0x01034400, 0x01034700, 0x01034800, 0x01034800, 0x01034800, 0x01034800, 0x01036400, "???? (UNIT)"],
 
+  // DC_NTE--  11/2000---  DC_V1-----  DC_V2-----  PC_NTE----  PC_V2-----  GC_NTE----  GC_V3-----  GC_EP3_NTE  GC_EP3----  XB_V3-----  BB_V4-----  NAME
   [0x02000000, 0x02000000, 0x02000000, 0x02000000, 0x02000000, 0x02000000, 0x02000000, 0x02000000, 0x02000000, 0x02000000, 0x02000000, 0x02000000, "Mag"],
   [0x02010000, 0x02010000, 0x02010000, 0x02010000, 0x02010000, 0x02010000, 0x02010000, 0x02010000, 0x02010000, 0x02010000, 0x02010000, 0x02010000, "Varuna"],
   [0x02020000, 0x02020000, 0x02020000, 0x02020000, 0x02020000, 0x02020000, 0x02020000, 0x02020000, 0x02020000, 0x02020000, 0x02020000, 0x02020000, "Mitra"],
@@ -841,10 +841,10 @@
   [0x02250000, 0x02250000, 0x02250000, 0x02250000, 0x02250000, 0x02250000, 0x02250000, 0x02250000, 0x02250000, 0x02250000, 0x02250000, 0x02250000, "Naraka"],
   [0x02260000, 0x02260000, 0x02260000, 0x02260000, 0x02260000, 0x02260000, 0x02260000, 0x02260000, 0x02260000, 0x02260000, 0x02260000, 0x02260000, "Madhu"],
   [0x02270000, 0x02270000, 0x02270000, 0x02270000, 0x02270000, 0x02270000, 0x02270000, 0x02270000, 0x02270000, 0x02270000, 0x02270000, 0x02270000, "Churel"],
-  [0x82000000, 0x82000000, 0x82000000, 0x02280000, 0x02280000, 0x02280000, 0x82000000, 0x02280000, 0x02280000, 0x02280000, 0x02280000, 0x02280000, "ROBOCHAO"],
+  [0x02280000, 0x02280000, 0x02280000, 0x02280000, 0x02280000, 0x02280000, 0x82000000, 0x02280000, 0x02280000, 0x02280000, 0x02280000, 0x02280000, "ROBOCHAO"],
-  [0x82000000, 0x82000000, 0x82000000, 0x02290000, 0x02290000, 0x02290000, 0x82000000, 0x02290000, 0x02290000, 0x02290000, 0x02290000, 0x02290000, "OPA-OPA"],
+  [0x02290000, 0x02290000, 0x02290000, 0x02290000, 0x02290000, 0x02290000, 0x82000000, 0x02290000, 0x02290000, 0x02290000, 0x02290000, 0x02290000, "OPA-OPA"],
-  [0x82000000, 0x82000000, 0x82000000, 0x022A0000, 0x022A0000, 0x022A0000, 0x82000000, 0x022A0000, 0x022A0000, 0x022A0000, 0x022A0000, 0x022A0000, "PIAN"],
+  [0x022A0000, 0x022A0000, 0x022A0000, 0x022A0000, 0x022A0000, 0x022A0000, 0x82000000, 0x022A0000, 0x022A0000, 0x022A0000, 0x022A0000, 0x022A0000, "PIAN"],
-  [0x82000000, 0x82000000, 0x82000000, 0x022B0000, 0x022B0000, 0x022B0000, 0x82000000, 0x022B0000, 0x022B0000, 0x022B0000, 0x022B0000, 0x022B0000, "CHAO"],
+  [0x022B0000, 0x022B0000, 0x022B0000, 0x022B0000, 0x022B0000, 0x022B0000, 0x82000000, 0x022B0000, 0x022B0000, 0x022B0000, 0x022B0000, 0x022B0000, "CHAO"],
   [0x82000000, 0x82000000, 0x82000000, 0x022C0000, 0x022C0000, 0x022C0000, 0x82000000, 0x022C0000, 0x022C0000, 0x022C0000, 0x022C0000, 0x022C0000, "CHU CHU"],
   [0x82000000, 0x82000000, 0x82000000, 0x022D0000, 0x022D0000, 0x022D0000, 0x82000000, 0x022D0000, 0x022D0000, 0x022D0000, 0x022D0000, 0x022D0000, "KAPU KAPU"],
   [0x82000000, 0x82000000, 0x82000000, 0x022E0000, 0x022E0000, 0x022E0000, 0x82000000, 0x022E0000, 0x022E0000, 0x022E0000, 0x022E0000, 0x022E0000, "ANGEL'S WING"],
@@ -883,8 +883,9 @@
   [0x82000000, 0x82000000, 0x82000000, 0x82000000, 0x82000000, 0x82000000, 0x82000000, 0x82000000, 0x82000000, 0x82000000, 0x82000000, 0x024F0000, "Cell of MAG 0505 (mag)"],
   [0x82000000, 0x82000000, 0x82000000, 0x82000000, 0x82000000, 0x82000000, 0x82000000, 0x82000000, 0x82000000, 0x82000000, 0x82000000, 0x02500000, "Cell of MAG 0506 (mag)"],
   [0x82000000, 0x82000000, 0x82000000, 0x82000000, 0x82000000, 0x82000000, 0x82000000, 0x82000000, 0x82000000, 0x82000000, 0x82000000, 0x02510000, "Cell of MAG 0507 (mag)"],
-  [0x82000000, 0x82000000, 0x82000000, 0x82000000, 0x82000000, 0x82000000, 0x02280000, 0x02420000, 0x02420000, 0x02420000, 0x02420000, 0x02520000, "???? (MAG)"],
+  [0x82000000, 0x82000000, 0x82000000, 0x02390000, 0x02390000, 0x02390000, 0x02280000, 0x02420000, 0x02420000, 0x02420000, 0x02420000, 0x02520000, "???? (MAG)"],
 
+  // DC_NTE--  11/2000---  DC_V1-----  DC_V2-----  PC_NTE----  PC_V2-----  GC_NTE----  GC_V3-----  GC_EP3_NTE  GC_EP3----  XB_V3-----  BB_V4-----  NAME
   [0x03000000, 0x03000000, 0x03000000, 0x03000000, 0x03000000, 0x03000000, 0x03000000, 0x03000000, 0x03000000, 0x03000000, 0x03000000, 0x03000000, "Monomate"],
   [0x03000100, 0x03000100, 0x03000100, 0x03000100, 0x03000100, 0x03000100, 0x03000100, 0x03000100, 0x03000100, 0x03000100, 0x03000100, 0x03000100, "Dimate"],
   [0x03000200, 0x03000200, 0x03000200, 0x03000200, 0x03000200, 0x03000200, 0x03000200, 0x03000200, 0x03000200, 0x03000200, 0x03000200, 0x03000200, "Trimate"],
@@ -1027,7 +1028,7 @@
   [0x830C0100, 0x830D0500, 0x830D0500, 0x830D0500, 0x830D0500, 0x830D0500, 0x830E0000, 0x03100200, 0x03100200, 0x03100200, 0x03100200, 0x03100200, "Photon Crystal"],
   [0x830C0100, 0x830D0500, 0x830D0500, 0x830D0500, 0x830D0500, 0x830D0500, 0x830E0000, 0x83100000, 0x83100000, 0x83100000, 0x83100000, 0x03100300, "Secret Ticket"],
   [0x830C0100, 0x830D0500, 0x830D0500, 0x830D0500, 0x830D0500, 0x830D0500, 0x830E0000, 0x83100000, 0x83100000, 0x83100000, 0x83100000, 0x03100400, "Photon Ticket"],
-  [0x830C0100, 0x830D0500, 0x830D0500, 0x030E0700, 0x030E0700, 0x030E0700, 0x830E0000, 0x03120000, 0x03120000, 0x03120000, 0x03120000, 0x03120000, "Weapons Bronze Badge"],
+  [0x830C0100, 0x830D0500, 0x830D0500, 0x030E0700, 0x030E0700, 0x030E0700, 0x03120000, 0x03120000, 0x03120000, 0x03120000, 0x03120000, 0x03120000, "Weapons Bronze Badge"],
   [0x830C0100, 0x830D0500, 0x830D0500, 0x030E0800, 0x030E0800, 0x030E0800, 0x830E0000, 0x03120100, 0x03120100, 0x03120100, 0x03120100, 0x03120100, "Weapons Silver Badge"],
   [0x830C0100, 0x830D0500, 0x830D0500, 0x030E0900, 0x030E0900, 0x030E0900, 0x830E0000, 0x03120200, 0x03120200, 0x03120200, 0x03120200, 0x03120200, "Weapons Gold Badge"],
   [0x830C0100, 0x830D0500, 0x830D0500, 0x030E0A00, 0x030E0A00, 0x030E0A00, 0x830E0000, 0x03120300, 0x03120300, 0x03120300, 0x03120300, 0x03120300, "Weapons Crystal Badge"],
@@ -1090,5 +1091,6 @@
   [0x830C0100, 0x830D0500, 0x830D0500, 0x830D0500, 0x830D0500, 0x830D0500, 0x830E0000, 0x830D0500, 0x830D0500, 0x830D0500, 0x830D0500, 0x03190100, "Team Points 1000"],
   [0x830C0100, 0x830D0500, 0x830D0500, 0x830D0500, 0x830D0500, 0x830D0500, 0x830E0000, 0x830D0500, 0x830D0500, 0x830D0500, 0x830D0500, 0x03190200, "Team Points 5000"],
   [0x830C0100, 0x830D0500, 0x830D0500, 0x830D0500, 0x830D0500, 0x830D0500, 0x830E0000, 0x830D0500, 0x830D0500, 0x830D0500, 0x830D0500, 0x03190300, "Team Points 10000"],
-  [0x830C0100, 0x830D0500, 0x830D0500, 0x030F0000, 0x030F0000, 0x030F0000, 0x03120000, 0x83170400, 0x83170400, 0x83170400, 0x83170400, 0x031A0000, "???? (TOOL)"],
+  [0x830C0100, 0x830D0500, 0x830D0500, 0x030F0000, 0x030F0000, 0x030F0000, 0x03130000, 0x03180000, 0x03180000, 0x03180000, 0x03180000, 0x031A0000, "???? (TOOL)"],
+  // DC_NTE--  11/2000---  DC_V1-----  DC_V2-----  PC_NTE----  PC_V2-----  GC_NTE----  GC_V3-----  GC_EP3_NTE  GC_EP3----  XB_V3-----  BB_V4-----  NAME
 ]

tests/item-parameter-table.test.sh

@@ -0,0 +1,73 @@
+#!/bin/sh
+
+set -e
+
+EXECUTABLE="$1"
+if [ -z "$EXECUTABLE" ]; then
+  EXECUTABLE="./newserv"
+fi
+
+DIR=tests/item-parameter-tables
+
+echo "... DC NTE"
+$EXECUTABLE decode-item-parameter-table --dc-nte $DIR/dc-nte.expected.bin --decompressed $DIR/dc-nte.json --hex
+$EXECUTABLE encode-item-parameter-table --dc-nte $DIR/dc-nte.json $DIR/dc-nte.encoded.bin --decompressed
+bindiff $DIR/dc-nte.expected.bin $DIR/dc-nte.encoded.bin
+
+echo "... DC 11/2000"
+$EXECUTABLE decode-item-parameter-table --dc-11-2000 $DIR/dc-11-2000.expected.bin --decompressed $DIR/dc-11-2000.json --hex
+$EXECUTABLE encode-item-parameter-table --dc-11-2000 $DIR/dc-11-2000.json $DIR/dc-11-2000.encoded.bin --decompressed
+bindiff $DIR/dc-11-2000.expected.bin $DIR/dc-11-2000.encoded.bin
+
+echo "... DC V1"
+$EXECUTABLE decode-item-parameter-table --dc-v1 $DIR/dc-v1.expected.bin --decompressed $DIR/dc-v1.json --hex
+$EXECUTABLE encode-item-parameter-table --dc-v1 $DIR/dc-v1.json $DIR/dc-v1.encoded.bin --decompressed
+bindiff $DIR/dc-v1.expected.bin $DIR/dc-v1.encoded.bin
+
+echo "... DC V2"
+$EXECUTABLE decode-item-parameter-table --dc-v2 $DIR/dc-v2.expected.bin --decompressed $DIR/dc-v2.json --hex
+$EXECUTABLE encode-item-parameter-table --dc-v2 $DIR/dc-v2.json $DIR/dc-v2.encoded.bin --decompressed
+bindiff $DIR/dc-v2.expected.bin $DIR/dc-v2.encoded.bin
+
+echo "... PC NTE"
+$EXECUTABLE decode-item-parameter-table --pc-nte $DIR/pc-nte.expected.bin --decompressed $DIR/pc-nte.json --hex
+$EXECUTABLE encode-item-parameter-table --pc-nte $DIR/pc-nte.json $DIR/pc-nte.encoded.bin --decompressed
+bindiff $DIR/pc-nte.expected.bin $DIR/pc-nte.encoded.bin
+
+echo "... PC V2"
+$EXECUTABLE decode-item-parameter-table --pc-v2 $DIR/pc-v2.expected.bin --decompressed $DIR/pc-v2.json --hex
+$EXECUTABLE encode-item-parameter-table --pc-v2 $DIR/pc-v2.json $DIR/pc-v2.encoded.bin --decompressed
+bindiff $DIR/pc-v2.expected.bin $DIR/pc-v2.encoded.bin
+
+echo "... GC NTE"
+$EXECUTABLE decode-item-parameter-table --gc-nte $DIR/gc-nte.expected.bin --decompressed $DIR/gc-nte.json --hex
+$EXECUTABLE encode-item-parameter-table --gc-nte $DIR/gc-nte.json $DIR/gc-nte.encoded.bin --decompressed
+bindiff $DIR/gc-nte.expected.bin $DIR/gc-nte.encoded.bin
+
+echo "... GC V3"
+$EXECUTABLE decode-item-parameter-table --gc-v3 $DIR/gc-v3.expected.bin --decompressed $DIR/gc-v3.json --hex
+$EXECUTABLE encode-item-parameter-table --gc-v3 $DIR/gc-v3.json $DIR/gc-v3.encoded.bin --decompressed
+bindiff $DIR/gc-v3.expected.bin $DIR/gc-v3.encoded.bin
+
+echo "... GC Ep3 NTE"
+$EXECUTABLE decode-item-parameter-table --gc-ep3-nte $DIR/gc-ep3-nte.expected.bin --decompressed $DIR/gc-ep3-nte.json --hex
+$EXECUTABLE encode-item-parameter-table --gc-ep3-nte $DIR/gc-ep3-nte.json $DIR/gc-ep3-nte.encoded.bin --decompressed
+bindiff $DIR/gc-ep3-nte.expected.bin $DIR/gc-ep3-nte.encoded.bin
+
+echo "... GC Ep3"
+$EXECUTABLE decode-item-parameter-table --gc-ep3 $DIR/gc-ep3.expected.bin --decompressed $DIR/gc-ep3.json --hex
+$EXECUTABLE encode-item-parameter-table --gc-ep3 $DIR/gc-ep3.json $DIR/gc-ep3.encoded.bin --decompressed
+bindiff $DIR/gc-ep3.expected.bin $DIR/gc-ep3.encoded.bin
+
+echo "... XB"
+$EXECUTABLE decode-item-parameter-table --xb-v3 $DIR/xb-v3.expected.bin --decompressed $DIR/xb-v3.json --hex
+$EXECUTABLE encode-item-parameter-table --xb-v3 $DIR/xb-v3.json $DIR/xb-v3.encoded.bin --decompressed
+bindiff $DIR/xb-v3.expected.bin $DIR/xb-v3.encoded.bin
+
+echo "... BB"
+$EXECUTABLE decode-item-parameter-table --bb-v4 $DIR/bb-v4.expected.bin --decompressed $DIR/bb-v4.json --hex
+$EXECUTABLE encode-item-parameter-table --bb-v4 $DIR/bb-v4.json $DIR/bb-v4.encoded.bin --decompressed
+bindiff $DIR/bb-v4.expected.bin $DIR/bb-v4.encoded.bin
+
+echo "... clean up"
+rm -f tests/item-parameter-tables/*.encoded.bin tests/item-parameter-tables/*.json