add ItemRT conversion action

src/AFSArchive.cc

@@ -9,7 +9,7 @@
 
 using namespace std;
 
-AFSArchive::AFSArchive(std::shared_ptr<const std::string> data)
+AFSArchive::AFSArchive(shared_ptr<const string> data)
     : data(data) {
   struct FileHeader {
     be_uint32_t magic;
@@ -23,7 +23,7 @@ AFSArchive::AFSArchive(std::shared_ptr<const std::string> data)
 
   StringReader r(*this->data);
   const auto& header = r.get<FileHeader>();
-  if (header.magic != 0x41465300) {
+  if (header.magic != 0x41465300) { // 'AFS\0'
     throw runtime_error("file is not an AFS archive");
   }
 
@@ -33,7 +33,7 @@ AFSArchive::AFSArchive(std::shared_ptr<const std::string> data)
   }
 }
 
-std::pair<const void*, size_t> AFSArchive::get(size_t index) const {
+pair<const void*, size_t> AFSArchive::get(size_t index) const {
   const auto& entry = this->entries.at(index);
   if (entry.offset > this->data->size()) {
     throw out_of_range("entry begins beyond end of archive");
@@ -45,7 +45,7 @@ std::pair<const void*, size_t> AFSArchive::get(size_t index) const {
   return make_pair(this->data->data() + entry.offset, entry.size);
 }
 
-std::string AFSArchive::get_copy(size_t index) const {
+string AFSArchive::get_copy(size_t index) const {
   auto ret = this->get(index);
   return string(reinterpret_cast<const char*>(ret.first), ret.second);
 }
@@ -54,3 +54,33 @@ StringReader AFSArchive::get_reader(size_t index) const {
   auto ret = this->get(index);
   return StringReader(ret.first, ret.second);
 }
+
+string AFSArchive::generate(const vector<string>& files, bool big_endian) {
+  return big_endian ? AFSArchive::generate_t<true>(files) : AFSArchive::generate_t<false>(files);
+}
+
+template <bool IsBigEndian>
+string AFSArchive::generate_t(const vector<string>& files) {
+  using U32T = typename std::conditional<IsBigEndian, be_uint32_t, le_uint32_t>::type;
+
+  StringWriter w;
+  w.put_u32b(0x41465300); // 'AFS\0'
+  w.put<U32T>(files.size());
+
+  // It seems entries are aligned to 0x800-byte boundaries, and the file's
+  // header is always 0x80000 (!) bytes, most of which is unused
+  uint32_t data_offset = 0x80000;
+  for (const auto& file : files) {
+    w.put<U32T>(data_offset);
+    w.put<U32T>(file.size());
+    data_offset = (data_offset + file.size() + 0x7FF) & (~0x7FF);
+  }
+
+  w.extend_to(0x80000);
+  for (const auto& file : files) {
+    w.write(file);
+    w.extend_to((w.size() + 0x7FF) & (~0x7FF));
+  }
+
+  return std::move(w.str());
+}

src/AFSArchive.hh

@@ -25,7 +25,12 @@ public:
   std::string get_copy(size_t index) const;
   StringReader get_reader(size_t index) const;
 
+  static std::string generate(const std::vector<std::string>& files, bool big_endian);
+
 private:
+  template <bool IsBigEndian>
+  static std::string generate_t(const std::vector<std::string>& files);
+
   std::shared_ptr<const std::string> data;
   std::vector<Entry> entries;
 };

src/EnemyType.cc

@@ -1053,3 +1053,35 @@ uint8_t rare_table_index_for_enemy_type(EnemyType enemy_type) {
       throw runtime_error(string_printf("%s does not have a rare table entry", name_for_enum(enemy_type)));
   }
 }
+
+const vector<EnemyType>& enemy_types_for_rare_table_index(Episode episode, uint8_t rt_index) {
+  const auto& generate_table = +[](Episode episode) -> vector<vector<EnemyType>> {
+    vector<vector<EnemyType>> ret;
+    for (size_t z = 0; z < static_cast<size_t>(EnemyType::MAX_ENEMY_TYPE); z++) {
+      EnemyType t = static_cast<EnemyType>(z);
+      try {
+        uint8_t rt_index = rare_table_index_for_enemy_type(t);
+        if (enemy_type_valid_for_episode(episode, t)) {
+          if (rt_index >= ret.size()) {
+            ret.resize(rt_index + 1);
+          }
+          ret[rt_index].emplace_back(t);
+        }
+      } catch (const exception&) {
+      }
+    }
+    return ret;
+  };
+
+  static array<vector<vector<EnemyType>>, 5> data;
+  auto& ret = data.at(static_cast<size_t>(episode));
+  if (ret.empty()) {
+    ret = generate_table(episode);
+  }
+  try {
+    return ret.at(rt_index);
+  } catch (const out_of_range&) {
+    static const vector<EnemyType> empty_vec;
+    return empty_vec;
+  }
+}

src/EnemyType.hh

@@ -143,3 +143,4 @@ EnemyType enum_for_name<EnemyType>(const char* name);
 bool enemy_type_valid_for_episode(Episode episode, EnemyType enemy_type);
 uint8_t battle_param_index_for_enemy_type(Episode episode, EnemyType enemy_type);
 uint8_t rare_table_index_for_enemy_type(EnemyType enemy_type);
+const std::vector<EnemyType>& enemy_types_for_rare_table_index(Episode episode, uint8_t rt_index);

src/GSLArchive.cc

@@ -74,3 +74,34 @@ StringReader GSLArchive::get_reader(const string& name) const {
     throw out_of_range("GSL does not contain file: " + name);
   }
 }
+
+string GSLArchive::generate(const unordered_map<string, string>& files, bool big_endian) {
+  return big_endian ? GSLArchive::generate_t<true>(files) : GSLArchive::generate_t<false>(files);
+}
+
+template <bool IsBigEndian>
+string GSLArchive::generate_t(const unordered_map<string, string>& files) {
+  StringWriter w;
+
+  // Make sure there's enough space for a blank header entry before any file's
+  // data pages begin
+  uint32_t data_start_offset = ((sizeof(GSLHeaderEntry<IsBigEndian>) * (files.size() + 1)) + 0x7FF) & (~0x7FF);
+  uint32_t data_offset = data_start_offset;
+  for (const auto& file : files) {
+    GSLHeaderEntry<IsBigEndian> entry;
+    entry.filename.encode(file.first);
+    entry.offset = data_offset >> 11;
+    entry.size = file.second.size();
+    entry.unused = 0;
+    w.put(entry);
+    data_offset = (data_offset + file.second.size() + 0x7FF) & (~0x7FF);
+  }
+  w.extend_to(data_start_offset);
+
+  for (const auto& file : files) {
+    w.write(file.second);
+    w.extend_to((w.size() + 0x7FF) & (~0x7FF));
+  }
+
+  return std::move(w.str());
+}

src/GSLArchive.hh

@@ -23,9 +23,13 @@ public:
   std::string get_copy(const std::string& name) const;
   StringReader get_reader(const std::string& name) const;
 
+  static std::string generate(const std::unordered_map<std::string, std::string>& files, bool big_endian);
+
 private:
   template <bool IsBigEndian>
   void load_t();
+  template <bool IsBigEndian>
+  static std::string generate_t(const std::unordered_map<std::string, std::string>& files);
 
   std::shared_ptr<const std::string> data;
 

src/Main.cc

@@ -253,15 +253,16 @@ The actions are:\n\
   encode-unicode-text-set [INPUT-FILENAME [OUTPUT-FILENAME]]\n\
     Decode a Unicode text set (e.g. unitxt_e.prs) to JSON for easy editing, or\n\
     encode a JSON file to a Unicode text set.\n\
-  format-rare-item-set [--json] [INPUT-FILENAME]\n\
+  convert-rare-item-set INPUT-FILENAME [OUTPUT-FILENAME]\n\
-    Print the contents of a rare item table in a human-readable format. If\n\
+    If OUTPUT-FILENAME is not given, print the contents of a rare item table in\n\
-    --json is given, the input is parsed as a JSON rare item set (see\n\
+    a human-readable format. Otherwise, convert the input rare item set to a\n\
-    system/blueburst/rare-table.json for an example of this format). If --json\n\
+    different format and write it to OUTPUT-FILENAME. Both filenames must end\n\
-    is not given, the input is parsed as a REL rare item set.\n\
+    in one of the following extensions:\n\
-  convert-itemrt-rel-to-json [INPUT-FILENAME [OUTPUT-FILENAME]]\n\
+      .json (newserv JSON rare item table)\n\
-  convert-itemrt-gsl-to-json [INPUT-FILENAME [OUTPUT-FILENAME]]\n\
+      .gsl (PSO BB little-endian GSL archive)\n\
-    Convert a REL or GSL rare table to a JSON rare item set. The resulting JSON\n\
+      .gslb (PSO GC big-endian GSL archive)\n\
-    has the same structure as system/blueburst/rare-table.json.\n\
+      .afs (PSO V2 little-endian AFS archive)\n\
+      .rel (Schtserv rare table; cannot be used in output filename)\n\
   generate-dc-serial-number [--domain=DOMAIN] [--subdomain=SUBDOMAIN]\n\
     Generate a PSO DC serial number. DOMAIN should be 0 for Japanese, 1 for\n\
     USA, or 2 for Europe. SUBDOMAIN should be 0 for v1, or 1 for v2.\n\
@@ -323,10 +324,7 @@ enum class Behavior {
   ENCODE_TEXT_ARCHIVE,
   DECODE_UNICODE_TEXT_SET,
   ENCODE_UNICODE_TEXT_SET,
-  FORMAT_RARE_ITEM_SET,
+  CONVERT_RARE_ITEM_SET,
-  CONVERT_ITEMRT_REL_TO_JSON,
-  CONVERT_ITEMRT_GSL_TO_JSON,
-  CONVERT_ITEMRT_AFS_TO_JSON,
   SHOW_EP3_MAPS,
   SHOW_EP3_CARDS,
   GENERATE_EP3_CARDS_HTML,
@@ -371,10 +369,7 @@ static bool behavior_takes_input_filename(Behavior b) {
       (b == Behavior::DECODE_QUEST_FILE) ||
       (b == Behavior::ENCODE_QST) ||
       (b == Behavior::DISASSEMBLE_QUEST_SCRIPT) ||
-      (b == Behavior::FORMAT_RARE_ITEM_SET) ||
+      (b == Behavior::CONVERT_RARE_ITEM_SET) ||
-      (b == Behavior::CONVERT_ITEMRT_REL_TO_JSON) ||
-      (b == Behavior::CONVERT_ITEMRT_GSL_TO_JSON) ||
-      (b == Behavior::CONVERT_ITEMRT_AFS_TO_JSON) ||
       (b == Behavior::EXTRACT_AFS) ||
       (b == Behavior::EXTRACT_GSL) ||
       (b == Behavior::EXTRACT_BML) ||
@@ -414,9 +409,7 @@ static bool behavior_takes_output_filename(Behavior b) {
       (b == Behavior::ENCODE_GVM) ||
       (b == Behavior::ENCODE_QST) ||
       (b == Behavior::DISASSEMBLE_QUEST_SCRIPT) ||
-      (b == Behavior::CONVERT_ITEMRT_REL_TO_JSON) ||
+      (b == Behavior::CONVERT_RARE_ITEM_SET) ||
-      (b == Behavior::CONVERT_ITEMRT_GSL_TO_JSON) ||
-      (b == Behavior::CONVERT_ITEMRT_AFS_TO_JSON) ||
       (b == Behavior::EXTRACT_AFS) ||
       (b == Behavior::EXTRACT_GSL) ||
       (b == Behavior::EXTRACT_BML) ||
@@ -448,10 +441,8 @@ int main(int argc, char** argv) {
   ssize_t compression_level = 0;
   bool expect_decompressed = false;
   bool compress_optimal = false;
-  bool json = false;
   bool download = false;
   bool one_line = false;
-  bool names = false;
   const char* find_decryption_seed_ciphertext = nullptr;
   vector<const char*> find_decryption_seed_plaintexts;
   const char* input_filename = nullptr;
@@ -469,8 +460,6 @@ int main(int argc, char** argv) {
       num_threads = strtoull(&argv[x][10], nullptr, 0);
     } else if (!strcmp(argv[x], "--one-line")) {
       one_line = true;
-    } else if (!strcmp(argv[x], "--names")) {
-      names = true;
     } else if (!strcmp(argv[x], "--download")) {
       download = true;
     } else if (!strcmp(argv[x], "--patch")) {
@@ -543,8 +532,6 @@ int main(int argc, char** argv) {
       skip_little_endian = true;
     } else if (!strcmp(argv[x], "--skip-big-endian")) {
       skip_big_endian = true;
-    } else if (!strcmp(argv[x], "--json")) {
-      json = true;
     } else if (!strcmp(argv[x], "--require-basic-credentials")) {
       replay_require_basic_credentials = true;
     } else if (!strncmp(argv[x], "--root-addr=", 12)) {
@@ -630,14 +617,8 @@ int main(int argc, char** argv) {
           behavior = Behavior::DISASSEMBLE_QUEST_SCRIPT;
         } else if (!strcmp(argv[x], "cat-client")) {
           behavior = Behavior::CAT_CLIENT;
-        } else if (!strcmp(argv[x], "format-rare-item-set")) {
+        } else if (!strcmp(argv[x], "convert-rare-item-set")) {
-          behavior = Behavior::FORMAT_RARE_ITEM_SET;
+          behavior = Behavior::CONVERT_RARE_ITEM_SET;
-        } else if (!strcmp(argv[x], "convert-itemrt-rel-to-json")) {
-          behavior = Behavior::CONVERT_ITEMRT_REL_TO_JSON;
-        } else if (!strcmp(argv[x], "convert-itemrt-gsl-to-json")) {
-          behavior = Behavior::CONVERT_ITEMRT_GSL_TO_JSON;
-        } else if (!strcmp(argv[x], "convert-itemrt-afs-to-json")) {
-          behavior = Behavior::CONVERT_ITEMRT_AFS_TO_JSON;
         } else if (!strcmp(argv[x], "show-ep3-maps")) {
           behavior = Behavior::SHOW_EP3_MAPS;
         } else if (!strcmp(argv[x], "show-ep3-cards")) {
@@ -749,20 +730,12 @@ int main(int argc, char** argv) {
         filename += ".json";
       } else if (behavior == Behavior::DISASSEMBLE_QUEST_SCRIPT) {
         filename += ".txt";
-      } else if ((behavior == Behavior::CONVERT_ITEMRT_REL_TO_JSON) ||
-          (behavior == Behavior::CONVERT_ITEMRT_GSL_TO_JSON) ||
-          (behavior == Behavior::CONVERT_ITEMRT_AFS_TO_JSON)) {
-        filename += ".json";
       } else {
         filename += ".dec";
       }
       save_file(filename, data, size);
 
-    } else if (isatty(fileno(stdout)) &&
+    } else if (isatty(fileno(stdout)) && (behavior != Behavior::DISASSEMBLE_QUEST_SCRIPT)) {
-        (behavior != Behavior::DISASSEMBLE_QUEST_SCRIPT) &&
-        (behavior != Behavior::CONVERT_ITEMRT_REL_TO_JSON) &&
-        (behavior != Behavior::CONVERT_ITEMRT_GSL_TO_JSON) &&
-        (behavior != Behavior::CONVERT_ITEMRT_AFS_TO_JSON)) {
       // If stdout is a terminal and the data is not known to be text, use
       // print_data to write the result
       print_data(stdout, data, size);
@@ -1550,221 +1523,50 @@ int main(int argc, char** argv) {
       break;
     }
 
-    case Behavior::FORMAT_RARE_ITEM_SET: {
+    case Behavior::CONVERT_RARE_ITEM_SET: {
       auto name_index = make_shared<ItemNameIndex>(
           JSON::parse(load_file("system/item-tables/names-v2.json")),
           JSON::parse(load_file("system/item-tables/names-v3.json")),
           JSON::parse(load_file("system/item-tables/names-v4.json")));
 
+      if (!input_filename || !strcmp(input_filename, "-")) {
+        throw runtime_error("input filename must be given");
+      }
+
       shared_ptr<string> data(new string(read_input_data()));
       shared_ptr<RareItemSet> rs;
-      if (json) {
+      if (ends_with(input_filename, ".json")) {
-        rs.reset(new JSONRareItemSet(JSON::parse(read_input_data()), cli_version, name_index));
+        rs.reset(new RareItemSet(JSON::parse(*data), cli_version, name_index));
+      } else if (ends_with(input_filename, ".gsl")) {
+        rs.reset(new RareItemSet(GSLArchive(data, false), false));
+      } else if (ends_with(input_filename, ".gslb")) {
+        rs.reset(new RareItemSet(GSLArchive(data, true), true));
+      } else if (ends_with(input_filename, ".afs")) {
+        rs.reset(new RareItemSet(AFSArchive(data)));
+      } else if (ends_with(input_filename, ".rel")) {
+        rs.reset(new RareItemSet(*data, true));
       } else {
-        rs.reset(new RELRareItemSet(data));
+        throw runtime_error("cannot determine input format; use a filename ending with .json, .gsl, .gslb, .afs, or .rel");
       }
 
-      auto format_drop = [&](const RareItemSet::ExpandedDrop& r) -> string {
+      if (!output_filename || !strcmp(output_filename, "-")) {
-        ItemData item;
+        rs->print_all_collections(stdout, cli_version, name_index);
-        item.data1[0] = r.item_code[0];
+      } else if (ends_with(output_filename, ".json")) {
-        item.data1[1] = r.item_code[1];
+        string data = rs->serialize_json(cli_version, name_index);
-        item.data1[2] = r.item_code[2];
+        write_output_data(data.data(), data.size());
-        string name = name_index->describe_item(cli_version, item);
+      } else if (ends_with(output_filename, ".gsl")) {
-
+        string data = rs->serialize_gsl(big_endian);
-        auto frac = reduce_fraction<uint64_t>(r.probability, 0x100000000);
+        write_output_data(data.data(), data.size());
-        return string_printf(
+      } else if (ends_with(output_filename, ".gslb")) {
-            "(%08" PRIX32 " => %" PRIu64 "/%" PRIu64 ") %02hhX%02hhX%02hhX (%s)",
+        string data = rs->serialize_gsl(true);
-            r.probability, frac.first, frac.second, r.item_code[0], r.item_code[1], r.item_code[2], name.c_str());
+        write_output_data(data.data(), data.size());
-      };
+      } else if (ends_with(output_filename, ".afs")) {
-
+        string data = rs->serialize_afs();
-      auto print_collection = [&](GameMode mode, Episode episode, uint8_t difficulty, uint8_t section_id) -> void {
+        write_output_data(data.data(), data.size());
-        string secid_name = name_for_section_id(section_id);
-        fprintf(stdout, "%s %s %s %s\n",
-            name_for_mode(mode),
-            name_for_episode(episode),
-            name_for_difficulty(difficulty),
-            secid_name.c_str());
-
-        fprintf(stdout, "  Monster rares:\n");
-        for (size_t z = 0; z < 0x65; z++) {
-          string enemy_types;
-          for (size_t w = 0; w < static_cast<size_t>(EnemyType::MAX_ENEMY_TYPE); w++) {
-            auto enemy_type = static_cast<EnemyType>(w);
-            try {
-              if (rare_table_index_for_enemy_type(enemy_type) == z &&
-                  enemy_type_valid_for_episode(episode, enemy_type)) {
-                enemy_types += name_for_enum(enemy_type);
-                enemy_types += ",";
-              }
-            } catch (const exception&) {
-            }
-          }
-          if (!enemy_types.empty()) {
-            enemy_types.resize(enemy_types.size() - 1);
-          }
-          for (const auto& spec : rs->get_enemy_specs(mode, episode, difficulty, section_id, z)) {
-            string s = format_drop(spec);
-            fprintf(stdout, "    %02zX: %s (%s)\n", z, s.c_str(), enemy_types.c_str());
-          }
-        }
-
-        fprintf(stdout, "  Box rares:\n");
-        for (size_t area = 0; area < 0x12; area++) {
-          for (const auto& spec : rs->get_box_specs(mode, episode, difficulty, section_id, area)) {
-            string s = format_drop(spec);
-            fprintf(stdout, "    (area %02zX) %s\n", area, s.c_str());
-          }
-        }
-      };
-
-      static const vector<Episode> episodes = {
-          Episode::EP1,
-          Episode::EP2,
-          Episode::EP4,
-      };
-      for (Episode episode : episodes) {
-        for (uint8_t difficulty = 0; difficulty < 4; difficulty++) {
-          for (uint8_t section_id = 0; section_id < 10; section_id++) {
-            print_collection(GameMode::NORMAL, episode, difficulty, section_id);
-          }
-        }
-      }
-
-      break;
-    }
-
-    case Behavior::CONVERT_ITEMRT_REL_TO_JSON:
-    case Behavior::CONVERT_ITEMRT_GSL_TO_JSON:
-    case Behavior::CONVERT_ITEMRT_AFS_TO_JSON: {
-      auto name_index = make_shared<ItemNameIndex>(
-          JSON::parse(load_file("system/item-tables/names-v2.json")),
-          JSON::parse(load_file("system/item-tables/names-v3.json")),
-          JSON::parse(load_file("system/item-tables/names-v4.json")));
-
-      shared_ptr<string> data(new string(read_input_data()));
-      unique_ptr<RareItemSet> rs;
-      if (behavior == Behavior::CONVERT_ITEMRT_GSL_TO_JSON) {
-        rs.reset(new GSLRareItemSet(data, big_endian));
-      } else if (behavior == Behavior::CONVERT_ITEMRT_AFS_TO_JSON) {
-        rs.reset(new AFSRareItemSet(data));
       } else {
-        rs.reset(new RELRareItemSet(data));
+        throw runtime_error("cannot determine output format; use a filename ending with .json, .gsl, .gslb, or .afs");
       }
 
-      // Compute the mapping of {rt_index: EnemyType} for each episode
-      const auto& generate_table = +[](Episode episode) -> vector<vector<EnemyType>> {
-        vector<vector<EnemyType>> ret;
-        for (size_t z = 0; z < static_cast<size_t>(EnemyType::MAX_ENEMY_TYPE); z++) {
-          EnemyType t = static_cast<EnemyType>(z);
-          try {
-            uint8_t rt_index = rare_table_index_for_enemy_type(t);
-            if (enemy_type_valid_for_episode(episode, t)) {
-              if (rt_index >= ret.size()) {
-                ret.resize(rt_index + 1);
-              }
-              ret[rt_index].emplace_back(t);
-            }
-          } catch (const exception&) {
-          }
-        }
-        return ret;
-      };
-
-      auto episodes_dict = JSON::dict();
-      static const array<pair<Episode, vector<vector<EnemyType>>>, 3> episodes = {
-          make_pair(Episode::EP1, generate_table(Episode::EP1)),
-          make_pair(Episode::EP2, generate_table(Episode::EP2)),
-          make_pair(Episode::EP4, generate_table(Episode::EP4)),
-      };
-      for (const auto& episode_it : episodes) {
-        Episode episode = episode_it.first;
-        const auto& rt_index_to_enemy_type = episode_it.second;
-        auto difficulty_dict = JSON::dict();
-        for (uint8_t difficulty = 0; difficulty < 4; difficulty++) {
-          auto section_id_dict = JSON::dict();
-          for (uint8_t section_id = 0; section_id < 10; section_id++) {
-            auto collection_dict = JSON::dict();
-
-            for (size_t rt_index = 0; rt_index < rt_index_to_enemy_type.size(); rt_index++) {
-              const auto& enemy_types = rt_index_to_enemy_type[rt_index];
-              if (enemy_types.empty()) {
-                continue;
-              }
-
-              for (const auto& spec : rs->get_enemy_specs(GameMode::NORMAL, episode, difficulty, section_id, rt_index)) {
-                uint32_t primary_identifier = (spec.item_code[0] << 16) | (spec.item_code[1] << 8) | spec.item_code[2];
-                if (primary_identifier == 0) {
-                  continue;
-                }
-
-                JSON id_json;
-                if (names) {
-                  ItemData data;
-                  data.data1[0] = spec.item_code[0];
-                  data.data1[1] = spec.item_code[1];
-                  data.data1[2] = spec.item_code[2];
-                  id_json = name_index->describe_item(cli_version, data);
-                } else {
-                  id_json = primary_identifier;
-                }
-
-                auto frac = reduce_fraction<uint64_t>(spec.probability, 0x100000000);
-                auto specs_json = JSON::list({JSON::list({string_printf("%" PRIu64 "/%" PRIu64, frac.first, frac.second), std::move(id_json)})});
-                for (const auto& enemy_type : enemy_types) {
-                  if (enemy_type_valid_for_episode(episode, enemy_type)) {
-                    collection_dict.emplace(name_for_enum(enemy_type), std::move(specs_json));
-                  }
-                }
-              }
-            }
-
-            for (size_t area = 0; area < 0x12; area++) {
-              auto area_list = JSON::list();
-
-              for (const auto& spec : rs->get_box_specs(GameMode::NORMAL, episode, difficulty, section_id, area)) {
-                uint32_t primary_identifier = (spec.item_code[0] << 16) | (spec.item_code[1] << 8) | spec.item_code[2];
-                if (primary_identifier == 0) {
-                  continue;
-                }
-
-                JSON id_json;
-                if (names) {
-                  ItemData data;
-                  data.data1[0] = spec.item_code[0];
-                  data.data1[1] = spec.item_code[1];
-                  data.data1[2] = spec.item_code[2];
-                  id_json = name_index->describe_item(cli_version, data);
-                } else {
-                  id_json = primary_identifier;
-                }
-
-                auto frac = reduce_fraction<uint64_t>(spec.probability, 0x100000000);
-                area_list.emplace_back(JSON::list({string_printf("%" PRIu64 "/%" PRIu64, frac.first, frac.second), std::move(id_json)}));
-              }
-
-              if (!area_list.empty()) {
-                collection_dict.emplace(
-                    string_printf("Box-%s", name_for_area(episode, area)),
-                    std::move(area_list));
-              }
-            }
-
-            if (!collection_dict.empty()) {
-              section_id_dict.emplace(name_for_section_id(section_id), std::move(collection_dict));
-            }
-          }
-          difficulty_dict.emplace(token_name_for_difficulty(difficulty), std::move(section_id_dict));
-        }
-        episodes_dict.emplace(token_name_for_episode(episode), std::move(difficulty_dict));
-      }
-
-      auto root_json = JSON::dict({{"Normal", std::move(episodes_dict)}});
-      string json_data = root_json.serialize(
-          JSON::SerializeOption::FORMAT |
-          JSON::SerializeOption::HEX_INTEGERS |
-          JSON::SerializeOption::SORT_DICT_KEYS);
-
-      write_output_data(json_data.data(), json_data.size());
       break;
     }
 

src/RareItemSet.cc

@@ -1,6 +1,7 @@
 #include "RareItemSet.hh"
 
 #include <phosg/Filesystem.hh>
+#include <phosg/Math.hh>
 #include <phosg/Random.hh>
 
 #include "BattleParamsIndex.hh"
@@ -9,7 +10,33 @@
 
 using namespace std;
 
-uint32_t RareItemSet::PackedDrop::expand_rate(uint8_t pc) {
+string RareItemSet::ExpandedDrop::str() const {
+  auto frac = reduce_fraction<uint64_t>(this->probability, 0x100000000);
+  return string_printf(
+      "(%08" PRIX32 " => %" PRIu64 "/%" PRIu64 ") %02hhX%02hhX%02hhX",
+      this->probability, frac.first, frac.second, this->item_code[0], this->item_code[1], this->item_code[2]);
+}
+
+string RareItemSet::ExpandedDrop::str(GameVersion version, shared_ptr<const ItemNameIndex> name_index) const {
+  ItemData item;
+  item.data1[0] = this->item_code[0];
+  item.data1[1] = this->item_code[1];
+  item.data1[2] = this->item_code[2];
+
+  string ret = this->str();
+  ret += " (";
+  ret += name_index->describe_item(version, item);
+  ret += ")";
+  return ret;
+}
+
+uint32_t RareItemSet::expand_rate(uint8_t pc) {
+  // pc = bits SSSSS VVV
+  //   S = shift + 4 (so actual shift is 0-27)
+  //   V = value - 7 (so actual value is 7-14)
+  // take the bits 00000000 00000000 00000000 00000010
+  // shift left by shift (0-27)
+  // multiply by value
   int8_t shift = ((pc >> 3) & 0x1F) - 4;
   if (shift < 0) {
     shift = 0;
@@ -17,93 +44,202 @@ uint32_t RareItemSet::PackedDrop::expand_rate(uint8_t pc) {
   return ((2 << shift) * ((pc & 7) + 7));
 }
 
-RareItemSet::ExpandedDrop::ExpandedDrop() : probability(0) {
+uint8_t RareItemSet::compress_rate(uint32_t probability) {
-  this->item_code[0] = 0;
+  // I'm too lazy to figure out the reverse bitwise math, so we just compute all
-  this->item_code[1] = 0;
+  // the expansions and take the closest one
-  this->item_code[2] = 0;
+  static std::map<uint32_t, uint8_t> inverse_map;
-}
+  if (inverse_map.empty()) {
-
+    for (size_t z = 0; z < 0x100; z++) {
-RareItemSet::ExpandedDrop::ExpandedDrop(const PackedDrop& d)
+      inverse_map.emplace(RareItemSet::expand_rate(z), z);
-    : probability(PackedDrop::expand_rate(d.probability)) {
-  this->item_code[0] = d.item_code[0];
-  this->item_code[1] = d.item_code[1];
-  this->item_code[2] = d.item_code[2];
-}
-
-std::vector<RareItemSet::ExpandedDrop> GSLRareItemSet::Table::get_enemy_specs(uint8_t rt_index) const {
-  vector<ExpandedDrop> ret;
-  if (this->monster_rares[rt_index].item_code[0] != 0 ||
-      this->monster_rares[rt_index].item_code[1] != 0 ||
-      this->monster_rares[rt_index].item_code[2] != 0) {
-    ret.emplace_back(this->monster_rares[rt_index]);
-  }
-  return ret;
-}
-
-std::vector<RareItemSet::ExpandedDrop> GSLRareItemSet::Table::get_box_specs(uint8_t area) const {
-  vector<ExpandedDrop> ret;
-  for (size_t z = 0; z < 0x1E; z++) {
-    if (this->box_areas[z] == area) {
-      ret.emplace_back(this->box_rares[z]);
     }
   }
+
+  auto it = inverse_map.lower_bound(probability);
+  if (it == inverse_map.end()) {
+    // The expanded probability is less likely than the least likely value
+    return inverse_map.rbegin()->second;
+  } else if (it->first == probability) {
+    // The expanded probability is exactly equal to this entry
+    return it->second;
+  } else if (it == inverse_map.begin()) {
+    // The expanded probability more likely than the most likely value
+    return it->second;
+  } else {
+    // The expanded probability is between two entries; choose the closer one
+    auto prev_it = it;
+    prev_it--;
+    int32_t prev_diff = static_cast<int32_t>(prev_it->first - probability);
+    int32_t next_diff = static_cast<int32_t>(it->first - probability);
+    return (prev_diff < next_diff) ? prev_it->second : it->second;
+  }
+}
+
+RareItemSet::ParsedRELData::PackedDrop::PackedDrop(const ExpandedDrop& exp)
+    : probability(RareItemSet::compress_rate(exp.probability)),
+      item_code(exp.item_code) {}
+
+RareItemSet::ExpandedDrop RareItemSet::ParsedRELData::PackedDrop::expand() const {
+  return ExpandedDrop{
+      .probability = RareItemSet::expand_rate(this->probability),
+      .item_code = this->item_code,
+  };
+}
+
+template <bool IsBigEndian>
+void RareItemSet::ParsedRELData::parse_t(StringReader r) {
+  using U32T = typename std::conditional<IsBigEndian, be_uint32_t, le_uint32_t>::type;
+
+  uint32_t root_offset = r.pget<U32T>(r.size() - 0x10);
+  const auto& root = r.pget<Offsets<IsBigEndian>>(root_offset);
+
+  StringReader monsters_r = r.sub(root.monster_rares_offset);
+  for (size_t z = 0; z < 0x65; z++) {
+    const auto& d = monsters_r.get<PackedDrop>();
+    this->monster_rares.emplace_back(d.expand());
+  }
+
+  StringReader box_areas_r = r.sub(root.box_areas_offset, root.box_count * sizeof(uint8_t));
+  StringReader box_drops_r = r.sub(root.box_rares_offset, root.box_count * sizeof(PackedDrop));
+  for (size_t z = 0; z < root.box_count; z++) {
+    uint8_t area = box_areas_r.get_u8();
+    const auto& drop = box_drops_r.get<PackedDrop>();
+    if (!drop.item_code.is_filled_with(0)) {
+      auto& box_rare = this->box_rares.emplace_back();
+      box_rare.area = area;
+      box_rare.drop = drop.expand();
+    }
+  }
+}
+
+template <bool IsBigEndian>
+std::string RareItemSet::ParsedRELData::serialize_t() const {
+  using U32T = typename std::conditional<IsBigEndian, be_uint32_t, le_uint32_t>::type;
+  using U16T = typename std::conditional<IsBigEndian, be_uint16_t, le_uint16_t>::type;
+
+  static const PackedDrop empty_drop;
+
+  Offsets<IsBigEndian> root;
+  root.box_count = this->box_rares.size();
+
+  StringWriter w;
+  root.monster_rares_offset = w.size();
+  for (const auto& drop : this->monster_rares) {
+    w.put(PackedDrop(drop));
+  }
+  while (w.size() < root.monster_rares_offset + 0x65 * sizeof(PackedDrop)) {
+    w.put(empty_drop);
+  }
+  root.box_areas_offset = w.size();
+  for (const auto& drop : this->box_rares) {
+    w.put_u8(drop.area);
+  }
+  root.box_rares_offset = w.size();
+  for (const auto& drop : this->box_rares) {
+    w.put(PackedDrop(drop.drop));
+  }
+  while (w.size() & 3) {
+    w.put_u8(0);
+  }
+  uint32_t root_offset = w.size();
+  w.put(root);
+  while (w.size() & 0x1F) {
+    w.put_u8(0);
+  }
+  uint32_t relocations_offset = w.size();
+  w.put<U16T>(root_offset >> 2);
+  w.put<U16T>(2);
+  w.put<U16T>(1);
+  while (w.size() & 0x1F) {
+    w.put_u8(0);
+  }
+  w.put<U32T>(relocations_offset);
+  w.put<U32T>(3); // num_relocations
+  w.put<U32T>(1); // TODO: What is this used for?
+  w.put<U32T>(0);
+  w.put<U32T>(root_offset);
+  w.put<U32T>(0);
+  w.put<U32T>(0);
+  w.put<U32T>(0);
+  return std::move(w.str());
+}
+
+RareItemSet::ParsedRELData::ParsedRELData(StringReader r, bool big_endian) {
+  if (big_endian) {
+    this->parse_t<true>(r);
+  } else {
+    this->parse_t<false>(r);
+  }
+}
+
+RareItemSet::ParsedRELData::ParsedRELData(const SpecCollection& collection) {
+  for (const auto& specs : collection.rt_index_to_specs) {
+    ExpandedDrop effective_spec;
+    for (const auto& spec : specs) {
+      if (effective_spec.item_code.is_filled_with(0)) {
+        effective_spec = spec;
+      } else if ((effective_spec.probability != spec.probability) ||
+          (effective_spec.item_code != spec.item_code)) {
+        throw runtime_error("monster spec cannot be converted to ItemRT format");
+      }
+    }
+    if (!effective_spec.item_code.is_filled_with(0)) {
+      this->monster_rares.emplace_back(specs.empty() ? ExpandedDrop() : specs[0]);
+    }
+  }
+
+  if (collection.box_area_to_specs.size() > 0xFF) {
+    throw runtime_error("area value too high");
+  }
+  for (uint8_t area = 0; area < collection.box_area_to_specs.size(); area++) {
+    for (const auto& spec : collection.box_area_to_specs[area]) {
+      this->box_rares.emplace_back(BoxRare{.area = area, .drop = spec});
+    }
+  }
+}
+
+std::string RareItemSet::ParsedRELData::serialize(bool big_endian) const {
+  if (big_endian) {
+    return this->serialize_t<true>();
+  } else {
+    return this->serialize_t<false>();
+  }
+}
+
+RareItemSet::SpecCollection RareItemSet::ParsedRELData::as_collection() const {
+  SpecCollection ret;
+  for (size_t z = 0; z < this->monster_rares.size(); z++) {
+    const auto& drop = this->monster_rares[z];
+    if (drop.item_code.is_filled_with(0)) {
+      continue;
+    }
+    if (z >= ret.rt_index_to_specs.size()) {
+      ret.rt_index_to_specs.resize(z + 1);
+    }
+    ret.rt_index_to_specs[z].emplace_back(drop);
+  }
+  for (const auto& drop : this->box_rares) {
+    if (drop.drop.item_code.is_filled_with(0)) {
+      continue;
+    }
+    if (drop.area >= ret.box_area_to_specs.size()) {
+      ret.box_area_to_specs.resize(drop.area + 1);
+    }
+    ret.box_area_to_specs[drop.area].emplace_back(drop.drop);
+  }
   return ret;
 }
 
-uint16_t RareItemSet::key_for_params(GameMode mode, Episode episode, uint8_t difficulty, uint8_t secid) {
+RareItemSet::RareItemSet(const AFSArchive& afs) {
-  if (difficulty > 3) {
-    throw logic_error("incorrect difficulty");
-  }
-  if (secid > 10) {
-    throw logic_error("incorrect section id");
-  }
-
-  uint16_t key = ((difficulty & 3) << 4) | (secid & 0x0F);
-  switch (mode) {
-    case GameMode::NORMAL:
-      break;
-    case GameMode::BATTLE:
-      key |= 0x0040;
-      break;
-    case GameMode::CHALLENGE:
-      key |= 0x0080;
-      break;
-    case GameMode::SOLO:
-      key |= 0x00C0;
-      break;
-    default:
-      throw logic_error("invalid episode in RareItemSet");
-  }
-  switch (episode) {
-    case Episode::EP1:
-      break;
-    case Episode::EP2:
-      key |= 0x0100;
-      break;
-    case Episode::EP4:
-      key |= 0x0200;
-      break;
-    default:
-      throw logic_error("invalid episode in RareItemSet");
-  }
-  return key;
-}
-
-AFSRareItemSet::AFSRareItemSet(shared_ptr<const string> data)
-    : afs(data) {
   const array<GameMode, 4> modes = {GameMode::NORMAL, GameMode::BATTLE, GameMode::CHALLENGE, GameMode::SOLO};
   for (GameMode mode : modes) {
     for (size_t difficulty = 0; difficulty < 4; difficulty++) {
       for (size_t section_id = 0; section_id < 10; section_id++) {
         try {
           size_t index = difficulty * 10 + section_id;
-          auto entry = this->afs.get(index);
+          ParsedRELData rel(afs.get_reader(index), false);
-          if (entry.second < sizeof(Table)) {
+          this->collections.emplace(
-            throw runtime_error(string_printf("table %zu is too small", index));
-          }
-          this->tables.emplace(
               this->key_for_params(mode, Episode::EP1, difficulty, section_id),
-              reinterpret_cast<const Table*>(entry.first));
+              rel.as_collection());
         } catch (const out_of_range&) {
         }
       }
@@ -111,45 +247,27 @@ AFSRareItemSet::AFSRareItemSet(shared_ptr<const string> data)
   }
 }
 
-std::vector<RareItemSet::ExpandedDrop> AFSRareItemSet::get_enemy_specs(
+string RareItemSet::gsl_entry_name_for_table(GameMode mode, Episode episode, uint8_t difficulty, uint8_t section_id) {
-    GameMode mode, Episode episode, uint8_t difficulty, uint8_t secid, uint8_t rt_index) const {
+  return string_printf("ItemRT%s%s%c%1hhu.rel",
-  try {
+      ((mode == GameMode::CHALLENGE) ? "c" : ""),
-    return this->tables.at(this->key_for_params(mode, episode, difficulty, secid))->get_enemy_specs(rt_index);
+      ((episode == Episode::EP2) ? "l" : ""),
-  } catch (const out_of_range&) {
+      tolower(abbreviation_for_difficulty(difficulty)), // One of "nhvu"
-    return {};
+      section_id);
-  }
 }
 
-std::vector<RareItemSet::ExpandedDrop> AFSRareItemSet::get_box_specs(
+RareItemSet::RareItemSet(const GSLArchive& gsl, bool is_big_endian) {
-    GameMode mode, Episode episode, uint8_t difficulty, uint8_t secid, uint8_t area) const {
-  try {
-    return this->tables.at(this->key_for_params(mode, episode, difficulty, secid))->get_box_specs(area);
-  } catch (const out_of_range&) {
-    return {};
-  }
-}
-
-GSLRareItemSet::GSLRareItemSet(shared_ptr<const string> data, bool is_big_endian)
-    : gsl(data, is_big_endian) {
   const array<Episode, 2> episodes = {Episode::EP1, Episode::EP2};
   const array<GameMode, 4> modes = {GameMode::NORMAL, GameMode::BATTLE, GameMode::CHALLENGE, GameMode::SOLO};
   for (GameMode mode : modes) {
     for (Episode episode : episodes) {
       for (size_t difficulty = 0; difficulty < 4; difficulty++) {
         for (size_t section_id = 0; section_id < 10; section_id++) {
-          string filename = string_printf("ItemRT%s%s%c%1zu.rel",
-              ((mode == GameMode::CHALLENGE) ? "c" : ""),
-              ((episode == Episode::EP2) ? "l" : ""),
-              tolower(abbreviation_for_difficulty(difficulty)), // One of "nhvu"
-              section_id);
           try {
-            auto entry = this->gsl.get(filename);
+            string filename = this->gsl_entry_name_for_table(mode, episode, difficulty, section_id);
-            if (entry.second < sizeof(Table)) {
+            ParsedRELData rel(gsl.get_reader(filename), is_big_endian);
-              throw runtime_error(string_printf("table %s is too small", filename.c_str()));
+            this->collections.emplace(
-            }
-            this->tables.emplace(
                 this->key_for_params(mode, episode, difficulty, section_id),
-                reinterpret_cast<const Table*>(entry.first));
+                rel.as_collection());
           } catch (const out_of_range&) {
           }
         }
@@ -158,71 +276,27 @@ GSLRareItemSet::GSLRareItemSet(shared_ptr<const string> data, bool is_big_endian
   }
 }
 
-std::vector<RareItemSet::ExpandedDrop> GSLRareItemSet::get_enemy_specs(
+RareItemSet::RareItemSet(const string& rel_data, bool is_big_endian) {
-    GameMode mode, Episode episode, uint8_t difficulty, uint8_t secid, uint8_t rt_index) const {
+  // Tables are 0x280 bytes in size in this format, laid out sequentially
-  try {
+  StringReader r(rel_data);
-    return this->tables.at(this->key_for_params(mode, episode, difficulty, secid))->get_enemy_specs(rt_index);
+  array<Episode, 3> episodes = {Episode::EP1, Episode::EP2, Episode::EP4};
-  } catch (const out_of_range&) {
+  for (size_t ep_index = 0; ep_index < episodes.size(); ep_index++) {
-    return {};
+    for (size_t difficulty = 0; difficulty < 4; difficulty++) {
+      for (size_t section_id = 0; section_id < 10; section_id++) {
+        try {
+          size_t index = (ep_index * 40) + difficulty * 10 + section_id;
+          ParsedRELData rel(r.sub(0x280 * index, 0x280), is_big_endian);
+          this->collections.emplace(
+              this->key_for_params(GameMode::NORMAL, episodes[ep_index], difficulty, section_id),
+              rel.as_collection());
+        } catch (const out_of_range&) {
+        }
+      }
+    }
   }
 }
 
-std::vector<RareItemSet::ExpandedDrop> GSLRareItemSet::get_box_specs(
+RareItemSet::RareItemSet(const JSON& json, GameVersion version, shared_ptr<const ItemNameIndex> name_index) {
-    GameMode mode, Episode episode, uint8_t difficulty, uint8_t secid, uint8_t area) const {
-  try {
-    return this->tables.at(this->key_for_params(mode, episode, difficulty, secid))->get_box_specs(area);
-  } catch (const out_of_range&) {
-    return {};
-  }
-}
-
-RELRareItemSet::RELRareItemSet(shared_ptr<const string> data) : data(data) {
-  if (this->data->size() != sizeof(Table) * 10 * 4 * 3) {
-    throw runtime_error("data file size is incorrect");
-  }
-}
-
-std::vector<RareItemSet::ExpandedDrop> RELRareItemSet::get_enemy_specs(
-    GameMode mode, Episode episode, uint8_t difficulty, uint8_t secid, uint8_t rt_index) const {
-  return this->get_table(mode, episode, difficulty, secid).get_enemy_specs(rt_index);
-}
-
-std::vector<RareItemSet::ExpandedDrop> RELRareItemSet::get_box_specs(
-    GameMode mode, Episode episode, uint8_t difficulty, uint8_t secid, uint8_t area) const {
-  return this->get_table(mode, episode, difficulty, secid).get_box_specs(area);
-}
-
-const RELRareItemSet::Table& RELRareItemSet::get_table(
-    GameMode mode, Episode episode, uint8_t difficulty, uint8_t secid) const {
-  (void)mode; // TODO: Shouldn't we check for challenge mode somewhere?
-
-  if (difficulty > 3) {
-    throw logic_error("incorrect difficulty");
-  }
-  if (secid > 10) {
-    throw logic_error("incorrect section id");
-  }
-
-  size_t ep_index;
-  switch (episode) {
-    case Episode::EP1:
-      ep_index = 0;
-      break;
-    case Episode::EP2:
-      ep_index = 1;
-      break;
-    case Episode::EP4:
-      ep_index = 2;
-      break;
-    default:
-      throw invalid_argument("incorrect episode");
-  }
-
-  const auto* tables = reinterpret_cast<const Table*>(this->data->data());
-  return tables[(ep_index * 10 * 4) + (difficulty * 10) + secid];
-}
-
-JSONRareItemSet::JSONRareItemSet(const JSON& json, GameVersion version, shared_ptr<const ItemNameIndex> name_index) {
   for (const auto& mode_it : json.as_dict()) {
     static const unordered_map<string, GameMode> mode_keys(
         {{"Normal", GameMode::NORMAL}, {"Battle", GameMode::BATTLE}, {"Challenge", GameMode::CHALLENGE}, {"Solo", GameMode::SOLO}});
@@ -303,22 +377,259 @@ JSONRareItemSet::JSONRareItemSet(const JSON& json, GameVersion version, shared_p
   }
 }
 
-std::vector<RareItemSet::ExpandedDrop> JSONRareItemSet::get_enemy_specs(
+std::string RareItemSet::serialize_afs() const {
+  vector<string> files;
+  for (uint8_t difficulty = 0; difficulty < 4; difficulty++) {
+    for (uint8_t section_id = 0; section_id < 10; section_id++) {
+      ParsedRELData rel(this->get_collection(GameMode::NORMAL, Episode::EP1, difficulty, section_id));
+      files.emplace_back(rel.serialize(false));
+    }
+  }
+  return AFSArchive::generate(files, false);
+}
+
+std::string RareItemSet::serialize_gsl(bool big_endian) const {
+  unordered_map<string, string> files;
+
+  static const std::array<Episode, 2> episodes = {Episode::EP1, Episode::EP2};
+  for (Episode episode : episodes) {
+    for (uint8_t difficulty = 0; difficulty < 4; difficulty++) {
+      for (uint8_t section_id = 0; section_id < 10; section_id++) {
+        try {
+          string filename = this->gsl_entry_name_for_table(GameMode::NORMAL, episode, difficulty, section_id);
+          ParsedRELData rel(this->get_collection(GameMode::NORMAL, episode, difficulty, section_id));
+          files.emplace(filename, rel.serialize(big_endian));
+        } catch (const out_of_range&) {
+          // Collection does not exist; skip it
+        }
+      }
+    }
+  }
+
+  for (uint8_t difficulty = 0; difficulty < 4; difficulty++) {
+    for (uint8_t section_id = 0; section_id < 10; section_id++) {
+      try {
+        string filename = this->gsl_entry_name_for_table(GameMode::CHALLENGE, Episode::EP1, difficulty, section_id);
+        ParsedRELData rel(this->get_collection(GameMode::CHALLENGE, Episode::EP1, difficulty, section_id));
+        files.emplace(filename, rel.serialize(big_endian));
+      } catch (const out_of_range&) {
+        // Collection does not exist; skip it
+      }
+    }
+  }
+  return GSLArchive::generate(files, big_endian);
+}
+
+std::string RareItemSet::serialize_json(GameVersion version, shared_ptr<const ItemNameIndex> name_index) const {
+  auto modes_dict = JSON::dict();
+  static const array<GameMode, 4> modes = {GameMode::NORMAL, GameMode::BATTLE, GameMode::CHALLENGE, GameMode::SOLO};
+  for (const auto& mode : modes) {
+    auto episodes_dict = JSON::dict();
+    static const array<Episode, 3> episodes = {Episode::EP1, Episode::EP2, Episode::EP4};
+    for (const auto& episode : episodes) {
+      auto difficulty_dict = JSON::dict();
+      for (uint8_t difficulty = 0; difficulty < 4; difficulty++) {
+        auto section_id_dict = JSON::dict();
+        for (uint8_t section_id = 0; section_id < 10; section_id++) {
+          auto collection_dict = JSON::dict();
+          for (size_t rt_index = 0; rt_index < 0x80; rt_index++) {
+            const auto& enemy_types = enemy_types_for_rare_table_index(episode, rt_index);
+            if (enemy_types.empty()) {
+              continue;
+            }
+
+            for (const auto& spec : this->get_enemy_specs(GameMode::NORMAL, episode, difficulty, section_id, rt_index)) {
+              uint32_t primary_identifier = (spec.item_code[0] << 16) | (spec.item_code[1] << 8) | spec.item_code[2];
+              if (primary_identifier == 0) {
+                continue;
+              }
+
+              auto frac = reduce_fraction<uint64_t>(spec.probability, 0x100000000);
+              auto spec_json = JSON::list({string_printf("%" PRIu64 "/%" PRIu64, frac.first, frac.second), primary_identifier});
+              if (name_index) {
+                ItemData data;
+                data.data1[0] = spec.item_code[0];
+                data.data1[1] = spec.item_code[1];
+                data.data1[2] = spec.item_code[2];
+                spec_json.emplace_back(name_index->describe_item(version, data));
+              }
+              for (const auto& enemy_type : enemy_types) {
+                if (enemy_type_valid_for_episode(episode, enemy_type)) {
+                  JSON this_spec_json = spec_json;
+                  collection_dict.emplace(name_for_enum(enemy_type), JSON::list()).first->second->emplace_back(std::move(this_spec_json));
+                }
+              }
+            }
+          }
+
+          for (size_t area = 0; area < 0x12; area++) {
+            auto area_list = JSON::list();
+
+            for (const auto& spec : this->get_box_specs(GameMode::NORMAL, episode, difficulty, section_id, area)) {
+              uint32_t primary_identifier = (spec.item_code[0] << 16) | (spec.item_code[1] << 8) | spec.item_code[2];
+              if (primary_identifier == 0) {
+                continue;
+              }
+
+              auto frac = reduce_fraction<uint64_t>(spec.probability, 0x100000000);
+              area_list.emplace_back(JSON::list({string_printf("%" PRIu64 "/%" PRIu64, frac.first, frac.second), std::move(primary_identifier)}));
+              if (name_index) {
+                ItemData data;
+                data.data1[0] = spec.item_code[0];
+                data.data1[1] = spec.item_code[1];
+                data.data1[2] = spec.item_code[2];
+                area_list.back().emplace_back(name_index->describe_item(version, data));
+              }
+            }
+
+            if (!area_list.empty()) {
+              collection_dict.emplace(
+                  string_printf("Box-%s", name_for_area(episode, area)),
+                  std::move(area_list));
+            }
+          }
+
+          if (!collection_dict.empty()) {
+            section_id_dict.emplace(name_for_section_id(section_id), std::move(collection_dict));
+          }
+        }
+        difficulty_dict.emplace(token_name_for_difficulty(difficulty), std::move(section_id_dict));
+      }
+      episodes_dict.emplace(token_name_for_episode(episode), std::move(difficulty_dict));
+    }
+    modes_dict.emplace(name_for_mode(mode), std::move(episodes_dict));
+  }
+
+  return modes_dict.serialize(JSON::SerializeOption::FORMAT | JSON::SerializeOption::HEX_INTEGERS | JSON::SerializeOption::SORT_DICT_KEYS);
+}
+
+void RareItemSet::print_collection(
+    FILE* stream,
+    GameVersion version,
+    GameMode mode,
+    Episode episode,
+    uint8_t difficulty,
+    uint8_t section_id,
+    shared_ptr<const ItemNameIndex> name_index) const {
+  const SpecCollection* collection;
+  try {
+    collection = &this->get_collection(mode, episode, difficulty, section_id);
+  } catch (const out_of_range&) {
+    return;
+  }
+
+  string secid_name = name_for_section_id(section_id);
+  fprintf(stream, "%s %s %s %s\n",
+      name_for_mode(mode),
+      name_for_episode(episode),
+      name_for_difficulty(difficulty),
+      secid_name.c_str());
+
+  fprintf(stream, "  Monster rares:\n");
+  for (size_t z = 0; z < collection->rt_index_to_specs.size(); z++) {
+    string enemy_types_str;
+    const auto& enemy_types = enemy_types_for_rare_table_index(episode, z);
+    for (EnemyType enemy_type : enemy_types) {
+      enemy_types_str += name_for_enum(enemy_type);
+      enemy_types_str.push_back(',');
+    }
+    if (!enemy_types_str.empty()) {
+      enemy_types_str.resize(enemy_types_str.size() - 1);
+    }
+
+    for (const auto& spec : collection->rt_index_to_specs[z]) {
+      string s = name_index ? spec.str(version, name_index) : spec.str();
+      fprintf(stream, "    %02zX: %s (%s)\n", z, s.c_str(), enemy_types_str.c_str());
+    }
+  }
+
+  fprintf(stream, "  Box rares:\n");
+  for (size_t area = 0; area < collection->box_area_to_specs.size(); area++) {
+    for (const auto& spec : collection->box_area_to_specs[area]) {
+      string s = name_index ? spec.str(version, name_index) : spec.str();
+      fprintf(stream, "    (area %02zX) %s\n", area, s.c_str());
+    }
+  }
+}
+
+void RareItemSet::print_all_collections(
+    FILE* stream, GameVersion version, std::shared_ptr<const ItemNameIndex> name_index) const {
+  static const array<GameMode, 4> modes = {GameMode::NORMAL, GameMode::BATTLE, GameMode::CHALLENGE, GameMode::SOLO};
+  static const array<Episode, 3> episodes = {Episode::EP1, Episode::EP2, Episode::EP4};
+  for (GameMode mode : modes) {
+    for (Episode episode : episodes) {
+      for (uint8_t difficulty = 0; difficulty < 4; difficulty++) {
+        for (uint8_t section_id = 0; section_id < 10; section_id++) {
+          try {
+            this->print_collection(stream, version, mode, episode, difficulty, section_id, name_index);
+          } catch (const out_of_range& e) {
+          }
+        }
+      }
+    }
+  }
+}
+
+std::vector<RareItemSet::ExpandedDrop> RareItemSet::get_enemy_specs(
     GameMode mode, Episode episode, uint8_t difficulty, uint8_t secid, uint8_t rt_index) const {
   try {
-    return this->collections.at(this->key_for_params(mode, episode, difficulty, secid)).rt_index_to_specs.at(rt_index);
+    return this->get_collection(mode, episode, difficulty, secid).rt_index_to_specs.at(rt_index);
   } catch (const out_of_range&) {
     static const std::vector<ExpandedDrop> empty_vector;
     return empty_vector;
   }
 }
 
-std::vector<RareItemSet::ExpandedDrop> JSONRareItemSet::get_box_specs(
+std::vector<RareItemSet::ExpandedDrop> RareItemSet::get_box_specs(
     GameMode mode, Episode episode, uint8_t difficulty, uint8_t secid, uint8_t area) const {
   try {
-    return this->collections.at(this->key_for_params(mode, episode, difficulty, secid)).box_area_to_specs.at(area);
+    return this->get_collection(mode, episode, difficulty, secid).box_area_to_specs.at(area);
   } catch (const out_of_range&) {
     static const std::vector<ExpandedDrop> empty_vector;
     return empty_vector;
   }
 }
+
+const RareItemSet::SpecCollection& RareItemSet::get_collection(
+    GameMode mode, Episode episode, uint8_t difficulty, uint8_t secid) const {
+  return this->collections.at(this->key_for_params(mode, episode, difficulty, secid));
+}
+
+uint16_t RareItemSet::key_for_params(GameMode mode, Episode episode, uint8_t difficulty, uint8_t secid) {
+  if (difficulty > 3) {
+    throw logic_error("incorrect difficulty");
+  }
+  if (secid > 10) {
+    throw logic_error("incorrect section id");
+  }
+
+  uint16_t key = ((difficulty & 3) << 4) | (secid & 0x0F);
+  switch (mode) {
+    case GameMode::NORMAL:
+      break;
+    case GameMode::BATTLE:
+      key |= 0x0040;
+      break;
+    case GameMode::CHALLENGE:
+      key |= 0x0080;
+      break;
+    case GameMode::SOLO:
+      key |= 0x00C0;
+      break;
+    default:
+      throw logic_error("invalid episode in RareItemSet");
+  }
+  switch (episode) {
+    case Episode::EP1:
+      break;
+    case Episode::EP2:
+      key |= 0x0100;
+      break;
+    case Episode::EP4:
+      key |= 0x0200;
+      break;
+    default:
+      throw logic_error("invalid episode in RareItemSet");
+  }
+  return key;
+}

src/RareItemSet.hh

@@ -17,111 +17,92 @@
 
 class RareItemSet {
 public:
-  struct PackedDrop {
-    uint8_t probability;
-    uint8_t item_code[3];
-
-    static uint32_t expand_rate(uint8_t pc);
-  } __attribute__((packed));
-
   struct ExpandedDrop {
-    uint32_t probability;
+    uint32_t probability = 0;
-    uint8_t item_code[3];
+    parray<uint8_t, 3> item_code;
 
-    ExpandedDrop();
+    std::string str() const;
-    explicit ExpandedDrop(const PackedDrop&);
+    std::string str(GameVersion version, std::shared_ptr<const ItemNameIndex> name_index) const;
   };
 
-  struct Table {
+  RareItemSet();
-    // 0x280 in size; describes one difficulty, section ID, and episode
+  RareItemSet(const AFSArchive& afs);
-    // TODO: It looks like this structure can actually vary. In PSOGC, these all
+  RareItemSet(const GSLArchive& gsl, bool is_big_endian);
-    // appear to be the same size/format, but that's probably not strictly
+  RareItemSet(const std::string& rel, bool is_big_endian);
-    // required to be the case.
+  RareItemSet(const JSON& json, GameVersion version, std::shared_ptr<const ItemNameIndex> name_index = nullptr);
-    /* 0000 */ parray<PackedDrop, 0x65> monster_rares;
+  ~RareItemSet() = default;
-    /* 0194 */ parray<uint8_t, 0x1E> box_areas;
-    /* 01B2 */ parray<PackedDrop, 0x1E> box_rares;
-    /* 022A */ parray<uint8_t, 2> unknown_a1;
-    /* 022C */ be_uint32_t monster_rares_offset; // == 0x0000
-    /* 0230 */ be_uint32_t box_count; // == 0x1E
-    /* 0234 */ be_uint32_t box_areas_offset; // == 0x0194
-    /* 0238 */ be_uint32_t box_rares_offset; // == 0x01B2
-    /* 023C */ be_uint32_t unused_offset1;
-    /* 0240 */ parray<be_uint16_t, 0x10> unknown_a2;
-    /* 0260 */ be_uint32_t unknown_a2_offset;
-    /* 0264 */ be_uint32_t unknown_a2_count;
-    /* 0268 */ be_uint32_t unknown_a3;
-    /* 026C */ be_uint32_t unknown_a4;
-    /* 0270 */ be_uint32_t offset_table_offset; // == 0x022C
-    /* 0274 */ parray<be_uint32_t, 3> unknown_a5;
-    /* 0280 */
 
-    std::vector<ExpandedDrop> get_enemy_specs(uint8_t rt_index) const;
+  std::vector<ExpandedDrop> get_enemy_specs(GameMode mode, Episode episode, uint8_t difficulty, uint8_t secid, uint8_t rt_index) const;
-    std::vector<ExpandedDrop> get_box_specs(uint8_t area) const;
+  std::vector<ExpandedDrop> get_box_specs(GameMode mode, Episode episode, uint8_t difficulty, uint8_t secid, uint8_t area) const;
-  } __attribute__((packed));
 
-  virtual ~RareItemSet() = default;
+  std::string serialize_afs() const;
+  std::string serialize_gsl(bool big_endian) const;
+  std::string serialize_json(GameVersion version, std::shared_ptr<const ItemNameIndex> name_index = nullptr) const;
 
-  virtual std::vector<ExpandedDrop> get_enemy_specs(GameMode mode, Episode episode, uint8_t difficulty, uint8_t secid, uint8_t rt_index) const = 0;
+  void print_collection(
-  virtual std::vector<ExpandedDrop> get_box_specs(GameMode mode, Episode episode, uint8_t difficulty, uint8_t secid, uint8_t area) const = 0;
+      FILE* stream,
+      GameVersion version,
+      GameMode mode,
+      Episode episode,
+      uint8_t difficulty,
+      uint8_t section_id,
+      std::shared_ptr<const ItemNameIndex> name_index = nullptr) const;
+  void print_all_collections(FILE* stream, GameVersion version, std::shared_ptr<const ItemNameIndex> name_index = nullptr) const;
 
 protected:
-  RareItemSet() = default;
-
-  static uint16_t key_for_params(GameMode mode, Episode episode, uint8_t difficulty, uint8_t secid);
-};
-
-class AFSRareItemSet : public RareItemSet {
-public:
-  AFSRareItemSet(std::shared_ptr<const std::string> data);
-  virtual ~AFSRareItemSet() = default;
-  virtual std::vector<ExpandedDrop> get_enemy_specs(GameMode mode, Episode episode, uint8_t difficulty, uint8_t secid, uint8_t rt_index) const;
-  virtual std::vector<ExpandedDrop> get_box_specs(GameMode mode, Episode episode, uint8_t difficulty, uint8_t secid, uint8_t area) const;
-
-private:
-  std::unordered_map<uint16_t, const Table*> tables;
-
-  AFSArchive afs;
-};
-
-class GSLRareItemSet : public RareItemSet {
-public:
-  GSLRareItemSet(std::shared_ptr<const std::string> data, bool is_big_endian);
-  virtual ~GSLRareItemSet() = default;
-  virtual std::vector<ExpandedDrop> get_enemy_specs(GameMode mode, Episode episode, uint8_t difficulty, uint8_t secid, uint8_t rt_index) const;
-  virtual std::vector<ExpandedDrop> get_box_specs(GameMode mode, Episode episode, uint8_t difficulty, uint8_t secid, uint8_t area) const;
-
-private:
-  std::unordered_map<uint16_t, const Table*> tables;
-
-  GSLArchive gsl;
-};
-
-class RELRareItemSet : public RareItemSet {
-public:
-  RELRareItemSet(std::shared_ptr<const std::string> data);
-  virtual ~RELRareItemSet() = default;
-  virtual std::vector<ExpandedDrop> get_enemy_specs(GameMode mode, Episode episode, uint8_t difficulty, uint8_t secid, uint8_t rt_index) const;
-  virtual std::vector<ExpandedDrop> get_box_specs(GameMode mode, Episode episode, uint8_t difficulty, uint8_t secid, uint8_t area) const;
-
-private:
-  std::shared_ptr<const std::string> data;
-
-  const Table& get_table(GameMode mode, Episode episode, uint8_t difficulty, uint8_t secid) const;
-};
-
-class JSONRareItemSet : public RareItemSet {
-public:
-  explicit JSONRareItemSet(const JSON& json, GameVersion version, std::shared_ptr<const ItemNameIndex> name_index = nullptr);
-  virtual ~JSONRareItemSet() = default;
-
-  virtual std::vector<ExpandedDrop> get_enemy_specs(GameMode mode, Episode episode, uint8_t difficulty, uint8_t secid, uint8_t rt_index) const;
-  virtual std::vector<ExpandedDrop> get_box_specs(GameMode mode, Episode episode, uint8_t difficulty, uint8_t secid, uint8_t area) const;
-
-private:
   struct SpecCollection {
     std::vector<std::vector<ExpandedDrop>> rt_index_to_specs;
     std::vector<std::vector<ExpandedDrop>> box_area_to_specs;
   };
 
+  struct ParsedRELData {
+    struct PackedDrop {
+      uint8_t probability = 0;
+      parray<uint8_t, 3> item_code;
+
+      PackedDrop() = default;
+      explicit PackedDrop(const ExpandedDrop&);
+      ExpandedDrop expand() const;
+    } __attribute__((packed));
+
+    template <bool IsBigEndian>
+    struct Offsets {
+      using U32T = typename std::conditional<IsBigEndian, be_uint32_t, le_uint32_t>::type;
+      /* 00 */ U32T monster_rares_offset; // -> parray<PackedDrop, 0x65>
+      /* 04 */ U32T box_count; // Usually 30 (0x1E)
+      /* 08 */ U32T box_areas_offset; // -> parray<uint8_t, 0x1E>
+      /* 0C */ U32T box_rares_offset; // -> parray<PackedDrop, 0x1E>
+      /* 10 */
+    } __attribute__((packed));
+
+    struct BoxRare {
+      uint8_t area;
+      ExpandedDrop drop;
+    };
+
+    std::vector<ExpandedDrop> monster_rares;
+    std::vector<BoxRare> box_rares;
+
+    ParsedRELData() = default;
+    ParsedRELData(StringReader r, bool big_endian);
+    explicit ParsedRELData(const SpecCollection& collection);
+    std::string serialize(bool big_endian) const;
+
+    template <bool IsBigEndian>
+    void parse_t(StringReader r);
+    template <bool IsBigEndian>
+    std::string serialize_t() const;
+
+    SpecCollection as_collection() const;
+  };
+
   std::unordered_map<uint16_t, SpecCollection> collections;
+
+  const SpecCollection& get_collection(GameMode mode, Episode episode, uint8_t difficulty, uint8_t secid) const;
+
+  static std::string gsl_entry_name_for_table(GameMode mode, Episode episode, uint8_t difficulty, uint8_t section_id);
+  static uint16_t key_for_params(GameMode mode, Episode episode, uint8_t difficulty, uint8_t secid);
+
+  static uint32_t expand_rate(uint8_t pc);
+  static uint8_t compress_rate(uint32_t probability);
 };

src/ServerState.cc

@@ -923,40 +923,38 @@ void ServerState::load_item_tables() {
 
     if (ends_with(filename, "-v2.json")) {
       config_log.info("Loading v2 JSON rare item table %s", filename.c_str());
-      this->rare_item_sets.emplace(basename, new JSONRareItemSet(JSON::parse(load_file(path)), GameVersion::PC, this->item_name_index));
+      this->rare_item_sets.emplace(basename, new RareItemSet(JSON::parse(load_file(path)), GameVersion::PC, this->item_name_index));
     } else if (ends_with(filename, "-v3.json")) {
       config_log.info("Loading v3 JSON rare item table %s", filename.c_str());
-      this->rare_item_sets.emplace(basename, new JSONRareItemSet(JSON::parse(load_file(path)), GameVersion::GC, this->item_name_index));
+      this->rare_item_sets.emplace(basename, new RareItemSet(JSON::parse(load_file(path)), GameVersion::GC, this->item_name_index));
     } else if (ends_with(filename, "-v4.json")) {
       config_log.info("Loading v4 JSON rare item table %s", filename.c_str());
-      this->rare_item_sets.emplace(basename, new JSONRareItemSet(JSON::parse(load_file(path)), GameVersion::BB, this->item_name_index));
+      this->rare_item_sets.emplace(basename, new RareItemSet(JSON::parse(load_file(path)), GameVersion::BB, this->item_name_index));
 
     } else if (ends_with(filename, ".afs")) {
       config_log.info("Loading AFS rare item table %s", filename.c_str());
       shared_ptr<string> data(new string(load_file(path)));
-      this->rare_item_sets.emplace(basename, new AFSRareItemSet(data));
+      this->rare_item_sets.emplace(basename, new RareItemSet(AFSArchive(data)));
 
     } else if (ends_with(filename, ".gsl")) {
       config_log.info("Loading GSL rare item table %s", filename.c_str());
       shared_ptr<string> data(new string(load_file(path)));
-      this->rare_item_sets.emplace(basename, new GSLRareItemSet(data, false));
+      this->rare_item_sets.emplace(basename, new RareItemSet(GSLArchive(data, false), false));
 
     } else if (ends_with(filename, ".gslb")) {
       config_log.info("Loading GSL rare item table %s", filename.c_str());
       shared_ptr<string> data(new string(load_file(path)));
-      this->rare_item_sets.emplace(basename, new GSLRareItemSet(data, true));
+      this->rare_item_sets.emplace(basename, new RareItemSet(GSLArchive(data, true), true));
 
-    } else if (ends_with(filename, ".reg")) {
+    } else if (ends_with(filename, ".rel")) {
       config_log.info("Loading REL rare item table %s", filename.c_str());
-      shared_ptr<string> data(new string(load_file(path)));
+      this->rare_item_sets.emplace(basename, new RareItemSet(load_file(path), true));
-      this->rare_item_sets.emplace(basename, new RELRareItemSet(data));
     }
   }
 
   if (!this->rare_item_sets.count("rare-table-v4")) {
     config_log.info("rare-table-v4 rare item set is not available; loading from BB data");
-    shared_ptr<string> data(new string(load_file("system/blueburst/ItemRT.rel")));
+    this->rare_item_sets.emplace("rare-table-v4", new RareItemSet(load_file("system/blueburst/ItemRT.rel"), true));
-    this->rare_item_sets.emplace("rare-table-v4", new RELRareItemSet(data));
   }
 
   config_log.info("Loading v2 common item table");

system/item-tables/names-v3.json

@@ -1172,7 +1172,7 @@
   "030E03": "Blue-black stone",
   "030E04": "Syncesta",
   "030E05": "Magic Water",
-  "030E06": "Parasitic cell Type D ",
+  "030E06": "Parasitic cell Type D",
   "030E07": "magic rock \"Heart Key\"",
   "030E08": "magic rock \"Moola\"",
   "030E09": "Star Amplifier",
@@ -1201,7 +1201,7 @@
   "030E20": "Amplifier of Zonde",
   "030E21": "Amplifier of Gizonde",
   "030E22": "Amplifier of Razonde",
-  "030E23": "Amplifier of Red ",
+  "030E23": "Amplifier of Red",
   "030E24": "Amplifier of Blue",
   "030E25": "Amplifier of Yellow",
   "030E26": "Heart of KAPU KAPU",

system/item-tables/names-v4.json

@@ -1398,7 +1398,7 @@
   "030E03": "Blue-black stone",
   "030E04": "Syncesta",
   "030E05": "Magic Water",
-  "030E06": "Parasitic cell Type D ",
+  "030E06": "Parasitic cell Type D",
   "030E07": "magic rock \"Heart Key\"",
   "030E08": "magic rock \"Moola\"",
   "030E09": "Star Amplifier",
@@ -1427,7 +1427,7 @@
   "030E20": "Amplifier of Zonde",
   "030E21": "Amplifier of Gizonde",
   "030E22": "Amplifier of Razonde",
-  "030E23": "Amplifier of Red ",
+  "030E23": "Amplifier of Red",
   "030E24": "Amplifier of Blue",
   "030E25": "Amplifier of Yellow",
   "030E26": "Heart of KAPU KAPU",