autogenerate ep3 map list, so new maps can be dropped in easily

CMakeLists.txt

@@ -36,6 +36,7 @@ add_executable(newserv
   src/Client.cc
   src/Compression.cc
   src/DNSServer.cc
+  src/Episode3.cc
   src/FileContentsCache.cc
   src/IPFrameInfo.cc
   src/IPStackSimulator.cc

src/CommandFormats.hh

@@ -1215,9 +1215,7 @@ struct S_RankUpdate_GC_Ep3_B7 {
 
 // B8 (S->C): Update card definitions (Episode 3)
 // Contents is a single little-endian le_uint32_t specifying the size of the
-// (PRS-compressed) data, followed immediately by the data. newserv sends the
-// system/ep3/cardupdate.mnr file verbatim using this command when an Episode 3
-// client connects to the login server.
+// (PRS-compressed) data, followed immediately by the data.
 // Note: BB has a handler for B8, but (as of yet) I don't know what it does. It
 // almost certainly doesn't do the same thing as the Ep3 B8 command.
 

src/Episode3.cc

@@ -0,0 +1,157 @@
+#include "Episode3.hh"
+
+#include <stdint.h>
+
+#include <phosg/Filesystem.hh>
+
+#include "Compression.hh"
+#include "Text.hh"
+
+using namespace std;
+
+
+
+Ep3DataIndex::Ep3DataIndex(const string& directory) {
+  try {
+    this->compressed_card_definitions = load_file(directory + "/cardupdate.mnr");
+    string data = prs_decompress(this->compressed_card_definitions);
+    // There's a footer after the card definitions, but we ignore it
+    if (data.size() % sizeof(Ep3CardDefinition) != sizeof(Ep3CardDefinitionsFooter)) {
+      throw runtime_error(string_printf(
+          "decompressed card update file size %zX is not aligned with card definition size %zX (%zX extra bytes)",
+          data.size(), sizeof(Ep3CardDefinition), data.size() % sizeof(Ep3CardDefinition)));
+    }
+    const auto* defs = reinterpret_cast<const Ep3CardDefinition*>(data.data());
+    size_t max_cards = data.size() / sizeof(Ep3CardDefinition);
+    for (size_t x = 0; x < max_cards; x++) {
+      // The last card entry has the build date and some other metadata (and
+      // isn't a real card, obviously), so skip it. Seems like the card ID is
+      // always a large number that won't fit in a uint16_t, so we use that to
+      // determine if the entry is a real card or not.
+      if (defs[x].card_id & 0xFFFF0000) {
+        continue;
+      }
+      shared_ptr<Ep3CardDefinition> shared_def(new Ep3CardDefinition(defs[x]));
+      if (!this->card_definitions.emplace(shared_def->card_id, shared_def).second) {
+        throw runtime_error(string_printf(
+            "duplicate card id: %08" PRIX32, shared_def->card_id.load()));
+      }
+
+      // TODO: remove debugging code here
+      // string a2str = format_data_string(defs[x].unknown_a2.data(), sizeof(defs[x].unknown_a2));
+      // string a4str = format_data_string(defs[x].unknown_a4.data(), sizeof(defs[x].unknown_a4));
+      // fprintf(stderr, "[debug] %-20s = %04X %s %04X %s\n", defs[x].name.data(), defs[x].unused.load(), a2str.c_str(), defs[x].unknown_a3.load(), a4str.c_str());
+    }
+
+    log(INFO, "Indexed %zu Episode 3 card definitions", this->card_definitions.size());
+  } catch (const exception& e) {
+    log(WARNING, "Failed to load Episode 3 card update: %s", e.what());
+  }
+
+  for (const auto& filename : list_directory(directory)) {
+    if (ends_with(filename, ".mnm")) {
+      try {
+        string compressed_data = load_file(directory + "/" + filename);
+        // There's a small header (Ep3CompressedMapHeader) before the compressed
+        // data, which we ignore
+        string data_to_decompress = compressed_data.substr(8);
+        string data = prs_decompress(data_to_decompress);
+        if (data.size() != sizeof(Ep3Map)) {
+          throw runtime_error(string_printf(
+              "decompressed data size is incorrect (expected %zu bytes, read %zu bytes)",
+              sizeof(Ep3Map), data.size()));
+        }
+
+        shared_ptr<MapEntry> entry(new MapEntry(
+            {*reinterpret_cast<const Ep3Map*>(data.data()), move(compressed_data)}));
+        if (!this->maps.emplace(entry->map.map_number, entry).second) {
+          throw runtime_error("duplicate map number");
+        }
+        string name = entry->map.name;
+        log(INFO, "Indexed Episode 3 map %s (%08" PRIX32 "; %s)",
+            filename.c_str(), entry->map.map_number.load(), name.c_str());
+
+      } catch (const exception& e) {
+        log(WARNING, "Failed to index Episode 3 map %s: %s",
+            filename.c_str(), e.what());
+      }
+    }
+  }
+
+  // TODO: Write a version of prs_compress that takes iovecs (or something
+  // similar) so we can eliminate all this string copying here. At least this
+  // only happens once at load time...
+  StringWriter entries_w;
+  StringWriter strings_w;
+
+  for (const auto& map_it : this->maps) {
+    Ep3MapList::Entry e;
+    const auto& map = map_it.second->map;
+    e.map_x = map.map_x;
+    e.map_y = map.map_y;
+    e.scene_data2 = map.scene_data2;
+    e.map_number = map.map_number.load();
+    e.width = map.width;
+    e.height = map.height;
+    e.map_tiles = map.map_tiles;
+    e.modification_tiles = map.modification_tiles;
+
+    e.name_offset = strings_w.size();
+    strings_w.write(map.name.data(), map.name.len());
+    strings_w.put_u8(0);
+    e.location_name_offset = strings_w.size();
+    strings_w.write(map.location_name.data(), map.location_name.len());
+    strings_w.put_u8(0);
+    e.quest_name_offset = strings_w.size();
+    strings_w.write(map.quest_name.data(), map.quest_name.len());
+    strings_w.put_u8(0);
+    e.description_offset = strings_w.size();
+    strings_w.write(map.description.data(), map.description.len());
+    strings_w.put_u8(0);
+
+    e.unknown_a2 = 0xFF000000;
+
+    entries_w.put(e);
+  }
+
+  Ep3MapList header;
+  header.num_maps = this->maps.size();
+  header.unknown_a1 = 0;
+  header.strings_offset = entries_w.size();
+  header.total_size = sizeof(Ep3MapList) + entries_w.size() + strings_w.size();
+
+  StringWriter w;
+  w.put(header);
+  w.write(entries_w.str());
+  w.write(strings_w.str());
+
+  StringWriter compressed_w;
+  compressed_w.put_u32b(w.str().size());
+  compressed_w.write(prs_compress(w.str()));
+  this->compressed_map_list = move(compressed_w.str());
+  log(INFO, "Generated Episode 3 compressed map list (%zu -> %zu bytes)",
+      w.size(), this->compressed_map_list.size());
+}
+
+const string& Ep3DataIndex::get_compressed_card_definitions() const {
+  if (this->compressed_card_definitions.empty()) {
+    throw runtime_error("card definitions are not available");
+  }
+  return this->compressed_card_definitions;
+}
+
+shared_ptr<const Ep3CardDefinition> Ep3DataIndex::get_card_definition(
+    uint32_t id) const {
+  return this->card_definitions.at(id);
+}
+
+const string& Ep3DataIndex::get_compressed_map_list() const {
+  if (this->compressed_map_list.empty()) {
+    throw runtime_error("map list is not available");
+  }
+  return this->compressed_map_list;
+}
+
+shared_ptr<const Ep3DataIndex::MapEntry> Ep3DataIndex::get_map(uint32_t id) const {
+  return this->maps.at(id);
+}

src/Episode3.hh

@@ -2,17 +2,57 @@
 
 #include <stdint.h>
 
+#include <string>
+#include <map>
+#include <memory>
+#include <unordered_map>
 #include <phosg/Encoding.hh>
 
 #include "Text.hh"
 
 
 
-struct Ep3Deck {
-  // TODO: are the last 4 bytes actually part of this? They don't seem to be
-  // used for anything else, but the game limits the name to 14 chars + a
-  // language marker, which equals exactly 0x10 characters.
+struct Ep3CardDefinition {
+  struct Stat {
+    be_uint16_t code;
+    uint8_t type;
+    uint8_t stat;
+  } __attribute__((packed));
+  be_uint32_t card_id;
+  parray<uint8_t, 0x40> jp_name;
+  uint8_t type;
+  uint8_t cost;
+  be_uint16_t unused;
+  Stat hp;
+  Stat ap;
+  Stat tp;
+  Stat mv;
+  parray<uint8_t, 8> left_colors;
+  parray<uint8_t, 8> right_colors;
+  parray<uint8_t, 8> top_colors;
+  parray<be_uint32_t, 8> range;
+  parray<uint8_t, 0x10> unknown_a2;
   ptext<char, 0x14> name;
+  ptext<char, 0x0B> jp_short_name;
+  ptext<char, 0x07> short_name;
+  be_uint16_t unknown_a3; // Could be has_abilities?
+  parray<uint8_t, 0x60> unknown_a4;
+} __attribute__((packed));
+
+struct Ep3CardDefinitionsFooter {
+  be_uint32_t num_cards1;
+  be_uint32_t unknown_a1;
+  be_uint32_t num_cards2;
+  be_uint32_t unknown_a2[11];
+  be_uint32_t unknown_offset_a3;
+  be_uint32_t unknown_a4[3];
+  be_uint32_t footer_offset;
+  be_uint32_t unknown_a5[3];
+} __attribute__((packed));
+
+struct Ep3Deck {
+  ptext<char, 0x10> name;
+  be_uint32_t client_id; // 0-3
   // List of card IDs. The card count is the number of nonzero entries here
   // before a zero entry (or 50 if no entries are nonzero). The first card ID is
   // the SC card, which the game implicitly subtracts from the limit - so a
@@ -76,8 +116,8 @@ struct Ep3MapList {
   struct Entry { // Should be 0x220 bytes in total
     // These 3 fields probably include the location ID (scenery to load) and the
     // music ID
-    be_uint16_t scene_data0;
-    be_uint16_t scene_data1;
+    be_uint16_t map_x;
+    be_uint16_t map_y;
     be_uint16_t scene_data2;
     be_uint16_t map_number;
     // Text offsets are from the beginning of the strings block after all map
@@ -98,7 +138,13 @@ struct Ep3MapList {
   // char strings[...EOF]; // Null-terminated strings, pointed to by offsets in Entry structs
 } __attribute__((packed));
 
-struct Ep3Map { // .mnm file format (after decompression)
+struct Ep3CompressedMapHeader { // .mnm file format
+  le_uint32_t map_number;
+  le_uint32_t compressed_data_size;
+  // Compressed data immediately follows (which decompresses to an Ep3Map)
+} __attribute__((packed));
+
+struct Ep3Map { // .mnm format (after decompressing and discarding the header)
   /* 0000 */ be_uint32_t unknown_a1;
   /* 0004 */ be_uint32_t map_number;
   /* 0008 */ uint8_t width;
@@ -139,22 +185,22 @@ struct Ep3Map { // .mnm file format (after decompression)
   /* 1D68 */ parray<uint8_t, 0x74> unknown_a6;
   /* 1DDC */ Ep3BattleRules default_rules;
   /* 1DEC */ parray<uint8_t, 4> unknown_a7;
-  /* 1DF0 */ ptext<char, 0x14> map_name;
+  /* 1DF0 */ ptext<char, 0x14> name;
   /* 1E04 */ ptext<char, 0x14> location_name;
   /* 1E18 */ ptext<char, 0x3C> quest_name; // Same a location_name for non-quest maps
   /* 1E54 */ ptext<char, 0x190> description;
-  /* 1FE4 */ be_uint16_t scene_data0;
-  /* 1FE6 */ be_uint16_t scene_data1;
+  /* 1FE4 */ be_uint16_t map_x;
+  /* 1FE6 */ be_uint16_t map_y;
   struct NPCDeck {
     ptext<char, 0x18> name;
-    parray<be_uint32_t, 0x20> card_ids; // Last one appears to always be FFFF
+    parray<be_uint16_t, 0x20> card_ids; // Last one appears to always be FFFF
   } __attribute__((packed));
   /* 1FE8 */ NPCDeck npc_decks[3]; // Unused if name[0] == 0
   struct NPCCharacter {
     parray<be_uint16_t, 2> unknown_a1;
     parray<uint8_t, 4> unknown_a2;
     ptext<char, 0x10> name;
-    parray<be_uint16_t, 0x80> unknown_a3;
+    parray<be_uint16_t, 0x7E> unknown_a3;
   } __attribute__((packed));
   /* 20F0 */ parray<NPCCharacter, 3> npc_chars; // Unused if name[0] == 0
   /* 242C */ parray<uint8_t, 0x14> unknown_a8; // Always FF?
@@ -171,3 +217,26 @@ struct Ep3Map { // .mnm file format (after decompression)
   /* 59B2 */ parray<be_uint16_t, 0x33> unknown_a9;
   /* 5A18 */
 } __attribute__((packed));
+
+class Ep3DataIndex {
+public:
+  explicit Ep3DataIndex(const std::string& directory);
+
+  const std::string& get_compressed_card_definitions() const;
+  std::shared_ptr<const Ep3CardDefinition> get_card_definition(uint32_t id) const;
+
+  struct MapEntry {
+    Ep3Map map;
+    std::string compressed_data;
+  };
+
+  const std::string& get_compressed_map_list() const;
+  std::shared_ptr<const MapEntry> get_map(uint32_t id) const;
+
+private:
+  std::string compressed_card_definitions;
+  std::unordered_map<uint32_t, std::shared_ptr<Ep3CardDefinition>> card_definitions;
+
+  std::string compressed_map_list;
+  std::map<uint32_t, std::shared_ptr<MapEntry>> maps;
+};

src/Main.cc

@@ -416,6 +416,9 @@ int main(int argc, char** argv) {
   log(INFO, "Loading level table");
   state->level_table.reset(new LevelTable("system/blueburst/PlyLevelTbl.prs", true));
 
+  log(INFO, "Collecting Episode 3 data");
+  state->ep3_data_index.reset(new Ep3DataIndex("system/ep3"));
+
   log(INFO, "Collecting quest metadata");
   state->quest_index.reset(new QuestIndex("system/quests"));
 

src/ReceiveCommands.cc

@@ -104,7 +104,7 @@ void process_login_complete(shared_ptr<ServerState> s, shared_ptr<Client> c) {
     // On the login server, send the ep3 updates and the main menu or welcome
     // message
     if (c->flags & Client::Flag::EPISODE_3) {
-      send_ep3_card_list_update(c);
+      send_ep3_card_list_update(s, c);
       send_ep3_rank_update(c);
     }
 
@@ -436,10 +436,10 @@ void process_ep3_server_data_request(shared_ptr<ServerState> s, shared_ptr<Clien
   switch (cmds[1].byte[0]) {
     // phase 1: map select
     case 0x40:
-      send_ep3_map_list(l);
+      send_ep3_map_list(s, l);
       break;
     case 0x41:
-      send_ep3_map_data(l, cmds[4].dword);
+      send_ep3_map_data(s, l, cmds[4].dword);
       break;
     /*// phase 2: deck/name entry
     case 0x13:

src/SendCommands.cc

@@ -1298,13 +1298,12 @@ void send_give_experience(shared_ptr<Lobby> l, shared_ptr<Client> c,
 ////////////////////////////////////////////////////////////////////////////////
 // ep3 only commands
 
-// sends the (PRS-compressed) card list to the client
-void send_ep3_card_list_update(shared_ptr<Client> c) {
-  auto file_data = file_cache.get("system/ep3/cardupdate.mnr");
+void send_ep3_card_list_update(shared_ptr<ServerState> s, shared_ptr<Client> c) {
+  const auto& data = s->ep3_data_index->get_compressed_card_definitions();
 
   StringWriter w;
-  w.put_u32l(file_data->size());
-  w.write(*file_data);
+  w.put_u32l(data.size());
+  w.write(data);
 
   send_command(c, 0xB8, 0x00, w.str());
 }
@@ -1317,31 +1316,30 @@ void send_ep3_rank_update(shared_ptr<Client> c) {
 }
 
 // sends the map list (used for battle setup) to all players in a game
-void send_ep3_map_list(shared_ptr<Lobby> l) {
-  auto file_data = file_cache.get("system/ep3/maplist.mnr");
+void send_ep3_map_list(shared_ptr<ServerState> s, shared_ptr<Lobby> l) {
+  const auto& data = s->ep3_data_index->get_compressed_map_list();
 
-  string data(16, '\0');
-  PSOSubcommand* subs = reinterpret_cast<PSOSubcommand*>(data.data());
+  string cmd_data(16, '\0');
+  PSOSubcommand* subs = reinterpret_cast<PSOSubcommand*>(cmd_data.data());
   subs[0].dword = 0x000000B6;
-  subs[1].dword = (file_data->size() + 0x14 + 3) & 0xFFFFFFFC;
+  subs[1].dword = (data.size() + 0x14 + 3) & 0xFFFFFFFC;
   subs[2].dword = 0x00000040;
-  subs[3].dword = file_data->size();
-  data += *file_data;
+  subs[3].dword = data.size();
+  cmd_data += data;
 
-  send_command(l, 0x6C, 0x00, data);
+  send_command(l, 0x6C, 0x00, cmd_data);
 }
 
 // sends the map data for the chosen map to all players in the game
-void send_ep3_map_data(shared_ptr<Lobby> l, uint32_t map_id) {
-  string filename = string_printf("system/ep3/map%08" PRIX32 ".mnm", map_id);
-  auto file_data = file_cache.get(filename);
+void send_ep3_map_data(shared_ptr<ServerState> s, shared_ptr<Lobby> l, uint32_t map_id) {
+  auto entry = s->ep3_data_index->get_map(map_id);
 
   string data(12, '\0');
   PSOSubcommand* subs = reinterpret_cast<PSOSubcommand*>(data.data());
   subs[0].dword = 0x000000B6;
-  subs[1].dword = (19 + file_data->size()) & 0xFFFFFFFC;
+  subs[1].dword = (19 + entry->compressed_data.size()) & 0xFFFFFFFC;
   subs[2].dword = 0x00000041;
-  data += *file_data;
+  data += entry->compressed_data;
 
   send_command(l, 0x6C, 0x00, data);
 }

src/SendCommands.hh

@@ -206,10 +206,13 @@ void send_shop(std::shared_ptr<Client> c, uint8_t shop_type);
 void send_level_up(std::shared_ptr<Lobby> l, std::shared_ptr<Client> c);
 void send_give_experience(std::shared_ptr<Lobby> l, std::shared_ptr<Client> c,
     uint32_t amount);
-void send_ep3_card_list_update(std::shared_ptr<Client> c);
+void send_ep3_card_list_update(
+    std::shared_ptr<ServerState> s, std::shared_ptr<Client> c);
 void send_ep3_rank_update(std::shared_ptr<Client> c);
-void send_ep3_map_list(std::shared_ptr<Lobby> l);
-void send_ep3_map_data(std::shared_ptr<Lobby> l, uint32_t map_id);
+void send_ep3_map_list(
+    std::shared_ptr<ServerState> s, std::shared_ptr<Lobby> l);
+void send_ep3_map_data(
+    std::shared_ptr<ServerState> s, std::shared_ptr<Lobby> l, uint32_t map_id);
 
 enum class QuestFileType {
   ONLINE = 0,

src/ServerState.hh

@@ -45,6 +45,7 @@ struct ServerState {
   bool allow_unregistered_users;
   RunShellBehavior run_shell_behavior;
   std::vector<std::shared_ptr<const PSOBBEncryption::KeyFile>> bb_private_keys;
+  std::shared_ptr<const Ep3DataIndex> ep3_data_index;
   std::shared_ptr<const QuestIndex> quest_index;
   std::shared_ptr<const LevelTable> level_table;
   std::shared_ptr<const BattleParamTable> battle_params;