implement BB system, guild card, and stream files in proxy save files option

README.md

@@ -545,7 +545,8 @@ There are many options available when starting a proxy session. All options are
     * Online quests and download quests (saved as .bin/.dat files)
     * GBA games (saved as .gba files)
     * Patches (saved as .bin files and disassembled as .txt files)
-    * Player data from BB sessions (saved as .psochar files)
+    * Player, system, and Guild Card data from BB sessions (saved as .psochar, .psosys, .psosysteam, and .psocard files)
+    * Stream file data from BB sessions (saved as ItemPMT, BattleParamEntry, ItemMagEdit, and PlyLevelTbl files)
     * Episode 3 online quests and maps (saved as .mnmd files)
     * Episode 3 download quests (saved as .mnm files)
     * Episode 3 card definitions (saved as .mnr files)

src/CommandFormats.hh

@@ -3342,6 +3342,7 @@ struct S_StreamFileIndexEntry_BB_01EB {
 } __packed_ws__(S_StreamFileIndexEntry_BB_01EB, 0x4C);
 
 // 02EB (S->C): Send stream file chunk (BB)
+// The command may be shorter than this structure for the last chunk.
 
 struct S_StreamFileChunk_BB_02EB {
   le_uint32_t chunk_index = 0;

src/ProxyCommands.cc

@@ -669,9 +669,14 @@ static asio::awaitable<HandlerResult> C_B_E0(shared_ptr<Client> c, Channel::Mess
 
 static asio::awaitable<HandlerResult> S_B_E2(shared_ptr<Client> c, Channel::Message& msg) {
   if (c->check_flag(Client::Flag::PROXY_SAVE_FILES)) {
-    string output_filename = std::format("system.{}.psosys", phosg::now());
-    phosg::save_object_file<PSOBBBaseSystemFile>(output_filename, msg.check_size_t<PSOBBBaseSystemFile>());
-    c->log.info_f("Wrote system file to {}", output_filename);
+    const auto& cmd = msg.check_size_t<S_SyncSystemFile_BB_E2>();
+    uint64_t ts = phosg::now();
+    string system_filename = std::format("system.{}.psosys", ts);
+    string team_membership_filename = std::format("system.{}.psosysteam", ts);
+    phosg::save_object_file(system_filename, cmd.system_file);
+    phosg::save_object_file(team_membership_filename, cmd.team_membership);
+    c->log.info_f("Wrote system file to {}", system_filename);
+    c->log.info_f("Wrote team membership to {}", team_membership_filename);
   }
   co_return HandlerResult::FORWARD;
 }
@@ -688,6 +693,96 @@ static asio::awaitable<HandlerResult> S_B_E7(shared_ptr<Client> c, Channel::Mess
   co_return HandlerResult::FORWARD;
 }
 
+static asio::awaitable<HandlerResult> S_B_DC(shared_ptr<Client> c, Channel::Message& msg) {
+  if (c->check_flag(Client::Flag::PROXY_SAVE_FILES) && (msg.command == 0x02DC)) {
+    const auto& cmd = msg.check_size_t<S_GuildCardFileChunk_02DC>(8, sizeof(S_GuildCardFileChunk_02DC));
+    size_t chunk_size = msg.data.size() - 8;
+    size_t chunk_offset = cmd.chunk_index * 0x6800;
+    if (chunk_offset >= sizeof(PSOBBGuildCardFile)) {
+      throw std::runtime_error("Guild Card file chunk offset out of range");
+    }
+    if (chunk_offset + chunk_size > sizeof(PSOBBGuildCardFile)) {
+      throw std::runtime_error("Guild Card file chunk extends beyond end of file");
+    }
+
+    if (!c->proxy_session->bb_guild_card_data) {
+      c->proxy_session->bb_guild_card_data = std::make_shared<PSOBBGuildCardFile>();
+    }
+    memcpy(
+        reinterpret_cast<uint8_t*>(c->proxy_session->bb_guild_card_data.get()) + chunk_offset,
+        cmd.data.data(), chunk_size);
+  }
+  co_return HandlerResult::FORWARD;
+}
+
+static asio::awaitable<HandlerResult> C_B_DC(shared_ptr<Client> c, Channel::Message& msg) {
+  if (c->check_flag(Client::Flag::PROXY_SAVE_FILES) && (msg.command == 0x03DC)) {
+    const auto& cmd = msg.check_size_t<C_GuildCardDataRequest_BB_03DC>();
+    if ((cmd.cont == 0) && c->proxy_session->bb_guild_card_data) {
+      string output_filename = std::format("guildcard.{}.psocard", phosg::now());
+      phosg::save_object_file(output_filename, *c->proxy_session->bb_guild_card_data);
+      c->log.info_f("Wrote Guild Card data to {}", output_filename);
+    }
+  }
+  co_return HandlerResult::FORWARD;
+}
+
+static asio::awaitable<HandlerResult> S_B_EB(shared_ptr<Client> c, Channel::Message& msg) {
+  if (c->check_flag(Client::Flag::PROXY_SAVE_FILES)) {
+    if (msg.command == 0x01EB) {
+      const auto* entries = &msg.check_size_t<S_StreamFileIndexEntry_BB_01EB>(
+          sizeof(S_StreamFileIndexEntry_BB_01EB) * msg.flag);
+      c->proxy_session->bb_stream_file_entries.clear();
+      size_t total_size = 0;
+      for (size_t z = 0; z < msg.flag; z++) {
+        c->proxy_session->bb_stream_file_entries.emplace_back(entries[z]);
+        total_size += entries[z].size;
+      }
+      c->proxy_session->bb_stream_file_data.clear();
+      c->proxy_session->bb_stream_file_data.resize(total_size, '\xFF');
+      c->proxy_session->bb_stream_file_data_received = 0;
+
+    } else if (msg.command == 0x02EB) {
+      const auto& cmd = msg.check_size_t<S_StreamFileChunk_BB_02EB>(4, sizeof(S_StreamFileChunk_BB_02EB));
+      size_t chunk_offset = cmd.chunk_index * 0x6800;
+      size_t chunk_size = msg.data.size() - 4;
+      if (chunk_offset >= c->proxy_session->bb_stream_file_data.size()) {
+        throw std::runtime_error("Stream file chunk offset out of range");
+      }
+      if (chunk_offset + chunk_size > c->proxy_session->bb_stream_file_data.size()) {
+        throw std::runtime_error(std::format(
+            "Stream file chunk extends beyond end of file (received 0x{:X} bytes at offset 0x{:X}; limit is 0x{:X})",
+            chunk_size, chunk_offset, c->proxy_session->bb_stream_file_data.size()));
+      }
+      memcpy(c->proxy_session->bb_stream_file_data.data() + chunk_offset, cmd.data.data(), chunk_size);
+      c->proxy_session->bb_stream_file_data_received += chunk_size;
+      if (c->proxy_session->bb_stream_file_data_received == c->proxy_session->bb_stream_file_data.size()) {
+        string output_prefix = std::format("streamfile.{}.", phosg::now());
+        for (const auto& entry : c->proxy_session->bb_stream_file_entries) {
+          std::string filename = entry.filename.decode();
+          std::string sanitized_filename = filename;
+          for (char& ch : sanitized_filename) {
+            if (((ch < '0') || (ch > '9')) && ((ch < 'A') || (ch > 'Z')) && ((ch < 'a') || (ch > 'z')) && (ch != '.')) {
+              ch = '_';
+            }
+          }
+          if (entry.offset >= c->proxy_session->bb_stream_file_data.size()) {
+            c->log.warning_f("BB stream file entry {} begins beyond end of data", filename);
+          } else if (entry.offset + entry.size > c->proxy_session->bb_stream_file_data.size()) {
+            c->log.warning_f("BB stream file entry {} ends beyond end of data", filename);
+          } else {
+            std::string output_filename = output_prefix + sanitized_filename;
+            auto f = phosg::fopen_unique(output_filename, "wb");
+            phosg::fwritex(f.get(), c->proxy_session->bb_stream_file_data.data() + entry.offset, entry.size);
+            c->log.info_f("Wrote stream file entry {}", output_filename);
+          }
+        }
+      }
+    }
+  }
+  co_return HandlerResult::FORWARD;
+}
+
 template <typename CmdT>
 static asio::awaitable<HandlerResult> S_C4(shared_ptr<Client> c, Channel::Message& msg) {
   bool modified = false;
@@ -2356,7 +2451,7 @@ static on_message_t handlers[0x100][NUM_VERSIONS][2] = {
 /* D9 */ {{S_invalid,     nullptr}, {S_invalid,     nullptr}, {S_invalid,     nullptr},  {S_invalid,     nullptr},     {S_invalid,        nullptr},     {S_invalid,        nullptr},      {S_invalid,     nullptr},      {S_invalid,     nullptr},      {S_invalid,        nullptr},     {S_invalid,        C_GX_D9},      {S_invalid,        C_GX_D9},      {S_invalid,        C_GX_D9},      {S_invalid,     C_GX_D9},      {S_invalid,    C_B_D9}},
 /* DA */ {{S_invalid,     nullptr}, {S_invalid,     nullptr}, {S_invalid,     nullptr},  {S_invalid,     nullptr},     {S_invalid,        nullptr},     {S_invalid,        nullptr},      {S_invalid,     nullptr},      {S_invalid,     nullptr},      {S_invalid,        nullptr},     {S_V3_BB_DA,       nullptr},      {S_V3_BB_DA,       nullptr},      {S_V3_BB_DA,       nullptr},      {S_V3_BB_DA,    nullptr},      {S_V3_BB_DA,   nullptr}},
 /* DB */ {{S_invalid,     nullptr}, {S_invalid,     nullptr}, {S_invalid,     nullptr},  {S_invalid,     nullptr},     {S_invalid,        nullptr},     {S_invalid,        nullptr},      {S_invalid,     nullptr},      {S_invalid,     nullptr},      {S_invalid,        nullptr},     {S_invalid,        nullptr},      {S_invalid,        nullptr},      {S_invalid,        nullptr},      {S_invalid,     nullptr},      {S_invalid,    nullptr}},
-/* DC */ {{S_invalid,     nullptr}, {S_invalid,     nullptr}, {S_invalid,     nullptr},  {S_invalid,     nullptr},     {S_invalid,        nullptr},     {S_invalid,        nullptr},      {S_invalid,     nullptr},      {S_invalid,     nullptr},      {S_invalid,        nullptr},     {nullptr,          nullptr},      {nullptr,          nullptr},      {nullptr,          nullptr},      {S_invalid,     nullptr},      {nullptr,      nullptr}},
+/* DC */ {{S_invalid,     nullptr}, {S_invalid,     nullptr}, {S_invalid,     nullptr},  {S_invalid,     nullptr},     {S_invalid,        nullptr},     {S_invalid,        nullptr},      {S_invalid,     nullptr},      {S_invalid,     nullptr},      {S_invalid,        nullptr},     {nullptr,          nullptr},      {nullptr,          nullptr},      {nullptr,          nullptr},      {S_invalid,     nullptr},      {S_B_DC,       C_B_DC}},
 /* DD */ {{S_invalid,     nullptr}, {S_invalid,     nullptr}, {S_invalid,     nullptr},  {S_invalid,     nullptr},     {S_invalid,        nullptr},     {S_invalid,        nullptr},      {S_invalid,     nullptr},      {S_invalid,     nullptr},      {S_invalid,        nullptr},     {S_invalid,        nullptr},      {S_invalid,        nullptr},      {S_invalid,        nullptr},      {S_invalid,     nullptr},      {nullptr,      nullptr}},
 /* DE */ {{S_invalid,     nullptr}, {S_invalid,     nullptr}, {S_invalid,     nullptr},  {S_invalid,     nullptr},     {S_invalid,        nullptr},     {S_invalid,        nullptr},      {S_invalid,     nullptr},      {S_invalid,     nullptr},      {S_invalid,        nullptr},     {S_invalid,        nullptr},      {S_invalid,        nullptr},      {S_invalid,        nullptr},      {S_invalid,     nullptr},      {nullptr,      nullptr}},
 /* DF */ {{S_invalid,     nullptr}, {S_invalid,     nullptr}, {S_invalid,     nullptr},  {S_invalid,     nullptr},     {S_invalid,        nullptr},     {S_invalid,        nullptr},      {S_invalid,     nullptr},      {S_invalid,     nullptr},      {S_invalid,        nullptr},     {S_invalid,        nullptr},      {S_invalid,        nullptr},      {S_invalid,        nullptr},      {S_invalid,     nullptr},      {S_invalid,    nullptr}},
@@ -2372,7 +2467,7 @@ static on_message_t handlers[0x100][NUM_VERSIONS][2] = {
 /* E8 */ {{S_invalid,     nullptr}, {S_invalid,     nullptr}, {S_invalid,     nullptr},  {S_invalid,     nullptr},     {S_invalid,        nullptr},     {S_invalid,        nullptr},      {S_invalid,     nullptr},      {S_invalid,     nullptr},      {S_invalid,        nullptr},     {S_E8,             nullptr},      {S_E8,             nullptr},      {S_E8,             nullptr},      {S_invalid,     nullptr},      {nullptr,      nullptr}},
 /* E9 */ {{S_invalid,     nullptr}, {S_invalid,     nullptr}, {S_invalid,     nullptr},  {S_invalid,     nullptr},     {S_invalid,        nullptr},     {S_invalid,        nullptr},      {S_invalid,     nullptr},      {S_invalid,     nullptr},      {S_invalid,        nullptr},     {S_66_69_E9,       nullptr},      {S_66_69_E9,       nullptr},      {S_66_69_E9,       nullptr},      {S_invalid,     nullptr},      {nullptr,      nullptr}},
 /* EA */ {{S_invalid,     nullptr}, {S_invalid,     nullptr}, {S_invalid,     nullptr},  {S_invalid,     nullptr},     {S_invalid,        nullptr},     {S_invalid,        nullptr},      {S_invalid,     nullptr},      {S_invalid,     nullptr},      {S_invalid,        nullptr},     {nullptr,          nullptr},      {nullptr,          nullptr},      {nullptr,          nullptr},      {S_invalid,     nullptr},      {nullptr,      nullptr}},
-/* EB */ {{S_invalid,     nullptr}, {S_invalid,     nullptr}, {S_invalid,     nullptr},  {S_invalid,     nullptr},     {S_invalid,        nullptr},     {S_invalid,        nullptr},      {S_invalid,     nullptr},      {S_invalid,     nullptr},      {S_invalid,        nullptr},     {S_DG_65_67_68_EB, nullptr},      {S_DG_65_67_68_EB, nullptr},      {S_DG_65_67_68_EB, nullptr},      {S_invalid,     nullptr},      {nullptr,      nullptr}},
+/* EB */ {{S_invalid,     nullptr}, {S_invalid,     nullptr}, {S_invalid,     nullptr},  {S_invalid,     nullptr},     {S_invalid,        nullptr},     {S_invalid,        nullptr},      {S_invalid,     nullptr},      {S_invalid,     nullptr},      {S_invalid,        nullptr},     {S_DG_65_67_68_EB, nullptr},      {S_DG_65_67_68_EB, nullptr},      {S_DG_65_67_68_EB, nullptr},      {S_invalid,     nullptr},      {S_B_EB,       nullptr}},
 /* EC */ {{S_invalid,     nullptr}, {S_invalid,     nullptr}, {S_invalid,     nullptr},  {S_invalid,     nullptr},     {S_invalid,        nullptr},     {S_invalid,        nullptr},      {S_invalid,     nullptr},      {S_invalid,     nullptr},      {S_invalid,        nullptr},     {S_invalid,        nullptr},      {S_invalid,        nullptr},      {S_invalid,        nullptr},      {S_invalid,     nullptr},      {S_invalid,    nullptr}},
 /* ED */ {{S_invalid,     nullptr}, {S_invalid,     nullptr}, {S_invalid,     nullptr},  {S_invalid,     nullptr},     {S_invalid,        nullptr},     {S_invalid,        nullptr},      {S_invalid,     nullptr},      {S_invalid,     nullptr},      {S_invalid,        nullptr},     {nullptr,          nullptr},      {nullptr,          nullptr},      {nullptr,          nullptr},      {S_invalid,     nullptr},      {nullptr,      nullptr}},
 /* EE */ {{S_invalid,     nullptr}, {S_invalid,     nullptr}, {S_invalid,     nullptr},  {S_invalid,     nullptr},     {S_invalid,        nullptr},     {S_invalid,        nullptr},      {S_invalid,     nullptr},      {S_invalid,     nullptr},      {S_invalid,        nullptr},     {nullptr,          nullptr},      {nullptr,          nullptr},      {nullptr,          nullptr},      {S_invalid,     nullptr},      {nullptr,      nullptr}},
@@ -2408,7 +2503,7 @@ static on_message_t get_handler(Version version, bool from_server, uint8_t comma
 }
 
 asio::awaitable<void> on_proxy_command(shared_ptr<Client> c, bool from_server, unique_ptr<Channel::Message> msg) {
-  auto fn = get_handler(c->version(), from_server, msg->command);
+  auto fn = get_handler(c->version(), from_server, msg->command & 0xFF);
   try {
     auto res = co_await fn(c, *msg);
     if (res == HandlerResult::FORWARD) {

src/ProxySession.hh

@@ -8,8 +8,10 @@
 #include <vector>
 
 #include "Channel.hh"
+#include "CommandFormats.hh"
 #include "ItemCreator.hh"
 #include "Map.hh"
+#include "SaveFileFormats.hh"
 
 struct ServerState;
 
@@ -78,6 +80,10 @@ struct ProxySession {
     std::string data;
   };
   std::unordered_map<std::string, SavingFile> saving_files;
+  std::shared_ptr<PSOBBGuildCardFile> bb_guild_card_data; // Only used if save files is enabled
+  std::vector<S_StreamFileIndexEntry_BB_01EB> bb_stream_file_entries; // Only used if save files is enabled
+  std::string bb_stream_file_data; // Only used if save files is enabled
+  size_t bb_stream_file_data_received = 0;
 
   void set_drop_mode(std::shared_ptr<ServerState> s, Version version, int64_t override_random_seed, ProxyDropMode new_mode);
 

src/ReceiveCommands.cc

@@ -1490,11 +1490,9 @@ static asio::awaitable<void> on_93_BB(shared_ptr<Client> c, Channel::Message& ms
     co_return;
 
   } else if (s->proxy_destination_bb.has_value()) {
-    // Start a proxy session immediately if there's a destination set. Two things to watch out for:
-    // - Ignore the persistent config if this is the first data server connection, to prevent quick reconnects from
-    //   incorrectly reusing the old session's state.
-    // - We don't send 00E6 (send_client_init_bb) in this case. This is because the login command is resent to the
-    //   remote server, and we forward its response back to the client directly.
+    // Start a proxy session immediately if there's a destination set. We don't send 00E6 (send_client_init_bb) in this
+    // case. This is because the login command is resent to the remote server, and we forward its response back to the
+    // client directly.
     const auto& [host, port] = *s->proxy_destination_bb;
     co_await start_proxy_session(c, host, port, c->bb_connection_phase != 0);
     c->proxy_session->remote_client_config_data = c->bb_client_config;