write battle/challenge records structures

README.md

@@ -64,7 +64,9 @@ Current known issues / missing features / things to do:
 - Implement private and overflow lobbies.
 - Enforce client-side size limits (e.g. for 60/62 commands) on the server side as well. (For 60/62 specifically, perhaps transform them to 6C/6D if needed.)
 - Encapsulate BB server-side random state and make replays deterministic.
-- Make a looser form of item tracking that can be used on non-BB versions when quests replace player inventories, like in battle and challenge modes.
+- Implement character and inventory replacement for battle and challenge modes.
+- Implement the C5 (battle/challenge records) command.
+- Implement choice search.
 - Episode 3 bugs
     - Fix behavior when joining a spectator team after the beginning of a battle.
     - Disconnecting during a match turns you into a COM if there are other humans in the match, even if the match is part of a tournament. This may be incorrect behavior for tournaments.

src/CommandFormats.hh

@@ -1098,6 +1098,101 @@ struct C_OpenFileConfirmation_44_A6 {
 // the client crashes! Essentially, it reflects the saved state of the player's
 // character rather than the live state.
 
+struct PlayerRecordsEntry_DC {
+  /* 00 */ le_uint32_t client_id;
+  /* 04 */ PlayerRecordsDC_Challenge challenge;
+  /* A4 */ PlayerRecords_Battle<false> battle;
+  /* BC */
+} __packed__;
+
+struct PlayerRecordsEntry_PC {
+  /* 00 */ le_uint32_t client_id;
+  /* 04 */ PlayerRecordsPC_Challenge challenge;
+  /* DC */ PlayerRecords_Battle<false> battle;
+  /* F4 */
+} __packed__;
+
+struct PlayerRecordsEntry_V3 {
+  /* 0000 */ le_uint32_t client_id;
+  /* 0004 */ PlayerRecordsV3_Challenge<false> challenge;
+  /* 0104 */ PlayerRecords_Battle<false> battle;
+  /* 011C */
+} __packed__;
+
+struct PlayerRecordsEntry_BB {
+  /* 0000 */ le_uint32_t client_id;
+  /* 0004 */ PlayerRecordsBB_Challenge challenge;
+  /* 0144 */ PlayerRecords_Battle<false> battle;
+  /* 015C */
+} __packed__;
+
+struct C_CharacterData_DCv1_61_98 {
+  /* 0000 */ PlayerInventory inventory;
+  /* 034C */ PlayerDispDataDCPCV3 disp;
+  /* 041C */
+} __attribute__((packed));
+
+struct C_CharacterData_DCv2_61_98 {
+  /* 0000 */ PlayerInventory inventory;
+  /* 034C */ PlayerDispDataDCPCV3 disp;
+  /* 041C */ PlayerRecordsEntry_DC records;
+  /* 04D8 */ ChoiceSearchConfig<le_uint16_t> choice_search_config;
+  /* 04F0 */
+} __attribute__((packed));
+
+struct C_CharacterData_PC_61_98 {
+  /* 0000 */ PlayerInventory inventory;
+  /* 034C */ PlayerDispDataDCPCV3 disp;
+  /* 041C */ PlayerRecordsEntry_PC records;
+  /* 0510 */ ChoiceSearchConfig<le_uint16_t> choice_search_config;
+  /* 0528 */ parray<le_uint32_t, 0x1E> blocked_senders;
+  /* 05A0 */ le_uint32_t auto_reply_enabled;
+  // The auto-reply message can be up to 0x200 characters. If it's shorter than
+  // that, the client truncates the command after the first null value (rounded
+  // up to the next 4-byte boundary).
+  /* 05A4 */ char16_t auto_reply[0];
+} __attribute__((packed));
+
+struct C_CharacterData_V3_61_98 {
+  /* 0000 */ PlayerInventory inventory;
+  /* 034C */ PlayerDispDataDCPCV3 disp;
+  /* 041C */ PlayerRecordsEntry_V3 records;
+  /* 0538 */ ChoiceSearchConfig<le_uint16_t> choice_search_config;
+  /* 0550 */ ptext<char, 0xAC> info_board;
+  /* 05FC */ parray<le_uint32_t, 0x1E> blocked_senders;
+  /* 0674 */ le_uint32_t auto_reply_enabled;
+  // The auto-reply message can be up to 0x200 bytes. If it's shorter than that,
+  // the client truncates the command after the first zero byte (rounded up to
+  // the next 4-byte boundary).
+  /* 0678 */ char auto_reply[0];
+} __attribute__((packed));
+
+struct C_CharacterData_GC_Ep3_61_98 {
+  /* 0000 */ PlayerInventory inventory;
+  /* 034C */ PlayerDispDataDCPCV3 disp;
+  /* 041C */ PlayerRecordsEntry_V3 records;
+  /* 0538 */ ChoiceSearchConfig<le_uint16_t> choice_search_config;
+  /* 0550 */ ptext<char, 0xAC> info_board;
+  /* 05FC */ parray<le_uint32_t, 0x1E> blocked_senders;
+  /* 0674 */ le_uint32_t auto_reply_enabled;
+  /* 0678 */ ptext<char, 0xAC> auto_reply;
+  /* 0724 */ Episode3::PlayerConfig ep3_config;
+  /* 2A74 */
+} __attribute__((packed));
+
+struct C_CharacterData_BB_61_98 {
+  /* 0000 */ PlayerInventory inventory;
+  /* 034C */ PlayerDispDataBB disp;
+  /* 04DC */ PlayerRecordsEntry_BB records;
+  /* 0638 */ ChoiceSearchConfig<le_uint16_t> choice_search_config;
+  /* 0650 */ ptext<char16_t, 0xAC> info_board;
+  /* 07A8 */ parray<le_uint32_t, 0x1E> blocked_senders;
+  /* 0820 */ le_uint32_t auto_reply_enabled;
+  // Like on V3, the client truncates the command if the auto reply message is
+  // shorter than 0x200 bytes.
+  /* 082C */ char16_t auto_reply[0];
+} __attribute__((packed));
+
 // 62: Target command
 // Internal name: SndPsoData2
 // When a client sends this command, the server should forward it to the player
@@ -2370,17 +2465,12 @@ struct S_ChoiceSearchResultEntry_V3_C4 {
   parray<uint8_t, 0x58> unused2;
 } __packed__;
 
-// C5 (S->C): Challenge rank update (DCv2 and later versions)
+// C5 (S->C): Player records update (DCv2 and later versions)
 // Internal name: RcvChallengeData
-// header.flag = entry count
+// Command is a list of PlayerRecordsEntry structures; header.flag specifies
+// the entry count.
 // The server sends this command when a player joins a lobby to update the
 // challenge mode records of all the present players.
-// Entry format is PlayerChallengeDataV3 or PlayerChallengeDataBB.
-// newserv currently doesn't send this command at all because the V3 and
-// BB formats aren't fully documented.
-// TODO: Figure out where the text is in those formats, write appropriate
-// conversion functions, and implement the command. Don't forget to overwrite
-// the client_id field in each entry before sending.
 
 // C6 (C->S): Set blocked senders list (V3/BB)
 // The command always contains the same number of entries, even if the entries
@@ -2974,10 +3064,35 @@ struct C_CreateSpectatorTeam_GC_Ep3_E7 {
 // E7 (S->C): Unknown (Episode 3)
 // Same format as E2 command.
 
-// E7: Save or load full player data (BB)
-// See export_bb_player_data() in Player.cc for format.
+// E7: Save or load full character data (BB)
 // TODO: Verify full breakdown from send_E7 in BB disassembly.
 
+struct SC_SyncCharacterSaveFile_BB_00E7 {
+  /* 0000 */ PlayerInventory inventory; // From player data
+  /* 034C */ PlayerDispDataBB disp; // From player data
+  /* 04DC */ le_uint32_t unknown_a1;
+  /* 04E0 */ le_uint32_t creation_timestamp;
+  /* 04E4 */ le_uint32_t signature; // == 0xA205B064 (see SaveFileFormats.hh)
+  /* 04E8 */ le_uint32_t play_time_seconds;
+  /* 04EC */ le_uint32_t option_flags; // account
+  /* 04F0 */ parray<uint8_t, 0x0208> quest_data1; // player
+  /* 06F8 */ PlayerBank bank; // player
+  /* 19C0 */ GuildCardBB guild_card;
+  /* 1AC8 */ le_uint32_t unknown_a3;
+  /* 1ACC */ parray<uint8_t, 0x04E0> symbol_chats; // account
+  /* 1FAC */ parray<uint8_t, 0x0A40> shortcuts; // account
+  /* 29EC */ ptext<char16_t, 0x00AC> auto_reply; // player
+  /* 2B44 */ ptext<char16_t, 0x00AC> info_board; // player
+  /* 2C9C */ PlayerRecords_Battle<false> battle_records;
+  /* 2CB4 */ parray<uint8_t, 4> unknown_a4;
+  /* 2CB8 */ PlayerRecordsBB_Challenge challenge_records;
+  /* 2DF8 */ parray<uint8_t, 0x0028> tech_menu_config; // player
+  /* 2E20 */ parray<uint8_t, 0x002C> unknown_a6;
+  /* 2E4C */ parray<uint8_t, 0x0058> quest_data2; // player
+  /* 2EA4 */ KeyAndTeamConfigBB key_config; // account
+  /* 3994 */
+} __attribute__((packed));
+
 // E8 (S->C): Join spectator team (Episode 3)
 // header.flag = player count (including spectators)
 

src/Main.cc

@@ -144,6 +144,9 @@ The actions are:\n\
     same way, but if it is not given, newserv will try all possible basis\n\
     values and return the one that results in the greatest number of zero bytes\n\
     in the output.\n\
+  encrypt-challenge-data [INPUT-FILENAME [OUTPUT-FILENAME]]\n\
+  decrypt-challenge-data [INPUT-FILENAME [OUTPUT-FILENAME]]\n\
+    Encrypt or decrypt data using the challenge mode trivial algorithm.\n\
   encrypt-gci-save CRYPT-OPTION INPUT-FILENAME [OUTPUT-FILENAME]\n\
   decrypt-gci-save CRYPT-OPTION INPUT-FILENAME [OUTPUT-FILENAME]\n\
     Encrypt or decrypt a character or Guild Card file in GCI format. If\n\
@@ -252,6 +255,8 @@ enum class Behavior {
   DECRYPT_DATA,
   ENCRYPT_TRIVIAL_DATA,
   DECRYPT_TRIVIAL_DATA,
+  ENCRYPT_CHALLENGE_DATA,
+  DECRYPT_CHALLENGE_DATA,
   ENCRYPT_GCI_SAVE,
   DECRYPT_GCI_SAVE,
   FIND_DECRYPTION_SEED,
@@ -289,6 +294,8 @@ static bool behavior_takes_input_filename(Behavior b) {
       (b == Behavior::DECRYPT_DATA) ||
       (b == Behavior::ENCRYPT_TRIVIAL_DATA) ||
       (b == Behavior::DECRYPT_TRIVIAL_DATA) ||
+      (b == Behavior::ENCRYPT_CHALLENGE_DATA) ||
+      (b == Behavior::DECRYPT_CHALLENGE_DATA) ||
       (b == Behavior::DECRYPT_GCI_SAVE) ||
       (b == Behavior::SALVAGE_GCI) ||
       (b == Behavior::ENCRYPT_GCI_SAVE) ||
@@ -318,6 +325,8 @@ static bool behavior_takes_output_filename(Behavior b) {
       (b == Behavior::DECRYPT_DATA) ||
       (b == Behavior::ENCRYPT_TRIVIAL_DATA) ||
       (b == Behavior::DECRYPT_TRIVIAL_DATA) ||
+      (b == Behavior::ENCRYPT_CHALLENGE_DATA) ||
+      (b == Behavior::DECRYPT_CHALLENGE_DATA) ||
       (b == Behavior::DECRYPT_GCI_SAVE) ||
       (b == Behavior::ENCRYPT_GCI_SAVE) ||
       (b == Behavior::DISASSEMBLE_QUEST_SCRIPT) ||
@@ -471,6 +480,10 @@ int main(int argc, char** argv) {
           behavior = Behavior::ENCRYPT_TRIVIAL_DATA;
         } else if (!strcmp(argv[x], "decrypt-trivial-data")) {
           behavior = Behavior::DECRYPT_TRIVIAL_DATA;
+        } else if (!strcmp(argv[x], "encrypt-challenge-data")) {
+          behavior = Behavior::ENCRYPT_CHALLENGE_DATA;
+        } else if (!strcmp(argv[x], "decrypt-challenge-data")) {
+          behavior = Behavior::DECRYPT_CHALLENGE_DATA;
         } else if (!strcmp(argv[x], "decrypt-gci-save")) {
           behavior = Behavior::DECRYPT_GCI_SAVE;
         } else if (!strcmp(argv[x], "encrypt-gci-save")) {
@@ -813,6 +826,17 @@ int main(int argc, char** argv) {
       break;
     }
 
+    case Behavior::ENCRYPT_CHALLENGE_DATA:
+    case Behavior::DECRYPT_CHALLENGE_DATA: {
+      string data = read_input_data();
+      const uint8_t* u8data = reinterpret_cast<const uint8_t*>(data.data());
+      string result = (behavior == Behavior::DECRYPT_CHALLENGE_DATA)
+          ? decrypt_challenge_rank_text(u8data, data.size())
+          : encrypt_challenge_rank_text(u8data, data.size());
+      write_output_data(result.data(), result.size());
+      break;
+    }
+
     case Behavior::ENCRYPT_GCI_SAVE:
     case Behavior::DECRYPT_GCI_SAVE: {
       uint32_t round1_seed;

src/PSOEncryption.cc

@@ -885,3 +885,44 @@ void decrypt_trivial_gci_data(void* data, size_t size, uint8_t basis) {
     bytes[z] ^= key;
   }
 }
+
+template <typename StringT, bool IsEncrypt>
+StringT crypt_challenge_rank_text(const void* src, size_t size) {
+  if (size == 0) {
+    return StringT();
+  }
+
+  StringT ret;
+  StringReader r(src, size);
+  typename StringT::value_type prev = 0;
+  while (!r.eof()) {
+    typename StringT::value_type ch = r.get<typename StringT::value_type>();
+    if (ch == 0) {
+      break;
+    }
+    if (ret.empty()) {
+      ret.push_back(ch ^ 0x7F);
+    } else {
+      ret.push_back(IsEncrypt ? ((ch - prev) ^ 0x7F) : ((ch ^ 0x7F) + ret.back()));
+    }
+    prev = ch;
+  }
+
+  return ret;
+}
+
+string encrypt_challenge_rank_text(const uint8_t* src, size_t size) {
+  return crypt_challenge_rank_text<string, true>(src, size);
+}
+
+string decrypt_challenge_rank_text(const uint8_t* src, size_t size) {
+  return crypt_challenge_rank_text<string, false>(src, size);
+}
+
+u16string encrypt_challenge_rank_text(const char16_t* src, size_t size) {
+  return crypt_challenge_rank_text<u16string, true>(src, size);
+}
+
+u16string decrypt_challenge_rank_text(const char16_t* src, size_t size) {
+  return crypt_challenge_rank_text<u16string, false>(src, size);
+}

src/PSOEncryption.hh

@@ -240,3 +240,8 @@ private:
 };
 
 void decrypt_trivial_gci_data(void* data, size_t size, uint8_t basis);
+
+std::string decrypt_challenge_rank_text(const uint8_t* data, size_t size);
+std::string encrypt_challenge_rank_text(const uint8_t* data, size_t size);
+std::u16string decrypt_challenge_rank_text(const le_uint16_t* data, size_t size);
+std::u16string encrypt_challenge_rank_text(const le_uint16_t* data, size_t size);

src/Player.cc

@@ -368,82 +368,6 @@ void ClientGameData::save_player_data() {
   }
 }
 
-void ClientGameData::import_player(const PSOPlayerDataDCPC& pd) {
-  auto player = this->player();
-  player->inventory = pd.inventory;
-  player->disp = pd.disp.to_bb();
-  // TODO: Add these fields to the command structure so we can parse them
-  // info_board = pd.info_board;
-  // blocked_senders = pd.blocked_senders;
-  // auto_reply = pd.auto_reply;
-}
-
-void ClientGameData::import_player(const PSOPlayerDataV3& gc, bool is_gc) {
-  auto account = this->account();
-  auto player = this->player();
-  player->inventory = gc.inventory;
-  if (is_gc) {
-    for (size_t z = 0; z < 30; z++) {
-      player->inventory.items[z].data.bswap_data2_if_mag();
-    }
-  }
-  player->disp = gc.disp.to_bb();
-  player->info_board = gc.info_board;
-  account->blocked_senders = gc.blocked_senders;
-  if (gc.auto_reply_enabled) {
-    player->auto_reply = gc.auto_reply;
-  } else {
-    player->auto_reply.clear(0);
-  }
-}
-
-void ClientGameData::import_player(const PSOPlayerDataBB& bb) {
-  auto account = this->account();
-  auto player = this->player();
-  // Note: we don't copy the inventory and disp here because we already have
-  // them (we sent the player data to the client in the first place)
-  player->info_board = bb.info_board;
-  account->blocked_senders = bb.blocked_senders;
-  if (bb.auto_reply_enabled) {
-    player->auto_reply = bb.auto_reply;
-  } else {
-    player->auto_reply.clear(0);
-  }
-}
-
-PlayerBB ClientGameData::export_player_bb() {
-  auto account = this->account();
-  auto player = this->player();
-
-  PlayerBB ret;
-  ret.inventory = player->inventory;
-  ret.disp = player->disp;
-  ret.unknown.clear(0);
-  ret.option_flags = account->option_flags;
-  ret.quest_data1 = player->quest_data1;
-  ret.bank = player->bank;
-  ret.guild_card_number = this->guild_card_number;
-  ret.name = player->disp.name;
-  ret.team_name = account->team_name;
-  ret.guild_card_description = player->guild_card_description;
-  ret.reserved1 = 0;
-  ret.reserved2 = 0;
-  ret.section_id = player->disp.visual.section_id;
-  ret.char_class = player->disp.visual.char_class;
-  ret.unknown3 = 0;
-  ret.symbol_chats = account->symbol_chats;
-  ret.shortcuts = account->shortcuts;
-  ret.auto_reply = player->auto_reply;
-  ret.info_board = player->info_board;
-  ret.unknown5.clear(0);
-  ret.challenge_data = player->challenge_data;
-  ret.tech_menu_config = player->tech_menu_config;
-  ret.unknown6.clear(0);
-  ret.quest_data2 = player->quest_data2;
-  ret.key_config = account->key_config;
-  return ret;
-}
-
 void PlayerLobbyDataPC::clear() {
   this->player_tag = 0;
   this->guild_card = 0;

src/Player.hh

@@ -182,13 +182,14 @@ struct PlayerDispDataBBPreview {
 
 // BB player appearance and stats data
 struct PlayerDispDataBB {
-  PlayerStats stats;
-  PlayerVisualConfig visual;
-  ptext<char16_t, 0x0C> name;
-  le_uint32_t play_time;
-  uint32_t unknown_a3;
-  parray<uint8_t, 0xE8> config;
-  parray<uint8_t, 0x14> technique_levels;
+  /* 0000 */ PlayerStats stats;
+  /* 0024 */ PlayerVisualConfig visual;
+  /* 0074 */ ptext<char16_t, 0x0C> name;
+  /* 008C */ le_uint32_t play_time;
+  /* 0090 */ uint32_t unknown_a3;
+  /* 0094 */ parray<uint8_t, 0xE8> config;
+  /* 017C */ parray<uint8_t, 0x14> technique_levels;
+  /* 0190 */
 
   PlayerDispDataBB() noexcept;
 
@@ -217,14 +218,15 @@ struct GuildCardV3 {
 
 // BB guild card format
 struct GuildCardBB {
-  le_uint32_t guild_card_number;
-  ptext<char16_t, 0x18> name;
-  ptext<char16_t, 0x10> team_name;
-  ptext<char16_t, 0x58> description;
-  uint8_t present; // should be 1 if guild card entry exists
-  uint8_t language;
-  uint8_t section_id;
-  uint8_t char_class;
+  /* 0000 */ le_uint32_t guild_card_number;
+  /* 0004 */ ptext<char16_t, 0x18> name;
+  /* 0034 */ ptext<char16_t, 0x10> team_name;
+  /* 0054 */ ptext<char16_t, 0x58> description;
+  /* 0104 */ uint8_t present; // should be 1 if guild card entry exists
+  /* 0105 */ uint8_t language;
+  /* 0106 */ uint8_t section_id;
+  /* 0107 */ uint8_t char_class;
+  /* 0108 */
 
   GuildCardBB() noexcept;
   void clear();
@@ -324,39 +326,84 @@ struct PlayerLobbyDataBB {
   void clear();
 } __attribute__((packed));
 
-struct PlayerChallengeDataV3 {
-  le_uint32_t client_id;
-  struct {
-    le_uint16_t unknown_a1;
-    parray<uint8_t, 2> unknown_a2; // Possibly unused
-    parray<le_uint32_t, 0x17> unknown_a3;
-    struct {
-      parray<uint8_t, 4> unknown_a1;
-      le_uint16_t unknown_a2;
-      parray<uint8_t, 2> unknown_a3;
-      parray<le_uint32_t, 5> unknown_a4;
-      parray<uint8_t, 0x34> unknown_a5;
-    } __attribute__((packed)) unknown_a4; // 0x50 bytes
-    struct {
-      parray<uint8_t, 4> unknown_a1;
-      parray<le_uint32_t, 3> unknown_a2;
-    } __attribute__((packed)) unknown_a5; // 0x10 bytes
-    struct UnknownPair {
-      le_uint32_t unknown_a1;
-      le_uint32_t unknown_a2;
-    } __attribute__((packed));
-    parray<UnknownPair, 3> unknown_a6; // 0x18 bytes
-    parray<uint8_t, 0x28> unknown_a7;
-  } __attribute__((packed)) unknown_a1; // 0x100 bytes
-  // On Episode 3, unknown_a2[0] is win count, [1] is loss count, and [4] is
-  // disconnect count
-  parray<le_uint16_t, 8> unknown_a2;
-  parray<le_uint32_t, 2> unknown_a3;
-} __attribute__((packed)); // 0x11C bytes
+template <bool IsWideChar>
+struct PlayerRecordsDCPC_Challenge {
+  using CharT = typename std::conditional<IsWideChar, char16_t, char>::type;
+  using CharBinT = typename std::conditional<IsWideChar, le_uint16_t, uint8_t>::type;
 
-struct PlayerChallengeDataBB {
-  le_uint32_t client_id;
-  parray<uint8_t, 0x158> unknown_a1;
+  /* 00 */ le_uint16_t title_color;
+  /* 02 */ parray<uint8_t, 2> unknown_a0;
+  /* 04 */ parray<CharBinT, 0x0C> rank_title; // Encrypted; see decrypt_challenge_rank_text
+  /* 10 */ parray<uint8_t, 0x24> unknown_a1; // TODO: This might be online or offline times
+  /* 34 */ le_uint16_t unknown_a2;
+  /* 36 */ le_uint16_t grave_deaths;
+  /* 38 */ parray<le_uint32_t, 5> grave_coords_time;
+  /* 4C */ ptext<CharT, 0x14> grave_team;
+  /* 60 */ ptext<CharT, 0x18> grave_message;
+  /* 78 */ parray<le_uint32_t, 9> times_ep1; // TODO: This might be offline times
+  /* 9C */ parray<uint8_t, 4> unknown_a3;
+  /* A0 */
+} __attribute__((packed));
+
+struct PlayerRecordsDC_Challenge : PlayerRecordsDCPC_Challenge<false> {
+} __attribute__((packed));
+struct PlayerRecordsPC_Challenge : PlayerRecordsDCPC_Challenge<true> {
+} __attribute__((packed));
+
+template <bool IsBigEndian>
+struct PlayerRecordsV3_Challenge {
+  using U16T = typename std::conditional<IsBigEndian, be_uint16_t, le_uint16_t>::type;
+  using U32T = typename std::conditional<IsBigEndian, be_uint32_t, le_uint32_t>::type;
+
+  // Offsets are (1) relative to start of C5 entry, and (2) relative to start
+  // of save file structure
+  /* 0000:001C */ U16T title_color; // XRGB1555
+  /* 0002:001E */ parray<uint8_t, 2> unknown_a2; // Probably actually unused
+  /* 0004:0020 */ parray<U32T, 9> times_ep1_online;
+  /* 0028:0044 */ parray<U32T, 5> times_ep2_online;
+  /* 003C:0058 */ parray<U32T, 9> times_ep1_offline;
+  /* 0060:007C */ parray<uint8_t, 4> unknown_a3;
+  /* 0064:0080 */ U16T grave_deaths;
+  /* 0066:0082 */ parray<uint8_t, 2> unknown_a4; // Probably actually unused
+  /* 0068:0084 */ parray<U32T, 5> grave_coords_time;
+  /* 007C:0098 */ ptext<char, 0x14> grave_team;
+  /* 0090:00AC */ ptext<char, 0x20> grave_message;
+  /* 00B0:00CC */ parray<uint8_t, 4> unknown_a5;
+  /* 00B4:00D0 */ parray<U32T, 9> unknown_a6;
+  /* 00D8:00F4 */ parray<uint8_t, 0x0C> rank_title; // Encrypted; see decrypt_challenge_rank_text
+  /* 00E4:0100 */ parray<uint8_t, 0x1C> unknown_a7;
+  /* 0100:011C */
+} __attribute__((packed));
+
+struct PlayerRecordsBB_Challenge {
+  // TODO: Figure out the rest of this structure. Probably it's very similar to
+  // the V3 structure, but it's a bit larger due to various text fields.
+  /* 0000 */ le_uint16_t title_color; // XRGB1555
+  /* 0002 */ parray<uint8_t, 2> unknown_a2; // Probably actually unused
+  /* 0004 */ parray<le_uint32_t, 9> times_ep1;
+  /* 0028 */ parray<le_uint32_t, 5> times_ep2;
+  /* 003C */ parray<le_uint32_t, 9> times_ep1_offline;
+  /* 0060 */ parray<uint8_t, 4> unknown_a3;
+  /* 0064 */ le_uint16_t grave_deaths;
+  /* 0066 */ parray<uint8_t, 2> unknown_a4; // Probably actually unused
+  /* 0068 */ parray<le_uint32_t, 5> grave_coords_time;
+  /* 007C */ ptext<char16_t, 0x14> grave_team;
+  /* 00A4 */ ptext<char16_t, 0x20> grave_message;
+  /* 00E4 */ parray<uint8_t, 4> unknown_a5;
+  /* 00E8 */ parray<le_uint32_t, 9> unknown_a6;
+  /* 010C */ parray<le_uint16_t, 0x0C> rank_title; // Encrypted; see decrypt_challenge_rank_text
+  /* 0124 */ parray<uint8_t, 0x1C> unknown_a7;
+  /* 0140 */
+} __attribute__((packed));
+
+template <bool IsBigEndian>
+struct PlayerRecords_Battle {
+  using U16T = typename std::conditional<IsBigEndian, be_uint16_t, le_uint16_t>::type;
+  // On Episode 3, battle_place_counts[0] is win count and [1] is loss count
+  /* 00 */ parray<U16T, 4> place_counts;
+  /* 08 */ U16T disconnect_count;
+  /* 0A */ parray<uint8_t, 0x0E> unknown_a1;
+  /* 18 */
 } __attribute__((packed));
 
 template <typename ItemIDT>
@@ -370,82 +417,12 @@ struct ChoiceSearchConfig {
   parray<Entry, 5> entries;
 } __attribute__((packed));
 
-struct PSOPlayerDataDCPC { // For command 61
-  PlayerInventory inventory;
-  PlayerDispDataDCPCV3 disp;
-} __attribute__((packed));
-
-struct PSOPlayerDataV3 { // For command 61
-  PlayerInventory inventory;
-  PlayerDispDataDCPCV3 disp;
-  PlayerChallengeDataV3 challenge_data;
-  ChoiceSearchConfig<le_uint16_t> choice_search_config;
-  ptext<char, 0xAC> info_board;
-  parray<le_uint32_t, 0x1E> blocked_senders;
-  le_uint32_t auto_reply_enabled;
-  // The auto-reply message can be up to 0x200 bytes. If it's shorter than that,
-  // the client truncates the command after the first zero byte (rounded up to
-  // the next 4-byte boundary).
-  char auto_reply[0];
-} __attribute__((packed));
-
-struct PSOPlayerDataGCEp3 { // For command 61
-  PlayerInventory inventory;
-  PlayerDispDataDCPCV3 disp;
-  PlayerChallengeDataV3 challenge_data;
-  ChoiceSearchConfig<le_uint16_t> choice_search_config;
-  ptext<char, 0xAC> info_board;
-  parray<le_uint32_t, 0x1E> blocked_senders;
-  le_uint32_t auto_reply_enabled;
-  ptext<char, 0xAC> auto_reply;
-  Episode3::PlayerConfig ep3_config;
-} __attribute__((packed));
-
-struct PSOPlayerDataBB { // For command 61
-  PlayerInventory inventory;
-  PlayerDispDataBB disp;
-  PlayerChallengeDataBB challenge_data;
-  ChoiceSearchConfig<le_uint16_t> choice_search_config;
-  ptext<char16_t, 0xAC> info_board;
-  parray<le_uint32_t, 0x1E> blocked_senders;
-  le_uint32_t auto_reply_enabled;
-  char16_t auto_reply[0];
-} __attribute__((packed));
-
-struct PlayerBB { // Used in 00E7 command
-  PlayerInventory inventory; // 0000-034C; player
-  PlayerDispDataBB disp; // 034C-04DC; player
-  parray<uint8_t, 0x0010> unknown; // 04DC-04EC; not saved
-  le_uint32_t option_flags; // 04EC-04F0; account
-  parray<uint8_t, 0x0208> quest_data1; // 04F0-06F8; player
-  PlayerBank bank; // 06F8-19C0; player
-  le_uint32_t guild_card_number; // 19C0-19C4; player
-  ptext<char16_t, 0x18> name; // 19C4-19F4; player
-  ptext<char16_t, 0x10> team_name; // 19F4-1A14; player
-  ptext<char16_t, 0x58> guild_card_description; // 1A14-1AC4; player
-  uint8_t reserved1; // 1AC4-1AC5; player
-  uint8_t reserved2; // 1AC5-1AC6; player
-  uint8_t section_id; // 1AC6-1AC7; player
-  uint8_t char_class; // 1AC7-1AC8; player
-  le_uint32_t unknown3; // 1AC8-1ACC; not saved
-  parray<uint8_t, 0x04E0> symbol_chats; // 1ACC-1FAC; account
-  parray<uint8_t, 0x0A40> shortcuts; // 1FAC-29EC; account
-  ptext<char16_t, 0x00AC> auto_reply; // 29EC-2B44; player
-  ptext<char16_t, 0x00AC> info_board; // 2B44-2C9C; player
-  parray<uint8_t, 0x001C> unknown5; // 2C9C-2CB8; not saved
-  parray<uint8_t, 0x0140> challenge_data; // 2CB8-2DF8; player
-  parray<uint8_t, 0x0028> tech_menu_config; // 2DF8-2E20; player
-  parray<uint8_t, 0x002C> unknown6; // 2E20-2E4C; not saved
-  parray<uint8_t, 0x0058> quest_data2; // 2E4C-2EA4; player
-  KeyAndTeamConfigBB key_config; // 2EA4-3994; account
-} __attribute__((packed));
-
 struct SavedPlayerDataBB { // .nsc file format
   ptext<char, 0x40> signature;
   PlayerDispDataBBPreview preview;
   ptext<char16_t, 0x00AC> auto_reply;
   PlayerBank bank;
-  parray<uint8_t, 0x0140> challenge_data;
+  PlayerRecordsBB_Challenge challenge_records;
   PlayerDispDataBB disp;
   ptext<char16_t, 0x0058> guild_card_description;
   ptext<char16_t, 0x00AC> info_board;
@@ -453,6 +430,9 @@ struct SavedPlayerDataBB { // .nsc file format
   parray<uint8_t, 0x0208> quest_data1;
   parray<uint8_t, 0x0058> quest_data2;
   parray<uint8_t, 0x0028> tech_menu_config;
+  // TODO: We don't save battle records in this structure, which is wrong.
+  // Make a new save file format that doesn't have the super-long signature,
+  // and also can save battle records.
 
   void add_item(const PlayerInventoryItem& item);
   PlayerInventoryItem remove_item(
@@ -524,13 +504,6 @@ public:
   void load_player_data();
   // Note: This function is not const because it updates the player's play time.
   void save_player_data();
-
-  void import_player(const PSOPlayerDataDCPC& pd);
-  void import_player(const PSOPlayerDataV3& pd, bool is_gc);
-  void import_player(const PSOPlayerDataBB& pd);
-  // Note: this function is not const because it can cause player and account
-  // data to be loaded
-  PlayerBB export_player_bb();
 };
 
 uint32_t compute_guild_card_checksum(const void* data, size_t size);

src/ProxyCommands.cc

@@ -1017,7 +1017,7 @@ static HandlerResult C_GXB_61(shared_ptr<ServerState>,
   // return MODIFIED if so.
 
   if (session.version == GameVersion::BB) {
-    auto& pd = check_size_t<PSOPlayerDataBB>(data, 0xFFFF);
+    auto& pd = check_size_t<C_CharacterData_BB_61_98>(data, 0xFFFF);
     if (session.options.enable_chat_filter) {
       add_color_inplace(pd.info_board.data(), pd.info_board.size());
     }
@@ -1034,9 +1034,9 @@ static HandlerResult C_GXB_61(shared_ptr<ServerState>,
     }
 
   } else {
-    PSOPlayerDataV3* pd;
+    C_CharacterData_V3_61_98* pd;
     if (flag == 4) { // Episode 3
-      auto& ep3_pd = check_size_t<PSOPlayerDataGCEp3>(data);
+      auto& ep3_pd = check_size_t<C_CharacterData_GC_Ep3_61_98>(data);
       if (ep3_pd.ep3_config.is_encrypted) {
         decrypt_trivial_gci_data(
             &ep3_pd.ep3_config.card_counts,
@@ -1046,9 +1046,9 @@ static HandlerResult C_GXB_61(shared_ptr<ServerState>,
         ep3_pd.ep3_config.basis = 0;
         modified = true;
       }
-      pd = reinterpret_cast<PSOPlayerDataV3*>(&ep3_pd);
+      pd = reinterpret_cast<C_CharacterData_V3_61_98*>(&ep3_pd);
     } else {
-      pd = &check_size_t<PSOPlayerDataV3>(data, 0xFFFF);
+      pd = &check_size_t<C_CharacterData_V3_61_98>(data, 0xFFFF);
     }
     if (session.options.enable_chat_filter) {
       add_color_inplace(pd->info_board.data(), pd->info_board.size());

src/QuestScript.cc

@@ -644,8 +644,8 @@ static const QuestScriptOpcodeDefinition opcode_defs[] = {
     {0xF872, "ba_set_time_limit", {INT32}, {}, V2, V2},
     {0xF872, "ba_set_time_limit", {}, {INT32}, V3, V4},
     {0xF873, "dark_falz_is_dead", {REG}, {}, V2, V4},
-    {0xF874, nullptr, {INT32, CSTRING}, {}, V2, V2}, // TODO (DX) - Similar to A0, but does something with the two strings in non-4P challenge mode
-    {0xF874, nullptr, {}, {INT32, CSTRING}, V3, V4}, // TODO (DX)
+    {0xF874, "set_cmode_rank_override", {INT32, CSTRING}, {}, V2, V2}, // argA is an XRGB8888 color, argB is two strings separated by \t or \n: the rank text to check for, and the rank text that should replace it if found
+    {0xF874, "set_cmode_rank_override", {}, {INT32, CSTRING}, V3, V4},
     {0xF875, "enable_stealth_suit_effect", {REG}, {}, V2, V4},
     {0xF876, "disable_stealth_suit_effect", {REG}, {}, V2, V4},
     {0xF877, "enable_techs", {REG}, {}, V2, V4},

src/ReceiveCommands.cc

@@ -2435,33 +2435,88 @@ static void on_61_98(shared_ptr<ServerState> s, shared_ptr<Client> c,
     uint16_t command, uint32_t flag, const string& data) {
 
   switch (c->version()) {
-    case GameVersion::DC:
+    case GameVersion::DC: {
+      if (c->flags & Client::Flag::IS_DC_V1) {
+        const auto& pd = check_size_t<C_CharacterData_DCv1_61_98>(data);
+        auto player = c->game_data.player();
+        player->inventory = pd.inventory;
+        player->disp = pd.disp.to_bb();
+      } else {
+        const auto& pd = check_size_t<C_CharacterData_DCv2_61_98>(data, 0xFFFF);
+        auto player = c->game_data.player();
+        player->inventory = pd.inventory;
+        player->disp = pd.disp.to_bb();
+        // TODO: Parse pd.records and send C5 at an appropriate time
+        // TODO: Parse choice search config
+      }
+      break;
+    }
     case GameVersion::PC: {
-      const auto& pd = check_size_t<PSOPlayerDataDCPC>(data, 0xFFFF);
-      c->game_data.import_player(pd);
+      const auto& pd = check_size_t<C_CharacterData_PC_61_98>(data, 0xFFFF);
+      auto player = c->game_data.player();
+      auto account = c->game_data.account();
+      player->inventory = pd.inventory;
+      player->disp = pd.disp.to_bb();
+      // TODO: Parse pd.records and send C5 at an appropriate time
+      // TODO: Parse choice search config
+      account->blocked_senders = pd.blocked_senders;
+      if (pd.auto_reply_enabled) {
+        player->auto_reply = pd.auto_reply;
+      } else {
+        player->auto_reply.clear(0);
+      }
       break;
     }
     case GameVersion::GC:
     case GameVersion::XB: {
-      const PSOPlayerDataV3* pd;
+      const C_CharacterData_V3_61_98* cmd;
       if (flag == 4) { // Episode 3
         if (!(c->flags & Client::Flag::IS_EPISODE_3)) {
           throw runtime_error("non-Episode 3 client sent Episode 3 player data");
         }
-        const auto* pd3 = &check_size_t<PSOPlayerDataGCEp3>(data);
+        const auto* pd3 = &check_size_t<C_CharacterData_GC_Ep3_61_98>(data);
         c->game_data.ep3_config.reset(new Episode3::PlayerConfig(pd3->ep3_config));
-        pd = reinterpret_cast<const PSOPlayerDataV3*>(pd3);
+        cmd = reinterpret_cast<const C_CharacterData_V3_61_98*>(pd3);
       } else {
-        pd = &check_size_t<PSOPlayerDataV3>(data,
-            sizeof(PSOPlayerDataV3) + c->game_data.player()->auto_reply.bytes());
+        cmd = &check_size_t<C_CharacterData_V3_61_98>(data, 0xFFFF);
       }
-      c->game_data.import_player(*pd, c->version() == GameVersion::GC);
+
+      auto account = c->game_data.account();
+      auto player = c->game_data.player();
+      player->inventory = cmd->inventory;
+      if (c->version() == GameVersion::GC) {
+        for (size_t z = 0; z < 30; z++) {
+          player->inventory.items[z].data.bswap_data2_if_mag();
+        }
+      }
+      player->disp = cmd->disp.to_bb();
+      // TODO: Parse cmd->records and send C5 at an appropriate time
+      // TODO: Parse choice search config
+      player->info_board = cmd->info_board;
+      account->blocked_senders = cmd->blocked_senders;
+      if (cmd->auto_reply_enabled) {
+        player->auto_reply = cmd->auto_reply;
+      } else {
+        player->auto_reply.clear(0);
+      }
+
       break;
     }
     case GameVersion::BB: {
-      const auto& pd = check_size_t<PSOPlayerDataBB>(data,
-          sizeof(PSOPlayerDataBB) + c->game_data.player()->auto_reply.bytes());
-      c->game_data.import_player(pd);
+      const auto& cmd = check_size_t<C_CharacterData_BB_61_98>(data, 0xFFFF);
+      auto account = c->game_data.account();
+      auto player = c->game_data.player();
+      // Note: we don't copy the inventory and disp here because we already have
+      // them (we sent the player data to the client in the first place)
+      // TODO: Parse pd.records and send C5 at an appropriate time
+      // TODO: Parse choice search config
+      player->info_board = cmd.info_board;
+      account->blocked_senders = cmd.blocked_senders;
+      if (cmd.auto_reply_enabled) {
+        player->auto_reply = cmd.auto_reply;
+      } else {
+        player->auto_reply.clear(0);
+      }
       break;
     }
     default:
@@ -2901,10 +2956,14 @@ static void on_xxED_BB(shared_ptr<ServerState>, shared_ptr<Client> c,
 
 static void on_00E7_BB(shared_ptr<ServerState>, shared_ptr<Client> c,
     uint16_t, uint32_t, const string& data) {
-  const auto& cmd = check_size_t<PlayerBB>(data);
+  const auto& cmd = check_size_t<SC_SyncCharacterSaveFile_BB_00E7>(data);
 
   // We only trust the player's quest data and challenge data.
-  c->game_data.player()->challenge_data = cmd.challenge_data;
+  // TODO: In the future, we shouldn't even need to trust these fields. We
+  // should instead verify our copy of the player against what the client sent,
+  // and alert on anything that's out of sync.
+  // TODO: In the future, we should save battle records here too.
+  c->game_data.player()->challenge_records = cmd.challenge_records;
   c->game_data.player()->quest_data1 = cmd.quest_data1;
   c->game_data.player()->quest_data2 = cmd.quest_data2;
 }

src/SaveFileFormats.hh

@@ -125,14 +125,17 @@ struct PSOGCSaveFileChatShortcutEntry {
 struct PSOGCCharacterFile {
   /* 00000 */ be_uint32_t checksum;
   struct Character {
-    /* 0000 */ PlayerInventory inventory;
-    /* 034C */ PlayerDispDataDCPCV3 disp;
-    /* 041C */ be_uint32_t unknown_a1;
-    /* 0420 */ be_uint32_t creation_timestamp;
+    // This structure is internally split into two by the game. The offsets here
+    // are relative to the start of this structure (first column), and relative
+    // to the start of the second internal structure (second column).
+    /* 0000:---- */ PlayerInventory inventory;
+    /* 034C:---- */ PlayerDispDataDCPCV3 disp;
+    /* 041C:0000 */ be_uint32_t unknown_a1;
+    /* 0420:0004 */ be_uint32_t creation_timestamp;
     // The signature field holds the value 0xA205B064, which is 2718281828 in
     // decimal - approximately e * 10^9. It's unknown why Sega chose this value.
-    /* 0424 */ be_uint32_t signature;
-    /* 0428 */ be_uint32_t play_time_seconds;
+    /* 0424:0008 */ be_uint32_t signature;
+    /* 0428:000C */ be_uint32_t play_time_seconds;
     // This field is a collection of several flags and small values. The known
     // fields are:
     //   ------zA BCDEFG-- HHHIIIJJ KLMNOPQR
@@ -151,19 +154,21 @@ struct PSOGCCharacterFile {
     //   P = Cursor position (0 = saved; 1 = non-saved)
     //   Q = Button config (0 = normal; 1 = L/R reversed)
     //   R = Map direction (0 = non-fixed; 1 = fixed)
-    /* 042C */ be_uint32_t option_flags;
-    /* 0430 */ be_uint32_t save_count;
-    /* 0434 */ parray<uint8_t, 0x230> unknown_a4;
-    /* 0664 */ PlayerBank bank;
-    /* 192C */ GuildCardV3 guild_card;
-    /* 19BC */ parray<PSOGCSaveFileSymbolChatEntry, 12> symbol_chats;
-    /* 1DDC */ parray<PSOGCSaveFileChatShortcutEntry, 20> chat_shortcuts;
-    /* 246C */ ptext<char, 0xAC> auto_reply;
-    /* 2518 */ ptext<char, 0xAC> info_board;
-    /* 25C4 */ parray<uint8_t, 0x11C> unknown_a5;
-    /* 26E0 */ parray<be_uint16_t, 20> tech_menu_shortcut_entries;
-    /* 2708 */ parray<uint8_t, 0x90> unknown_a6;
-    /* 2798 */
+    /* 042C:0010 */ be_uint32_t option_flags;
+    /* 0430:0014 */ be_uint32_t save_count;
+    /* 0434:0018 */ parray<uint8_t, 0x230> unknown_a4;
+    /* 0664:0248 */ PlayerBank bank;
+    /* 192C:1510 */ GuildCardV3 guild_card;
+    /* 19BC:15A0 */ parray<PSOGCSaveFileSymbolChatEntry, 12> symbol_chats;
+    /* 1DDC:19C0 */ parray<PSOGCSaveFileChatShortcutEntry, 20> chat_shortcuts;
+    /* 246C:2050 */ ptext<char, 0xAC> auto_reply;
+    /* 2518:20FC */ ptext<char, 0xAC> info_board;
+    /* 25C4:21A8 */ PlayerRecords_Battle<true> battle_records;
+    /* 25DC:21C0 */ parray<uint8_t, 4> unknown_a2;
+    /* 25E0:21C4 */ PlayerRecordsV3_Challenge<true> challenge_records;
+    /* 26E0:22C4 */ parray<be_uint16_t, 20> tech_menu_shortcut_entries;
+    /* 2708:22EC */ parray<uint8_t, 0x90> unknown_a6;
+    /* 2798:237C */
   } __attribute__((packed));
   /* 00004 */ parray<Character, 7> characters;
   /* 1152C */ ptext<char, 0x10> serial_number; // As %08X (not decimal)

src/SendCommands.cc

@@ -606,7 +606,43 @@ void send_approve_player_choice_bb(shared_ptr<Client> c) {
 }
 
 void send_complete_player_bb(shared_ptr<Client> c) {
-  send_command_t(c, 0x00E7, 0x00000000, c->game_data.export_player_bb());
+  auto account = c->game_data.account();
+  auto player = c->game_data.player();
+
+  SC_SyncCharacterSaveFile_BB_00E7 cmd;
+  cmd.inventory = player->inventory;
+  cmd.disp = player->disp;
+  cmd.unknown_a1 = 0;
+  cmd.creation_timestamp = 0;
+  cmd.signature = 0xA205B064;
+  cmd.play_time_seconds = 0; // TODO: Can we just use the same value as in disp?
+  cmd.option_flags = account->option_flags;
+  cmd.quest_data1 = player->quest_data1;
+  cmd.bank = player->bank;
+  cmd.guild_card.guild_card_number = c->game_data.guild_card_number;
+  cmd.guild_card.name = player->disp.name;
+  cmd.guild_card.team_name = account->team_name;
+  cmd.guild_card.description = player->guild_card_description;
+  cmd.guild_card.present = 1;
+  cmd.guild_card.language = cmd.inventory.language;
+  cmd.guild_card.section_id = player->disp.visual.section_id;
+  cmd.guild_card.char_class = player->disp.visual.char_class;
+  cmd.unknown_a3 = 0;
+  cmd.symbol_chats = account->symbol_chats;
+  cmd.shortcuts = account->shortcuts;
+  cmd.auto_reply = player->auto_reply;
+  cmd.info_board = player->info_board;
+  cmd.battle_records.place_counts.clear(0);
+  cmd.battle_records.disconnect_count = 0;
+  cmd.battle_records.unknown_a1.clear(0);
+  cmd.unknown_a4 = 0;
+  cmd.challenge_records = player->challenge_records;
+  cmd.tech_menu_config = player->tech_menu_config;
+  cmd.unknown_a6.clear(0);
+  cmd.quest_data2 = player->quest_data2;
+  cmd.key_config = account->key_config;
+
+  send_command_t(c, 0x00E7, 0x00000000, cmd);
 }
 
 ////////////////////////////////////////////////////////////////////////////////