implement save file decryption/encryption

CMakeLists.txt

@@ -89,6 +89,7 @@ add_executable(newserv
   src/ReceiveCommands.cc
   src/ReceiveSubcommands.cc
   src/ReplaySession.cc
+  src/SaveFileFormats.cc
   src/SendCommands.cc
   src/Server.cc
   src/ServerShell.cc
@@ -125,14 +126,23 @@ foreach(TestCase IN ITEMS ${TestCases})
 endforeach()
 
 add_test(
-    NAME compression-prs
+    NAME "compression-prs"
     WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
     COMMAND ${CMAKE_SOURCE_DIR}/test-compression.sh prs ${CMAKE_BINARY_DIR}/newserv)
 add_test(
-    NAME compression-bc0
+    NAME "compression-bc0"
     WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
     COMMAND ${CMAKE_SOURCE_DIR}/test-compression.sh bc0 ${CMAKE_BINARY_DIR}/newserv)
 
+add_test(
+    NAME "decode-vms"
+    WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+    COMMAND ${CMAKE_SOURCE_DIR}/test-decode-vms.sh ${CMAKE_BINARY_DIR}/newserv)
+
+add_test(
+    NAME "decode-gci"
+    WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+    COMMAND ${CMAKE_SOURCE_DIR}/test-decode-gci.sh ${CMAKE_BINARY_DIR}/newserv)
 
 # Installation configuration
 

README.md

@@ -367,6 +367,7 @@ newserv has many CLI options, which can be used to access functionality other th
 * Compute the decompressed size of compressed PRS data without decompressing it
 * Encrypt or decrypt data using any PSO version's network encryption scheme
 * Encrypt or decrypt data using Episode 3's trivial scheme
+* Encrypt or decrypt PSO GC save data (.gci files)
 * Run a brute-force search for a decryption seed
 * Decode Shift-JIS text to UTF-16
 * Convert quests in .gci, .vms, .dlq, or .qst format to .bin/.dat format

src/BMLArchive.cc

@@ -10,33 +10,37 @@ using namespace std;
 
 
 
-template <typename LongT>
+template <bool IsBigEndian>
 struct BMLHeader {
+  using U32T = typename std::conditional<IsBigEndian, be_uint32_t, le_uint32_t>::type;
+
   parray<uint8_t, 0x04> unknown_a1;
-  LongT num_entries;
+  U32T num_entries;
   parray<uint8_t, 0x38> unknown_a2;
 } __attribute__((packed));
 
-template <typename LongT>
+template <bool IsBigEndian>
 struct BMLHeaderEntry {
+  using U32T = typename std::conditional<IsBigEndian, be_uint32_t, le_uint32_t>::type;
+
   ptext<char, 0x20> filename;
-  LongT compressed_size;
+  U32T compressed_size;
   parray<uint8_t, 0x04> unknown_a1;
-  LongT decompressed_size;
+  U32T decompressed_size;
-  LongT compressed_gvm_size;
+  U32T compressed_gvm_size;
-  LongT decompressed_gvm_size;
+  U32T decompressed_gvm_size;
   parray<uint8_t, 0x0C> unknown_a2;
 } __attribute__((packed));
 
-template <typename LongT>
+template <bool IsBigEndian>
 void BMLArchive::load_t() {
   StringReader r(*this->data);
 
-  const auto& header = r.get<BMLHeader<LongT>>();
+  const auto& header = r.get<BMLHeader<IsBigEndian>>();
 
   size_t offset = 0x800;
   while (this->entries.size() < header.num_entries) {
-    const auto& entry = r.get<BMLHeaderEntry<LongT>>();
+    const auto& entry = r.get<BMLHeaderEntry<IsBigEndian>>();
 
     if (offset + entry.compressed_size > this->data->size()) {
       throw runtime_error("BML data entry extends beyond end of data");
@@ -58,9 +62,9 @@ void BMLArchive::load_t() {
 BMLArchive::BMLArchive(shared_ptr<const string> data, bool big_endian)
   : data(data) {
   if (big_endian) {
-    this->load_t<be_uint32_t>();
+    this->load_t<true>();
   } else {
-    this->load_t<le_uint32_t>();
+    this->load_t<false>();
   }
 }
 

src/BMLArchive.hh

@@ -29,7 +29,7 @@ public:
   StringReader get_reader(const std::string& name) const;
 
 private:
-  template <typename LongT>
+  template <bool IsBigEndian>
   void load_t();
 
   std::shared_ptr<const std::string> data;

src/CommandFormats.hh

@@ -1905,8 +1905,11 @@ struct S_ExecuteCode_B2 {
   // The code immediately follows, ending with an S_ExecuteCode_Footer_B2
 } __packed__;
 
-template <typename LongT>
+template <bool IsBigEndian>
 struct S_ExecuteCode_Footer_B2 {
+  using U16T = typename std::conditional<IsBigEndian, be_uint16_t, le_uint16_t>:: type;
+  using U32T = typename std::conditional<IsBigEndian, be_uint32_t, le_uint32_t>:: type;
+
   // Relocations is a list of words (le_uint16_t on DC/PC/XB/BB, be_uint16_t on
   // GC) containing the number of doublewords (uint32_t) to skip for each
   // relocation. The relocation pointer starts immediately after the
@@ -1925,19 +1928,18 @@ struct S_ExecuteCode_Footer_B2 {
   // relocations_offset points there, so those 12 bytes may also be omitted from
   // the command entirely (without changing code_size - so code_size would
   // technically extend beyond the end of the B2 command).
-  LongT relocations_offset = 0; // Relative to code base (after checksum_size)
+  U32T relocations_offset = 0; // Relative to code base (after checksum_size)
-  LongT num_relocations = 0;
+  U32T num_relocations = 0;
-  parray<LongT, 2> unused1;
+  parray<U32T, 2> unused1;
   // entrypoint_offset is doubly indirect - it points to a pointer to a 32-bit
   // value that itself is the actual entrypoint. This is presumably done so the
   // entrypoint can be optionally relocated.
-  LongT entrypoint_addr_offset = 0; // Relative to code base (after checksum_size).
+  U32T entrypoint_addr_offset = 0; // Relative to code base (after checksum_size).
-  parray<LongT, 3> unused2;
+  parray<U32T, 3> unused2;
 } __packed__;
 
-struct S_ExecuteCode_Footer_GC_B2 : S_ExecuteCode_Footer_B2<be_uint32_t> { } __packed__;
+struct S_ExecuteCode_Footer_GC_B2 : S_ExecuteCode_Footer_B2<true> { } __packed__;
-struct S_ExecuteCode_Footer_DC_PC_XB_BB_B2
+struct S_ExecuteCode_Footer_DC_PC_XB_BB_B2 : S_ExecuteCode_Footer_B2<false> { } __packed__;
-    : S_ExecuteCode_Footer_B2<le_uint32_t> { } __packed__;
 
 // B3 (C->S): Execute code and/or checksum memory result
 // Not used on versions that don't support the B2 command (see above).

src/CommonItemSet.cc

@@ -7,7 +7,7 @@
 CommonItemSet::CommonItemSet(shared_ptr<const string> data)
   : gsl(data, true) { }
 
-const CommonItemSet::BETable& CommonItemSet::get_table(
+const CommonItemSet::Table<true>& CommonItemSet::get_table(
     Episode episode, GameMode mode, uint8_t difficulty, uint8_t secid) const {
   // TODO: What should we do for Ep4?
   string filename = string_printf("ItemPT%s%s%c%1d.rel",
@@ -16,12 +16,12 @@ const CommonItemSet::BETable& CommonItemSet::get_table(
       tolower(abbreviation_for_difficulty(difficulty)),
       secid);
   auto data = this->gsl.get(filename);
-  if (data.second < sizeof(BETable)) {
+  if (data.second < sizeof(Table<true>)) {
     throw runtime_error(string_printf(
         "ItemPT entry %s is too small (received %zX bytes, expected %zX bytes)",
-        filename.c_str(), data.second, sizeof(BETable)));
+        filename.c_str(), data.second, sizeof(Table<true>)));
   }
-  return *reinterpret_cast<const BETable*>(data.first);
+  return *reinterpret_cast<const Table<true>*>(data.first);
 }
 
 

src/CommonItemSet.hh

@@ -23,9 +23,10 @@ public:
   //       (is_ep2 ? "l" : ""),
   //       char_for_difficulty(difficulty), // One of "nhvu"
   //       section_id);
-  // For GC, use be_uint16_t/be_uint32_t; for other platforms use le variants
+  template <bool IsBigEndian>
-  template <typename U16T, typename U32T>
   struct Table {
+    using U16T = typename std::conditional<IsBigEndian, be_uint16_t, le_uint16_t>::type;
+    using U32T = typename std::conditional<IsBigEndian, be_uint32_t, le_uint32_t>::type;
     // This data structure uses index probability tables in multiple places. An
     // index probability table is a table where each entry holds the probability
     // that that entry's index is used. For example, if the armor slot count
@@ -240,10 +241,7 @@ public:
 
   CommonItemSet(std::shared_ptr<const std::string> data);
 
-  using BETable = Table<be_uint16_t, be_uint32_t>;
+  const Table<true>& get_table(
-  using LETable = Table<le_uint16_t, le_uint32_t>;
-
-  const BETable& get_table(
       Episode episode, GameMode mode, uint8_t difficulty, uint8_t secid) const;
 
 private:

src/Episode3/DataIndex.cc

@@ -10,6 +10,7 @@
 
 #include "../Loggers.hh"
 #include "../Compression.hh"
+#include "../PSOEncryption.hh"
 #include "../Text.hh"
 #include "../Quest.hh"
 
@@ -1712,4 +1713,33 @@ shared_ptr<const COMDeckDefinition> DataIndex::random_com_deck() const {
 
 
 
+void PlayerConfig::decrypt() {
+  if (!this->is_encrypted) {
+    return;
+  }
+  decrypt_trivial_gci_data(
+      &this->card_counts,
+      offsetof(PlayerConfig, decks) - offsetof(PlayerConfig, card_counts),
+      this->basis);
+  this->is_encrypted = 0;
+  this->basis = 0;
+}
+
+void PlayerConfig::encrypt(uint8_t basis) {
+  if (this->is_encrypted) {
+    if (this->basis == basis) {
+      return;
+    }
+    this->decrypt();
+  }
+  decrypt_trivial_gci_data(
+      &this->card_counts,
+      offsetof(PlayerConfig, decks) - offsetof(PlayerConfig, card_counts),
+      basis);
+  this->is_encrypted = 1;
+  this->basis = basis;
+}
+
+
+
 } // namespace Episode3

src/Episode3/DataIndex.hh

@@ -582,8 +582,8 @@ struct DeckDefinition {
 } __attribute__((packed)); // 0x84 bytes in total
 
 struct PlayerConfig {
-  // The first offsets in the comments in this struct are relative to start of
+  // The first offsets in the comments in this struct are relative to the start
-  // 61/98 command; the second are relative to the start of the
+  // of the 61/98 command; the second are relative to the start of the
   // Ep3PlayerDataSegment structure in the reverse-engineering project.
   // TODO: Fill in the unknown fields here by looking around callsites of
   // get_player_data_segment
@@ -626,6 +626,9 @@ struct PlayerConfig {
   /* 299C:2120 */ ptext<char, 0x10> name;
   /* 29AC:2130 */ parray<uint8_t, 0xCC> unknown_a11;
   /* 2A78:21FC */
+
+  void decrypt();
+  void encrypt(uint8_t basis);
 } __attribute__((packed));
 
 enum class HPType : uint8_t {

src/FunctionCompiler.cc

@@ -40,7 +40,7 @@ const char* name_for_architecture(CompiledFunctionCode::Architecture arch) {
 
 
 
-template <typename FooterT, typename U16T>
+template <typename FooterT>
 string CompiledFunctionCode::generate_client_command_t(
       const unordered_map<string, uint32_t>& label_writes,
       const string& suffix) const {
@@ -71,7 +71,7 @@ string CompiledFunctionCode::generate_client_command_t(
 
   footer.relocations_offset = w.size();
   for (uint16_t delta : this->relocation_deltas) {
-    w.put<U16T>(delta);
+    w.put<typename FooterT::U16T>(delta);
   }
   if (this->relocation_deltas.size() & 1) {
     w.put_u16(0);
@@ -85,10 +85,10 @@ string CompiledFunctionCode::generate_client_command(
       const unordered_map<string, uint32_t>& label_writes,
       const string& suffix) const {
   if (this->arch == Architecture::POWERPC) {
-    return this->generate_client_command_t<S_ExecuteCode_Footer_GC_B2, be_uint16_t>(
+    return this->generate_client_command_t<S_ExecuteCode_Footer_GC_B2>(
         label_writes, suffix);
   } else if ((this->arch == Architecture::X86) || (this->arch == Architecture::SH4)) {
-    return this->generate_client_command_t<S_ExecuteCode_Footer_DC_PC_XB_BB_B2, le_uint16_t>(
+    return this->generate_client_command_t<S_ExecuteCode_Footer_DC_PC_XB_BB_B2>(
         label_writes, suffix);
   } else {
     throw logic_error("invalid architecture");

src/FunctionCompiler.hh

@@ -37,7 +37,7 @@ struct CompiledFunctionCode {
 
   bool is_big_endian() const;
 
-  template <typename FooterT, typename U16T>
+  template <typename FooterT>
   std::string generate_client_command_t(
         const std::unordered_map<std::string, uint32_t>& label_writes,
         const std::string& suffix) const;

src/GSLArchive.cc

@@ -10,20 +10,22 @@ using namespace std;
 
 
 
-template <typename LongT>
+template <bool IsBigEndian>
 struct GSLHeaderEntry {
+  using U32T = typename std::conditional<IsBigEndian, be_uint32_t, le_uint32_t>::type;
+
   ptext<char, 0x20> filename;
-  LongT offset; // In pages, so actual offset is this * 0x800
+  U32T offset; // In pages, so actual offset is this * 0x800
-  LongT size;
+  U32T size;
   uint64_t unused;
 } __attribute__((packed));
 
-template <typename LongT>
+template <bool IsBigEndian>
 void GSLArchive::load_t() {
   StringReader r(*this->data);
   uint64_t min_data_offset = 0xFFFFFFFFFFFFFFFF;
   while (r.where() < min_data_offset) {
-    const auto& entry = r.get<GSLHeaderEntry<LongT>>();
+    const auto& entry = r.get<GSLHeaderEntry<IsBigEndian>>();
     if (entry.filename.len() == 0) {
       break;
     }
@@ -38,9 +40,9 @@ void GSLArchive::load_t() {
 GSLArchive::GSLArchive(shared_ptr<const string> data, bool big_endian)
   : data(data) {
   if (big_endian) {
-    this->load_t<be_uint32_t>();
+    this->load_t<true>();
   } else {
-    this->load_t<le_uint32_t>();
+    this->load_t<false>();
   }
 }
 

src/GSLArchive.hh

@@ -26,7 +26,7 @@ public:
   StringReader get_reader(const std::string& name) const;
 
 private:
-  template <typename LongT>
+  template <bool IsBigEndian>
   void load_t();
 
   std::shared_ptr<const std::string> data;

src/ItemCreator.hh

@@ -86,7 +86,7 @@ private:
   std::shared_ptr<const ToolRandomSet> tool_random_set;
   std::shared_ptr<const WeaponRandomSet> weapon_random_set;
   std::shared_ptr<const ItemParameterTable> item_parameter_table;
-  const CommonItemSet::Table<be_uint16_t, be_uint32_t>* pt;
+  const CommonItemSet::Table<true>* pt;
   const RareItemSet::Table* rt;
   std::shared_ptr<const Restrictions> restrictions;
 

src/Main.cc

@@ -24,6 +24,7 @@
 #include "ProxyServer.hh"
 #include "PSOGCObjectGraph.hh"
 #include "ReplaySession.hh"
+#include "SaveFileFormats.hh"
 #include "SendCommands.hh"
 #include "Server.hh"
 #include "ServerShell.hh"
@@ -124,6 +125,12 @@ The actions are:\n\
     trivial algorithm. If SEED is given, it should be specified as one hex\n\
     byte. If SEED is not given, newserv will try all possible seeds and return\n\
     the one that results in the greatest number of zero bytes in the output.\n\
+  encrypt-gci-save CRYPT-OPTION INPUT-FILENAME [OUTPUT-FILENAME]\n\
+  decrypt-gci-save CRYPT-OPTION INPUT-FILENAME [OUTPUT-FILENAME]\n\
+    Encrypt or decrypt a character or Guild Card file. If encrypting, the\n\
+    checksum is also recomputed and stored in the encrypted file. CRYPT-OPTION\n\
+    is required; it can be either --sys=SYSTEM-FILENAME or --seed=ROUND1-SEED\n\
+    (specified in hex).\n\
   find-decryption-seed <OPTIONS...>\n\
     Perform a brute-force search for a decryption seed of the given data. The\n\
     ciphertext is specified with the --encrypted=DATA option and the expected\n\
@@ -197,6 +204,8 @@ enum class Behavior {
   ENCRYPT_DATA,
   DECRYPT_DATA,
   DECRYPT_TRIVIAL_DATA,
+  ENCRYPT_GCI_SAVE,
+  DECRYPT_GCI_SAVE,
   FIND_DECRYPTION_SEED,
   DECODE_QUEST_FILE,
   DECODE_SJIS,
@@ -220,6 +229,8 @@ static bool behavior_takes_input_filename(Behavior b) {
          (b == Behavior::ENCRYPT_DATA) ||
          (b == Behavior::DECRYPT_DATA) ||
          (b == Behavior::DECRYPT_TRIVIAL_DATA) ||
+         (b == Behavior::DECRYPT_GCI_SAVE) ||
+         (b == Behavior::ENCRYPT_GCI_SAVE) ||
          (b == Behavior::DECODE_QUEST_FILE) ||
          (b == Behavior::DECODE_SJIS) ||
          (b == Behavior::FORMAT_ITEMRT_ENTRY) ||
@@ -239,6 +250,8 @@ static bool behavior_takes_output_filename(Behavior b) {
          (b == Behavior::ENCRYPT_DATA) ||
          (b == Behavior::DECRYPT_DATA) ||
          (b == Behavior::DECRYPT_TRIVIAL_DATA) ||
+         (b == Behavior::DECRYPT_GCI_SAVE) ||
+         (b == Behavior::ENCRYPT_GCI_SAVE) ||
          (b == Behavior::DECODE_SJIS) ||
          (b == Behavior::EXTRACT_GSL) ||
          (b == Behavior::EXTRACT_BML);
@@ -267,6 +280,7 @@ int main(int argc, char** argv) {
   vector<const char*> find_decryption_seed_plaintexts;
   const char* input_filename = nullptr;
   const char* output_filename = nullptr;
+  const char* system_filename = nullptr;
   const char* replay_required_access_key = "";
   const char* replay_required_password = "";
   uint32_t root_object_address = 0;
@@ -291,6 +305,8 @@ int main(int argc, char** argv) {
       cli_version = GameVersion::BB;
     } else if (!strncmp(argv[x], "--seed=", 7)) {
       seed = &argv[x][7];
+    } else if (!strncmp(argv[x], "--sys=", 6)) {
+      system_filename = &argv[x][6];
     } else if (!strncmp(argv[x], "--key=", 6)) {
       key_file_name = &argv[x][6];
     } else if (!strncmp(argv[x], "--encrypted=", 12)) {
@@ -337,6 +353,10 @@ int main(int argc, char** argv) {
           behavior = Behavior::DECRYPT_DATA;
         } else if (!strcmp(argv[x], "decrypt-trivial-data")) {
           behavior = Behavior::DECRYPT_TRIVIAL_DATA;
+        } else if (!strcmp(argv[x], "decrypt-gci-save")) {
+          behavior = Behavior::DECRYPT_GCI_SAVE;
+        } else if (!strcmp(argv[x], "encrypt-gci-save")) {
+          behavior = Behavior::ENCRYPT_GCI_SAVE;
         } else if (!strcmp(argv[x], "find-decryption-seed")) {
           behavior = Behavior::FIND_DECRYPTION_SEED;
         } else if (!strcmp(argv[x], "decode-sjis")) {
@@ -408,13 +428,29 @@ int main(int argc, char** argv) {
     } else if (!output_filename && input_filename && strcmp(input_filename, "-")) {
       string filename = input_filename;
       if (behavior == Behavior::COMPRESS_PRS) {
-        if (ends_with(filename, ".bind") || ends_with(filename, ".datd") || ends_with(filename, ".mnmd")) {
+        if (ends_with(filename, ".bind") ||
+            ends_with(filename, ".datd") ||
+            ends_with(filename, ".mnmd")) {
           filename.resize(filename.size() - 1);
         } else {
           filename += ".prs";
         }
       } else if (behavior == Behavior::DECOMPRESS_PRS) {
-        if (ends_with(filename, ".bin") || ends_with(filename, ".dat") || ends_with(filename, ".mnm")) {
+        if (ends_with(filename, ".bin") ||
+            ends_with(filename, ".dat") ||
+            ends_with(filename, ".mnm")) {
+          filename += "d";
+        } else {
+          filename += ".dec";
+        }
+      } else if (behavior == Behavior::ENCRYPT_GCI_SAVE) {
+        if (ends_with(filename, ".gcid")) {
+          filename.resize(filename.size() - 1);
+        } else {
+          filename += ".gci";
+        }
+      } else if (behavior == Behavior::DECRYPT_GCI_SAVE) {
+        if (ends_with(filename, ".gci")) {
           filename += "d";
         } else {
           filename += ".dec";
@@ -575,6 +611,76 @@ int main(int argc, char** argv) {
       break;
     }
 
+    case Behavior::ENCRYPT_GCI_SAVE:
+    case Behavior::DECRYPT_GCI_SAVE: {
+      uint32_t round1_seed;
+      if (system_filename) {
+        string system_data = load_file(system_filename);
+        StringReader r(system_data);
+        const auto& header = r.get<PSOGCIFileHeader>();
+        header.check();
+        const auto& system = r.get<PSOGCSystemFile>();
+        round1_seed = system.creation_internet_time;
+      } else if (!seed.empty()) {
+        round1_seed = stoul(seed, nullptr, 16);
+      } else {
+        // TODO: We can support brute-forcing character file encryption, but I'm
+        // lazy and this would probably not be useful for anyone.
+        throw runtime_error("either --sys or --seed must be given");
+      }
+
+      bool is_decrypt = (behavior == Behavior::DECRYPT_GCI_SAVE);
+
+      auto data = read_input_data();
+      StringReader r(data);
+      const auto& header = r.get<PSOGCIFileHeader>();
+      header.check();
+
+      size_t data_start_offset = r.where();
+
+      auto process_file = [&]<typename StructT>() {
+        if (is_decrypt) {
+          const void* data_section = r.getv(header.data_size);
+          auto decrypted = decrypt_gci_fixed_size_file_data_section<StructT>(
+              data_section, header.data_size, round1_seed);
+          *reinterpret_cast<StructT*>(data.data() + data_start_offset) = decrypted;
+        } else {
+          const auto& s = r.get<StructT>();
+          auto encrypted = encrypt_gci_fixed_size_file_data_section<StructT>(
+              s, round1_seed);
+          if (data_start_offset + encrypted.size() > data.size()) {
+            throw runtime_error("encrypted result exceeds file size");
+          }
+          memcpy(data.data() + data_start_offset, encrypted.data(), encrypted.size());
+        }
+      };
+
+      if (header.data_size == sizeof(PSOGCGuildCardFile)) {
+        process_file.template operator()<PSOGCGuildCardFile>();
+      } else if (header.is_ep12() && (header.data_size == sizeof(PSOGCCharacterFile))) {
+        process_file.template operator()<PSOGCCharacterFile>();
+      } else if (header.is_ep3() && (header.data_size == sizeof(PSOGCEp3CharacterFile))) {
+        auto* charfile = reinterpret_cast<PSOGCEp3CharacterFile*>(data.data() + data_start_offset);
+        if (!is_decrypt) {
+          for (size_t z = 0; z < charfile->characters.size(); z++) {
+            charfile->characters[z].ep3_config.encrypt(random_object<uint8_t>());
+          }
+        }
+        process_file.template operator()<PSOGCEp3CharacterFile>();
+        if (is_decrypt) {
+          for (size_t z = 0; z < charfile->characters.size(); z++) {
+            charfile->characters[z].ep3_config.decrypt();
+          }
+        }
+      } else {
+        throw runtime_error("unrecognized save type");
+      }
+
+      write_output_data(data.data(), data.size());
+
+      break;
+    }
+
     case Behavior::FIND_DECRYPTION_SEED: {
       if (find_decryption_seed_plaintexts.empty() || !find_decryption_seed_ciphertext) {
         throw runtime_error("both --encrypted and --decrypted must be specified");

src/Map.cc

@@ -461,14 +461,16 @@ vector<PSOEnemy> parse_map(
 
 SetDataTable::SetDataTable(shared_ptr<const string> data, bool big_endian) {
   if (big_endian) {
-    this->load_table_t<be_uint32_t>(data);
+    this->load_table_t<true>(data);
   } else {
-    this->load_table_t<le_uint32_t>(data);
+    this->load_table_t<false>(data);
   }
 }
 
-template <typename U32T>
+template <bool IsBigEndian>
 void SetDataTable::load_table_t(shared_ptr<const string> data) {
+  using U32T = typename conditional<IsBigEndian, be_uint32_t, le_uint32_t>::type;
+
   StringReader r(*data);
 
   struct Footer {

src/Map.hh

@@ -113,7 +113,7 @@ public:
   void print(FILE* stream) const;
 
 private:
-  template <typename U32T>
+  template <bool IsBigEndian>
   void load_table_t(std::shared_ptr<const std::string> data);
 
   // Indexes are [area_id][variation1][variation2]

src/PSOEncryption.cc

@@ -38,8 +38,10 @@ uint32_t PSOLFGEncryption::next(bool advance) {
   return ret;
 }
 
-template <typename LongT>
+template <bool IsBigEndian>
 void PSOLFGEncryption::encrypt_t(void* vdata, size_t size, bool advance) {
+  using U32T = typename std::conditional<IsBigEndian, be_uint32_t, le_uint32_t>::type;
+
   if (size & 3) {
     throw invalid_argument("size must be a multiple of 4");
   }
@@ -48,18 +50,44 @@ void PSOLFGEncryption::encrypt_t(void* vdata, size_t size, bool advance) {
   }
   size >>= 2;
 
-  LongT* data = reinterpret_cast<LongT*>(vdata);
+  U32T* data = reinterpret_cast<U32T*>(vdata);
   for (size_t x = 0; x < size; x++) {
     data[x] ^= this->next(advance);
   }
 }
 
+template <bool IsBigEndian>
+void PSOLFGEncryption::encrypt_minus_t(void* vdata, size_t size, bool advance) {
+  using U32T = typename std::conditional<IsBigEndian, be_uint32_t, le_uint32_t>::type;
+
+  if (size & 3) {
+    throw invalid_argument("size must be a multiple of 4");
+  }
+  if (!advance && (size != 4)) {
+    throw logic_error("cannot peek-encrypt/decrypt with size > 4");
+  }
+  size >>= 2;
+
+  U32T* data = reinterpret_cast<U32T*>(vdata);
+  for (size_t x = 0; x < size; x++) {
+    data[x] = this->next(advance) - data[x];
+  }
+}
+
 void PSOLFGEncryption::encrypt(void* vdata, size_t size, bool advance) {
-  this->encrypt_t<le_uint32_t>(vdata, size, advance);
+  this->encrypt_t<false>(vdata, size, advance);
 }
 
 void PSOLFGEncryption::encrypt_big_endian(void* vdata, size_t size, bool advance) {
-  this->encrypt_t<be_uint32_t>(vdata, size, advance);
+  this->encrypt_t<true>(vdata, size, advance);
+}
+
+void PSOLFGEncryption::encrypt_minus(void* vdata, size_t size, bool advance) {
+  this->encrypt_minus_t<false>(vdata, size, advance);
+}
+
+void PSOLFGEncryption::encrypt_big_endian_minus(void* vdata, size_t size, bool advance) {
+  this->encrypt_minus_t<true>(vdata, size, advance);
 }
 
 void PSOLFGEncryption::encrypt_both_endian(

src/PSOEncryption.hh

@@ -45,16 +45,20 @@ class PSOLFGEncryption : public PSOEncryption {
 public:
   virtual void encrypt(void* data, size_t size, bool advance = true);
   void encrypt_big_endian(void* data, size_t size, bool advance = true);
+  void encrypt_minus(void* data, size_t size, bool advance = true);
+  void encrypt_big_endian_minus(void* data, size_t size, bool advance = true);
   void encrypt_both_endian(void* le_data, void* be_data, size_t size, bool advance = true);
 
+  template <bool IsBigEndian>
+  void encrypt_t(void* data, size_t size, bool advance = true);
+  template <bool IsBigEndian>
+  void encrypt_minus_t(void* data, size_t size, bool advance = true);
+
   uint32_t next(bool advance = true);
 
 protected:
   explicit PSOLFGEncryption(uint32_t seed, size_t stream_length, size_t end_offset);
 
-  template <typename LongT>
-  void encrypt_t(void* data, size_t size, bool advance);
-
   virtual void update_stream() = 0;
 
   std::vector<uint32_t> stream;

src/Player.hh

@@ -204,14 +204,15 @@ struct PlayerDispDataBB {
 // TODO: Is this the same for XB as it is for GC? (This struct is based on the
 // GC format)
 struct GuildCardV3 {
-  le_uint32_t player_tag;
+  /* 00 */ le_uint32_t player_tag;
-  le_uint32_t guild_card_number;
+  /* 04 */ le_uint32_t guild_card_number;
-  ptext<char, 0x18> name;
+  /* 08 */ ptext<char, 0x18> name;
-  ptext<char, 0x6C> description;
+  /* 20 */ ptext<char, 0x6C> description;
-  uint8_t present; // should be 1
+  /* 8C */ uint8_t present; // should be 1
-  uint8_t language;
+  /* 8D */ uint8_t language;
-  uint8_t section_id;
+  /* 8E */ uint8_t section_id;
-  uint8_t char_class;
+  /* 8F */ uint8_t char_class;
+  /* 90 */ 
 
   GuildCardV3() noexcept;
 } __attribute__((packed));

src/Quest.cc

@@ -12,6 +12,7 @@
 #include <phosg/Tools.hh>
 
 #include "Loggers.hh"
+#include "SaveFileFormats.hh"
 #include "CommandFormats.hh"
 #include "Compression.hh"
 #include "PSOEncryption.hh"
@@ -23,81 +24,10 @@ using namespace std;
 
 // GCI decoding logic
 
-struct ShuffleTables {
+template <bool IsBigEndian>
-  uint8_t forward_table[0x100];
+struct PSOMemCardDLQFileEncryptedHeader {
-  uint8_t reverse_table[0x100];
+  using U32T = typename std::conditional<IsBigEndian, be_uint32_t, le_uint32_t>::type;
 
-  ShuffleTables(PSOV2Encryption& crypt) {
-    for (size_t x = 0; x < 0x100; x++) {
-      this->forward_table[x] = x;
-    }
-
-    int32_t r28 = 0xFF;
-    uint8_t* r31 = &this->forward_table[0xFF];
-    while (r28 >= 0) {
-      uint32_t r3 = this->pseudorand(crypt, r28 + 1);
-      if (r3 >= 0x100) {
-        throw logic_error("bad r3");
-      }
-      uint8_t t = this->forward_table[r3];
-      this->forward_table[r3] = *r31;
-      *r31 = t;
-
-      this->reverse_table[t] = r28;
-      r31--;
-      r28--;
-    }
-  }
-
-  static uint32_t pseudorand(PSOV2Encryption& crypt, uint32_t prev) {
-    return (((prev & 0xFFFF) * ((crypt.next() >> 16) & 0xFFFF)) >> 16) & 0xFFFF;
-  }
-
-  void shuffle(void* vdest, const void* vsrc, size_t size, bool reverse) {
-    uint8_t* dest = reinterpret_cast<uint8_t*>(vdest);
-    const uint8_t* src = reinterpret_cast<const uint8_t*>(vsrc);
-    const uint8_t* table = reverse ? this->reverse_table : this->forward_table;
-
-    for (size_t block_offset = 0; block_offset < (size & 0xFFFFFF00); block_offset += 0x100) {
-      for (size_t z = 0; z < 0x100; z++) {
-        dest[block_offset + table[z]] = src[block_offset + z];
-      }
-    }
-
-    // Any remaining bytes that don't fill an entire block are not shuffled
-    memcpy(&dest[size & 0xFFFFFF00], &src[size & 0xFFFFFF00], size & 0xFF);
-  }
-};
-
-struct PSOGCIFileHeader {
-  parray<char, 4> game_id; // 'GPOE', 'GPSP', etc.
-  parray<char, 2> developer_id; // '8P' for Sega
-  parray<uint8_t, 0x3A> remaining_gci_header; // There is a structure for this but we don't use it
-  ptext<char, 0x1C> game_name; // e.g. "PSO EPISODE I & II" or "PSO EPISODE III"
-  be_uint32_t embedded_seed; // Used in some of Ralf's quest packs
-  ptext<char, 0x20> quest_name;
-  parray<uint8_t, 0x2000> banner_and_icon;
-  // data_size specifies the number of bytes in the encrypted section, including
-  // the encrypted header (below) and all encrypted data after it.
-  be_uint32_t data_size;
-  // To compute checksum, set checksum to zero, then compute the CRC32 of all
-  // fields in this struct starting with game_name. (Yes, including the checksum
-  // field, which is temporarily zero.)
-  be_uint32_t checksum;
-
-  bool checksum_correct() const {
-    uint32_t cs = crc32(&this->game_name, sizeof(this->game_name));
-    cs = crc32(&this->embedded_seed, sizeof(this->embedded_seed), cs);
-    cs = crc32(&this->quest_name, sizeof(this->quest_name), cs);
-    cs = crc32(&this->banner_and_icon, sizeof(this->banner_and_icon), cs);
-    cs = crc32(&this->data_size, sizeof(this->data_size), cs);
-    cs = crc32("\0\0\0\0", 4, cs);
-    return (cs == this->checksum);
-  }
-} __attribute__((packed));
-
-template <typename U32T>
-struct PSOMemCardFileEncryptedHeader {
   U32T round2_seed;
   // To compute checksum, set checksum to zero, then compute the CRC32 of the
   // entire data section, including this header struct (but not the unencrypted
@@ -108,50 +38,27 @@ struct PSOMemCardFileEncryptedHeader {
   // Data follows here.
 } __attribute__((packed));
 
-struct PSOVMSFileEncryptedHeader : PSOMemCardFileEncryptedHeader<le_uint32_t> { } __attribute__((packed));
+struct PSOVMSDLQFileEncryptedHeader : PSOMemCardDLQFileEncryptedHeader<false> { } __attribute__((packed));
-struct PSOGCIFileEncryptedHeader : PSOMemCardFileEncryptedHeader<be_uint32_t> { } __attribute__((packed));
+struct PSOGCIDLQFileEncryptedHeader : PSOMemCardDLQFileEncryptedHeader<true> { } __attribute__((packed));
 
 template <bool IsBigEndian>
-string decrypt_gci_or_vms_v2_data_section(
+string decrypt_gci_or_vms_v2_download_quest_data_section(
     const void* data_section, size_t size, uint32_t seed) {
-
+  string decrypted = decrypt_gci_or_vms_v2_data_section<IsBigEndian>(
-  string decrypted(size, '\0');
+      data_section, size, seed);
-  {
-    PSOV2Encryption shuf_crypt(seed);
-    ShuffleTables shuf(shuf_crypt);
-    shuf.shuffle(decrypted.data(), data_section, size, true);
-  }
 
   size_t orig_size = decrypted.size();
   decrypted.resize((decrypted.size() + 3) & (~3));
 
-  PSOV2Encryption crypt(seed);
-  if (IsBigEndian) {
-    auto* be_dwords = reinterpret_cast<be_uint32_t*>(decrypted.data());
-    for (size_t z = 0; z < decrypted.size() / sizeof(be_uint32_t); z++) {
-      be_dwords[z] = crypt.next() - be_dwords[z];
-    }
-  } else {
-    auto* le_dwords = reinterpret_cast<le_uint32_t*>(decrypted.data());
-    for (size_t z = 0; z < decrypted.size() / sizeof(le_uint32_t); z++) {
-      le_dwords[z] = crypt.next() - le_dwords[z];
-    }
-  }
-
   // Note: Other PSO save files have the round 2 seed at the end of the data,
   // not at the beginning. Presumably they did this because the system,
   // character, and Guild Card files are a constant size, but download quest
   // files can vary in size.
-  using HeaderT = typename conditional<IsBigEndian, PSOMemCardFileEncryptedHeader<be_uint32_t>, PSOMemCardFileEncryptedHeader<le_uint32_t>>::type;
+  using HeaderT = PSOMemCardDLQFileEncryptedHeader<IsBigEndian>;
   auto* header = reinterpret_cast<HeaderT*>(decrypted.data());
   PSOV2Encryption round2_crypt(header->round2_seed);
-  if (IsBigEndian) {
+  round2_crypt.encrypt_t<IsBigEndian>(
-    round2_crypt.encrypt_big_endian(
+      decrypted.data() + 4, (decrypted.size() - 4));
-        decrypted.data() + 4, (decrypted.size() - 4));
-  } else {
-    round2_crypt.decrypt(
-        decrypted.data() + 4, (decrypted.size() - 4));
-  }
 
   if (header->decompressed_size & 0xFFF00000) {
     throw runtime_error(string_printf(
@@ -168,14 +75,17 @@ string decrypt_gci_or_vms_v2_data_section(
         expected_crc, actual_crc));
   }
 
+  // Unlike the above rounds, round 3 is always little-endian (it corresponds to
+  // the round of encryption done on the server before sending the file to the
+  // client in the first place)
   PSOV2Encryption(header->round3_seed).decrypt(
       decrypted.data() + sizeof(HeaderT),
       decrypted.size() - sizeof(HeaderT));
   decrypted.resize(orig_size);
 
-  // Some GCI files have decompressed_size fields that are 8 bytes smaller than
+  // Some download quest GCI files have decompressed_size fields that are 8
-  // the actual decompressed size of the data. They seem to work fine, so we
+  // bytes smaller than the actual decompressed size of the data. They seem to
-  // accept both cases as correct.
+  // work fine, so we accept both cases as correct.
   size_t decompressed_size = prs_decompress_size(
       decrypted.data() + sizeof(HeaderT),
       decrypted.size() - sizeof(HeaderT));
@@ -212,13 +122,13 @@ string decrypt_vms_v1_data_section(const void* data_section, size_t size) {
 }
 
 template <bool IsBigEndian>
-string find_seed_and_decrypt_gci_or_vms_v2_data_section(
+string find_seed_and_decrypt_gci_or_vms_v2_download_quest_data_section(
     const void* data_section, size_t size, size_t num_threads) {
   mutex result_lock;
   string result;
   uint64_t result_seed = parallel_range<uint64_t>([&](uint64_t seed, size_t) {
     try {
-      string ret = decrypt_gci_or_vms_v2_data_section<IsBigEndian>(
+      string ret = decrypt_gci_or_vms_v2_download_quest_data_section<IsBigEndian>(
           data_section, size, seed);
       lock_guard<mutex> g(result_lock);
       result = move(ret);
@@ -703,32 +613,20 @@ string Quest::decode_gci(
 
   StringReader r(data);
   const auto& header = r.get<PSOGCIFileHeader>();
-  if (!header.checksum_correct()) {
+  header.check();
-    throw runtime_error("GCI file unencrypted header checksum is incorrect");
-  }
 
-  if (header.developer_id[0] != '8' || header.developer_id[1] != 'P') {
+  if (header.is_ep12()) {
-    throw runtime_error("GCI file is not for a Sega game");
+    const auto& dlq_header = r.get<PSOGCIDLQFileEncryptedHeader>(false);
-  }
-  if (header.game_id[0] != 'G') {
-    throw runtime_error("GCI file is not for a GameCube game");
-  }
-  if (header.game_id[1] != 'P') {
-    throw runtime_error("GCI file is not for Phantasy Star Online");
-  }
-
-  if (header.game_id[2] == 'O') { // Episodes 1&2 (GPO*)
-    const auto& encrypted_header = r.get<PSOGCIFileEncryptedHeader>(false);
     // Unencrypted GCI files appear to always have zeroes in these fields.
     // Encrypted GCI files are highly unlikely to have zeroes in ALL of these
     // fields, so assume it's encrypted if any of them are nonzero.
-    if (encrypted_header.round2_seed || encrypted_header.checksum || encrypted_header.round3_seed) {
+    if (dlq_header.round2_seed || dlq_header.checksum || dlq_header.round3_seed) {
       if (known_seed >= 0) {
-        return decrypt_gci_or_vms_v2_data_section<true>(
+        return decrypt_gci_or_vms_v2_download_quest_data_section<true>(
             r.getv(header.data_size), header.data_size, known_seed);
 
       } else if (header.embedded_seed != 0) {
-        return decrypt_gci_or_vms_v2_data_section<true>(
+        return decrypt_gci_or_vms_v2_download_quest_data_section<true>(
             r.getv(header.data_size), header.data_size, header.embedded_seed);
 
       } else {
@@ -738,16 +636,16 @@ string Quest::decode_gci(
         if (find_seed_num_threads == 0) {
           find_seed_num_threads = thread::hardware_concurrency();
         }
-        return find_seed_and_decrypt_gci_or_vms_v2_data_section<true>(
+        return find_seed_and_decrypt_gci_or_vms_v2_download_quest_data_section<true>(
             r.getv(header.data_size), header.data_size, find_seed_num_threads);
       }
 
     } else { // Unencrypted GCI format
-      r.skip(sizeof(PSOGCIFileEncryptedHeader));
+      r.skip(sizeof(PSOGCIDLQFileEncryptedHeader));
-      string compressed_data = r.readx(header.data_size - sizeof(PSOGCIFileEncryptedHeader));
+      string compressed_data = r.readx(header.data_size - sizeof(PSOGCIDLQFileEncryptedHeader));
       size_t decompressed_bytes = prs_decompress_size(compressed_data);
 
-      size_t expected_decompressed_bytes = encrypted_header.decompressed_size - 8;
+      size_t expected_decompressed_bytes = dlq_header.decompressed_size - 8;
       if (decompressed_bytes < expected_decompressed_bytes) {
         throw runtime_error(string_printf(
             "GCI decompressed data is smaller than expected size (have 0x%zX bytes, expected 0x%zX bytes)",
@@ -810,7 +708,7 @@ string Quest::decode_vms(
   } catch (const exception& e) { }
 
   if (known_seed >= 0) {
-    return decrypt_gci_or_vms_v2_data_section<false>(
+    return decrypt_gci_or_vms_v2_download_quest_data_section<false>(
         data_section, header.data_size, known_seed);
 
   } else {
@@ -820,7 +718,7 @@ string Quest::decode_vms(
     if (find_seed_num_threads == 0) {
       find_seed_num_threads = thread::hardware_concurrency();
     }
-    return find_seed_and_decrypt_gci_or_vms_v2_data_section<false>(
+    return find_seed_and_decrypt_gci_or_vms_v2_download_quest_data_section<false>(
         data_section, header.data_size, find_seed_num_threads);
   }
 }

src/SaveFileFormats.cc

@@ -0,0 +1,89 @@
+#include "SaveFileFormats.hh"
+
+#include <string>
+#include <stdexcept>
+
+using namespace std;
+
+
+
+ShuffleTables::ShuffleTables(PSOV2Encryption& crypt) {
+  for (size_t x = 0; x < 0x100; x++) {
+    this->forward_table[x] = x;
+  }
+
+  int32_t r28 = 0xFF;
+  uint8_t* r31 = &this->forward_table[0xFF];
+  while (r28 >= 0) {
+    uint32_t r3 = this->pseudorand(crypt, r28 + 1);
+    if (r3 >= 0x100) {
+      throw logic_error("bad r3");
+    }
+    uint8_t t = this->forward_table[r3];
+    this->forward_table[r3] = *r31;
+    *r31 = t;
+
+    this->reverse_table[t] = r28;
+    r31--;
+    r28--;
+  }
+}
+
+uint32_t ShuffleTables::pseudorand(PSOV2Encryption& crypt, uint32_t prev) {
+  return (((prev & 0xFFFF) * ((crypt.next() >> 16) & 0xFFFF)) >> 16) & 0xFFFF;
+}
+
+void ShuffleTables::shuffle(void* vdest, const void* vsrc, size_t size, bool reverse) const {
+  uint8_t* dest = reinterpret_cast<uint8_t*>(vdest);
+  const uint8_t* src = reinterpret_cast<const uint8_t*>(vsrc);
+  const uint8_t* table = reverse ? this->reverse_table : this->forward_table;
+
+  for (size_t block_offset = 0; block_offset < (size & 0xFFFFFF00); block_offset += 0x100) {
+    for (size_t z = 0; z < 0x100; z++) {
+      dest[block_offset + table[z]] = src[block_offset + z];
+    }
+  }
+
+  // Any remaining bytes that don't fill an entire block are not shuffled
+  memcpy(&dest[size & 0xFFFFFF00], &src[size & 0xFFFFFF00], size & 0xFF);
+}
+
+
+
+bool PSOGCIFileHeader::checksum_correct() const {
+  uint32_t cs = crc32(&this->game_name, this->game_name.bytes());
+  cs = crc32(&this->embedded_seed, sizeof(this->embedded_seed), cs);
+  cs = crc32(&this->file_name, this->file_name.bytes(), cs);
+  cs = crc32(&this->banner, this->banner.bytes(), cs);
+  cs = crc32(&this->icon, this->icon.bytes(), cs);
+  cs = crc32(&this->data_size, sizeof(this->data_size), cs);
+  cs = crc32("\0\0\0\0", 4, cs); // this->checksum (treated as zero)
+  return (cs == this->checksum);
+}
+
+void PSOGCIFileHeader::check() const {
+  if (!this->checksum_correct()) {
+    throw runtime_error("GCI file unencrypted header checksum is incorrect");
+  }
+  if (this->developer_id[0] != '8' || this->developer_id[1] != 'P') {
+    throw runtime_error("GCI file is not for a Sega game");
+  }
+  if (this->game_id[0] != 'G') {
+    throw runtime_error("GCI file is not for a GameCube game");
+  }
+  if (this->game_id[1] != 'P') {
+    throw runtime_error("GCI file is not for Phantasy Star Online");
+  }
+  if ((this->game_id[1] != 'P') ||
+      ((this->game_id[2] != 'S') && (this->game_id[2] != 'O'))) {
+    throw runtime_error("GCI file is not for Phantasy Star Online");
+  }
+}
+
+bool PSOGCIFileHeader::is_ep12() const {
+  return (this->game_id[2] == 'O');
+}
+
+bool PSOGCIFileHeader::is_ep3() const {
+  return (this->game_id[2] == 'S');
+}

src/SaveFileFormats.hh

@@ -0,0 +1,229 @@
+#pragma once
+
+#include <stdint.h>
+
+#include <string>
+#include <phosg/Encoding.hh>
+#include <phosg/Hash.hh>
+#include <phosg/Strings.hh>
+#include <phosg/Random.hh>
+
+#include "PSOEncryption.hh"
+#include "Text.hh"
+#include "Player.hh"
+
+
+
+struct ShuffleTables {
+  uint8_t forward_table[0x100];
+  uint8_t reverse_table[0x100];
+
+  ShuffleTables(PSOV2Encryption& crypt);
+
+  static uint32_t pseudorand(PSOV2Encryption& crypt, uint32_t prev);
+
+  void shuffle(void* vdest, const void* vsrc, size_t size, bool reverse) const;
+};
+
+
+
+struct PSOGCIFileHeader {
+  /* 0000 */ parray<char, 4> game_id; // 'GPOE', 'GPSP', etc.
+  /* 0004 */ parray<char, 2> developer_id; // '8P' for Sega
+  // There is a structure for this part of the header, but we don't use it
+  /* 0006 */ parray<uint8_t, 0x3A> remaining_gci_header;
+  // game_name is e.g. "PSO EPISODE I & II" or "PSO EPISODE III"
+  /* 0040 */ ptext<char, 0x1C> game_name;
+  /* 005C */ be_uint32_t embedded_seed; // Used in some of Ralf's quest packs
+  /* 0060 */ ptext<char, 0x20> file_name;
+  /* 0080 */ parray<uint8_t, 0x1800> banner;
+  /* 1880 */ parray<uint8_t, 0x800> icon;
+  // data_size specifies the number of bytes remaining in the file. In all cases
+  // except for the system file, this data is encrypted.
+  /* 2080 */ be_uint32_t data_size;
+  // To compute checksum, set checksum to zero, then compute the CRC32 of all
+  // fields in this struct starting with gci_header.game_name. (Yes, including
+  // the checksum field, which is temporarily zero.) See checksum_correct below.
+  /* 2084 */ be_uint32_t checksum;
+
+  bool checksum_correct() const;
+  void check() const;
+
+  bool is_ep12() const;
+  bool is_ep3() const;
+} __attribute__((packed));
+
+struct PSOGCSystemFile {
+  /* 0000 */ be_uint32_t checksum;
+  /* 0004 */ be_uint16_t unknown_a1;
+  /* 0006 */ uint8_t unknown_a2;
+  /* 0007 */ uint8_t language;
+  /* 0008 */ be_uint32_t unknown_a3;
+  /* 000C */ be_uint16_t unknown_a4;
+  /* 000E */ be_uint16_t unknown_a5;
+  /* 0010 */ parray<uint8_t, 0x100> unknown_a6;
+  /* 0110 */ parray<uint8_t, 8> unknown_a7;
+  /* 0118 */ be_uint32_t creation_internet_time; // Character file round1 seed
+  /* 011C */
+} __attribute__((packed));
+
+struct PSOGCEp3SystemFile {
+  /* 0000 */ PSOGCSystemFile base;
+  /* 011C */ int8_t unknown_a1;
+  /* 011D */ parray<uint8_t, 11> unknown_a2;
+  /* 0128 */ be_uint32_t unknown_a3;
+  /* 012C */
+} __attribute__((packed));
+
+struct PSOGCCharacterFile {
+  /* 00000 */ be_uint32_t checksum;
+  /* 00004 */ parray<uint8_t, 0x11564> unknown_a1; // TODO
+  /* 11568 */ be_uint32_t round2_seed;
+  /* 1156C */
+} __attribute__((packed));
+
+struct PSOGCEp3CharacterFile {
+  /* 00000 */ be_uint32_t checksum; // crc32 of this field (as 0) through end of struct
+  struct Character {
+    /* 0000 */ PlayerInventory inventory;
+    /* 034C */ PlayerDispDataDCPCV3 disp;
+    /* 041C */ be_uint32_t unknown_a1;
+    /* 0420 */ be_uint32_t save_token; // Sent in 96 command
+    /* 0424 */ parray<be_uint32_t, 3> unknown_a2;
+    /* 0430 */ be_uint32_t save_count; // Sent in 96 command
+    /* 0434 */ parray<uint8_t, 0x498> unknown_a3;
+    /* 08CC */ GuildCardV3 guild_card;
+    struct SymbolChatEntry {
+      /* 00 */ be_uint32_t present;
+      /* 04 */ ptext<char, 0x18> name;
+      /* 1C */ be_uint16_t unused;
+      /* 1E */ uint8_t flags;
+      /* 1F */ uint8_t face_spec;
+      struct CornerObject {
+        uint8_t type;
+        uint8_t flags_color;
+      } __attribute__((packed));
+      /* 20 */ parray<CornerObject, 4> corner_objects;
+      struct FacePart {
+        uint8_t type;
+        uint8_t x;
+        uint8_t y;
+        uint8_t flags;
+      } __attribute__((packed));
+      /* 28 */ parray<FacePart, 12> face_parts;
+      /* 58 */
+    } __attribute__((packed));
+    /* 095C */ parray<SymbolChatEntry, 12> symbol_chats;
+    struct ChatShortcut {
+      /* 00 */ be_uint32_t present_type;
+      /* 04 */ parray<uint8_t, 0x50> definition;
+      /* 54 */
+    } __attribute__((packed));
+    /* 0D7C */ parray<ChatShortcut, 20> chat_shortcuts;
+    /* 140C */ parray<uint8_t, 0xAC> unknown_a4;
+    /* 14B8 */ ptext<char, 0xAC> info_board;
+    /* 1564 */ parray<uint8_t, 0xF4> unknown_a5;
+    /* 1658 */ Episode3::PlayerConfig ep3_config;
+    /* 39A8 */ be_uint32_t unknown_a7;
+    /* 39AC */ be_uint32_t unknown_a8;
+    /* 39B0 */ be_uint32_t unknown_a9;
+    /* 39B4 */
+  } __attribute__((packed));
+  /* 00004 */ parray<Character, 7> characters;
+  /* 193F0 */ ptext<char, 0x10> serial_number; // As %08X (not decimal)
+  /* 19400 */ ptext<char, 0x10> access_key;
+  /* 19410 */ ptext<char, 0x10> password;
+  /* 19420 */ be_uint32_t unknown_a1;
+  /* 19424 */ be_uint32_t unknown_a2;
+  /* 19428 */ be_uint32_t unknown_a3;
+  /* 1942C */ parray<be_uint32_t, 0x20> unknown_a4;
+  /* 194AC */ be_uint32_t round2_seed;
+  /* 194B0 */
+} __attribute__((packed));
+
+struct PSOGCGuildCardFile {
+  /* 0000 */ be_uint32_t checksum;
+  /* 0004 */ parray<uint8_t, 0xE284> unknown_a1;
+  /* E288 */ be_uint32_t round2_seed;
+  /* E28C */
+} __attribute__((packed));
+
+
+
+template <bool IsBigEndian>
+std::string decrypt_gci_or_vms_v2_data_section(
+    const void* data_section, size_t size, uint32_t round1_seed) {
+
+  std::string decrypted(size, '\0');
+  PSOV2Encryption shuf_crypt(round1_seed);
+  ShuffleTables shuf(shuf_crypt);
+  shuf.shuffle(decrypted.data(), data_section, size, true);
+
+  size_t orig_size = decrypted.size();
+  decrypted.resize((decrypted.size() + 3) & (~3));
+
+  PSOV2Encryption round1_crypt(round1_seed);
+  round1_crypt.encrypt_minus_t<IsBigEndian>(decrypted.data(), decrypted.size());
+
+  decrypted.resize(orig_size);
+  return decrypted;
+}
+
+template <bool IsBigEndian>
+std::string encrypt_gci_or_vms_v2_data_section(
+    const void* data_section, size_t size, uint32_t round1_seed) {
+  std::string encrypted(reinterpret_cast<const char*>(data_section), size);
+  encrypted.resize((encrypted.size() + 3) & (~3));
+
+  PSOV2Encryption crypt(round1_seed);
+  crypt.encrypt_minus_t<IsBigEndian>(encrypted.data(), encrypted.size());
+
+  std::string ret(size, '\0');
+  PSOV2Encryption shuf_crypt(round1_seed);
+  ShuffleTables shuf(shuf_crypt);
+  shuf.shuffle(ret.data(), encrypted.data(), size, false);
+
+  return ret;
+}
+
+template <typename StructT>
+StructT decrypt_gci_fixed_size_file_data_section(
+    const void* data_section, size_t size, uint32_t round1_seed) {
+  std::string decrypted = decrypt_gci_or_vms_v2_data_section<true>(
+      data_section, size, round1_seed);
+
+  if (decrypted.size() < sizeof(StructT)) {
+    throw std::runtime_error("file too small for structure");
+  }
+  StructT ret = *reinterpret_cast<const StructT*>(decrypted.data());
+
+  PSOV2Encryption round2_crypt(ret.round2_seed);
+  round2_crypt.encrypt_big_endian(&ret, offsetof(StructT, round2_seed));
+
+  uint32_t expected_crc = ret.checksum;
+  ret.checksum = 0;
+  uint32_t actual_crc = crc32(&ret, sizeof(ret));
+  ret.checksum = expected_crc;
+  if (expected_crc != actual_crc) {
+    throw std::runtime_error(string_printf(
+        "incorrect decrypted data section checksum: expected %08" PRIX32 "; received %08" PRIX32,
+        expected_crc, actual_crc));
+  }
+
+  return ret;
+}
+
+template <typename StructT>
+std::string encrypt_gci_fixed_size_file_data_section(
+    const StructT& s, uint32_t round1_seed) {
+  StructT encrypted = s;
+  encrypted.checksum = 0;
+  encrypted.round2_seed = random_object<uint32_t>();
+  encrypted.checksum = crc32(&encrypted, sizeof(encrypted));
+
+  PSOV2Encryption round2_crypt(encrypted.round2_seed);
+  round2_crypt.encrypt_big_endian(&encrypted, offsetof(StructT, round2_seed));
+
+  return encrypt_gci_or_vms_v2_data_section<true>(
+      &encrypted, sizeof(StructT), round1_seed);
+}

test-decode-gci.sh

@@ -0,0 +1,36 @@
+#!/bin/sh
+
+set -e
+
+EXECUTABLE="$1"
+if [ "$EXECUTABLE" == "" ]; then
+  EXECUTABLE="./newserv"
+fi
+
+echo "... decode GCIEpisode3.gci"
+$EXECUTABLE decode-gci tests/GCIEpisode3.gci
+diff tests/GCIEpisode3.dec tests/GCIEpisode3.gci.dec
+echo "... decode GCIWithoutEncryption.gci"
+$EXECUTABLE decode-gci tests/GCIWithoutEncryption.gci
+diff tests/GCIWithoutEncryption.dec tests/GCIWithoutEncryption.gci.dec
+echo "... decode GCIWithEmbeddedKey.gci"
+$EXECUTABLE decode-gci tests/GCIWithEmbeddedKey.gci
+diff tests/GCIWithEmbeddedKey.dec tests/GCIWithEmbeddedKey.gci.dec
+echo "... decode GCIWithoutEmbeddedKey.gci"
+$EXECUTABLE decode-gci tests/GCIWithoutEmbeddedKey.gci --seed=1705B11E
+diff tests/GCIWithoutEmbeddedKey.dec tests/GCIWithoutEmbeddedKey.gci.dec
+
+echo "... re-encrypt GCICharFile.gci"
+./newserv encrypt-gci-save --sys=tests/GCISystemFile.gci tests/GCICharFile.gcid tests/GCICharFile.gci
+./newserv decrypt-gci-save --sys=tests/GCISystemFile.gci tests/GCICharFile.gci tests/GCICharFile-redec.gcid
+hexdump -vC tests/GCICharFile.gcid > tests/GCICharFile.gcid.hex
+hexdump -vC tests/GCICharFile-redec.gcid > tests/GCICharFile-redec.gcid.hex
+# There should be differences on two lines: the checksum and the round2 seed
+NUM_DIFF_LINES=$(diff -y --suppress-common-lines tests/GCICharFile.gcid.hex tests/GCICharFile-redec.gcid.hex | wc -l)
+if [[ $NUM_DIFF_LINES -ne 2 ]]; then
+  diff -U3 tests/GCICharFile.gcid.hex tests/GCICharFile-redec.gcid.hex
+  exit 1
+fi
+
+echo "... clean up"
+rm tests/*.gci.dec tests/GCICharFile.gci tests/GCICharFile-redec.gcid tests/*.hex

test-decode-vms.sh

@@ -0,0 +1,18 @@
+#!/bin/sh
+
+set -e
+
+EXECUTABLE="$1"
+if [ "$EXECUTABLE" == "" ]; then
+  EXECUTABLE="./newserv"
+fi
+
+echo "... decode LionelV1.vms"
+$EXECUTABLE decode-vms tests/LionelV1.vms
+diff tests/LionelV1.dec tests/LionelV1.vms.dec
+echo "... decode LionelV2.vms"
+$EXECUTABLE decode-vms tests/LionelV2.vms --seed=D0231610
+diff tests/LionelV2.dec tests/LionelV2.vms.dec
+
+echo "... clean up"
+rm tests/*.vms.dec