add GCI decryption function

src/Main.cc

@@ -8,6 +8,7 @@
 #include <phosg/Network.hh>
 #include <phosg/Strings.hh>
 #include <set>
+#include <thread>
 #include <unordered_map>
 
 #include "CatSession.hh"
@@ -16,6 +17,7 @@
 #include "Loggers.hh"
 #include "NetworkAddresses.hh"
 #include "ProxyServer.hh"
+#include "PSOEncryptionSeedFinder.hh"
 #include "ReplaySession.hh"
 #include "SendCommands.hh"
 #include "Server.hh"
@@ -223,38 +225,60 @@ When options are given, newserv will do things other than running the server.\n\
 Specifically:\n\
   --decrypt-data\n\
   --encrypt-data\n\
-      If either of these options is given, newserv will read from stdin and\n\
-      write the encrypted or decrypted result to stdout. By default, PSO GC\n\
-      encryption is used, but this can be overridden with --pc or --bb. The\n\
-      --seed option specifies the encryption seed (4 hex bytes for PC or GC, or\n\
-      48 hex bytes for BB). For BB, the --key option is required as well, and\n\
-      should refer to a .nsk file in system/blueburst/keys (without the\n\
-      directory or .nsk extension).\n\
+      Read from stdin, encrypt or decrypt the data, and write the result to\n\
+      stdout. By default, PSO V3 encryption is used, but this can be overridden\n\
+      with --pc or --bb. The --seed option specifies the encryption seed (4 hex\n\
+      bytes for PC or GC, or 48 hex bytes for BB). For BB, the --key option is\n\
+      required as well, and refers to a .nsk file in system/blueburst/keys\n\
+      (without the directory or .nsk extension). For non-BB ciphers, the\n\
+      --big-endian option applies the cipher masks as big-endian instead of\n\
+      little-endian, which is necessary for some GameCube file formats.\n\
+  --find-decryption-seed\n\
+      Perform a brute-force search for a decryption seed of the given data.\n\
+      The ciphertext is specified with the --encrypted= option and the expected\n\
+      plaintext is specified with the --decrypted= option. The plaintext may\n\
+      include unmatched bytes (specified with the ? operator), but overall it\n\
+      must be the same length as the ciphertext. By default, this option uses\n\
+      PSO V3 encryption, but this can be overridden with --pc. (BB encryption\n\
+      seeds are too long to be searched for with this function.) By default,\n\
+      the number of worker threads is equal the the number of CPU cores in the\n\
+      system, but this can be overridden with the --threads= option. To use a\n\
+      rainbow table instead of computing the cipherstreams inline, use the\n\
+      --rainbow-table=FILENAME option.\n\
+  --generate-rainbow-table=FILENAME\n\
+      Generate a decryption table for V3 encryption (or V2 if --pc is given).\n\
+      The --match-length= option must be given, which specifies the match\n\
+      length for the table. The total table size is the match length * 4 GB.\n\
+      As for --encrypt-data, the --big-endian option specifies that the table\n\
+      uses big-endian encryption. As for --find-decryption-seed, the --threads\n\
+      option specifies the parallelism for generating the table.\n\
   --decode-sjis\n\
-      If this option is given, newserv applies its text decoding algorithm to\n\
-      the data on stdin, producing little-endian UTF-16 data on stdout.\n\
+      Apply newserv\'s text decoding algorithm to the data on stdin, producing\n\
+      little-endian UTF-16 data on stdout.\n\
   --decode-gci=FILENAME\n\
   --decode-dlq=FILENAME\n\
   --decode-qst=FILENAME\n\
-      If any of these options are given, newserv will decode the given quest\n\
-      file into a compressed, unencrypted .bin or .dat file (or in the case of)\n\
-      --decode-qst, both a .bin and a .dat file).\n\
+      Decode the given quest file into a compressed, unencrypted .bin or .dat\n\
+      file (or in the case of --decode-qst, both a .bin and a .dat file). The\n\
+      --decode-gci option can be used to decrypt encrypted GCI files. If you\n\
+      know the player\'s serial number who generated the GCI file, use the\n\
+      --seed= option and give the serial number (as a hex-encoded integer). If\n\
+      you don\'t know the serial number, newserv will find it via a brute-force\n\
+      search, but this will likely take a long time.\n\
   --cat-client=ADDR:PORT\n\
-      If this option is given, newserv will behave as if it's a PSO client, and\n\
-      will connect to the given server. It will then print all the received\n\
-      commands to stdout, and forward any commands typed into stdin to the\n\
-      remote server. It is assumed that the input and output are terminals, so\n\
-      all commands are hex-encoded. The --patch, --dc, --pc, --gc, and --bb\n\
-      options can be used to select the command format and encryption. If --bb\n\
-      is used, the --key option is also required (as in --decrypt-data above).\n\
+      Connect to the given server and simulate a PSO client. newserv will then\n\
+      print all the received commands to stdout, and forward any commands typed\n\
+      into stdin to the remote server. It is assumed that the input and output\n\
+      are terminals, so all commands are hex-encoded. The --patch, --dc, --pc,\n\
+      --gc, and --bb options can be used to select the command format and\n\
+      encryption. If --bb is used, the --key option is also required (as in\n\
+      --decrypt-data above).\n\
   --replay-log=FILENAME\n\
-      This option makes newserv replay terminal log as if it were a client\n\
-      session. This is used for regression testing, to make sure client\n\
-      sessions are repeatable and code changes don\'t affect existing (working)\n\
-      functionality.\n\
+      Replay a terminal log as if it were a client session. This is used for\n\
+      regression testing, to make sure client sessions are repeatable and code\n\
+      changes don\'t affect existing (working) functionality.\n\
   --extract-gsl=FILENAME\n\
-      This option makes newserv extract all files from a GSL archive and place\n\
-      them in the current directory.\n\
+      Extract all files from a GSL archive into the current directory.\n\
 \n\
 A few options apply to multiple modes described above:\n\
   --parse-data\n\
@@ -269,6 +293,8 @@ enum class Behavior {
   RUN_SERVER = 0,
   DECRYPT_DATA,
   ENCRYPT_DATA,
+  FIND_DECRYPTION_SEED,
+  GENERATE_RAINBOW_TABLE,
   DECODE_QUEST_FILE,
   DECODE_SJIS,
   EXTRACT_GSL,
@@ -290,8 +316,15 @@ int main(int argc, char** argv) {
   string seed;
   string key_file_name;
   const char* config_filename = "system/config.json";
+  string rainbow_table_filename;
   bool parse_data = false;
-  bool byteswap_data = false;
+  bool big_endian = false;
+  bool skip_little_endian = false;
+  bool skip_big_endian = false;
+  size_t num_threads = 0;
+  size_t match_length = 0;
+  const char* find_decryption_seed_ciphertext = nullptr;
+  vector<const char*> find_decryption_seed_plaintexts;
   const char* replay_log_filename = nullptr;
   const char* extract_gsl_filename = nullptr;
   const char* replay_required_access_key = "";
@@ -305,6 +338,11 @@ int main(int argc, char** argv) {
       behavior = Behavior::DECRYPT_DATA;
     } else if (!strcmp(argv[x], "--encrypt-data")) {
       behavior = Behavior::ENCRYPT_DATA;
+    } else if (!strcmp(argv[x], "--find-decryption-seed")) {
+      behavior = Behavior::FIND_DECRYPTION_SEED;
+    } else if (!strncmp(argv[x], "--generate-rainbow-table=", 25)) {
+      behavior = Behavior::GENERATE_RAINBOW_TABLE;
+      rainbow_table_filename = &argv[x][25];
     } else if (!strcmp(argv[x], "--decode-sjis")) {
       behavior = Behavior::DECODE_SJIS;
     } else if (!strncmp(argv[x], "--decode-gci=", 13)) {
@@ -322,6 +360,12 @@ int main(int argc, char** argv) {
     } else if (!strncmp(argv[x], "--cat-client=", 13)) {
       behavior = Behavior::CAT_CLIENT;
       cat_client_remote = make_sockaddr_storage(parse_netloc(&argv[x][13])).first;
+    } else if (!strncmp(argv[x], "--threads=", 10)) {
+      num_threads = strtoull(&argv[x][13], nullptr, 0);
+    } else if (!strncmp(argv[x], "--match-length=", 15)) {
+      match_length = strtoull(&argv[x][15], nullptr, 0);
+    } else if (!strncmp(argv[x], "--rainbow-table=", 16)) {
+      rainbow_table_filename = &argv[x][16];
     } else if (!strcmp(argv[x], "--patch")) {
       cli_version = GameVersion::PATCH;
     } else if (!strcmp(argv[x], "--dc")) {
@@ -338,10 +382,18 @@ int main(int argc, char** argv) {
       seed = &argv[x][7];
     } else if (!strncmp(argv[x], "--key=", 6)) {
       key_file_name = &argv[x][6];
+    } else if (!strncmp(argv[x], "--encrypted=", 12)) {
+      find_decryption_seed_ciphertext = &argv[x][12];
+    } else if (!strncmp(argv[x], "--decrypted=", 12)) {
+      find_decryption_seed_plaintexts.emplace_back(&argv[x][12]);
     } else if (!strcmp(argv[x], "--parse-data")) {
       parse_data = true;
-    } else if (!strcmp(argv[x], "--byteswap-data")) {
-      byteswap_data = true;
+    } else if (!strcmp(argv[x], "--big-endian")) {
+      big_endian = true;
+    } else if (!strcmp(argv[x], "--skip-little-endian")) {
+      skip_little_endian = true;
+    } else if (!strcmp(argv[x], "--skip-big-endian")) {
+      skip_big_endian = true;
     } else if (!strncmp(argv[x], "--replay-log=", 13)) {
       behavior = Behavior::REPLAY_LOG;
       replay_log_filename = &argv[x][13];
@@ -389,7 +441,7 @@ int main(int argc, char** argv) {
         data = parse_data_string(data);
       }
 
-      if (byteswap_data) {
+      if (big_endian) {
         uint32_t* dwords = reinterpret_cast<uint32_t*>(data.data());
         for (size_t x = 0; x < (data.size() >> 2); x++) {
           dwords[x] = bswap32(dwords[x]);
@@ -404,7 +456,7 @@ int main(int argc, char** argv) {
         throw logic_error("invalid behavior");
       }
 
-      if (byteswap_data) {
+      if (big_endian) {
         uint32_t* dwords = reinterpret_cast<uint32_t*>(data.data());
         for (size_t x = 0; x < (data.size() >> 2); x++) {
           dwords[x] = bswap32(dwords[x]);
@@ -421,9 +473,76 @@ int main(int argc, char** argv) {
       break;
     }
 
+    case Behavior::FIND_DECRYPTION_SEED: {
+      if (find_decryption_seed_plaintexts.empty() || !find_decryption_seed_ciphertext) {
+        throw runtime_error("both --encrypted and --decrypted must be specified");
+      }
+      if (cli_version == GameVersion::BB) {
+        throw runtime_error("--find-decryption-seed cannot be used for BB ciphers");
+      }
+
+      vector<pair<string, string>> plaintexts;
+      for (const auto& plaintext_ascii : find_decryption_seed_plaintexts) {
+        string mask;
+        string data = parse_data_string(plaintext_ascii, &mask);
+        plaintexts.emplace_back(move(data), move(mask));
+      }
+      string ciphertext = parse_data_string(find_decryption_seed_ciphertext);
+
+      if (num_threads == 0) {
+        num_threads = thread::hardware_concurrency();
+      }
+
+      PSOEncryptionSeedFinder finder(ciphertext, plaintexts, num_threads);
+      PSOEncryptionSeedFinder::ThreadResults results;
+      if (!rainbow_table_filename.empty()) {
+        results = finder.find_seed(rainbow_table_filename);
+      } else {
+        using Flag = PSOEncryptionSeedFinder::Flag;
+        uint64_t flags =
+            (((cli_version == GameVersion::GC) || (cli_version == GameVersion::XB)) ? Flag::V3 : 0) |
+            (skip_little_endian ? Flag::SKIP_LITTLE_ENDIAN : 0) |
+            (skip_big_endian ? Flag::SKIP_BIG_ENDIAN : 0);
+        results = finder.find_seed(flags);
+      }
+
+      log_info("Minimum differences: %zu", results.min_differences);
+      for (auto result : results.results) {
+        if (result.differences != results.min_differences) {
+          throw logic_error("incorrect difference count in result");
+        }
+        if (result.is_indeterminate) {
+          log_info("Example match: %08" PRIX32 " (%zu)",
+              result.seed, result.differences);
+        } else {
+          log_info("Example match: %08" PRIX32 " (%zu; %s, %s)",
+              result.seed,
+              result.differences,
+              result.is_v3 ? "v3" : "v2",
+              result.is_big_endian ? "big-endian" : "little-endian");
+        }
+      }
+      for (size_t z = 0; z < results.difference_histogram.size(); z++) {
+        log_info("(Difference histogram) %zu => %zu results",
+            z, results.difference_histogram[z]);
+      }
+      break;
+    }
+
+    case Behavior::GENERATE_RAINBOW_TABLE: {
+      if (num_threads == 0) {
+        num_threads = thread::hardware_concurrency();
+      }
+      bool is_v3 = ((cli_version == GameVersion::GC) || (cli_version == GameVersion::XB));
+      PSOEncryptionSeedFinder::generate_rainbow_table(
+          rainbow_table_filename, is_v3, big_endian, match_length, num_threads);
+      break;
+    }
+
     case Behavior::DECODE_QUEST_FILE:
       if (quest_file_type == QuestFileFormat::GCI) {
-        save_file(quest_filename + ".dec", Quest::decode_gci(quest_filename));
+        int64_t dec_seed = seed.empty() ? -1 : stoul(seed, nullptr, 16);
+        save_file(quest_filename + ".dec", Quest::decode_gci(quest_filename, num_threads, dec_seed));
       } else if (quest_file_type == QuestFileFormat::DLQ) {
         save_file(quest_filename + ".dec", Quest::decode_dlq(quest_filename));
       } else if (quest_file_type == QuestFileFormat::QST) {

src/PSOEncryption.cc

@@ -105,6 +105,25 @@ void PSOV2Encryption::encrypt_big_endian(void* vdata, size_t size, bool advance)
   this->encrypt_t<be_uint32_t>(vdata, size, advance);
 }
 
+void PSOV2Encryption::encrypt_both_endian(
+    void* le_vdata, void* be_vdata, size_t size, bool advance) {
+  if (size & 3) {
+    throw invalid_argument("size must be a multiple of 4");
+  }
+  if (!advance && (size != 4)) {
+    throw logic_error("cannot peek-encrypt/decrypt with size > 4");
+  }
+  size >>= 2;
+
+  le_uint32_t* le_data = reinterpret_cast<le_uint32_t*>(le_vdata);
+  be_uint32_t* be_data = reinterpret_cast<be_uint32_t*>(be_vdata);
+  for (size_t x = 0; x < size; x++) {
+    uint32_t key = this->next(advance);
+    le_data[x] ^= key;
+    be_data[x] ^= key;
+  }
+}
+
 PSOEncryption::Type PSOV2Encryption::type() const {
   return Type::V2;
 }
@@ -195,6 +214,27 @@ void PSOV3Encryption::encrypt_big_endian(void* vdata, size_t size, bool advance)
   this->encrypt_t<be_uint32_t>(vdata, size, advance);
 }
 
+// TODO: PSOV2Encryption an PSOV3Encryption should have a base class in common
+// that implements this function, because it's identical in both classes
+void PSOV3Encryption::encrypt_both_endian(
+    void* le_vdata, void* be_vdata, size_t size, bool advance) {
+  if (size & 3) {
+    throw invalid_argument("size must be a multiple of 4");
+  }
+  if (!advance && (size != 4)) {
+    throw logic_error("cannot peek-encrypt/decrypt with size > 4");
+  }
+  size >>= 2;
+
+  le_uint32_t* le_data = reinterpret_cast<le_uint32_t*>(le_vdata);
+  be_uint32_t* be_data = reinterpret_cast<be_uint32_t*>(be_vdata);
+  for (size_t x = 0; x < size; x++) {
+    uint32_t key = this->next(advance);
+    le_data[x] ^= key;
+    be_data[x] ^= key;
+  }
+}
+
 PSOEncryption::Type PSOV3Encryption::type() const {
   return Type::V3;
 }

src/PSOEncryption.hh

@@ -48,6 +48,7 @@ public:
 
   virtual void encrypt(void* data, size_t size, bool advance = true);
   void encrypt_big_endian(void* data, size_t size, bool advance = true);
+  void encrypt_both_endian(void* le_data, void* be_data, size_t size, bool advance = true);
 
   uint32_t next(bool advance = true);
 
@@ -69,6 +70,7 @@ public:
 
   virtual void encrypt(void* data, size_t size, bool advance = true);
   void encrypt_big_endian(void* data, size_t size, bool advance = true);
+  void encrypt_both_endian(void* le_data, void* be_data, size_t size, bool advance = true);
 
   uint32_t next(bool advance = true);
 

src/PSOEncryptionSeedFinder.cc

@@ -0,0 +1,359 @@
+#include "PSOEncryptionSeedFinder.hh"
+
+#include <atomic>
+#include <memory>
+#include <thread>
+#include <vector>
+#include <phosg/Time.hh>
+
+#include "PSOEncryption.hh"
+
+using namespace std;
+
+
+
+static size_t difference_match(const string& data1, const string& data2) {
+  if (data1.size() != data2.size()) {
+    return max<size_t>(data1.size(), data2.size());
+  }
+  size_t differences = 0;
+  for (size_t z = 0; z < data1.size(); z++) {
+    if (data1[z] != data2[z]) {
+      differences++;
+    }
+  }
+  return differences;
+}
+
+
+
+PSOEncryptionSeedFinder::PSOEncryptionSeedFinder(
+    const std::string& ciphertext,
+    const std::vector<std::pair<std::string, std::string>>& plaintexts,
+    size_t num_threads)
+  : ciphertext(ciphertext), plaintexts(plaintexts), num_threads(num_threads) {
+  if (num_threads == 0) {
+    throw logic_error("must use at least one thread");
+  }
+  if (this->ciphertext.empty() || (this->ciphertext.size() & 3)) {
+    throw runtime_error("ciphertext length must be a nonzero multiple of 4");
+  }
+  if (this->plaintexts.empty()) {
+    throw runtime_error("no plaintexts provided");
+  }
+  size_t plaintext_size = this->plaintexts[0].first.size();
+  for (const auto& plaintext : this->plaintexts) {
+    if (plaintext.first.size() != plaintext_size) {
+      throw runtime_error("plaintexts are not all the same size");
+    }
+    if (plaintext.first.size() != plaintext.second.size()) {
+      throw logic_error("plaintext and plaintext mask are not the same size");
+    }
+  }
+}
+
+
+
+PSOEncryptionSeedFinder::Result::Result(uint32_t seed, size_t differences)
+  : seed(seed),
+    differences(differences),
+    is_indeterminate(true),
+    is_big_endian(false),
+    is_v3(false) { }
+PSOEncryptionSeedFinder::Result::Result(
+    uint32_t seed, size_t differences, bool is_big_endian, bool is_v3)
+  : seed(seed),
+    differences(differences),
+    is_indeterminate(false),
+    is_big_endian(is_big_endian),
+    is_v3(is_v3) { }
+
+
+
+void PSOEncryptionSeedFinder::ThreadResults::add_result(const Result& res) {
+  if (res.differences < this->min_differences) {
+    this->results.clear();
+    this->min_differences = res.differences;
+  }
+  if ((res.differences == this->min_differences) && (this->results.size() < 10)) {
+    this->results.emplace_back(res);
+  }
+  if (this->difference_histogram.size() <= res.differences) {
+    this->difference_histogram.resize(res.differences + 1, 0);
+  }
+  this->difference_histogram[res.differences]++;
+}
+
+void PSOEncryptionSeedFinder::ThreadResults::combine_from(
+    const ThreadResults& other) {
+  if (this->min_differences > other.min_differences) {
+    this->min_differences = other.min_differences;
+    this->results = other.results;
+  } else if (this->min_differences == other.min_differences) {
+    this->results.insert(this->results.end(), other.results.begin(), other.results.end());
+  }
+  if (this->difference_histogram.size() < other.difference_histogram.size()) {
+    this->difference_histogram.resize(other.difference_histogram.size(), 0);
+  }
+  for (size_t z = 0; z < other.difference_histogram.size(); z++) {
+    this->difference_histogram[z] += other.difference_histogram[z];
+  }
+}
+
+
+
+PSOEncryptionSeedFinder::ThreadResults PSOEncryptionSeedFinder::find_seed(
+    uint64_t flags) {
+  // TODO: Use a specific logger here
+  log_info("Searching for decryption key (%s, %zu threads)",
+      (flags & Flag::V3) ? "v3" : "v2", this->num_threads);
+  return this->parallel_find_seed_t(
+      &PSOEncryptionSeedFinder::find_seed_without_rainbow_table_thread_fn,
+      this,
+      flags);
+}
+
+PSOEncryptionSeedFinder::ThreadResults PSOEncryptionSeedFinder::find_seed(
+    const string& rainbow_table_filename) {
+  size_t plaintext_size = this->plaintexts[0].first.size();
+
+  scoped_fd fd(rainbow_table_filename, O_RDONLY);
+  int64_t expected_rainbow_table_size = static_cast<int64_t>(plaintext_size) << 32;
+  if (fstat(fd).st_size != expected_rainbow_table_size) {
+    throw runtime_error("rainbow table size is incorrect");
+  }
+
+  // TODO: Use a specific logger here
+  log_info("Searching for decryption key (%zu threads) using rainbow table %s",
+      this->num_threads, rainbow_table_filename.c_str());
+  return this->parallel_find_seed_t(
+      &PSOEncryptionSeedFinder::find_seed_with_rainbow_table_thread_fn,
+      this,
+      static_cast<int>(fd),
+      0x1000);
+}
+
+void PSOEncryptionSeedFinder::generate_rainbow_table(
+    const std::string& filename,
+    bool is_v3,
+    bool is_big_endian,
+    size_t match_length,
+    size_t num_threads) {
+  if ((match_length == 0) || (match_length & 3)) {
+    throw runtime_error("match length must be a nonzero multiple of 4");
+  }
+  if (num_threads == 0) {
+    throw logic_error("must use at least one thread");
+  }
+
+  uint64_t file_size = static_cast<uint64_t>(match_length) << 32;
+  string file_size_str = format_size(file_size);
+
+  scoped_fd fd(filename, O_CREAT | O_WRONLY);
+  log_info("Allocating file space for rainbow table (match_length=%zu bytes => table size is %s)",
+      match_length, file_size_str.c_str());
+  if (ftruncate(fd, file_size) < 0) {
+    throw runtime_error("cannot allocate file space for table");
+  }
+
+  size_t page_size = 0x1000;
+
+  PSOEncryptionSeedFinder::parallel_all_seeds_t(
+      num_threads,
+      &PSOEncryptionSeedFinder::generate_rainbow_table_thread_fn,
+      static_cast<int>(fd),
+      match_length,
+      page_size,
+      is_v3,
+      is_big_endian);
+
+  log_info("Wrote %s to rainbow table %s\n", file_size_str.c_str(), filename.c_str());
+}
+
+void PSOEncryptionSeedFinder::parallel_all_seeds(
+    size_t num_threads, function<bool(uint32_t, size_t)> fn) {
+  PSOEncryptionSeedFinder::parallel_all_seeds_t(
+      num_threads,
+      &PSOEncryptionSeedFinder::parallel_all_seeds_thread_fn,
+      fn);
+}
+
+
+
+template <typename... ThreadArgTs>
+void PSOEncryptionSeedFinder::parallel_all_seeds_t(
+    size_t num_threads, ThreadArgTs... args) {
+  atomic<uint64_t> current_seed(0);
+  vector<thread> threads;
+  while (threads.size() < num_threads) {
+    threads.emplace_back(args..., ref(current_seed), threads.size());
+  }
+
+  uint64_t start_time = now();
+  uint64_t displayed_current_seed;
+  while ((displayed_current_seed = current_seed.load()) < 0x100000000) {
+
+    uint64_t elapsed_time = now() - start_time;
+    string elapsed_str = format_duration(elapsed_time);
+
+    string remaining_str;
+    if (displayed_current_seed) {
+      uint64_t total_time = (elapsed_time << 32) / displayed_current_seed;
+      uint64_t remaining_time = total_time - elapsed_time;
+      remaining_str = format_duration(remaining_time);
+    } else {
+      remaining_str = "...";
+    }
+
+    fprintf(stderr, "... %08" PRIX64 " (%s / -%s)\r", displayed_current_seed,
+        elapsed_str.c_str(), remaining_str.c_str());
+    usleep(1000000);
+  }
+
+  log_info("Waiting for worker threads to terminate\n");
+  for (auto& t : threads) {
+    t.join();
+  }
+}
+
+template <typename... ThreadArgTs>
+PSOEncryptionSeedFinder::ThreadResults PSOEncryptionSeedFinder::parallel_find_seed_t(
+    ThreadArgTs... args) {
+
+  vector<ThreadResults> all_thread_results;
+  all_thread_results.resize(this->num_threads);
+  this->parallel_all_seeds_t(this->num_threads, args..., ref(all_thread_results));
+
+  ThreadResults overall_results = all_thread_results[0];
+  for (const auto& thread_results : all_thread_results) {
+    overall_results.combine_from(thread_results);
+  }
+  return overall_results;
+}
+
+
+
+void PSOEncryptionSeedFinder::parallel_all_seeds_thread_fn(
+    function<bool(uint32_t, size_t)> fn,
+    atomic<uint64_t>& current_seed,
+    size_t thread_num) {
+  uint64_t seed;
+  while ((seed = current_seed.fetch_add(1)) < 0x100000000) {
+    if (fn(seed, thread_num)) {
+      current_seed = 0x100000000;
+    }
+  }
+}
+
+void PSOEncryptionSeedFinder::find_seed_without_rainbow_table_thread_fn(
+    uint64_t flags,
+    vector<ThreadResults>& all_results,
+    atomic<uint64_t>& current_seed,
+    size_t thread_num) {
+  size_t plaintext_size = this->plaintexts[0].first.size();
+
+  auto& results = all_results.at(thread_num);
+  results.results.clear();
+  results.min_differences = plaintext_size + 1;
+  results.difference_histogram.clear();
+
+  bool is_v3 = flags & Flag::V3;
+  bool skip_little_endian = flags & Flag::SKIP_LITTLE_ENDIAN;
+  bool skip_big_endian = flags & Flag::SKIP_BIG_ENDIAN;
+
+  uint64_t seed;
+  while ((seed = current_seed.fetch_add(1)) < 0x100000000) {
+    string be_decrypt_buf = this->ciphertext.substr(0, plaintext_size);
+    string le_decrypt_buf = this->ciphertext.substr(0, plaintext_size);
+    if (is_v3) {
+      PSOV3Encryption(seed).encrypt_both_endian(
+          le_decrypt_buf.data(),
+          be_decrypt_buf.data(),
+          be_decrypt_buf.size());
+    } else {
+      PSOV2Encryption(seed).encrypt_both_endian(
+          le_decrypt_buf.data(),
+          be_decrypt_buf.data(),
+          be_decrypt_buf.size());
+    }
+
+    for (const auto& plaintext : this->plaintexts) {
+      if (!skip_little_endian) {
+        size_t diff = difference_match(le_decrypt_buf, plaintext.first);
+        results.add_result(Result(seed, diff, false, is_v3));
+      } else if (!skip_big_endian) {
+        size_t diff = difference_match(be_decrypt_buf, plaintext.first);
+        results.add_result(Result(seed, diff, true, is_v3));
+      }
+    }
+    if (results.min_differences == 0) {
+      current_seed = 0x100000000;
+    }
+  }
+}
+
+void PSOEncryptionSeedFinder::find_seed_with_rainbow_table_thread_fn(
+    int fd,
+    size_t page_size,
+    vector<ThreadResults>& all_results,
+    atomic<uint64_t>& current_seed,
+    size_t thread_num) {
+  size_t plaintext_size = this->plaintexts[0].first.size();
+
+  auto& results = all_results.at(thread_num);
+  results.results.clear();
+  results.min_differences = plaintext_size + 1;
+  results.difference_histogram.clear();
+
+  uint64_t seed;
+  string rainbow_buf(page_size * plaintext_size, '\0');
+  while ((seed = current_seed.fetch_add(page_size)) < 0x100000000) {
+    preadx(fd, rainbow_buf.data(), rainbow_buf.size(), seed * plaintext_size);
+    for (size_t z = 0; z < page_size; z++) {
+      for (size_t x = 0; x < plaintext_size; x++) {
+        rainbow_buf[z * plaintext_size + x] ^= this->ciphertext[x];
+      }
+      for (const auto& plaintext : this->plaintexts) {
+        size_t diff = difference_match(
+            &rainbow_buf[z * plaintext_size], plaintext.first);
+        results.add_result(Result(seed, diff));
+      }
+      if (results.min_differences == 0) {
+        current_seed = 0x100000000;
+      }
+    }
+  }
+}
+
+void PSOEncryptionSeedFinder::generate_rainbow_table_thread_fn(
+    int fd,
+    size_t match_length,
+    size_t page_size,
+    bool is_v3,
+    bool is_big_endian,
+    atomic<uint64_t>& current_seed,
+    size_t) {
+  uint64_t seed;
+  string buf(match_length * page_size, '\0');
+  while ((seed = current_seed.fetch_add(page_size)) < 0x100000000) {
+    memset(buf.data(), 0, buf.size());
+    for (size_t z = 0; z < page_size; z++) {
+      if (is_v3) {
+        PSOV3Encryption crypt(seed + z);
+        if (is_big_endian) {
+          crypt.encrypt_big_endian(buf.data() + z * match_length, match_length);
+        } else {
+          crypt.encrypt(buf.data() + z * match_length, match_length);
+        }
+      } else {
+        PSOV2Encryption crypt(seed + z);
+        if (is_big_endian) {
+          crypt.encrypt_big_endian(buf.data() + z * match_length, match_length);
+        } else {
+          crypt.encrypt(buf.data() + z * match_length, match_length);
+        }
+      }
+    }
+    pwritex(fd, buf.data(), buf.size(), seed * match_length);
+  }
+}

src/PSOEncryptionSeedFinder.hh

@@ -0,0 +1,99 @@
+#pragma once
+
+#include <inttypes.h>
+#include <stddef.h>
+
+#include <atomic>
+#include <string>
+#include <thread>
+#include <utility>
+#include <vector>
+
+
+
+class PSOEncryptionSeedFinder {
+public:
+  PSOEncryptionSeedFinder(
+      const std::string& ciphertext,
+      const std::vector<std::pair<std::string, std::string>>& plaintexts,
+      size_t num_threads);
+  ~PSOEncryptionSeedFinder() = default;
+
+  enum Flag {
+    V3                 = 0x01,
+    SKIP_LITTLE_ENDIAN = 0x02,
+    SKIP_BIG_ENDIAN    = 0x04,
+  };
+
+  struct Result {
+    uint32_t seed;
+    size_t differences;
+    bool is_indeterminate;
+    bool is_big_endian;
+    bool is_v3;
+
+    Result(uint32_t seed, size_t differences);
+    Result(uint32_t seed, size_t differences, bool is_big_endian, bool is_v3);
+  };
+
+  struct ThreadResults {
+    std::vector<Result> results;
+    size_t min_differences;
+    std::vector<size_t> difference_histogram;
+
+    void add_result(const Result& res);
+    void combine_from(const ThreadResults& other);
+  };
+
+  ThreadResults find_seed(std::function<bool(uint32_t, size_t)> fn);
+
+  ThreadResults find_seed(uint64_t flags);
+  ThreadResults find_seed(const std::string& rainbow_table_filename);
+
+  static void generate_rainbow_table(
+      const std::string& filename,
+      bool is_v3,
+      bool is_big_endian,
+      size_t match_length,
+      size_t num_threads);
+
+  static void parallel_all_seeds(
+      size_t num_threads, std::function<bool(uint32_t, size_t)> fn);
+
+private:
+  template <typename... ThreadArgTs>
+  static void parallel_all_seeds_t(size_t num_threads, ThreadArgTs... args);
+  template <typename... ThreadArgTs>
+  ThreadResults parallel_find_seed_t(ThreadArgTs... args);
+
+
+  static void parallel_all_seeds_thread_fn(
+      std::function<bool(uint32_t, size_t)> fn,
+      std::atomic<uint64_t>& current_seed,
+      size_t thread_num);
+
+  void find_seed_without_rainbow_table_thread_fn(
+      uint64_t flags,
+      std::vector<ThreadResults>& all_results,
+      std::atomic<uint64_t>& current_seed,
+      size_t thread_num);
+  void find_seed_with_rainbow_table_thread_fn(
+      int fd,
+      size_t page_size,
+      std::vector<ThreadResults>& all_results,
+      std::atomic<uint64_t>& current_seed,
+      size_t thread_num);
+
+  static void generate_rainbow_table_thread_fn(
+      int fd,
+      size_t match_length,
+      size_t page_size,
+      bool is_v3,
+      bool is_big_endian,
+      std::atomic<uint64_t>& current_seed,
+      size_t thread_num);
+
+  std::string ciphertext;
+  std::vector<std::pair<std::string, std::string>> plaintexts;
+  size_t num_threads;
+};

src/Quest.cc

@@ -1,10 +1,12 @@
 #include "Quest.hh"
 
 #include <algorithm>
+#include <mutex>
 #include <string>
 #include <unordered_map>
 #include <phosg/Filesystem.hh>
 #include <phosg/Encoding.hh>
+#include <phosg/Hash.hh>
 #include <phosg/Random.hh>
 #include <phosg/Strings.hh>
 
@@ -12,15 +14,164 @@
 #include "CommandFormats.hh"
 #include "Compression.hh"
 #include "PSOEncryption.hh"
+#include "PSOEncryptionSeedFinder.hh"
 #include "Text.hh"
 
 using namespace std;
 
 
 
+// GCI decoding logic
+
+struct ShuffleTables {
+  uint8_t forward_table[0x100]; // table1 / 804FB9B8
+  uint8_t reverse_table[0x100]; // table2 / 804FBAB8
+
+  ShuffleTables(PSOV2Encryption& crypt) {
+    for (size_t x = 0; x < 0x100; x++) {
+      this->forward_table[x] = x;
+    }
+
+    int32_t r28 = 0xFF;
+    uint8_t* r31 = &this->forward_table[0xFF];
+    while (r28 >= 0) {
+      uint32_t r3 = this->pseudorand(crypt, r28 + 1);
+      if (r3 >= 0x100) {
+        throw logic_error("bad r3");
+      }
+      uint8_t t = this->forward_table[r3];
+      this->forward_table[r3] = *r31;
+      *r31 = t;
+
+      this->reverse_table[t] = r28;
+      r31--;
+      r28--;
+    }
+  }
+
+  static uint32_t pseudorand(PSOV2Encryption& crypt, uint32_t prev) {
+    return (((prev & 0xFFFF) * ((crypt.next() >> 16) & 0xFFFF)) >> 16) & 0xFFFF;
+  }
+
+  void shuffle(void* vdest, const void* vsrc, size_t size, bool reverse) {
+    uint8_t* dest = reinterpret_cast<uint8_t*>(vdest);
+    const uint8_t* src = reinterpret_cast<const uint8_t*>(vsrc);
+    const uint8_t* table = reverse ? this->reverse_table : this->forward_table;
+
+    for (size_t block_offset = 0; block_offset < (size & 0xFFFFFF00); block_offset += 0x100) {
+      for (size_t z = 0; z < 0x100; z++) {
+        dest[block_offset + table[z]] = src[block_offset + z];
+      }
+    }
+
+    // Any remaining bytes that don't fill an entire block are not shuffled
+    memcpy(&dest[size & 0xFFFFFF00], &src[size & 0xFFFFFF00], size & 0xFF);
+  }
+};
+
+struct PSOGCIFileHeader {
+  parray<uint8_t, 0x40> gci_header;
+  parray<uint8_t, 0x40> pso_header;
+  parray<uint8_t, 0x2000> banner_and_icon;
+  // data_size specifies the number of bytes in the encrypted section, including
+  // the encrypted header (below) and all encrypted data after it.
+  be_uint32_t data_size;
+  // To compute checksum, set checksum to zero, then compute the CRC32 of all
+  // fields in this struct except gci_header. (Yes, including the checksum
+  // field, which is temporarily zero.)
+  be_uint32_t checksum;
+
+  bool checksum_correct() const {
+    uint32_t cs = crc32(&this->pso_header, sizeof(this->pso_header));
+    cs = crc32(&this->banner_and_icon, sizeof(this->banner_and_icon), cs);
+    cs = crc32(&this->data_size, sizeof(this->data_size), cs);
+    cs = crc32("\0\0\0\0", 4, cs);
+    return (cs == this->checksum);
+  }
+} __attribute__((packed));
+
+struct PSOGCIFileEncryptedHeader {
+  be_uint32_t round2_seed;
+  // To compute checksum, set checksum to zero, then compute the CRC32 of the
+  // entire data section, including this header struct (but not the unencrypted
+  // header struct).
+  be_uint32_t checksum;
+  le_uint32_t decompressed_size;
+  le_uint32_t round3_seed;
+  // Data follows here.
+} __attribute__((packed));
+
+string decrypt_gci_data_section(
+    const void* data_section, size_t size, uint32_t seed) {
+  string decrypted(size, '\0');
+  {
+    PSOV2Encryption shuf_crypt(seed);
+    ShuffleTables shuf(shuf_crypt);
+    shuf.shuffle(decrypted.data(), data_section, size, true);
+  }
+
+  decrypted.resize((decrypted.size() + 3) & (~3));
+  auto* be_dwords = reinterpret_cast<be_uint32_t*>(decrypted.data());
+
+  PSOV2Encryption crypt(seed);
+  for (size_t z = 0; z < decrypted.size() / sizeof(be_uint32_t); z++) {
+    be_dwords[z] = crypt.next() - be_dwords[z];
+  }
+
+  auto* header = reinterpret_cast<PSOGCIFileEncryptedHeader*>(
+      decrypted.data());
+  PSOV2Encryption(header->round2_seed).encrypt_big_endian(
+      decrypted.data() + 4, (decrypted.size() - 4) & (~3));
+
+  uint32_t expected_crc = header->checksum;
+  header->checksum = 0;
+  uint32_t actual_crc = crc32(decrypted.data(), decrypted.size());
+  header->checksum = expected_crc;
+
+  if (expected_crc != actual_crc) {
+    throw runtime_error("incorrect decrypted data section checksum");
+  }
+
+  PSOV2Encryption(header->round3_seed).decrypt(
+      decrypted.data() + sizeof(PSOGCIFileEncryptedHeader),
+      decrypted.size() - sizeof(PSOGCIFileEncryptedHeader));
+
+  string ret = decrypted.substr(sizeof(PSOGCIFileEncryptedHeader));
+  if (prs_decompress_size(ret) != header->decompressed_size) {
+    throw runtime_error("decompressed size does not match size in header");
+  }
+
+  return ret;
+}
+
+string find_seed_and_decrypt_gci_data_section(
+    const void* data_section, size_t size, size_t num_threads) {
+  mutex result_lock;
+  string result;
+  PSOEncryptionSeedFinder::parallel_all_seeds(num_threads, [&](
+      uint32_t seed, size_t) {
+    try {
+      string ret = decrypt_gci_data_section(data_section, size, seed);
+      lock_guard<mutex> g(result_lock);
+      result = move(ret);
+      return true;
+    } catch (const runtime_error&) {
+      return false;
+    }
+  });
+
+  if (!result.empty()) {
+    return result;
+  } else {
+    throw runtime_error("no seed found");
+  }
+}
+
+
+
 struct PSODownloadQuestHeader {
   // When sending a DLQ to the client, this is the DECOMPRESSED size. When
-  // reading it from a GCI file, this is the COMPRESSED size.
+  // reading it from an (unencrypted) GCI file, this is the COMPRESSED size.
   le_uint32_t size;
   // Note: use PSO PC encryption, even for GC quests.
   le_uint32_t encryption_seed;
@@ -228,7 +379,7 @@ Quest::Quest(const string& bin_filename)
 
   // get the category from the second token if needed
   if (this->category == QuestCategory::UNKNOWN) {
-    static const unordered_map<std::string, QuestCategory> name_to_category({
+    static const unordered_map<string, QuestCategory> name_to_category({
       {"ret", QuestCategory::RETRIEVAL},
       {"ext", QuestCategory::EXTERMINATION},
       {"evt", QuestCategory::EVENT},
@@ -245,7 +396,7 @@ Quest::Quest(const string& bin_filename)
     tokens.erase(tokens.begin() + 1);
   }
 
-  static const unordered_map<std::string, GameVersion> name_to_version({
+  static const unordered_map<string, GameVersion> name_to_version({
     {"d1",  GameVersion::DC},
     {"dc",  GameVersion::DC},
     {"pc",  GameVersion::PC},
@@ -348,7 +499,7 @@ Quest::Quest(const string& bin_filename)
   }
 }
 
-static string basename_for_filename(const std::string& filename) {
+static string basename_for_filename(const string& filename) {
   size_t slash_pos = filename.rfind('/');
   if (slash_pos != string::npos) {
     return filename.substr(slash_pos + 1);
@@ -356,11 +507,11 @@ static string basename_for_filename(const std::string& filename) {
   return filename;
 }
 
-std::string Quest::bin_filename() const {
+string Quest::bin_filename() const {
   return basename_for_filename(this->file_basename + ".bin");
 }
 
-std::string Quest::dat_filename() const {
+string Quest::dat_filename() const {
   return basename_for_filename(this->file_basename + ".dat");
 }
 
@@ -374,7 +525,7 @@ shared_ptr<const string> Quest::bin_contents() const {
         this->bin_contents_ptr.reset(new string(prs_compress(load_file(this->file_basename + ".bind"))));
         break;
       case FileFormat::BIN_DAT_GCI:
-        this->bin_contents_ptr.reset(new string(this->decode_gci(this->file_basename + ".bin.gci")));
+        this->bin_contents_ptr.reset(new string(this->decode_gci(this->file_basename + ".bin.gci", false)));
         break;
       case FileFormat::BIN_DAT_DLQ:
         this->bin_contents_ptr.reset(new string(this->decode_dlq(this->file_basename + ".bin.dlq")));
@@ -402,7 +553,7 @@ shared_ptr<const string> Quest::dat_contents() const {
         this->dat_contents_ptr.reset(new string(prs_compress(load_file(this->file_basename + ".datd"))));
         break;
       case FileFormat::BIN_DAT_GCI:
-        this->dat_contents_ptr.reset(new string(this->decode_gci(this->file_basename + ".dat.gci")));
+        this->dat_contents_ptr.reset(new string(this->decode_gci(this->file_basename + ".dat.gci", false)));
         break;
       case FileFormat::BIN_DAT_DLQ:
         this->dat_contents_ptr.reset(new string(this->decode_dlq(this->file_basename + ".dat.dlq")));
@@ -420,51 +571,50 @@ shared_ptr<const string> Quest::dat_contents() const {
   return this->dat_contents_ptr;
 }
 
-string Quest::decode_gci(const string& filename) {
-
+string Quest::decode_gci(
+    const string& filename, ssize_t find_seed_num_threads, int64_t known_seed) {
   string data = load_file(filename);
-  if (data.size() < 0x2080 + sizeof(PSODownloadQuestHeader)) {
-    throw runtime_error(string_printf(
-        "GCI file is truncated before download quest header (have 0x%zX bytes)", data.size()));
-  }
-  PSODownloadQuestHeader* h = reinterpret_cast<PSODownloadQuestHeader*>(
-      data.data() + 0x2080);
 
-  string compressed_data_with_header = data.substr(0x2088, h->size);
-
-  // For now, we can only load unencrypted quests, unfortunately
-  // TODO: Figure out how GCI encryption works and implement it here.
-
-  // Unlike the DLQ header, this one is stored little-endian. The compressed
-  // data immediately follows this header.
-  struct DecryptedHeader {
-    uint32_t unknown1;
-    uint32_t unknown2;
-    uint32_t decompressed_size;
-    uint32_t unknown4;
-  } __attribute__((packed));
-  if (compressed_data_with_header.size() < sizeof(DecryptedHeader)) {
-    throw runtime_error("GCI file compressed data truncated during header");
-  }
-  DecryptedHeader* dh = reinterpret_cast<DecryptedHeader*>(
-      compressed_data_with_header.data());
-  if (dh->unknown1 || dh->unknown2 || dh->unknown4) {
-    throw runtime_error("GCI file appears to be encrypted");
+  StringReader r(data);
+  const auto& header = r.get<PSOGCIFileHeader>();
+  if (!header.checksum_correct()) {
+    throw runtime_error("GCI file unencrypted header checksum is incorrect");
   }
 
-  string data_to_decompress = compressed_data_with_header.substr(sizeof(DecryptedHeader));
-  size_t decompressed_bytes = prs_decompress_size(data_to_decompress);
+  const auto& encrypted_header = r.get<PSOGCIFileEncryptedHeader>(false);
+  // Unencrypted GCI files appear to always have zeroes in these fields.
+  // Encrypted GCI files are highly unlikely to have zeroes in ALL of these
+  // fields, so assume it's encrypted if any of them are nonzero.
+  if (encrypted_header.round2_seed || encrypted_header.checksum || encrypted_header.round3_seed) {
+    if (known_seed >= 0) {
+      return decrypt_gci_data_section(
+          r.getv(header.data_size), header.data_size, known_seed);
 
-  size_t expected_decompressed_bytes = dh->decompressed_size - 8;
-  if (decompressed_bytes < expected_decompressed_bytes) {
-    throw runtime_error(string_printf(
-        "GCI decompressed data is smaller than expected size (have 0x%zX bytes, expected 0x%zX bytes)",
-        decompressed_bytes, expected_decompressed_bytes));
+    } else {
+      if (find_seed_num_threads < 0) {
+        throw runtime_error("GCI file appears to be encrypted");
+      }
+      if (find_seed_num_threads == 0) {
+        find_seed_num_threads = thread::hardware_concurrency();
+      }
+      return find_seed_and_decrypt_gci_data_section(
+          r.getv(header.data_size), header.data_size, find_seed_num_threads);
+    }
+
+  } else { // Unencrypted GCI format
+    r.skip(sizeof(PSOGCIFileEncryptedHeader));
+    string compressed_data = r.read(r.remaining());
+    size_t decompressed_bytes = prs_decompress_size(compressed_data);
+
+    size_t expected_decompressed_bytes = encrypted_header.decompressed_size - 8;
+    if (decompressed_bytes < expected_decompressed_bytes) {
+      throw runtime_error(string_printf(
+          "GCI decompressed data is smaller than expected size (have 0x%zX bytes, expected 0x%zX bytes)",
+          decompressed_bytes, expected_decompressed_bytes));
+    }
+
+    return compressed_data;
   }
-
-  // The caller expects to get PRS-compressed data when calling bin_contents()
-  // and dat_contents(), so we shouldn't decompress it here.
-  return data_to_decompress;
 }
 
 string Quest::decode_dlq(const string& filename) {
@@ -609,7 +759,7 @@ pair<string, string> Quest::decode_qst(const string& filename) {
 
 
 
-QuestIndex::QuestIndex(const std::string& directory) : directory(directory) {
+QuestIndex::QuestIndex(const string& directory) : directory(directory) {
   auto filename_set = list_directory(this->directory);
   vector<string> filenames(filename_set.begin(), filename_set.end());
   sort(filenames.begin(), filenames.end());

src/Quest.hh

@@ -70,7 +70,10 @@ public:
 
   std::shared_ptr<Quest> create_download_quest() const;
 
-  static std::string decode_gci(const std::string& filename);
+  static std::string decode_gci(
+      const std::string& filename,
+      ssize_t find_seed_num_threads = -1,
+      int64_t known_seed = -1);
   static std::string decode_dlq(const std::string& filename);
   static std::pair<std::string, std::string> decode_qst(const std::string& filename);