From 1a3dd26cb311441ba0932dd7b9be228dafa3f0d3 Mon Sep 17 00:00:00 2001 From: Martin Michelsen Date: Sat, 10 Sep 2022 01:29:35 -0700 Subject: [PATCH] add GCI decryption function --- CMakeLists.txt | 1 + README.md | 3 +- src/Main.cc | 181 ++++++++++++++--- src/PSOEncryption.cc | 40 ++++ src/PSOEncryption.hh | 2 + src/PSOEncryptionSeedFinder.cc | 359 +++++++++++++++++++++++++++++++++ src/PSOEncryptionSeedFinder.hh | 99 +++++++++ src/Quest.cc | 246 +++++++++++++++++----- src/Quest.hh | 5 +- 9 files changed, 855 insertions(+), 81 deletions(-) create mode 100644 src/PSOEncryptionSeedFinder.cc create mode 100644 src/PSOEncryptionSeedFinder.hh diff --git a/CMakeLists.txt b/CMakeLists.txt index 7c99489f..2e54627e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -71,6 +71,7 @@ add_executable(newserv src/ProxyCommands.cc src/ProxyServer.cc src/PSOEncryption.cc + src/PSOEncryptionSeedFinder.cc src/PSOProtocol.cc src/Quest.cc src/RareItemSet.cc diff --git a/README.md b/README.md index 2809f4a9..0b811ad5 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,7 @@ newserv supports several versions of PSO. Specifically: This project is primarily for my own nostalgia; I offer no guarantees on how or when this project will advance. With that said, feel free to submit GitHub issues if you find bugs or have feature requests. I'd like to make the server as stable and complete as possible, but I can't promise that I'll respond to issues in a timely manner. Current known issues / missing features: +- Partially-encrypted GCI quest files probably don't work. Fix this. Also check that completely unencrypted GCI quest files actually work. - Support disconnect hooks to clean up state, like if a client disconnects during quest loading or a trade window execution. - Episode 3 battles aren't implemented. - PSOBB is not well-tested and likely will disconnect or misbehave when clients try to use unimplemented features. @@ -76,7 +77,7 @@ Standard quest files should be named like `q###-CATEGORY-VERSION.EXT`, battle qu There are multiple PSO quest formats out there; newserv supports most of them. Specifically, newserv can use quests in any of the following formats: - Compressed bin/dat format: These quests consist of two files with the same base name, a .bin file and a .dat file. (This is the format you'll get if you saved a quest with set-save-files.) - Uncompressed bin/dat format: These quests consist of two files with the same base name, a .bind file and a .datd file. -- Unencrypted GCI format: These quests also consist of a .bin and .dat file, but an encoding is applied on top of them. The filenames should end in .bin.gci and .dat.gci. (Note that there also exists an encrypted GCI format, which newserv does not support.) +- Unencrypted GCI format: These quests also consist of a .bin and .dat file, but an encoding is applied on top of them. The filenames should end in .bin.gci and .dat.gci. Note that there also exists an encrypted GCI format, which newserv does not support at runtime, but you can also use newserv to convert these files to bin/dat format and then use them with the server. Run `newserv --help` and see the `--decode-gci` option for more information. - Encrypted DLQ format: These quests also consist of a .bin and .dat file, but download quest encryption is applied on top of them. The filenames should end in .bin.dlq and .dat.dlq. - QST format: These quests consist of only a .qst file, which contains both the .bin and .dat files within it. diff --git a/src/Main.cc b/src/Main.cc index 5842c728..178b2946 100644 --- a/src/Main.cc +++ b/src/Main.cc @@ -8,6 +8,7 @@ #include #include #include +#include #include #include "CatSession.hh" @@ -16,6 +17,7 @@ #include "Loggers.hh" #include "NetworkAddresses.hh" #include "ProxyServer.hh" +#include "PSOEncryptionSeedFinder.hh" #include "ReplaySession.hh" #include "SendCommands.hh" #include "Server.hh" @@ -223,38 +225,60 @@ When options are given, newserv will do things other than running the server.\n\ Specifically:\n\ --decrypt-data\n\ --encrypt-data\n\ - If either of these options is given, newserv will read from stdin and\n\ - write the encrypted or decrypted result to stdout. By default, PSO GC\n\ - encryption is used, but this can be overridden with --pc or --bb. The\n\ - --seed option specifies the encryption seed (4 hex bytes for PC or GC, or\n\ - 48 hex bytes for BB). For BB, the --key option is required as well, and\n\ - should refer to a .nsk file in system/blueburst/keys (without the\n\ - directory or .nsk extension).\n\ + Read from stdin, encrypt or decrypt the data, and write the result to\n\ + stdout. By default, PSO V3 encryption is used, but this can be overridden\n\ + with --pc or --bb. The --seed option specifies the encryption seed (4 hex\n\ + bytes for PC or GC, or 48 hex bytes for BB). For BB, the --key option is\n\ + required as well, and refers to a .nsk file in system/blueburst/keys\n\ + (without the directory or .nsk extension). For non-BB ciphers, the\n\ + --big-endian option applies the cipher masks as big-endian instead of\n\ + little-endian, which is necessary for some GameCube file formats.\n\ + --find-decryption-seed\n\ + Perform a brute-force search for a decryption seed of the given data.\n\ + The ciphertext is specified with the --encrypted= option and the expected\n\ + plaintext is specified with the --decrypted= option. The plaintext may\n\ + include unmatched bytes (specified with the ? operator), but overall it\n\ + must be the same length as the ciphertext. By default, this option uses\n\ + PSO V3 encryption, but this can be overridden with --pc. (BB encryption\n\ + seeds are too long to be searched for with this function.) By default,\n\ + the number of worker threads is equal the the number of CPU cores in the\n\ + system, but this can be overridden with the --threads= option. To use a\n\ + rainbow table instead of computing the cipherstreams inline, use the\n\ + --rainbow-table=FILENAME option.\n\ + --generate-rainbow-table=FILENAME\n\ + Generate a decryption table for V3 encryption (or V2 if --pc is given).\n\ + The --match-length= option must be given, which specifies the match\n\ + length for the table. The total table size is the match length * 4 GB.\n\ + As for --encrypt-data, the --big-endian option specifies that the table\n\ + uses big-endian encryption. As for --find-decryption-seed, the --threads\n\ + option specifies the parallelism for generating the table.\n\ --decode-sjis\n\ - If this option is given, newserv applies its text decoding algorithm to\n\ - the data on stdin, producing little-endian UTF-16 data on stdout.\n\ + Apply newserv\'s text decoding algorithm to the data on stdin, producing\n\ + little-endian UTF-16 data on stdout.\n\ --decode-gci=FILENAME\n\ --decode-dlq=FILENAME\n\ --decode-qst=FILENAME\n\ - If any of these options are given, newserv will decode the given quest\n\ - file into a compressed, unencrypted .bin or .dat file (or in the case of)\n\ - --decode-qst, both a .bin and a .dat file).\n\ + Decode the given quest file into a compressed, unencrypted .bin or .dat\n\ + file (or in the case of --decode-qst, both a .bin and a .dat file). The\n\ + --decode-gci option can be used to decrypt encrypted GCI files. If you\n\ + know the player\'s serial number who generated the GCI file, use the\n\ + --seed= option and give the serial number (as a hex-encoded integer). If\n\ + you don\'t know the serial number, newserv will find it via a brute-force\n\ + search, but this will likely take a long time.\n\ --cat-client=ADDR:PORT\n\ - If this option is given, newserv will behave as if it's a PSO client, and\n\ - will connect to the given server. It will then print all the received\n\ - commands to stdout, and forward any commands typed into stdin to the\n\ - remote server. It is assumed that the input and output are terminals, so\n\ - all commands are hex-encoded. The --patch, --dc, --pc, --gc, and --bb\n\ - options can be used to select the command format and encryption. If --bb\n\ - is used, the --key option is also required (as in --decrypt-data above).\n\ + Connect to the given server and simulate a PSO client. newserv will then\n\ + print all the received commands to stdout, and forward any commands typed\n\ + into stdin to the remote server. It is assumed that the input and output\n\ + are terminals, so all commands are hex-encoded. The --patch, --dc, --pc,\n\ + --gc, and --bb options can be used to select the command format and\n\ + encryption. If --bb is used, the --key option is also required (as in\n\ + --decrypt-data above).\n\ --replay-log=FILENAME\n\ - This option makes newserv replay terminal log as if it were a client\n\ - session. This is used for regression testing, to make sure client\n\ - sessions are repeatable and code changes don\'t affect existing (working)\n\ - functionality.\n\ + Replay a terminal log as if it were a client session. This is used for\n\ + regression testing, to make sure client sessions are repeatable and code\n\ + changes don\'t affect existing (working) functionality.\n\ --extract-gsl=FILENAME\n\ - This option makes newserv extract all files from a GSL archive and place\n\ - them in the current directory.\n\ + Extract all files from a GSL archive into the current directory.\n\ \n\ A few options apply to multiple modes described above:\n\ --parse-data\n\ @@ -269,6 +293,8 @@ enum class Behavior { RUN_SERVER = 0, DECRYPT_DATA, ENCRYPT_DATA, + FIND_DECRYPTION_SEED, + GENERATE_RAINBOW_TABLE, DECODE_QUEST_FILE, DECODE_SJIS, EXTRACT_GSL, @@ -290,8 +316,15 @@ int main(int argc, char** argv) { string seed; string key_file_name; const char* config_filename = "system/config.json"; + string rainbow_table_filename; bool parse_data = false; - bool byteswap_data = false; + bool big_endian = false; + bool skip_little_endian = false; + bool skip_big_endian = false; + size_t num_threads = 0; + size_t match_length = 0; + const char* find_decryption_seed_ciphertext = nullptr; + vector find_decryption_seed_plaintexts; const char* replay_log_filename = nullptr; const char* extract_gsl_filename = nullptr; const char* replay_required_access_key = ""; @@ -305,6 +338,11 @@ int main(int argc, char** argv) { behavior = Behavior::DECRYPT_DATA; } else if (!strcmp(argv[x], "--encrypt-data")) { behavior = Behavior::ENCRYPT_DATA; + } else if (!strcmp(argv[x], "--find-decryption-seed")) { + behavior = Behavior::FIND_DECRYPTION_SEED; + } else if (!strncmp(argv[x], "--generate-rainbow-table=", 25)) { + behavior = Behavior::GENERATE_RAINBOW_TABLE; + rainbow_table_filename = &argv[x][25]; } else if (!strcmp(argv[x], "--decode-sjis")) { behavior = Behavior::DECODE_SJIS; } else if (!strncmp(argv[x], "--decode-gci=", 13)) { @@ -322,6 +360,12 @@ int main(int argc, char** argv) { } else if (!strncmp(argv[x], "--cat-client=", 13)) { behavior = Behavior::CAT_CLIENT; cat_client_remote = make_sockaddr_storage(parse_netloc(&argv[x][13])).first; + } else if (!strncmp(argv[x], "--threads=", 10)) { + num_threads = strtoull(&argv[x][13], nullptr, 0); + } else if (!strncmp(argv[x], "--match-length=", 15)) { + match_length = strtoull(&argv[x][15], nullptr, 0); + } else if (!strncmp(argv[x], "--rainbow-table=", 16)) { + rainbow_table_filename = &argv[x][16]; } else if (!strcmp(argv[x], "--patch")) { cli_version = GameVersion::PATCH; } else if (!strcmp(argv[x], "--dc")) { @@ -338,10 +382,18 @@ int main(int argc, char** argv) { seed = &argv[x][7]; } else if (!strncmp(argv[x], "--key=", 6)) { key_file_name = &argv[x][6]; + } else if (!strncmp(argv[x], "--encrypted=", 12)) { + find_decryption_seed_ciphertext = &argv[x][12]; + } else if (!strncmp(argv[x], "--decrypted=", 12)) { + find_decryption_seed_plaintexts.emplace_back(&argv[x][12]); } else if (!strcmp(argv[x], "--parse-data")) { parse_data = true; - } else if (!strcmp(argv[x], "--byteswap-data")) { - byteswap_data = true; + } else if (!strcmp(argv[x], "--big-endian")) { + big_endian = true; + } else if (!strcmp(argv[x], "--skip-little-endian")) { + skip_little_endian = true; + } else if (!strcmp(argv[x], "--skip-big-endian")) { + skip_big_endian = true; } else if (!strncmp(argv[x], "--replay-log=", 13)) { behavior = Behavior::REPLAY_LOG; replay_log_filename = &argv[x][13]; @@ -389,7 +441,7 @@ int main(int argc, char** argv) { data = parse_data_string(data); } - if (byteswap_data) { + if (big_endian) { uint32_t* dwords = reinterpret_cast(data.data()); for (size_t x = 0; x < (data.size() >> 2); x++) { dwords[x] = bswap32(dwords[x]); @@ -404,7 +456,7 @@ int main(int argc, char** argv) { throw logic_error("invalid behavior"); } - if (byteswap_data) { + if (big_endian) { uint32_t* dwords = reinterpret_cast(data.data()); for (size_t x = 0; x < (data.size() >> 2); x++) { dwords[x] = bswap32(dwords[x]); @@ -421,9 +473,76 @@ int main(int argc, char** argv) { break; } + case Behavior::FIND_DECRYPTION_SEED: { + if (find_decryption_seed_plaintexts.empty() || !find_decryption_seed_ciphertext) { + throw runtime_error("both --encrypted and --decrypted must be specified"); + } + if (cli_version == GameVersion::BB) { + throw runtime_error("--find-decryption-seed cannot be used for BB ciphers"); + } + + vector> plaintexts; + for (const auto& plaintext_ascii : find_decryption_seed_plaintexts) { + string mask; + string data = parse_data_string(plaintext_ascii, &mask); + plaintexts.emplace_back(move(data), move(mask)); + } + string ciphertext = parse_data_string(find_decryption_seed_ciphertext); + + if (num_threads == 0) { + num_threads = thread::hardware_concurrency(); + } + + PSOEncryptionSeedFinder finder(ciphertext, plaintexts, num_threads); + PSOEncryptionSeedFinder::ThreadResults results; + if (!rainbow_table_filename.empty()) { + results = finder.find_seed(rainbow_table_filename); + } else { + using Flag = PSOEncryptionSeedFinder::Flag; + uint64_t flags = + (((cli_version == GameVersion::GC) || (cli_version == GameVersion::XB)) ? Flag::V3 : 0) | + (skip_little_endian ? Flag::SKIP_LITTLE_ENDIAN : 0) | + (skip_big_endian ? Flag::SKIP_BIG_ENDIAN : 0); + results = finder.find_seed(flags); + } + + log_info("Minimum differences: %zu", results.min_differences); + for (auto result : results.results) { + if (result.differences != results.min_differences) { + throw logic_error("incorrect difference count in result"); + } + if (result.is_indeterminate) { + log_info("Example match: %08" PRIX32 " (%zu)", + result.seed, result.differences); + } else { + log_info("Example match: %08" PRIX32 " (%zu; %s, %s)", + result.seed, + result.differences, + result.is_v3 ? "v3" : "v2", + result.is_big_endian ? "big-endian" : "little-endian"); + } + } + for (size_t z = 0; z < results.difference_histogram.size(); z++) { + log_info("(Difference histogram) %zu => %zu results", + z, results.difference_histogram[z]); + } + break; + } + + case Behavior::GENERATE_RAINBOW_TABLE: { + if (num_threads == 0) { + num_threads = thread::hardware_concurrency(); + } + bool is_v3 = ((cli_version == GameVersion::GC) || (cli_version == GameVersion::XB)); + PSOEncryptionSeedFinder::generate_rainbow_table( + rainbow_table_filename, is_v3, big_endian, match_length, num_threads); + break; + } + case Behavior::DECODE_QUEST_FILE: if (quest_file_type == QuestFileFormat::GCI) { - save_file(quest_filename + ".dec", Quest::decode_gci(quest_filename)); + int64_t dec_seed = seed.empty() ? -1 : stoul(seed, nullptr, 16); + save_file(quest_filename + ".dec", Quest::decode_gci(quest_filename, num_threads, dec_seed)); } else if (quest_file_type == QuestFileFormat::DLQ) { save_file(quest_filename + ".dec", Quest::decode_dlq(quest_filename)); } else if (quest_file_type == QuestFileFormat::QST) { diff --git a/src/PSOEncryption.cc b/src/PSOEncryption.cc index c8b4254d..f12f7ad4 100644 --- a/src/PSOEncryption.cc +++ b/src/PSOEncryption.cc @@ -105,6 +105,25 @@ void PSOV2Encryption::encrypt_big_endian(void* vdata, size_t size, bool advance) this->encrypt_t(vdata, size, advance); } +void PSOV2Encryption::encrypt_both_endian( + void* le_vdata, void* be_vdata, size_t size, bool advance) { + if (size & 3) { + throw invalid_argument("size must be a multiple of 4"); + } + if (!advance && (size != 4)) { + throw logic_error("cannot peek-encrypt/decrypt with size > 4"); + } + size >>= 2; + + le_uint32_t* le_data = reinterpret_cast(le_vdata); + be_uint32_t* be_data = reinterpret_cast(be_vdata); + for (size_t x = 0; x < size; x++) { + uint32_t key = this->next(advance); + le_data[x] ^= key; + be_data[x] ^= key; + } +} + PSOEncryption::Type PSOV2Encryption::type() const { return Type::V2; } @@ -195,6 +214,27 @@ void PSOV3Encryption::encrypt_big_endian(void* vdata, size_t size, bool advance) this->encrypt_t(vdata, size, advance); } +// TODO: PSOV2Encryption an PSOV3Encryption should have a base class in common +// that implements this function, because it's identical in both classes +void PSOV3Encryption::encrypt_both_endian( + void* le_vdata, void* be_vdata, size_t size, bool advance) { + if (size & 3) { + throw invalid_argument("size must be a multiple of 4"); + } + if (!advance && (size != 4)) { + throw logic_error("cannot peek-encrypt/decrypt with size > 4"); + } + size >>= 2; + + le_uint32_t* le_data = reinterpret_cast(le_vdata); + be_uint32_t* be_data = reinterpret_cast(be_vdata); + for (size_t x = 0; x < size; x++) { + uint32_t key = this->next(advance); + le_data[x] ^= key; + be_data[x] ^= key; + } +} + PSOEncryption::Type PSOV3Encryption::type() const { return Type::V3; } diff --git a/src/PSOEncryption.hh b/src/PSOEncryption.hh index 275eee96..eef0fae9 100644 --- a/src/PSOEncryption.hh +++ b/src/PSOEncryption.hh @@ -48,6 +48,7 @@ public: virtual void encrypt(void* data, size_t size, bool advance = true); void encrypt_big_endian(void* data, size_t size, bool advance = true); + void encrypt_both_endian(void* le_data, void* be_data, size_t size, bool advance = true); uint32_t next(bool advance = true); @@ -69,6 +70,7 @@ public: virtual void encrypt(void* data, size_t size, bool advance = true); void encrypt_big_endian(void* data, size_t size, bool advance = true); + void encrypt_both_endian(void* le_data, void* be_data, size_t size, bool advance = true); uint32_t next(bool advance = true); diff --git a/src/PSOEncryptionSeedFinder.cc b/src/PSOEncryptionSeedFinder.cc new file mode 100644 index 00000000..94ae7acf --- /dev/null +++ b/src/PSOEncryptionSeedFinder.cc @@ -0,0 +1,359 @@ +#include "PSOEncryptionSeedFinder.hh" + +#include +#include +#include +#include +#include + +#include "PSOEncryption.hh" + +using namespace std; + + + +static size_t difference_match(const string& data1, const string& data2) { + if (data1.size() != data2.size()) { + return max(data1.size(), data2.size()); + } + size_t differences = 0; + for (size_t z = 0; z < data1.size(); z++) { + if (data1[z] != data2[z]) { + differences++; + } + } + return differences; +} + + + +PSOEncryptionSeedFinder::PSOEncryptionSeedFinder( + const std::string& ciphertext, + const std::vector>& plaintexts, + size_t num_threads) + : ciphertext(ciphertext), plaintexts(plaintexts), num_threads(num_threads) { + if (num_threads == 0) { + throw logic_error("must use at least one thread"); + } + if (this->ciphertext.empty() || (this->ciphertext.size() & 3)) { + throw runtime_error("ciphertext length must be a nonzero multiple of 4"); + } + if (this->plaintexts.empty()) { + throw runtime_error("no plaintexts provided"); + } + size_t plaintext_size = this->plaintexts[0].first.size(); + for (const auto& plaintext : this->plaintexts) { + if (plaintext.first.size() != plaintext_size) { + throw runtime_error("plaintexts are not all the same size"); + } + if (plaintext.first.size() != plaintext.second.size()) { + throw logic_error("plaintext and plaintext mask are not the same size"); + } + } +} + + + +PSOEncryptionSeedFinder::Result::Result(uint32_t seed, size_t differences) + : seed(seed), + differences(differences), + is_indeterminate(true), + is_big_endian(false), + is_v3(false) { } +PSOEncryptionSeedFinder::Result::Result( + uint32_t seed, size_t differences, bool is_big_endian, bool is_v3) + : seed(seed), + differences(differences), + is_indeterminate(false), + is_big_endian(is_big_endian), + is_v3(is_v3) { } + + + +void PSOEncryptionSeedFinder::ThreadResults::add_result(const Result& res) { + if (res.differences < this->min_differences) { + this->results.clear(); + this->min_differences = res.differences; + } + if ((res.differences == this->min_differences) && (this->results.size() < 10)) { + this->results.emplace_back(res); + } + if (this->difference_histogram.size() <= res.differences) { + this->difference_histogram.resize(res.differences + 1, 0); + } + this->difference_histogram[res.differences]++; +} + +void PSOEncryptionSeedFinder::ThreadResults::combine_from( + const ThreadResults& other) { + if (this->min_differences > other.min_differences) { + this->min_differences = other.min_differences; + this->results = other.results; + } else if (this->min_differences == other.min_differences) { + this->results.insert(this->results.end(), other.results.begin(), other.results.end()); + } + if (this->difference_histogram.size() < other.difference_histogram.size()) { + this->difference_histogram.resize(other.difference_histogram.size(), 0); + } + for (size_t z = 0; z < other.difference_histogram.size(); z++) { + this->difference_histogram[z] += other.difference_histogram[z]; + } +} + + + +PSOEncryptionSeedFinder::ThreadResults PSOEncryptionSeedFinder::find_seed( + uint64_t flags) { + // TODO: Use a specific logger here + log_info("Searching for decryption key (%s, %zu threads)", + (flags & Flag::V3) ? "v3" : "v2", this->num_threads); + return this->parallel_find_seed_t( + &PSOEncryptionSeedFinder::find_seed_without_rainbow_table_thread_fn, + this, + flags); +} + +PSOEncryptionSeedFinder::ThreadResults PSOEncryptionSeedFinder::find_seed( + const string& rainbow_table_filename) { + size_t plaintext_size = this->plaintexts[0].first.size(); + + scoped_fd fd(rainbow_table_filename, O_RDONLY); + int64_t expected_rainbow_table_size = static_cast(plaintext_size) << 32; + if (fstat(fd).st_size != expected_rainbow_table_size) { + throw runtime_error("rainbow table size is incorrect"); + } + + // TODO: Use a specific logger here + log_info("Searching for decryption key (%zu threads) using rainbow table %s", + this->num_threads, rainbow_table_filename.c_str()); + return this->parallel_find_seed_t( + &PSOEncryptionSeedFinder::find_seed_with_rainbow_table_thread_fn, + this, + static_cast(fd), + 0x1000); +} + +void PSOEncryptionSeedFinder::generate_rainbow_table( + const std::string& filename, + bool is_v3, + bool is_big_endian, + size_t match_length, + size_t num_threads) { + if ((match_length == 0) || (match_length & 3)) { + throw runtime_error("match length must be a nonzero multiple of 4"); + } + if (num_threads == 0) { + throw logic_error("must use at least one thread"); + } + + uint64_t file_size = static_cast(match_length) << 32; + string file_size_str = format_size(file_size); + + scoped_fd fd(filename, O_CREAT | O_WRONLY); + log_info("Allocating file space for rainbow table (match_length=%zu bytes => table size is %s)", + match_length, file_size_str.c_str()); + if (ftruncate(fd, file_size) < 0) { + throw runtime_error("cannot allocate file space for table"); + } + + size_t page_size = 0x1000; + + PSOEncryptionSeedFinder::parallel_all_seeds_t( + num_threads, + &PSOEncryptionSeedFinder::generate_rainbow_table_thread_fn, + static_cast(fd), + match_length, + page_size, + is_v3, + is_big_endian); + + log_info("Wrote %s to rainbow table %s\n", file_size_str.c_str(), filename.c_str()); +} + +void PSOEncryptionSeedFinder::parallel_all_seeds( + size_t num_threads, function fn) { + PSOEncryptionSeedFinder::parallel_all_seeds_t( + num_threads, + &PSOEncryptionSeedFinder::parallel_all_seeds_thread_fn, + fn); +} + + + +template +void PSOEncryptionSeedFinder::parallel_all_seeds_t( + size_t num_threads, ThreadArgTs... args) { + atomic current_seed(0); + vector threads; + while (threads.size() < num_threads) { + threads.emplace_back(args..., ref(current_seed), threads.size()); + } + + uint64_t start_time = now(); + uint64_t displayed_current_seed; + while ((displayed_current_seed = current_seed.load()) < 0x100000000) { + + uint64_t elapsed_time = now() - start_time; + string elapsed_str = format_duration(elapsed_time); + + string remaining_str; + if (displayed_current_seed) { + uint64_t total_time = (elapsed_time << 32) / displayed_current_seed; + uint64_t remaining_time = total_time - elapsed_time; + remaining_str = format_duration(remaining_time); + } else { + remaining_str = "..."; + } + + fprintf(stderr, "... %08" PRIX64 " (%s / -%s)\r", displayed_current_seed, + elapsed_str.c_str(), remaining_str.c_str()); + usleep(1000000); + } + + log_info("Waiting for worker threads to terminate\n"); + for (auto& t : threads) { + t.join(); + } +} + +template +PSOEncryptionSeedFinder::ThreadResults PSOEncryptionSeedFinder::parallel_find_seed_t( + ThreadArgTs... args) { + + vector all_thread_results; + all_thread_results.resize(this->num_threads); + this->parallel_all_seeds_t(this->num_threads, args..., ref(all_thread_results)); + + ThreadResults overall_results = all_thread_results[0]; + for (const auto& thread_results : all_thread_results) { + overall_results.combine_from(thread_results); + } + return overall_results; +} + + + +void PSOEncryptionSeedFinder::parallel_all_seeds_thread_fn( + function fn, + atomic& current_seed, + size_t thread_num) { + uint64_t seed; + while ((seed = current_seed.fetch_add(1)) < 0x100000000) { + if (fn(seed, thread_num)) { + current_seed = 0x100000000; + } + } +} + +void PSOEncryptionSeedFinder::find_seed_without_rainbow_table_thread_fn( + uint64_t flags, + vector& all_results, + atomic& current_seed, + size_t thread_num) { + size_t plaintext_size = this->plaintexts[0].first.size(); + + auto& results = all_results.at(thread_num); + results.results.clear(); + results.min_differences = plaintext_size + 1; + results.difference_histogram.clear(); + + bool is_v3 = flags & Flag::V3; + bool skip_little_endian = flags & Flag::SKIP_LITTLE_ENDIAN; + bool skip_big_endian = flags & Flag::SKIP_BIG_ENDIAN; + + uint64_t seed; + while ((seed = current_seed.fetch_add(1)) < 0x100000000) { + string be_decrypt_buf = this->ciphertext.substr(0, plaintext_size); + string le_decrypt_buf = this->ciphertext.substr(0, plaintext_size); + if (is_v3) { + PSOV3Encryption(seed).encrypt_both_endian( + le_decrypt_buf.data(), + be_decrypt_buf.data(), + be_decrypt_buf.size()); + } else { + PSOV2Encryption(seed).encrypt_both_endian( + le_decrypt_buf.data(), + be_decrypt_buf.data(), + be_decrypt_buf.size()); + } + + for (const auto& plaintext : this->plaintexts) { + if (!skip_little_endian) { + size_t diff = difference_match(le_decrypt_buf, plaintext.first); + results.add_result(Result(seed, diff, false, is_v3)); + } else if (!skip_big_endian) { + size_t diff = difference_match(be_decrypt_buf, plaintext.first); + results.add_result(Result(seed, diff, true, is_v3)); + } + } + if (results.min_differences == 0) { + current_seed = 0x100000000; + } + } +} + +void PSOEncryptionSeedFinder::find_seed_with_rainbow_table_thread_fn( + int fd, + size_t page_size, + vector& all_results, + atomic& current_seed, + size_t thread_num) { + size_t plaintext_size = this->plaintexts[0].first.size(); + + auto& results = all_results.at(thread_num); + results.results.clear(); + results.min_differences = plaintext_size + 1; + results.difference_histogram.clear(); + + uint64_t seed; + string rainbow_buf(page_size * plaintext_size, '\0'); + while ((seed = current_seed.fetch_add(page_size)) < 0x100000000) { + preadx(fd, rainbow_buf.data(), rainbow_buf.size(), seed * plaintext_size); + for (size_t z = 0; z < page_size; z++) { + for (size_t x = 0; x < plaintext_size; x++) { + rainbow_buf[z * plaintext_size + x] ^= this->ciphertext[x]; + } + for (const auto& plaintext : this->plaintexts) { + size_t diff = difference_match( + &rainbow_buf[z * plaintext_size], plaintext.first); + results.add_result(Result(seed, diff)); + } + if (results.min_differences == 0) { + current_seed = 0x100000000; + } + } + } +} + +void PSOEncryptionSeedFinder::generate_rainbow_table_thread_fn( + int fd, + size_t match_length, + size_t page_size, + bool is_v3, + bool is_big_endian, + atomic& current_seed, + size_t) { + uint64_t seed; + string buf(match_length * page_size, '\0'); + while ((seed = current_seed.fetch_add(page_size)) < 0x100000000) { + memset(buf.data(), 0, buf.size()); + for (size_t z = 0; z < page_size; z++) { + if (is_v3) { + PSOV3Encryption crypt(seed + z); + if (is_big_endian) { + crypt.encrypt_big_endian(buf.data() + z * match_length, match_length); + } else { + crypt.encrypt(buf.data() + z * match_length, match_length); + } + } else { + PSOV2Encryption crypt(seed + z); + if (is_big_endian) { + crypt.encrypt_big_endian(buf.data() + z * match_length, match_length); + } else { + crypt.encrypt(buf.data() + z * match_length, match_length); + } + } + } + pwritex(fd, buf.data(), buf.size(), seed * match_length); + } +} diff --git a/src/PSOEncryptionSeedFinder.hh b/src/PSOEncryptionSeedFinder.hh new file mode 100644 index 00000000..38713ac7 --- /dev/null +++ b/src/PSOEncryptionSeedFinder.hh @@ -0,0 +1,99 @@ +#pragma once + +#include +#include + +#include +#include +#include +#include +#include + + + +class PSOEncryptionSeedFinder { +public: + PSOEncryptionSeedFinder( + const std::string& ciphertext, + const std::vector>& plaintexts, + size_t num_threads); + ~PSOEncryptionSeedFinder() = default; + + enum Flag { + V3 = 0x01, + SKIP_LITTLE_ENDIAN = 0x02, + SKIP_BIG_ENDIAN = 0x04, + }; + + struct Result { + uint32_t seed; + size_t differences; + bool is_indeterminate; + bool is_big_endian; + bool is_v3; + + Result(uint32_t seed, size_t differences); + Result(uint32_t seed, size_t differences, bool is_big_endian, bool is_v3); + }; + + struct ThreadResults { + std::vector results; + size_t min_differences; + std::vector difference_histogram; + + void add_result(const Result& res); + void combine_from(const ThreadResults& other); + }; + + ThreadResults find_seed(std::function fn); + + ThreadResults find_seed(uint64_t flags); + ThreadResults find_seed(const std::string& rainbow_table_filename); + + static void generate_rainbow_table( + const std::string& filename, + bool is_v3, + bool is_big_endian, + size_t match_length, + size_t num_threads); + + static void parallel_all_seeds( + size_t num_threads, std::function fn); + +private: + template + static void parallel_all_seeds_t(size_t num_threads, ThreadArgTs... args); + template + ThreadResults parallel_find_seed_t(ThreadArgTs... args); + + + static void parallel_all_seeds_thread_fn( + std::function fn, + std::atomic& current_seed, + size_t thread_num); + + void find_seed_without_rainbow_table_thread_fn( + uint64_t flags, + std::vector& all_results, + std::atomic& current_seed, + size_t thread_num); + void find_seed_with_rainbow_table_thread_fn( + int fd, + size_t page_size, + std::vector& all_results, + std::atomic& current_seed, + size_t thread_num); + + static void generate_rainbow_table_thread_fn( + int fd, + size_t match_length, + size_t page_size, + bool is_v3, + bool is_big_endian, + std::atomic& current_seed, + size_t thread_num); + + std::string ciphertext; + std::vector> plaintexts; + size_t num_threads; +}; diff --git a/src/Quest.cc b/src/Quest.cc index ba73d29d..9b70beb0 100644 --- a/src/Quest.cc +++ b/src/Quest.cc @@ -1,10 +1,12 @@ #include "Quest.hh" #include +#include #include #include #include #include +#include #include #include @@ -12,15 +14,164 @@ #include "CommandFormats.hh" #include "Compression.hh" #include "PSOEncryption.hh" +#include "PSOEncryptionSeedFinder.hh" #include "Text.hh" using namespace std; +// GCI decoding logic + +struct ShuffleTables { + uint8_t forward_table[0x100]; // table1 / 804FB9B8 + uint8_t reverse_table[0x100]; // table2 / 804FBAB8 + + ShuffleTables(PSOV2Encryption& crypt) { + for (size_t x = 0; x < 0x100; x++) { + this->forward_table[x] = x; + } + + int32_t r28 = 0xFF; + uint8_t* r31 = &this->forward_table[0xFF]; + while (r28 >= 0) { + uint32_t r3 = this->pseudorand(crypt, r28 + 1); + if (r3 >= 0x100) { + throw logic_error("bad r3"); + } + uint8_t t = this->forward_table[r3]; + this->forward_table[r3] = *r31; + *r31 = t; + + this->reverse_table[t] = r28; + r31--; + r28--; + } + } + + static uint32_t pseudorand(PSOV2Encryption& crypt, uint32_t prev) { + return (((prev & 0xFFFF) * ((crypt.next() >> 16) & 0xFFFF)) >> 16) & 0xFFFF; + } + + void shuffle(void* vdest, const void* vsrc, size_t size, bool reverse) { + uint8_t* dest = reinterpret_cast(vdest); + const uint8_t* src = reinterpret_cast(vsrc); + const uint8_t* table = reverse ? this->reverse_table : this->forward_table; + + for (size_t block_offset = 0; block_offset < (size & 0xFFFFFF00); block_offset += 0x100) { + for (size_t z = 0; z < 0x100; z++) { + dest[block_offset + table[z]] = src[block_offset + z]; + } + } + + // Any remaining bytes that don't fill an entire block are not shuffled + memcpy(&dest[size & 0xFFFFFF00], &src[size & 0xFFFFFF00], size & 0xFF); + } +}; + +struct PSOGCIFileHeader { + parray gci_header; + parray pso_header; + parray banner_and_icon; + // data_size specifies the number of bytes in the encrypted section, including + // the encrypted header (below) and all encrypted data after it. + be_uint32_t data_size; + // To compute checksum, set checksum to zero, then compute the CRC32 of all + // fields in this struct except gci_header. (Yes, including the checksum + // field, which is temporarily zero.) + be_uint32_t checksum; + + bool checksum_correct() const { + uint32_t cs = crc32(&this->pso_header, sizeof(this->pso_header)); + cs = crc32(&this->banner_and_icon, sizeof(this->banner_and_icon), cs); + cs = crc32(&this->data_size, sizeof(this->data_size), cs); + cs = crc32("\0\0\0\0", 4, cs); + return (cs == this->checksum); + } +} __attribute__((packed)); + +struct PSOGCIFileEncryptedHeader { + be_uint32_t round2_seed; + // To compute checksum, set checksum to zero, then compute the CRC32 of the + // entire data section, including this header struct (but not the unencrypted + // header struct). + be_uint32_t checksum; + le_uint32_t decompressed_size; + le_uint32_t round3_seed; + // Data follows here. +} __attribute__((packed)); + +string decrypt_gci_data_section( + const void* data_section, size_t size, uint32_t seed) { + string decrypted(size, '\0'); + { + PSOV2Encryption shuf_crypt(seed); + ShuffleTables shuf(shuf_crypt); + shuf.shuffle(decrypted.data(), data_section, size, true); + } + + decrypted.resize((decrypted.size() + 3) & (~3)); + auto* be_dwords = reinterpret_cast(decrypted.data()); + + PSOV2Encryption crypt(seed); + for (size_t z = 0; z < decrypted.size() / sizeof(be_uint32_t); z++) { + be_dwords[z] = crypt.next() - be_dwords[z]; + } + + auto* header = reinterpret_cast( + decrypted.data()); + PSOV2Encryption(header->round2_seed).encrypt_big_endian( + decrypted.data() + 4, (decrypted.size() - 4) & (~3)); + + uint32_t expected_crc = header->checksum; + header->checksum = 0; + uint32_t actual_crc = crc32(decrypted.data(), decrypted.size()); + header->checksum = expected_crc; + + if (expected_crc != actual_crc) { + throw runtime_error("incorrect decrypted data section checksum"); + } + + PSOV2Encryption(header->round3_seed).decrypt( + decrypted.data() + sizeof(PSOGCIFileEncryptedHeader), + decrypted.size() - sizeof(PSOGCIFileEncryptedHeader)); + + string ret = decrypted.substr(sizeof(PSOGCIFileEncryptedHeader)); + if (prs_decompress_size(ret) != header->decompressed_size) { + throw runtime_error("decompressed size does not match size in header"); + } + + return ret; +} + +string find_seed_and_decrypt_gci_data_section( + const void* data_section, size_t size, size_t num_threads) { + mutex result_lock; + string result; + PSOEncryptionSeedFinder::parallel_all_seeds(num_threads, [&]( + uint32_t seed, size_t) { + try { + string ret = decrypt_gci_data_section(data_section, size, seed); + lock_guard g(result_lock); + result = move(ret); + return true; + } catch (const runtime_error&) { + return false; + } + }); + + if (!result.empty()) { + return result; + } else { + throw runtime_error("no seed found"); + } +} + + + struct PSODownloadQuestHeader { // When sending a DLQ to the client, this is the DECOMPRESSED size. When - // reading it from a GCI file, this is the COMPRESSED size. + // reading it from an (unencrypted) GCI file, this is the COMPRESSED size. le_uint32_t size; // Note: use PSO PC encryption, even for GC quests. le_uint32_t encryption_seed; @@ -228,7 +379,7 @@ Quest::Quest(const string& bin_filename) // get the category from the second token if needed if (this->category == QuestCategory::UNKNOWN) { - static const unordered_map name_to_category({ + static const unordered_map name_to_category({ {"ret", QuestCategory::RETRIEVAL}, {"ext", QuestCategory::EXTERMINATION}, {"evt", QuestCategory::EVENT}, @@ -245,7 +396,7 @@ Quest::Quest(const string& bin_filename) tokens.erase(tokens.begin() + 1); } - static const unordered_map name_to_version({ + static const unordered_map name_to_version({ {"d1", GameVersion::DC}, {"dc", GameVersion::DC}, {"pc", GameVersion::PC}, @@ -348,7 +499,7 @@ Quest::Quest(const string& bin_filename) } } -static string basename_for_filename(const std::string& filename) { +static string basename_for_filename(const string& filename) { size_t slash_pos = filename.rfind('/'); if (slash_pos != string::npos) { return filename.substr(slash_pos + 1); @@ -356,11 +507,11 @@ static string basename_for_filename(const std::string& filename) { return filename; } -std::string Quest::bin_filename() const { +string Quest::bin_filename() const { return basename_for_filename(this->file_basename + ".bin"); } -std::string Quest::dat_filename() const { +string Quest::dat_filename() const { return basename_for_filename(this->file_basename + ".dat"); } @@ -374,7 +525,7 @@ shared_ptr Quest::bin_contents() const { this->bin_contents_ptr.reset(new string(prs_compress(load_file(this->file_basename + ".bind")))); break; case FileFormat::BIN_DAT_GCI: - this->bin_contents_ptr.reset(new string(this->decode_gci(this->file_basename + ".bin.gci"))); + this->bin_contents_ptr.reset(new string(this->decode_gci(this->file_basename + ".bin.gci", false))); break; case FileFormat::BIN_DAT_DLQ: this->bin_contents_ptr.reset(new string(this->decode_dlq(this->file_basename + ".bin.dlq"))); @@ -402,7 +553,7 @@ shared_ptr Quest::dat_contents() const { this->dat_contents_ptr.reset(new string(prs_compress(load_file(this->file_basename + ".datd")))); break; case FileFormat::BIN_DAT_GCI: - this->dat_contents_ptr.reset(new string(this->decode_gci(this->file_basename + ".dat.gci"))); + this->dat_contents_ptr.reset(new string(this->decode_gci(this->file_basename + ".dat.gci", false))); break; case FileFormat::BIN_DAT_DLQ: this->dat_contents_ptr.reset(new string(this->decode_dlq(this->file_basename + ".dat.dlq"))); @@ -420,51 +571,50 @@ shared_ptr Quest::dat_contents() const { return this->dat_contents_ptr; } -string Quest::decode_gci(const string& filename) { - +string Quest::decode_gci( + const string& filename, ssize_t find_seed_num_threads, int64_t known_seed) { string data = load_file(filename); - if (data.size() < 0x2080 + sizeof(PSODownloadQuestHeader)) { - throw runtime_error(string_printf( - "GCI file is truncated before download quest header (have 0x%zX bytes)", data.size())); - } - PSODownloadQuestHeader* h = reinterpret_cast( - data.data() + 0x2080); - string compressed_data_with_header = data.substr(0x2088, h->size); - - // For now, we can only load unencrypted quests, unfortunately - // TODO: Figure out how GCI encryption works and implement it here. - - // Unlike the DLQ header, this one is stored little-endian. The compressed - // data immediately follows this header. - struct DecryptedHeader { - uint32_t unknown1; - uint32_t unknown2; - uint32_t decompressed_size; - uint32_t unknown4; - } __attribute__((packed)); - if (compressed_data_with_header.size() < sizeof(DecryptedHeader)) { - throw runtime_error("GCI file compressed data truncated during header"); - } - DecryptedHeader* dh = reinterpret_cast( - compressed_data_with_header.data()); - if (dh->unknown1 || dh->unknown2 || dh->unknown4) { - throw runtime_error("GCI file appears to be encrypted"); + StringReader r(data); + const auto& header = r.get(); + if (!header.checksum_correct()) { + throw runtime_error("GCI file unencrypted header checksum is incorrect"); } - string data_to_decompress = compressed_data_with_header.substr(sizeof(DecryptedHeader)); - size_t decompressed_bytes = prs_decompress_size(data_to_decompress); + const auto& encrypted_header = r.get(false); + // Unencrypted GCI files appear to always have zeroes in these fields. + // Encrypted GCI files are highly unlikely to have zeroes in ALL of these + // fields, so assume it's encrypted if any of them are nonzero. + if (encrypted_header.round2_seed || encrypted_header.checksum || encrypted_header.round3_seed) { + if (known_seed >= 0) { + return decrypt_gci_data_section( + r.getv(header.data_size), header.data_size, known_seed); - size_t expected_decompressed_bytes = dh->decompressed_size - 8; - if (decompressed_bytes < expected_decompressed_bytes) { - throw runtime_error(string_printf( - "GCI decompressed data is smaller than expected size (have 0x%zX bytes, expected 0x%zX bytes)", - decompressed_bytes, expected_decompressed_bytes)); + } else { + if (find_seed_num_threads < 0) { + throw runtime_error("GCI file appears to be encrypted"); + } + if (find_seed_num_threads == 0) { + find_seed_num_threads = thread::hardware_concurrency(); + } + return find_seed_and_decrypt_gci_data_section( + r.getv(header.data_size), header.data_size, find_seed_num_threads); + } + + } else { // Unencrypted GCI format + r.skip(sizeof(PSOGCIFileEncryptedHeader)); + string compressed_data = r.read(r.remaining()); + size_t decompressed_bytes = prs_decompress_size(compressed_data); + + size_t expected_decompressed_bytes = encrypted_header.decompressed_size - 8; + if (decompressed_bytes < expected_decompressed_bytes) { + throw runtime_error(string_printf( + "GCI decompressed data is smaller than expected size (have 0x%zX bytes, expected 0x%zX bytes)", + decompressed_bytes, expected_decompressed_bytes)); + } + + return compressed_data; } - - // The caller expects to get PRS-compressed data when calling bin_contents() - // and dat_contents(), so we shouldn't decompress it here. - return data_to_decompress; } string Quest::decode_dlq(const string& filename) { @@ -609,7 +759,7 @@ pair Quest::decode_qst(const string& filename) { -QuestIndex::QuestIndex(const std::string& directory) : directory(directory) { +QuestIndex::QuestIndex(const string& directory) : directory(directory) { auto filename_set = list_directory(this->directory); vector filenames(filename_set.begin(), filename_set.end()); sort(filenames.begin(), filenames.end()); diff --git a/src/Quest.hh b/src/Quest.hh index f2dbba15..e0d46c60 100644 --- a/src/Quest.hh +++ b/src/Quest.hh @@ -70,7 +70,10 @@ public: std::shared_ptr create_download_quest() const; - static std::string decode_gci(const std::string& filename); + static std::string decode_gci( + const std::string& filename, + ssize_t find_seed_num_threads = -1, + int64_t known_seed = -1); static std::string decode_dlq(const std::string& filename); static std::pair decode_qst(const std::string& filename);