From 8afc9522944f7591c192dc0b8055593713bffd6c Mon Sep 17 00:00:00 2001 From: Martin Michelsen Date: Wed, 21 Sep 2022 00:10:47 -0700 Subject: [PATCH] simplify decryption seed finder --- CMakeLists.txt | 1 - src/Main.cc | 119 +++++------ src/PSOEncryptionSeedFinder.cc | 359 --------------------------------- src/PSOEncryptionSeedFinder.hh | 100 --------- src/Quest.cc | 13 +- 5 files changed, 58 insertions(+), 534 deletions(-) delete mode 100644 src/PSOEncryptionSeedFinder.cc delete mode 100644 src/PSOEncryptionSeedFinder.hh diff --git a/CMakeLists.txt b/CMakeLists.txt index fdff97de..2a6b923d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -71,7 +71,6 @@ add_executable(newserv src/ProxyCommands.cc src/ProxyServer.cc src/PSOEncryption.cc - src/PSOEncryptionSeedFinder.cc src/PSOProtocol.cc src/Quest.cc src/RareItemSet.cc diff --git a/src/Main.cc b/src/Main.cc index b1b801ed..b0afe25a 100644 --- a/src/Main.cc +++ b/src/Main.cc @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -18,7 +19,6 @@ #include "Loggers.hh" #include "NetworkAddresses.hh" #include "ProxyServer.hh" -#include "PSOEncryptionSeedFinder.hh" #include "ReplaySession.hh" #include "SendCommands.hh" #include "Server.hh" @@ -246,16 +246,7 @@ Specifically:\n\ PSO V3 encryption, but this can be overridden with --pc. (BB encryption\n\ seeds are too long to be searched for with this function.) By default,\n\ the number of worker threads is equal the the number of CPU cores in the\n\ - system, but this can be overridden with the --threads= option. To use a\n\ - rainbow table instead of computing the cipherstreams inline, use the\n\ - --rainbow-table=FILENAME option.\n\ - --generate-rainbow-table=FILENAME\n\ - Generate a decryption table for V3 encryption (or V2 if --pc is given).\n\ - The --match-length= option must be given, which specifies the match\n\ - length for the table. The total table size is the match length * 4 GB.\n\ - As for --encrypt-data, the --big-endian option specifies that the table\n\ - uses big-endian encryption. As for --find-decryption-seed, the --threads\n\ - option specifies the parallelism for generating the table.\n\ + system, but this can be overridden with the --threads= option.\n\ --decode-sjis\n\ Apply newserv\'s text decoding algorithm to the data on stdin, producing\n\ little-endian UTF-16 data on stdout.\n\ @@ -300,7 +291,6 @@ enum class Behavior { DECRYPT_DATA, ENCRYPT_DATA, FIND_DECRYPTION_SEED, - GENERATE_RAINBOW_TABLE, DECODE_QUEST_FILE, DECODE_SJIS, EXTRACT_GSL, @@ -322,13 +312,11 @@ int main(int argc, char** argv) { string seed; string key_file_name; const char* config_filename = "system/config.json"; - string rainbow_table_filename; bool parse_data = false; bool big_endian = false; bool skip_little_endian = false; bool skip_big_endian = false; size_t num_threads = 0; - size_t match_length = 0; const char* find_decryption_seed_ciphertext = nullptr; vector find_decryption_seed_plaintexts; const char* replay_log_filename = nullptr; @@ -350,9 +338,6 @@ int main(int argc, char** argv) { behavior = Behavior::ENCRYPT_DATA; } else if (!strcmp(argv[x], "--find-decryption-seed")) { behavior = Behavior::FIND_DECRYPTION_SEED; - } else if (!strncmp(argv[x], "--generate-rainbow-table=", 25)) { - behavior = Behavior::GENERATE_RAINBOW_TABLE; - rainbow_table_filename = &argv[x][25]; } else if (!strcmp(argv[x], "--decode-sjis")) { behavior = Behavior::DECODE_SJIS; } else if (!strncmp(argv[x], "--decode-gci=", 13)) { @@ -371,11 +356,7 @@ int main(int argc, char** argv) { behavior = Behavior::CAT_CLIENT; cat_client_remote = make_sockaddr_storage(parse_netloc(&argv[x][13])).first; } else if (!strncmp(argv[x], "--threads=", 10)) { - num_threads = strtoull(&argv[x][13], nullptr, 0); - } else if (!strncmp(argv[x], "--match-length=", 15)) { - match_length = strtoull(&argv[x][15], nullptr, 0); - } else if (!strncmp(argv[x], "--rainbow-table=", 16)) { - rainbow_table_filename = &argv[x][16]; + num_threads = strtoull(&argv[x][10], nullptr, 0); } else if (!strcmp(argv[x], "--patch")) { cli_version = GameVersion::PATCH; } else if (!strcmp(argv[x], "--dc")) { @@ -516,61 +497,67 @@ int main(int argc, char** argv) { throw runtime_error("--find-decryption-seed cannot be used for BB ciphers"); } + size_t max_plaintext_size = 0; vector> plaintexts; for (const auto& plaintext_ascii : find_decryption_seed_plaintexts) { string mask; string data = parse_data_string(plaintext_ascii, &mask); + if (data.size() != mask.size()) { + throw logic_error("plaintext and mask are not the same size"); + } + max_plaintext_size = max(max_plaintext_size, data.size()); plaintexts.emplace_back(move(data), move(mask)); } string ciphertext = parse_data_string(find_decryption_seed_ciphertext); - if (num_threads == 0) { - num_threads = thread::hardware_concurrency(); - } - - PSOEncryptionSeedFinder finder(ciphertext, plaintexts, num_threads); - PSOEncryptionSeedFinder::ThreadResults results; - if (!rainbow_table_filename.empty()) { - results = finder.find_seed(rainbow_table_filename); - } else { - using Flag = PSOEncryptionSeedFinder::Flag; - uint64_t flags = - (((cli_version == GameVersion::GC) || (cli_version == GameVersion::XB)) ? Flag::V3 : 0) | - (skip_little_endian ? Flag::SKIP_LITTLE_ENDIAN : 0) | - (skip_big_endian ? Flag::SKIP_BIG_ENDIAN : 0); - results = finder.find_seed(flags); - } - - log_info("Minimum differences: %zu", results.min_differences); - for (auto result : results.results) { - if (result.differences != results.min_differences) { - throw logic_error("incorrect difference count in result"); + auto mask_match = +[](const void* a, const void* b, const void* m, size_t size) -> bool { + const uint8_t* a8 = reinterpret_cast(a); + const uint8_t* b8 = reinterpret_cast(b); + const uint8_t* m8 = reinterpret_cast(m); + for (size_t z = 0; z < size; z++) { + if ((a8[z] & m8[z]) != (b8[z] & m8[z])) { + return false; + } } - if (result.is_indeterminate) { - log_info("Example match: %08" PRIX32 " (%zu)", - result.seed, result.differences); - } else { - log_info("Example match: %08" PRIX32 " (%zu; %s, %s)", - result.seed, - result.differences, - result.is_v3 ? "v3" : "v2", - result.is_big_endian ? "big-endian" : "little-endian"); - } - } - for (size_t z = 0; z < results.difference_histogram.size(); z++) { - log_info("(Difference histogram) %zu => %zu results", - z, results.difference_histogram[z]); - } - break; - } + return true; + }; - case Behavior::GENERATE_RAINBOW_TABLE: { - if (num_threads == 0) { - num_threads = thread::hardware_concurrency(); - } bool is_v3 = ((cli_version == GameVersion::GC) || (cli_version == GameVersion::XB)); - PSOEncryptionSeedFinder::generate_rainbow_table( - rainbow_table_filename, is_v3, big_endian, match_length, num_threads); + uint64_t seed = parallel_range([&](uint64_t seed, size_t) -> bool { + string be_decrypt_buf = ciphertext.substr(0, max_plaintext_size); + string le_decrypt_buf = ciphertext.substr(0, max_plaintext_size); + if (is_v3) { + PSOV3Encryption(seed).encrypt_both_endian( + le_decrypt_buf.data(), + be_decrypt_buf.data(), + be_decrypt_buf.size()); + } else { + PSOV2Encryption(seed).encrypt_both_endian( + le_decrypt_buf.data(), + be_decrypt_buf.data(), + be_decrypt_buf.size()); + } + + for (const auto& plaintext : plaintexts) { + if (!skip_little_endian) { + if (mask_match(le_decrypt_buf.data(), plaintext.first.data(), plaintext.second.data(), plaintext.second.size())) { + return true; + } + } + if (!skip_big_endian) { + if (mask_match(be_decrypt_buf.data(), plaintext.first.data(), plaintext.second.data(), plaintext.second.size())) { + return true; + } + } + } + return false; + }, 0, 0x100000000, num_threads); + + if (seed < 0x100000000) { + log_info("Found seed %08" PRIX64, seed); + } else { + log_error("No seed found"); + } break; } diff --git a/src/PSOEncryptionSeedFinder.cc b/src/PSOEncryptionSeedFinder.cc deleted file mode 100644 index 2b511c22..00000000 --- a/src/PSOEncryptionSeedFinder.cc +++ /dev/null @@ -1,359 +0,0 @@ -#include "PSOEncryptionSeedFinder.hh" - -#include -#include -#include -#include -#include - -#include "PSOEncryption.hh" - -using namespace std; - - - -static size_t difference_match(const string& data1, const string& data2) { - if (data1.size() != data2.size()) { - return max(data1.size(), data2.size()); - } - size_t differences = 0; - for (size_t z = 0; z < data1.size(); z++) { - if (data1[z] != data2[z]) { - differences++; - } - } - return differences; -} - - - -PSOEncryptionSeedFinder::PSOEncryptionSeedFinder( - const std::string& ciphertext, - const std::vector>& plaintexts, - size_t num_threads) - : ciphertext(ciphertext), plaintexts(plaintexts), num_threads(num_threads) { - if (num_threads == 0) { - throw logic_error("must use at least one thread"); - } - if (this->ciphertext.empty() || (this->ciphertext.size() & 3)) { - throw runtime_error("ciphertext length must be a nonzero multiple of 4"); - } - if (this->plaintexts.empty()) { - throw runtime_error("no plaintexts provided"); - } - size_t plaintext_size = this->plaintexts[0].first.size(); - for (const auto& plaintext : this->plaintexts) { - if (plaintext.first.size() != plaintext_size) { - throw runtime_error("plaintexts are not all the same size"); - } - if (plaintext.first.size() != plaintext.second.size()) { - throw logic_error("plaintext and plaintext mask are not the same size"); - } - } -} - - - -PSOEncryptionSeedFinder::Result::Result(uint32_t seed, size_t differences) - : seed(seed), - differences(differences), - is_indeterminate(true), - is_big_endian(false), - is_v3(false) { } -PSOEncryptionSeedFinder::Result::Result( - uint32_t seed, size_t differences, bool is_big_endian, bool is_v3) - : seed(seed), - differences(differences), - is_indeterminate(false), - is_big_endian(is_big_endian), - is_v3(is_v3) { } - - - -void PSOEncryptionSeedFinder::ThreadResults::add_result(const Result& res) { - if (res.differences < this->min_differences) { - this->results.clear(); - this->min_differences = res.differences; - } - if ((res.differences == this->min_differences) && (this->results.size() < 10)) { - this->results.emplace_back(res); - } - if (this->difference_histogram.size() <= res.differences) { - this->difference_histogram.resize(res.differences + 1, 0); - } - this->difference_histogram[res.differences]++; -} - -void PSOEncryptionSeedFinder::ThreadResults::combine_from( - const ThreadResults& other) { - if (this->min_differences > other.min_differences) { - this->min_differences = other.min_differences; - this->results = other.results; - } else if (this->min_differences == other.min_differences) { - this->results.insert(this->results.end(), other.results.begin(), other.results.end()); - } - if (this->difference_histogram.size() < other.difference_histogram.size()) { - this->difference_histogram.resize(other.difference_histogram.size(), 0); - } - for (size_t z = 0; z < other.difference_histogram.size(); z++) { - this->difference_histogram[z] += other.difference_histogram[z]; - } -} - - - -PSOEncryptionSeedFinder::ThreadResults PSOEncryptionSeedFinder::find_seed( - uint64_t flags) { - // TODO: Use a specific logger here - log_info("Searching for decryption key (%s, %zu threads)", - (flags & Flag::V3) ? "v3" : "v2", this->num_threads); - return this->parallel_find_seed_t( - &PSOEncryptionSeedFinder::find_seed_without_rainbow_table_thread_fn, - this, - flags); -} - -PSOEncryptionSeedFinder::ThreadResults PSOEncryptionSeedFinder::find_seed( - const string& rainbow_table_filename) { - size_t plaintext_size = this->plaintexts[0].first.size(); - - scoped_fd fd(rainbow_table_filename, O_RDONLY); - int64_t expected_rainbow_table_size = static_cast(plaintext_size) << 32; - if (fstat(fd).st_size != expected_rainbow_table_size) { - throw runtime_error("rainbow table size is incorrect"); - } - - // TODO: Use a specific logger here - log_info("Searching for decryption key (%zu threads) using rainbow table %s", - this->num_threads, rainbow_table_filename.c_str()); - return this->parallel_find_seed_t( - &PSOEncryptionSeedFinder::find_seed_with_rainbow_table_thread_fn, - this, - static_cast(fd), - 0x1000); -} - -void PSOEncryptionSeedFinder::generate_rainbow_table( - const std::string& filename, - bool is_v3, - bool is_big_endian, - size_t match_length, - size_t num_threads) { - if ((match_length == 0) || (match_length & 3)) { - throw runtime_error("match length must be a nonzero multiple of 4"); - } - if (num_threads == 0) { - throw logic_error("must use at least one thread"); - } - - uint64_t file_size = static_cast(match_length) << 32; - string file_size_str = format_size(file_size); - - scoped_fd fd(filename, O_CREAT | O_WRONLY); - log_info("Allocating file space for rainbow table (match_length=%zu bytes => table size is %s)", - match_length, file_size_str.c_str()); - if (ftruncate(fd, file_size) < 0) { - throw runtime_error("cannot allocate file space for table"); - } - - size_t page_size = 0x1000; - - PSOEncryptionSeedFinder::parallel_all_seeds_t( - num_threads, - &PSOEncryptionSeedFinder::generate_rainbow_table_thread_fn, - static_cast(fd), - match_length, - page_size, - is_v3, - is_big_endian); - - log_info("Wrote %s to rainbow table %s\n", file_size_str.c_str(), filename.c_str()); -} - -void PSOEncryptionSeedFinder::parallel_all_seeds( - size_t num_threads, function fn) { - PSOEncryptionSeedFinder::parallel_all_seeds_t( - num_threads, - &PSOEncryptionSeedFinder::parallel_all_seeds_thread_fn, - fn); -} - - - -// TODO: Use phosg's parallel_range for this instead -template -void PSOEncryptionSeedFinder::parallel_all_seeds_t( - size_t num_threads, ThreadArgTs... args) { - atomic current_seed(0); - vector threads; - while (threads.size() < num_threads) { - threads.emplace_back(args..., ref(current_seed), threads.size()); - } - - uint64_t start_time = now(); - uint64_t displayed_current_seed; - while ((displayed_current_seed = current_seed.load()) < 0x100000000) { - - uint64_t elapsed_time = now() - start_time; - string elapsed_str = format_duration(elapsed_time); - - string remaining_str; - if (displayed_current_seed) { - uint64_t total_time = (elapsed_time << 32) / displayed_current_seed; - uint64_t remaining_time = total_time - elapsed_time; - remaining_str = format_duration(remaining_time); - } else { - remaining_str = "..."; - } - - fprintf(stderr, "... %08" PRIX64 " (%s / -%s)\r", displayed_current_seed, - elapsed_str.c_str(), remaining_str.c_str()); - usleep(1000000); - } - - for (auto& t : threads) { - t.join(); - } -} - -template -PSOEncryptionSeedFinder::ThreadResults PSOEncryptionSeedFinder::parallel_find_seed_t( - ThreadArgTs... args) { - - vector all_thread_results; - all_thread_results.resize(this->num_threads); - this->parallel_all_seeds_t(this->num_threads, args..., ref(all_thread_results)); - - ThreadResults overall_results = all_thread_results[0]; - for (const auto& thread_results : all_thread_results) { - overall_results.combine_from(thread_results); - } - return overall_results; -} - - - -void PSOEncryptionSeedFinder::parallel_all_seeds_thread_fn( - function fn, - atomic& current_seed, - size_t thread_num) { - uint64_t seed; - while ((seed = current_seed.fetch_add(1)) < 0x100000000) { - if (fn(seed, thread_num)) { - current_seed = 0x100000000; - } - } -} - -void PSOEncryptionSeedFinder::find_seed_without_rainbow_table_thread_fn( - uint64_t flags, - vector& all_results, - atomic& current_seed, - size_t thread_num) { - size_t plaintext_size = this->plaintexts[0].first.size(); - - auto& results = all_results.at(thread_num); - results.results.clear(); - results.min_differences = plaintext_size + 1; - results.difference_histogram.clear(); - - bool is_v3 = flags & Flag::V3; - bool skip_little_endian = flags & Flag::SKIP_LITTLE_ENDIAN; - bool skip_big_endian = flags & Flag::SKIP_BIG_ENDIAN; - - uint64_t seed; - while ((seed = current_seed.fetch_add(1)) < 0x100000000) { - string be_decrypt_buf = this->ciphertext.substr(0, plaintext_size); - string le_decrypt_buf = this->ciphertext.substr(0, plaintext_size); - if (is_v3) { - PSOV3Encryption(seed).encrypt_both_endian( - le_decrypt_buf.data(), - be_decrypt_buf.data(), - be_decrypt_buf.size()); - } else { - PSOV2Encryption(seed).encrypt_both_endian( - le_decrypt_buf.data(), - be_decrypt_buf.data(), - be_decrypt_buf.size()); - } - - for (const auto& plaintext : this->plaintexts) { - if (!skip_little_endian) { - size_t diff = difference_match(le_decrypt_buf, plaintext.first); - results.add_result(Result(seed, diff, false, is_v3)); - } else if (!skip_big_endian) { - size_t diff = difference_match(be_decrypt_buf, plaintext.first); - results.add_result(Result(seed, diff, true, is_v3)); - } - } - if (results.min_differences == 0) { - current_seed = 0x100000000; - } - } -} - -void PSOEncryptionSeedFinder::find_seed_with_rainbow_table_thread_fn( - int fd, - size_t page_size, - vector& all_results, - atomic& current_seed, - size_t thread_num) { - size_t plaintext_size = this->plaintexts[0].first.size(); - - auto& results = all_results.at(thread_num); - results.results.clear(); - results.min_differences = plaintext_size + 1; - results.difference_histogram.clear(); - - uint64_t seed; - string rainbow_buf(page_size * plaintext_size, '\0'); - while ((seed = current_seed.fetch_add(page_size)) < 0x100000000) { - preadx(fd, rainbow_buf.data(), rainbow_buf.size(), seed * plaintext_size); - for (size_t z = 0; z < page_size; z++) { - for (size_t x = 0; x < plaintext_size; x++) { - rainbow_buf[z * plaintext_size + x] ^= this->ciphertext[x]; - } - for (const auto& plaintext : this->plaintexts) { - size_t diff = difference_match( - &rainbow_buf[z * plaintext_size], plaintext.first); - results.add_result(Result(seed, diff)); - } - if (results.min_differences == 0) { - current_seed = 0x100000000; - } - } - } -} - -void PSOEncryptionSeedFinder::generate_rainbow_table_thread_fn( - int fd, - size_t match_length, - size_t page_size, - bool is_v3, - bool is_big_endian, - atomic& current_seed, - size_t) { - uint64_t seed; - string buf(match_length * page_size, '\0'); - while ((seed = current_seed.fetch_add(page_size)) < 0x100000000) { - memset(buf.data(), 0, buf.size()); - for (size_t z = 0; z < page_size; z++) { - if (is_v3) { - PSOV3Encryption crypt(seed + z); - if (is_big_endian) { - crypt.encrypt_big_endian(buf.data() + z * match_length, match_length); - } else { - crypt.encrypt(buf.data() + z * match_length, match_length); - } - } else { - PSOV2Encryption crypt(seed + z); - if (is_big_endian) { - crypt.encrypt_big_endian(buf.data() + z * match_length, match_length); - } else { - crypt.encrypt(buf.data() + z * match_length, match_length); - } - } - } - pwritex(fd, buf.data(), buf.size(), seed * match_length); - } -} diff --git a/src/PSOEncryptionSeedFinder.hh b/src/PSOEncryptionSeedFinder.hh deleted file mode 100644 index 16c0d3c0..00000000 --- a/src/PSOEncryptionSeedFinder.hh +++ /dev/null @@ -1,100 +0,0 @@ -#pragma once - -#include -#include - -#include -#include -#include -#include -#include -#include - - - -class PSOEncryptionSeedFinder { -public: - PSOEncryptionSeedFinder( - const std::string& ciphertext, - const std::vector>& plaintexts, - size_t num_threads); - ~PSOEncryptionSeedFinder() = default; - - enum Flag { - V3 = 0x01, - SKIP_LITTLE_ENDIAN = 0x02, - SKIP_BIG_ENDIAN = 0x04, - }; - - struct Result { - uint32_t seed; - size_t differences; - bool is_indeterminate; - bool is_big_endian; - bool is_v3; - - Result(uint32_t seed, size_t differences); - Result(uint32_t seed, size_t differences, bool is_big_endian, bool is_v3); - }; - - struct ThreadResults { - std::vector results; - size_t min_differences; - std::vector difference_histogram; - - void add_result(const Result& res); - void combine_from(const ThreadResults& other); - }; - - ThreadResults find_seed(std::function fn); - - ThreadResults find_seed(uint64_t flags); - ThreadResults find_seed(const std::string& rainbow_table_filename); - - static void generate_rainbow_table( - const std::string& filename, - bool is_v3, - bool is_big_endian, - size_t match_length, - size_t num_threads); - - static void parallel_all_seeds( - size_t num_threads, std::function fn); - -private: - template - static void parallel_all_seeds_t(size_t num_threads, ThreadArgTs... args); - template - ThreadResults parallel_find_seed_t(ThreadArgTs... args); - - - static void parallel_all_seeds_thread_fn( - std::function fn, - std::atomic& current_seed, - size_t thread_num); - - void find_seed_without_rainbow_table_thread_fn( - uint64_t flags, - std::vector& all_results, - std::atomic& current_seed, - size_t thread_num); - void find_seed_with_rainbow_table_thread_fn( - int fd, - size_t page_size, - std::vector& all_results, - std::atomic& current_seed, - size_t thread_num); - - static void generate_rainbow_table_thread_fn( - int fd, - size_t match_length, - size_t page_size, - bool is_v3, - bool is_big_endian, - std::atomic& current_seed, - size_t thread_num); - - std::string ciphertext; - std::vector> plaintexts; - size_t num_threads; -}; diff --git a/src/Quest.cc b/src/Quest.cc index 4fbf2b3e..955238a0 100644 --- a/src/Quest.cc +++ b/src/Quest.cc @@ -9,12 +9,12 @@ #include #include #include +#include #include "Loggers.hh" #include "CommandFormats.hh" #include "Compression.hh" #include "PSOEncryption.hh" -#include "PSOEncryptionSeedFinder.hh" #include "Text.hh" using namespace std; @@ -162,22 +162,19 @@ string find_seed_and_decrypt_gci_data_section( const void* data_section, size_t size, size_t num_threads) { mutex result_lock; string result; - uint32_t result_seed = 0xFFFFFFFF; - PSOEncryptionSeedFinder::parallel_all_seeds(num_threads, [&]( - uint32_t seed, size_t) { + uint64_t result_seed = parallel_range([&](uint64_t seed, size_t) { try { string ret = decrypt_gci_data_section(data_section, size, seed); lock_guard g(result_lock); result = move(ret); - result_seed = seed; return true; } catch (const runtime_error&) { return false; } - }); + }, 0, 0x100000000, num_threads); - if (!result.empty()) { - static_game_data_log.info("Found seed %08" PRIX32 " to decrypt GCI file", + if (!result.empty() && (result_seed < 0x100000000)) { + static_game_data_log.info("Found seed %08" PRIX64 " to decrypt GCI file", result_seed); return result; } else {