incentive/psopeeps-newserv
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

simplify decryption seed finder

Browse Source
This commit is contained in:
Martin Michelsen
2022-09-21 00:10:47 -07:00
parent 30426acbbe
commit 8afc952294
5 changed files with 58 additions and 534 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CMakeLists.txt
-1
View File
@@ -71,7 +71,6 @@ add_executable(newserv
src/ProxyCommands.cc
src/ProxyServer.cc
src/PSOEncryption.cc
src/PSOEncryptionSeedFinder.cc
src/PSOProtocol.cc
src/Quest.cc
src/RareItemSet.cc
src/Main.cc
+53 -66
View File
@@ -7,6 +7,7 @@
#include <phosg/JSON.hh>
#include <phosg/Network.hh>
#include <phosg/Strings.hh>
#include <phosg/Tools.hh>
#include <set>
#include <thread>
#include <unordered_map>
@@ -18,7 +19,6 @@
#include "Loggers.hh"
#include "NetworkAddresses.hh"
#include "ProxyServer.hh"
#include "PSOEncryptionSeedFinder.hh"
#include "ReplaySession.hh"
#include "SendCommands.hh"
#include "Server.hh"
@@ -246,16 +246,7 @@ Specifically:\n\
PSO V3 encryption, but this can be overridden with --pc. (BB encryption\n\
seeds are too long to be searched for with this function.) By default,\n\
the number of worker threads is equal the the number of CPU cores in the\n\
system, but this can be overridden with the --threads= option. To use a\n\
rainbow table instead of computing the cipherstreams inline, use the\n\
--rainbow-table=FILENAME option.\n\
--generate-rainbow-table=FILENAME\n\
Generate a decryption table for V3 encryption (or V2 if --pc is given).\n\
The --match-length= option must be given, which specifies the match\n\
length for the table. The total table size is the match length * 4 GB.\n\
As for --encrypt-data, the --big-endian option specifies that the table\n\
uses big-endian encryption. As for --find-decryption-seed, the --threads\n\
option specifies the parallelism for generating the table.\n\
system, but this can be overridden with the --threads= option.\n\
--decode-sjis\n\
Apply newserv\'s text decoding algorithm to the data on stdin, producing\n\
little-endian UTF-16 data on stdout.\n\
@@ -300,7 +291,6 @@ enum class Behavior {
DECRYPT_DATA,
ENCRYPT_DATA,
FIND_DECRYPTION_SEED,
GENERATE_RAINBOW_TABLE,
DECODE_QUEST_FILE,
DECODE_SJIS,
EXTRACT_GSL,
@@ -322,13 +312,11 @@ int main(int argc, char** argv) {
string seed;
string key_file_name;
const char* config_filename = "system/config.json";
string rainbow_table_filename;
bool parse_data = false;
bool big_endian = false;
bool skip_little_endian = false;
bool skip_big_endian = false;
size_t num_threads = 0;
size_t match_length = 0;
const char* find_decryption_seed_ciphertext = nullptr;
vector<const char*> find_decryption_seed_plaintexts;
const char* replay_log_filename = nullptr;
@@ -350,9 +338,6 @@ int main(int argc, char** argv) {
behavior = Behavior::ENCRYPT_DATA;
} else if (!strcmp(argv[x], "--find-decryption-seed")) {
behavior = Behavior::FIND_DECRYPTION_SEED;
} else if (!strncmp(argv[x], "--generate-rainbow-table=", 25)) {
behavior = Behavior::GENERATE_RAINBOW_TABLE;
rainbow_table_filename = &argv[x][25];
} else if (!strcmp(argv[x], "--decode-sjis")) {
behavior = Behavior::DECODE_SJIS;
} else if (!strncmp(argv[x], "--decode-gci=", 13)) {
@@ -371,11 +356,7 @@ int main(int argc, char** argv) {
behavior = Behavior::CAT_CLIENT;
cat_client_remote = make_sockaddr_storage(parse_netloc(&argv[x][13])).first;
} else if (!strncmp(argv[x], "--threads=", 10)) {
num_threads = strtoull(&argv[x][13], nullptr, 0);
} else if (!strncmp(argv[x], "--match-length=", 15)) {
match_length = strtoull(&argv[x][15], nullptr, 0);
} else if (!strncmp(argv[x], "--rainbow-table=", 16)) {
rainbow_table_filename = &argv[x][16];
num_threads = strtoull(&argv[x][10], nullptr, 0);
} else if (!strcmp(argv[x], "--patch")) {
cli_version = GameVersion::PATCH;
} else if (!strcmp(argv[x], "--dc")) {
@@ -516,61 +497,67 @@ int main(int argc, char** argv) {
throw runtime_error("--find-decryption-seed cannot be used for BB ciphers");
}
size_t max_plaintext_size = 0;
vector<pair<string, string>> plaintexts;
for (const auto& plaintext_ascii : find_decryption_seed_plaintexts) {
string mask;
string data = parse_data_string(plaintext_ascii, &mask);
if (data.size() != mask.size()) {
throw logic_error("plaintext and mask are not the same size");
}
max_plaintext_size = max<size_t>(max_plaintext_size, data.size());
plaintexts.emplace_back(move(data), move(mask));
}
string ciphertext = parse_data_string(find_decryption_seed_ciphertext);
if (num_threads == 0) {
num_threads = thread::hardware_concurrency();
}
PSOEncryptionSeedFinder finder(ciphertext, plaintexts, num_threads);
PSOEncryptionSeedFinder::ThreadResults results;
if (!rainbow_table_filename.empty()) {
results = finder.find_seed(rainbow_table_filename);
} else {
using Flag = PSOEncryptionSeedFinder::Flag;
uint64_t flags =
(((cli_version == GameVersion::GC) || (cli_version == GameVersion::XB)) ? Flag::V3 : 0) |
(skip_little_endian ? Flag::SKIP_LITTLE_ENDIAN : 0) |
(skip_big_endian ? Flag::SKIP_BIG_ENDIAN : 0);
results = finder.find_seed(flags);
}
log_info("Minimum differences: %zu", results.min_differences);
for (auto result : results.results) {
if (result.differences != results.min_differences) {
throw logic_error("incorrect difference count in result");
auto mask_match = +[](const void* a, const void* b, const void* m, size_t size) -> bool {
const uint8_t* a8 = reinterpret_cast<const uint8_t*>(a);
const uint8_t* b8 = reinterpret_cast<const uint8_t*>(b);
const uint8_t* m8 = reinterpret_cast<const uint8_t*>(m);
for (size_t z = 0; z < size; z++) {
if ((a8[z] & m8[z]) != (b8[z] & m8[z])) {
return false;
}
}
if (result.is_indeterminate) {
log_info("Example match: %08" PRIX32 " (%zu)",
result.seed, result.differences);
} else {
log_info("Example match: %08" PRIX32 " (%zu; %s, %s)",
result.seed,
result.differences,
result.is_v3 ? "v3" : "v2",
result.is_big_endian ? "big-endian" : "little-endian");
}
}
for (size_t z = 0; z < results.difference_histogram.size(); z++) {
log_info("(Difference histogram) %zu => %zu results",
z, results.difference_histogram[z]);
}
break;
}
return true;
};
case Behavior::GENERATE_RAINBOW_TABLE: {
if (num_threads == 0) {
num_threads = thread::hardware_concurrency();
}
bool is_v3 = ((cli_version == GameVersion::GC) || (cli_version == GameVersion::XB));
PSOEncryptionSeedFinder::generate_rainbow_table(
rainbow_table_filename, is_v3, big_endian, match_length, num_threads);
uint64_t seed = parallel_range<uint64_t>([&](uint64_t seed, size_t) -> bool {
string be_decrypt_buf = ciphertext.substr(0, max_plaintext_size);
string le_decrypt_buf = ciphertext.substr(0, max_plaintext_size);
if (is_v3) {
PSOV3Encryption(seed).encrypt_both_endian(
le_decrypt_buf.data(),
be_decrypt_buf.data(),
be_decrypt_buf.size());
} else {
PSOV2Encryption(seed).encrypt_both_endian(
le_decrypt_buf.data(),
be_decrypt_buf.data(),
be_decrypt_buf.size());
}
for (const auto& plaintext : plaintexts) {
if (!skip_little_endian) {
if (mask_match(le_decrypt_buf.data(), plaintext.first.data(), plaintext.second.data(), plaintext.second.size())) {
return true;
}
}
if (!skip_big_endian) {
if (mask_match(be_decrypt_buf.data(), plaintext.first.data(), plaintext.second.data(), plaintext.second.size())) {
return true;
}
}
}
return false;
}, 0, 0x100000000, num_threads);
if (seed < 0x100000000) {
log_info("Found seed %08" PRIX64, seed);
} else {
log_error("No seed found");
}
break;
}
src/PSOEncryptionSeedFinder.cc
-359
View File
@@ -1,359 +0,0 @@
#include "PSOEncryptionSeedFinder.hh"
#include <atomic>
#include <memory>
#include <thread>
#include <vector>
#include <phosg/Time.hh>
#include "PSOEncryption.hh"
using namespace std;
static size_t difference_match(const string& data1, const string& data2) {
if (data1.size() != data2.size()) {
return max<size_t>(data1.size(), data2.size());
}
size_t differences = 0;
for (size_t z = 0; z < data1.size(); z++) {
if (data1[z] != data2[z]) {
differences++;
}
}
return differences;
}
PSOEncryptionSeedFinder::PSOEncryptionSeedFinder(
const std::string& ciphertext,
const std::vector<std::pair<std::string, std::string>>& plaintexts,
size_t num_threads)
: ciphertext(ciphertext), plaintexts(plaintexts), num_threads(num_threads) {
if (num_threads == 0) {
throw logic_error("must use at least one thread");
}
if (this->ciphertext.empty() || (this->ciphertext.size() & 3)) {
throw runtime_error("ciphertext length must be a nonzero multiple of 4");
}
if (this->plaintexts.empty()) {
throw runtime_error("no plaintexts provided");
}
size_t plaintext_size = this->plaintexts[0].first.size();
for (const auto& plaintext : this->plaintexts) {
if (plaintext.first.size() != plaintext_size) {
throw runtime_error("plaintexts are not all the same size");
}
if (plaintext.first.size() != plaintext.second.size()) {
throw logic_error("plaintext and plaintext mask are not the same size");
}
}
}
PSOEncryptionSeedFinder::Result::Result(uint32_t seed, size_t differences)
: seed(seed),
differences(differences),
is_indeterminate(true),
is_big_endian(false),
is_v3(false) { }
PSOEncryptionSeedFinder::Result::Result(
uint32_t seed, size_t differences, bool is_big_endian, bool is_v3)
: seed(seed),
differences(differences),
is_indeterminate(false),
is_big_endian(is_big_endian),
is_v3(is_v3) { }
void PSOEncryptionSeedFinder::ThreadResults::add_result(const Result& res) {
if (res.differences < this->min_differences) {
this->results.clear();
this->min_differences = res.differences;
}
if ((res.differences == this->min_differences) && (this->results.size() < 10)) {
this->results.emplace_back(res);
}
if (this->difference_histogram.size() <= res.differences) {
this->difference_histogram.resize(res.differences + 1, 0);
}
this->difference_histogram[res.differences]++;
}
void PSOEncryptionSeedFinder::ThreadResults::combine_from(
const ThreadResults& other) {
if (this->min_differences > other.min_differences) {
this->min_differences = other.min_differences;
this->results = other.results;
} else if (this->min_differences == other.min_differences) {
this->results.insert(this->results.end(), other.results.begin(), other.results.end());
}
if (this->difference_histogram.size() < other.difference_histogram.size()) {
this->difference_histogram.resize(other.difference_histogram.size(), 0);
}
for (size_t z = 0; z < other.difference_histogram.size(); z++) {
this->difference_histogram[z] += other.difference_histogram[z];
}
}
PSOEncryptionSeedFinder::ThreadResults PSOEncryptionSeedFinder::find_seed(
uint64_t flags) {
// TODO: Use a specific logger here
log_info("Searching for decryption key (%s, %zu threads)",
(flags & Flag::V3) ? "v3" : "v2", this->num_threads);
return this->parallel_find_seed_t(
&PSOEncryptionSeedFinder::find_seed_without_rainbow_table_thread_fn,
this,
flags);
}
PSOEncryptionSeedFinder::ThreadResults PSOEncryptionSeedFinder::find_seed(
const string& rainbow_table_filename) {
size_t plaintext_size = this->plaintexts[0].first.size();
scoped_fd fd(rainbow_table_filename, O_RDONLY);
int64_t expected_rainbow_table_size = static_cast<int64_t>(plaintext_size) << 32;
if (fstat(fd).st_size != expected_rainbow_table_size) {
throw runtime_error("rainbow table size is incorrect");
}
// TODO: Use a specific logger here
log_info("Searching for decryption key (%zu threads) using rainbow table %s",
this->num_threads, rainbow_table_filename.c_str());
return this->parallel_find_seed_t(
&PSOEncryptionSeedFinder::find_seed_with_rainbow_table_thread_fn,
this,
static_cast<int>(fd),
0x1000);
}
void PSOEncryptionSeedFinder::generate_rainbow_table(
const std::string& filename,
bool is_v3,
bool is_big_endian,
size_t match_length,
size_t num_threads) {
if ((match_length == 0) || (match_length & 3)) {
throw runtime_error("match length must be a nonzero multiple of 4");
}
if (num_threads == 0) {
throw logic_error("must use at least one thread");
}
uint64_t file_size = static_cast<uint64_t>(match_length) << 32;
string file_size_str = format_size(file_size);
scoped_fd fd(filename, O_CREAT | O_WRONLY);
log_info("Allocating file space for rainbow table (match_length=%zu bytes => table size is %s)",
match_length, file_size_str.c_str());
if (ftruncate(fd, file_size) < 0) {
throw runtime_error("cannot allocate file space for table");
}
size_t page_size = 0x1000;
PSOEncryptionSeedFinder::parallel_all_seeds_t(
num_threads,
&PSOEncryptionSeedFinder::generate_rainbow_table_thread_fn,
static_cast<int>(fd),
match_length,
page_size,
is_v3,
is_big_endian);
log_info("Wrote %s to rainbow table %s\n", file_size_str.c_str(), filename.c_str());
}
void PSOEncryptionSeedFinder::parallel_all_seeds(
size_t num_threads, function<bool(uint32_t, size_t)> fn) {
PSOEncryptionSeedFinder::parallel_all_seeds_t(
num_threads,
&PSOEncryptionSeedFinder::parallel_all_seeds_thread_fn,
fn);
}
// TODO: Use phosg's parallel_range for this instead
template <typename... ThreadArgTs>
void PSOEncryptionSeedFinder::parallel_all_seeds_t(
size_t num_threads, ThreadArgTs... args) {
atomic<uint64_t> current_seed(0);
vector<thread> threads;
while (threads.size() < num_threads) {
threads.emplace_back(args..., ref(current_seed), threads.size());
}
uint64_t start_time = now();
uint64_t displayed_current_seed;
while ((displayed_current_seed = current_seed.load()) < 0x100000000) {
uint64_t elapsed_time = now() - start_time;
string elapsed_str = format_duration(elapsed_time);
string remaining_str;
if (displayed_current_seed) {
uint64_t total_time = (elapsed_time << 32) / displayed_current_seed;
uint64_t remaining_time = total_time - elapsed_time;
remaining_str = format_duration(remaining_time);
} else {
remaining_str = "...";
}
fprintf(stderr, "... %08" PRIX64 " (%s / -%s)\r", displayed_current_seed,
elapsed_str.c_str(), remaining_str.c_str());
usleep(1000000);
}
for (auto& t : threads) {
t.join();
}
}
template <typename... ThreadArgTs>
PSOEncryptionSeedFinder::ThreadResults PSOEncryptionSeedFinder::parallel_find_seed_t(
ThreadArgTs... args) {
vector<ThreadResults> all_thread_results;
all_thread_results.resize(this->num_threads);
this->parallel_all_seeds_t(this->num_threads, args..., ref(all_thread_results));
ThreadResults overall_results = all_thread_results[0];
for (const auto& thread_results : all_thread_results) {
overall_results.combine_from(thread_results);
}
return overall_results;
}
void PSOEncryptionSeedFinder::parallel_all_seeds_thread_fn(
function<bool(uint32_t, size_t)> fn,
atomic<uint64_t>& current_seed,
size_t thread_num) {
uint64_t seed;
while ((seed = current_seed.fetch_add(1)) < 0x100000000) {
if (fn(seed, thread_num)) {
current_seed = 0x100000000;
}
}
}
void PSOEncryptionSeedFinder::find_seed_without_rainbow_table_thread_fn(
uint64_t flags,
vector<ThreadResults>& all_results,
atomic<uint64_t>& current_seed,
size_t thread_num) {
size_t plaintext_size = this->plaintexts[0].first.size();
auto& results = all_results.at(thread_num);
results.results.clear();
results.min_differences = plaintext_size + 1;
results.difference_histogram.clear();
bool is_v3 = flags & Flag::V3;
bool skip_little_endian = flags & Flag::SKIP_LITTLE_ENDIAN;
bool skip_big_endian = flags & Flag::SKIP_BIG_ENDIAN;
uint64_t seed;
while ((seed = current_seed.fetch_add(1)) < 0x100000000) {
string be_decrypt_buf = this->ciphertext.substr(0, plaintext_size);
string le_decrypt_buf = this->ciphertext.substr(0, plaintext_size);
if (is_v3) {
PSOV3Encryption(seed).encrypt_both_endian(
le_decrypt_buf.data(),
be_decrypt_buf.data(),
be_decrypt_buf.size());
} else {
PSOV2Encryption(seed).encrypt_both_endian(
le_decrypt_buf.data(),
be_decrypt_buf.data(),
be_decrypt_buf.size());
}
for (const auto& plaintext : this->plaintexts) {
if (!skip_little_endian) {
size_t diff = difference_match(le_decrypt_buf, plaintext.first);
results.add_result(Result(seed, diff, false, is_v3));
} else if (!skip_big_endian) {
size_t diff = difference_match(be_decrypt_buf, plaintext.first);
results.add_result(Result(seed, diff, true, is_v3));
}
}
if (results.min_differences == 0) {
current_seed = 0x100000000;
}
}
}
void PSOEncryptionSeedFinder::find_seed_with_rainbow_table_thread_fn(
int fd,
size_t page_size,
vector<ThreadResults>& all_results,
atomic<uint64_t>& current_seed,
size_t thread_num) {
size_t plaintext_size = this->plaintexts[0].first.size();
auto& results = all_results.at(thread_num);
results.results.clear();
results.min_differences = plaintext_size + 1;
results.difference_histogram.clear();
uint64_t seed;
string rainbow_buf(page_size * plaintext_size, '\0');
while ((seed = current_seed.fetch_add(page_size)) < 0x100000000) {
preadx(fd, rainbow_buf.data(), rainbow_buf.size(), seed * plaintext_size);
for (size_t z = 0; z < page_size; z++) {
for (size_t x = 0; x < plaintext_size; x++) {
rainbow_buf[z * plaintext_size + x] ^= this->ciphertext[x];
}
for (const auto& plaintext : this->plaintexts) {
size_t diff = difference_match(
&rainbow_buf[z * plaintext_size], plaintext.first);
results.add_result(Result(seed, diff));
}
if (results.min_differences == 0) {
current_seed = 0x100000000;
}
}
}
}
void PSOEncryptionSeedFinder::generate_rainbow_table_thread_fn(
int fd,
size_t match_length,
size_t page_size,
bool is_v3,
bool is_big_endian,
atomic<uint64_t>& current_seed,
size_t) {
uint64_t seed;
string buf(match_length * page_size, '\0');
while ((seed = current_seed.fetch_add(page_size)) < 0x100000000) {
memset(buf.data(), 0, buf.size());
for (size_t z = 0; z < page_size; z++) {
if (is_v3) {
PSOV3Encryption crypt(seed + z);
if (is_big_endian) {
crypt.encrypt_big_endian(buf.data() + z * match_length, match_length);
} else {
crypt.encrypt(buf.data() + z * match_length, match_length);
}
} else {
PSOV2Encryption crypt(seed + z);
if (is_big_endian) {
crypt.encrypt_big_endian(buf.data() + z * match_length, match_length);
} else {
crypt.encrypt(buf.data() + z * match_length, match_length);
}
}
}
pwritex(fd, buf.data(), buf.size(), seed * match_length);
}
}
src/PSOEncryptionSeedFinder.hh
-100
View File
@@ -1,100 +0,0 @@
#pragma once
#include <inttypes.h>
#include <stddef.h>
#include <atomic>
#include <functional>
#include <string>
#include <thread>
#include <utility>
#include <vector>
class PSOEncryptionSeedFinder {
public:
PSOEncryptionSeedFinder(
const std::string& ciphertext,
const std::vector<std::pair<std::string, std::string>>& plaintexts,
size_t num_threads);
~PSOEncryptionSeedFinder() = default;
enum Flag {
V3 = 0x01,
SKIP_LITTLE_ENDIAN = 0x02,
SKIP_BIG_ENDIAN = 0x04,
};
struct Result {
uint32_t seed;
size_t differences;
bool is_indeterminate;
bool is_big_endian;
bool is_v3;
Result(uint32_t seed, size_t differences);
Result(uint32_t seed, size_t differences, bool is_big_endian, bool is_v3);
};
struct ThreadResults {
std::vector<Result> results;
size_t min_differences;
std::vector<size_t> difference_histogram;
void add_result(const Result& res);
void combine_from(const ThreadResults& other);
};
ThreadResults find_seed(std::function<bool(uint32_t, size_t)> fn);
ThreadResults find_seed(uint64_t flags);
ThreadResults find_seed(const std::string& rainbow_table_filename);
static void generate_rainbow_table(
const std::string& filename,
bool is_v3,
bool is_big_endian,
size_t match_length,
size_t num_threads);
static void parallel_all_seeds(
size_t num_threads, std::function<bool(uint32_t, size_t)> fn);
private:
template <typename... ThreadArgTs>
static void parallel_all_seeds_t(size_t num_threads, ThreadArgTs... args);
template <typename... ThreadArgTs>
ThreadResults parallel_find_seed_t(ThreadArgTs... args);
static void parallel_all_seeds_thread_fn(
std::function<bool(uint32_t, size_t)> fn,
std::atomic<uint64_t>& current_seed,
size_t thread_num);
void find_seed_without_rainbow_table_thread_fn(
uint64_t flags,
std::vector<ThreadResults>& all_results,
std::atomic<uint64_t>& current_seed,
size_t thread_num);
void find_seed_with_rainbow_table_thread_fn(
int fd,
size_t page_size,
std::vector<ThreadResults>& all_results,
std::atomic<uint64_t>& current_seed,
size_t thread_num);
static void generate_rainbow_table_thread_fn(
int fd,
size_t match_length,
size_t page_size,
bool is_v3,
bool is_big_endian,
std::atomic<uint64_t>& current_seed,
size_t thread_num);
std::string ciphertext;
std::vector<std::pair<std::string, std::string>> plaintexts;
size_t num_threads;
};
src/Quest.cc
+5 -8
View File
@@ -9,12 +9,12 @@
#include <phosg/Hash.hh>
#include <phosg/Random.hh>
#include <phosg/Strings.hh>
#include <phosg/Tools.hh>
#include "Loggers.hh"
#include "CommandFormats.hh"
#include "Compression.hh"
#include "PSOEncryption.hh"
#include "PSOEncryptionSeedFinder.hh"
#include "Text.hh"
using namespace std;
@@ -162,22 +162,19 @@ string find_seed_and_decrypt_gci_data_section(
const void* data_section, size_t size, size_t num_threads) {
mutex result_lock;
string result;
uint32_t result_seed = 0xFFFFFFFF;
PSOEncryptionSeedFinder::parallel_all_seeds(num_threads, [&](
uint32_t seed, size_t) {
uint64_t result_seed = parallel_range<uint64_t>([&](uint64_t seed, size_t) {
try {
string ret = decrypt_gci_data_section(data_section, size, seed);
lock_guard<mutex> g(result_lock);
result = move(ret);
result_seed = seed;
return true;
} catch (const runtime_error&) {
return false;
}
});
}, 0, 0x100000000, num_threads);
if (!result.empty()) {
static_game_data_log.info("Found seed %08" PRIX32 " to decrypt GCI file",
if (!result.empty() && (result_seed < 0x100000000)) {
static_game_data_log.info("Found seed %08" PRIX64 " to decrypt GCI file",
result_seed);
return result;
} else {