From 65c08667cc05ef158ccd08fcf904c69bf84cf191 Mon Sep 17 00:00:00 2001 From: Martin Michelsen Date: Sat, 10 Jun 2023 10:48:53 -0700 Subject: [PATCH] decode download quests during proxy save-files --- README.md | 3 ++- src/Episode3/DataIndex.cc | 4 +-- src/Main.cc | 8 +++--- src/ProxyCommands.cc | 38 +++++++++++++++++++------- src/ProxyServer.cc | 14 +++++++--- src/ProxyServer.hh | 10 ++++--- src/Quest.cc | 56 +++++++++++++++++++-------------------- src/Quest.hh | 9 ++++--- 8 files changed, 86 insertions(+), 56 deletions(-) diff --git a/README.md b/README.md index 585a2ca8..0b82c136 100644 --- a/README.md +++ b/README.md @@ -262,11 +262,12 @@ There are many options available when starting a proxy session. All options are * **Block events**: disables holiday events sent by the remote server. * **Block patches**: prevents any B2 (patch) commands from reaching the client. * **Save files**: saves copies of several kinds of files when they're sent by the remote server. The files are written to the current directory (which is usually the directory containing the system/ directory). These kinds of files can be saved: - * Online quests and download quests (saved as .bin/.dat files; for download quests, you may need to manually rename them to .bin.dlq/.dat.dlq to use them with newserv) + * Online quests and download quests (saved as .bin/.dat files) * GBA games (saved as .gba files) * Patches (saved as .bin files, and disassembled into PowerPC assembly if newserv is built with patch support) * Player data from BB sessions (saved as .bin files, which are not the same format as .nsc files) * Episode 3 online quests and maps (saved as .mnmd files) + * Episode 3 download quests (saved as .mnm files) * Episode 3 card definitions (saved as .mnr files) * Episode 3 media updates (saved as .gvm, .bml, or .bin files) diff --git a/src/Episode3/DataIndex.cc b/src/Episode3/DataIndex.cc index 9fad041b..e066e486 100644 --- a/src/Episode3/DataIndex.cc +++ b/src/Episode3/DataIndex.cc @@ -1618,9 +1618,9 @@ DataIndex::DataIndex(const string& directory, uint32_t behavior_flags) } else if (ends_with(filename, ".mnm") || ends_with(filename, ".bin")) { entry.reset(new MapEntry(load_file(dir + "/" + filename), is_quest)); } else if (ends_with(filename, ".gci")) { - entry.reset(new MapEntry(Quest::decode_gci(dir + "/" + filename), is_quest)); + entry.reset(new MapEntry(Quest::decode_gci_file(dir + "/" + filename), is_quest)); } else if (ends_with(filename, ".dlq")) { - entry.reset(new MapEntry(Quest::decode_dlq(dir + "/" + filename), is_quest)); + entry.reset(new MapEntry(Quest::decode_dlq_file(dir + "/" + filename), is_quest)); } if (entry.get()) { diff --git a/src/Main.cc b/src/Main.cc index fa2acea8..93a498d0 100644 --- a/src/Main.cc +++ b/src/Main.cc @@ -913,17 +913,17 @@ int main(int argc, char** argv) { string output_filename_base = input_filename; if (quest_file_type == QuestFileFormat::GCI) { int64_t dec_seed = seed.empty() ? -1 : stoul(seed, nullptr, 16); - auto decoded = Quest::decode_gci(input_filename, num_threads, dec_seed); + auto decoded = Quest::decode_gci_file(input_filename, num_threads, dec_seed); save_file(output_filename_base + ".dec", decoded); } else if (quest_file_type == QuestFileFormat::VMS) { int64_t dec_seed = seed.empty() ? -1 : stoul(seed, nullptr, 16); - auto decoded = Quest::decode_vms(input_filename, num_threads, dec_seed); + auto decoded = Quest::decode_vms_file(input_filename, num_threads, dec_seed); save_file(output_filename_base + ".dec", decoded); } else if (quest_file_type == QuestFileFormat::DLQ) { - auto decoded = Quest::decode_dlq(input_filename); + auto decoded = Quest::decode_dlq_file(input_filename); save_file(output_filename_base + ".dec", decoded); } else if (quest_file_type == QuestFileFormat::QST) { - auto data = Quest::decode_qst(input_filename); + auto data = Quest::decode_qst_file(input_filename); save_file(output_filename_base + ".bin", data.first); save_file(output_filename_base + ".dat", data.second); } else { diff --git a/src/ProxyCommands.cc b/src/ProxyCommands.cc index e0573f5a..0b5d402a 100644 --- a/src/ProxyCommands.cc +++ b/src/ProxyCommands.cc @@ -1104,10 +1104,25 @@ static HandlerResult S_44_A6(shared_ptr, string filename = cmd.filename; string output_filename; + bool is_download = (command == 0xA6); if (session.options.save_files) { - output_filename = string_printf("%s.%s.%" PRIu64, - filename.c_str(), - (command == 0xA6) ? "download" : "online", now()); + size_t extension_offset = filename.rfind('.'); + string basename, extension; + if (extension_offset != string::npos) { + basename = filename.substr(0, extension_offset); + extension = filename.substr(extension_offset); + if (extension == ".bin" && (session.newserv_client_config.cfg.flags & Client::Flag::IS_EPISODE_3)) { + extension += ".mnm"; + } + } else { + basename = filename; + } + output_filename = string_printf("%s.%s.%" PRIu64 "%s", + basename.c_str(), + is_download ? "download" : "online", + now(), + extension.c_str()); + for (size_t x = 0; x < output_filename.size(); x++) { if (output_filename[x] < 0x20 || output_filename[x] > 0x7E || output_filename[x] == '/') { output_filename[x] = '_'; @@ -1118,11 +1133,13 @@ static HandlerResult S_44_A6(shared_ptr, } } + // Episode 3 download quests aren't DLQ-encoded + bool decode_dlq = is_download && !(session.newserv_client_config.cfg.flags & Client::Flag::IS_EPISODE_3); ProxyServer::LinkedSession::SavingFile sf( - cmd.filename, output_filename, cmd.file_size); + cmd.filename, output_filename, cmd.file_size, decode_dlq); session.saving_files.emplace(cmd.filename, std::move(sf)); if (session.options.save_files) { - session.log.info("Opened file %s", output_filename.c_str()); + session.log.info("Saving %s from server to %s", filename.c_str(), output_filename.c_str()); } else { session.log.info("Tracking file %s", filename.c_str()); } @@ -1158,15 +1175,16 @@ static HandlerResult S_13_A7(shared_ptr, modified = true; } - if (sf->f.get()) { - session.log.info("Writing %" PRIu32 " bytes to %s", - cmd.data_size.load(), sf->output_filename.c_str()); - fwritex(sf->f.get(), cmd.data.data(), cmd.data_size); + if (!sf->output_filename.empty()) { + session.log.info("Adding %" PRIu32 " bytes to %s => %s", + cmd.data_size.load(), sf->basename.c_str(), sf->output_filename.c_str()); + sf->blocks.emplace_back(reinterpret_cast(cmd.data.data()), cmd.data_size); } sf->remaining_bytes -= cmd.data_size; if (sf->remaining_bytes == 0) { - session.log.info("Closing file %s", sf->output_filename.c_str()); + session.log.info("Writing file %s => %s", sf->basename.c_str(), sf->output_filename.c_str()); + sf->write(); session.saving_files.erase(cmd.filename); } diff --git a/src/ProxyServer.cc b/src/ProxyServer.cc index e85e2bba..039f6b9a 100644 --- a/src/ProxyServer.cc +++ b/src/ProxyServer.cc @@ -599,13 +599,19 @@ void ProxyServer::LinkedSession::connect() { ProxyServer::LinkedSession::SavingFile::SavingFile( const string& basename, const string& output_filename, - uint32_t remaining_bytes) + size_t remaining_bytes, + bool is_download) : basename(basename), output_filename(output_filename), - remaining_bytes(remaining_bytes) { - if (!this->output_filename.empty()) { - this->f = fopen_unique(this->output_filename, "wb"); + is_download(is_download), + remaining_bytes(remaining_bytes) {} + +void ProxyServer::LinkedSession::SavingFile::write() const { + string data = join(this->blocks); + if (is_download && (ends_with(this->basename, ".bin") || ends_with(this->basename, ".dat"))) { + data = Quest::decode_dlq_data(data); } + save_file(this->output_filename, data); } void ProxyServer::LinkedSession::dispatch_on_timeout( diff --git a/src/ProxyServer.hh b/src/ProxyServer.hh index ed3fea58..c4bb9228 100644 --- a/src/ProxyServer.hh +++ b/src/ProxyServer.hh @@ -97,13 +97,17 @@ public: struct SavingFile { std::string basename; std::string output_filename; - uint32_t remaining_bytes; - std::unique_ptr> f; + bool is_download; + size_t remaining_bytes; + std::deque blocks; SavingFile( const std::string& basename, const std::string& output_filename, - uint32_t remaining_bytes); + size_t remaining_bytes, + bool is_download); + + void write() const; }; std::unordered_map saving_files; diff --git a/src/Quest.cc b/src/Quest.cc index f173816a..e29707db 100644 --- a/src/Quest.cc +++ b/src/Quest.cc @@ -507,19 +507,19 @@ shared_ptr Quest::bin_contents() const { this->file_basename + (this->has_mnm_extension ? ".mnmd" : ".bind"))))); break; case FileFormat::BIN_DAT_GCI: - this->bin_contents_ptr.reset(new string(this->decode_gci( + this->bin_contents_ptr.reset(new string(this->decode_gci_file( this->file_basename + (this->has_mnm_extension ? ".mnm.gci" : ".bin.gci")))); break; case FileFormat::BIN_DAT_VMS: - this->bin_contents_ptr.reset(new string(this->decode_vms( + this->bin_contents_ptr.reset(new string(this->decode_vms_file( this->file_basename + (this->has_mnm_extension ? ".mnm.vms" : ".bin.vms")))); break; case FileFormat::BIN_DAT_DLQ: - this->bin_contents_ptr.reset(new string(this->decode_dlq( + this->bin_contents_ptr.reset(new string(this->decode_dlq_file( this->file_basename + (this->has_mnm_extension ? ".mnm.dlq" : ".bin.dlq")))); break; case FileFormat::QST: { - auto result = this->decode_qst(this->file_basename + ".qst"); + auto result = this->decode_qst_file(this->file_basename + ".qst"); this->bin_contents_ptr.reset(new string(std::move(result.first))); this->dat_contents_ptr.reset(new string(std::move(result.second))); break; @@ -544,16 +544,16 @@ shared_ptr Quest::dat_contents() const { this->dat_contents_ptr.reset(new string(prs_compress(load_file(this->file_basename + ".datd")))); break; case FileFormat::BIN_DAT_GCI: - this->dat_contents_ptr.reset(new string(this->decode_gci(this->file_basename + ".dat.gci"))); + this->dat_contents_ptr.reset(new string(this->decode_gci_file(this->file_basename + ".dat.gci"))); break; case FileFormat::BIN_DAT_VMS: - this->dat_contents_ptr.reset(new string(this->decode_vms(this->file_basename + ".dat.vms"))); + this->dat_contents_ptr.reset(new string(this->decode_vms_file(this->file_basename + ".dat.vms"))); break; case FileFormat::BIN_DAT_DLQ: - this->dat_contents_ptr.reset(new string(this->decode_dlq(this->file_basename + ".dat.dlq"))); + this->dat_contents_ptr.reset(new string(this->decode_dlq_file(this->file_basename + ".dat.dlq"))); break; case FileFormat::QST: { - auto result = this->decode_qst(this->file_basename + ".qst"); + auto result = this->decode_qst_file(this->file_basename + ".qst"); this->bin_contents_ptr.reset(new string(std::move(result.first))); this->dat_contents_ptr.reset(new string(std::move(result.second))); break; @@ -565,7 +565,7 @@ shared_ptr Quest::dat_contents() const { return this->dat_contents_ptr; } -string Quest::decode_gci( +string Quest::decode_gci_file( const string& filename, ssize_t find_seed_num_threads, int64_t known_seed) { string data = load_file(filename); @@ -649,7 +649,7 @@ string Quest::decode_gci( } } -string Quest::decode_vms( +string Quest::decode_vms_file( const string& filename, ssize_t find_seed_num_threads, int64_t known_seed) { string data = load_file(filename); @@ -682,32 +682,32 @@ string Quest::decode_vms( } } -string Quest::decode_dlq(const string& filename) { - uint32_t decompressed_size; - uint32_t key; - string data; - { - auto f = fopen_unique(filename, "rb"); - decompressed_size = freadx(f.get()); - key = freadx(f.get()); - data = read_all(f.get()); - } +string Quest::decode_dlq_data(const string& data) { + StringReader r(data); + uint32_t decompressed_size = r.get_u32l(); + uint32_t key = r.get_u32l(); // The compressed data size does not need to be a multiple of 4, but the V2 // encryption (which is used for all download quests, even in V3) requires the // data size to be a multiple of 4. We'll just temporarily stick a few bytes // on the end, then throw them away later if needed. + string decrypted = r.read(r.remaining()); PSOV2Encryption encr(key); size_t original_size = data.size(); - data.resize((data.size() + 3) & (~3)); - encr.decrypt(data); - data.resize(original_size); + decrypted.resize((decrypted.size() + 3) & (~3)); + encr.decrypt(decrypted); + decrypted.resize(original_size); - if (prs_decompress_size(data) != decompressed_size) { + if (prs_decompress_size(decrypted) != decompressed_size) { throw runtime_error("decompressed size does not match size in header"); } - return data; + return decrypted; +} + +string Quest::decode_dlq_file(const string& filename) { + auto f = fopen_unique(filename, "rb"); + return Quest::decode_dlq_data(read_all(f.get())); } template @@ -814,14 +814,14 @@ static pair decode_qst_t(FILE* f) { } if (subformat == Quest::FileFormat::BIN_DAT_DLQ) { - bin_contents = Quest::decode_dlq(bin_contents); - dat_contents = Quest::decode_dlq(dat_contents); + bin_contents = Quest::decode_dlq_file(bin_contents); + dat_contents = Quest::decode_dlq_file(dat_contents); } return make_pair(bin_contents, dat_contents); } -pair Quest::decode_qst(const string& filename) { +pair Quest::decode_qst_file(const string& filename) { auto f = fopen_unique(filename, "rb"); // QST files start with an open file command, but the format differs depending diff --git a/src/Quest.hh b/src/Quest.hh index 85b46470..42c2618d 100644 --- a/src/Quest.hh +++ b/src/Quest.hh @@ -82,16 +82,17 @@ public: std::shared_ptr create_download_quest() const; - static std::string decode_gci( + static std::string decode_gci_file( const std::string& filename, ssize_t find_seed_num_threads = -1, int64_t known_seed = -1); - static std::string decode_vms( + static std::string decode_vms_file( const std::string& filename, ssize_t find_seed_num_threads = -1, int64_t known_seed = -1); - static std::string decode_dlq(const std::string& filename); - static std::pair decode_qst(const std::string& filename); + static std::string decode_dlq_file(const std::string& filename); + static std::string decode_dlq_data(const std::string& filename); + static std::pair decode_qst_file(const std::string& filename); std::string export_qst(GameVersion version) const;