decode download quests during proxy save-files

README.md

@@ -262,11 +262,12 @@ There are many options available when starting a proxy session. All options are
 * **Block events**: disables holiday events sent by the remote server.
 * **Block patches**: prevents any B2 (patch) commands from reaching the client.
 * **Save files**: saves copies of several kinds of files when they're sent by the remote server. The files are written to the current directory (which is usually the directory containing the system/ directory). These kinds of files can be saved:
-    * Online quests and download quests (saved as .bin/.dat files; for download quests, you may need to manually rename them to .bin.dlq/.dat.dlq to use them with newserv)
+    * Online quests and download quests (saved as .bin/.dat files)
     * GBA games (saved as .gba files)
     * Patches (saved as .bin files, and disassembled into PowerPC assembly if newserv is built with patch support)
     * Player data from BB sessions (saved as .bin files, which are not the same format as .nsc files)
     * Episode 3 online quests and maps (saved as .mnmd files)
+    * Episode 3 download quests (saved as .mnm files)
     * Episode 3 card definitions (saved as .mnr files)
     * Episode 3 media updates (saved as .gvm, .bml, or .bin files)
 

src/Episode3/DataIndex.cc

@@ -1618,9 +1618,9 @@ DataIndex::DataIndex(const string& directory, uint32_t behavior_flags)
         } else if (ends_with(filename, ".mnm") || ends_with(filename, ".bin")) {
           entry.reset(new MapEntry(load_file(dir + "/" + filename), is_quest));
         } else if (ends_with(filename, ".gci")) {
-          entry.reset(new MapEntry(Quest::decode_gci(dir + "/" + filename), is_quest));
+          entry.reset(new MapEntry(Quest::decode_gci_file(dir + "/" + filename), is_quest));
         } else if (ends_with(filename, ".dlq")) {
-          entry.reset(new MapEntry(Quest::decode_dlq(dir + "/" + filename), is_quest));
+          entry.reset(new MapEntry(Quest::decode_dlq_file(dir + "/" + filename), is_quest));
         }
 
         if (entry.get()) {

src/Main.cc

@@ -913,17 +913,17 @@ int main(int argc, char** argv) {
       string output_filename_base = input_filename;
       if (quest_file_type == QuestFileFormat::GCI) {
         int64_t dec_seed = seed.empty() ? -1 : stoul(seed, nullptr, 16);
-        auto decoded = Quest::decode_gci(input_filename, num_threads, dec_seed);
+        auto decoded = Quest::decode_gci_file(input_filename, num_threads, dec_seed);
         save_file(output_filename_base + ".dec", decoded);
       } else if (quest_file_type == QuestFileFormat::VMS) {
         int64_t dec_seed = seed.empty() ? -1 : stoul(seed, nullptr, 16);
-        auto decoded = Quest::decode_vms(input_filename, num_threads, dec_seed);
+        auto decoded = Quest::decode_vms_file(input_filename, num_threads, dec_seed);
         save_file(output_filename_base + ".dec", decoded);
       } else if (quest_file_type == QuestFileFormat::DLQ) {
-        auto decoded = Quest::decode_dlq(input_filename);
+        auto decoded = Quest::decode_dlq_file(input_filename);
         save_file(output_filename_base + ".dec", decoded);
       } else if (quest_file_type == QuestFileFormat::QST) {
-        auto data = Quest::decode_qst(input_filename);
+        auto data = Quest::decode_qst_file(input_filename);
         save_file(output_filename_base + ".bin", data.first);
         save_file(output_filename_base + ".dat", data.second);
       } else {

src/ProxyCommands.cc

@@ -1104,10 +1104,25 @@ static HandlerResult S_44_A6(shared_ptr<ServerState>,
 
   string filename = cmd.filename;
   string output_filename;
+  bool is_download = (command == 0xA6);
   if (session.options.save_files) {
-    output_filename = string_printf("%s.%s.%" PRIu64,
+    size_t extension_offset = filename.rfind('.');
-        filename.c_str(),
+    string basename, extension;
-        (command == 0xA6) ? "download" : "online", now());
+    if (extension_offset != string::npos) {
+      basename = filename.substr(0, extension_offset);
+      extension = filename.substr(extension_offset);
+      if (extension == ".bin" && (session.newserv_client_config.cfg.flags & Client::Flag::IS_EPISODE_3)) {
+        extension += ".mnm";
+      }
+    } else {
+      basename = filename;
+    }
+    output_filename = string_printf("%s.%s.%" PRIu64 "%s",
+        basename.c_str(),
+        is_download ? "download" : "online",
+        now(),
+        extension.c_str());
+
     for (size_t x = 0; x < output_filename.size(); x++) {
       if (output_filename[x] < 0x20 || output_filename[x] > 0x7E || output_filename[x] == '/') {
         output_filename[x] = '_';
@@ -1118,11 +1133,13 @@ static HandlerResult S_44_A6(shared_ptr<ServerState>,
     }
   }
 
+  // Episode 3 download quests aren't DLQ-encoded
+  bool decode_dlq = is_download && !(session.newserv_client_config.cfg.flags & Client::Flag::IS_EPISODE_3);
   ProxyServer::LinkedSession::SavingFile sf(
-      cmd.filename, output_filename, cmd.file_size);
+      cmd.filename, output_filename, cmd.file_size, decode_dlq);
   session.saving_files.emplace(cmd.filename, std::move(sf));
   if (session.options.save_files) {
-    session.log.info("Opened file %s", output_filename.c_str());
+    session.log.info("Saving %s from server to %s", filename.c_str(), output_filename.c_str());
   } else {
     session.log.info("Tracking file %s", filename.c_str());
   }
@@ -1158,15 +1175,16 @@ static HandlerResult S_13_A7(shared_ptr<ServerState>,
     modified = true;
   }
 
-  if (sf->f.get()) {
+  if (!sf->output_filename.empty()) {
-    session.log.info("Writing %" PRIu32 " bytes to %s",
+    session.log.info("Adding %" PRIu32 " bytes to %s => %s",
-        cmd.data_size.load(), sf->output_filename.c_str());
+        cmd.data_size.load(), sf->basename.c_str(), sf->output_filename.c_str());
-    fwritex(sf->f.get(), cmd.data.data(), cmd.data_size);
+    sf->blocks.emplace_back(reinterpret_cast<const char*>(cmd.data.data()), cmd.data_size);
   }
   sf->remaining_bytes -= cmd.data_size;
 
   if (sf->remaining_bytes == 0) {
-    session.log.info("Closing file %s", sf->output_filename.c_str());
+    session.log.info("Writing file %s => %s", sf->basename.c_str(), sf->output_filename.c_str());
+    sf->write();
     session.saving_files.erase(cmd.filename);
   }
 

src/ProxyServer.cc

@@ -599,13 +599,19 @@ void ProxyServer::LinkedSession::connect() {
 ProxyServer::LinkedSession::SavingFile::SavingFile(
     const string& basename,
     const string& output_filename,
-    uint32_t remaining_bytes)
+    size_t remaining_bytes,
+    bool is_download)
     : basename(basename),
       output_filename(output_filename),
-      remaining_bytes(remaining_bytes) {
+      is_download(is_download),
-  if (!this->output_filename.empty()) {
+      remaining_bytes(remaining_bytes) {}
-    this->f = fopen_unique(this->output_filename, "wb");
+
+void ProxyServer::LinkedSession::SavingFile::write() const {
+  string data = join(this->blocks);
+  if (is_download && (ends_with(this->basename, ".bin") || ends_with(this->basename, ".dat"))) {
+    data = Quest::decode_dlq_data(data);
   }
+  save_file(this->output_filename, data);
 }
 
 void ProxyServer::LinkedSession::dispatch_on_timeout(

src/ProxyServer.hh

@@ -97,13 +97,17 @@ public:
     struct SavingFile {
       std::string basename;
       std::string output_filename;
-      uint32_t remaining_bytes;
+      bool is_download;
-      std::unique_ptr<FILE, std::function<void(FILE*)>> f;
+      size_t remaining_bytes;
+      std::deque<std::string> blocks;
 
       SavingFile(
           const std::string& basename,
           const std::string& output_filename,
-          uint32_t remaining_bytes);
+          size_t remaining_bytes,
+          bool is_download);
+
+      void write() const;
     };
     std::unordered_map<std::string, SavingFile> saving_files;
 

src/Quest.cc

@@ -507,19 +507,19 @@ shared_ptr<const string> Quest::bin_contents() const {
             this->file_basename + (this->has_mnm_extension ? ".mnmd" : ".bind")))));
         break;
       case FileFormat::BIN_DAT_GCI:
-        this->bin_contents_ptr.reset(new string(this->decode_gci(
+        this->bin_contents_ptr.reset(new string(this->decode_gci_file(
             this->file_basename + (this->has_mnm_extension ? ".mnm.gci" : ".bin.gci"))));
         break;
       case FileFormat::BIN_DAT_VMS:
-        this->bin_contents_ptr.reset(new string(this->decode_vms(
+        this->bin_contents_ptr.reset(new string(this->decode_vms_file(
             this->file_basename + (this->has_mnm_extension ? ".mnm.vms" : ".bin.vms"))));
         break;
       case FileFormat::BIN_DAT_DLQ:
-        this->bin_contents_ptr.reset(new string(this->decode_dlq(
+        this->bin_contents_ptr.reset(new string(this->decode_dlq_file(
             this->file_basename + (this->has_mnm_extension ? ".mnm.dlq" : ".bin.dlq"))));
         break;
       case FileFormat::QST: {
-        auto result = this->decode_qst(this->file_basename + ".qst");
+        auto result = this->decode_qst_file(this->file_basename + ".qst");
         this->bin_contents_ptr.reset(new string(std::move(result.first)));
         this->dat_contents_ptr.reset(new string(std::move(result.second)));
         break;
@@ -544,16 +544,16 @@ shared_ptr<const string> Quest::dat_contents() const {
         this->dat_contents_ptr.reset(new string(prs_compress(load_file(this->file_basename + ".datd"))));
         break;
       case FileFormat::BIN_DAT_GCI:
-        this->dat_contents_ptr.reset(new string(this->decode_gci(this->file_basename + ".dat.gci")));
+        this->dat_contents_ptr.reset(new string(this->decode_gci_file(this->file_basename + ".dat.gci")));
         break;
       case FileFormat::BIN_DAT_VMS:
-        this->dat_contents_ptr.reset(new string(this->decode_vms(this->file_basename + ".dat.vms")));
+        this->dat_contents_ptr.reset(new string(this->decode_vms_file(this->file_basename + ".dat.vms")));
         break;
       case FileFormat::BIN_DAT_DLQ:
-        this->dat_contents_ptr.reset(new string(this->decode_dlq(this->file_basename + ".dat.dlq")));
+        this->dat_contents_ptr.reset(new string(this->decode_dlq_file(this->file_basename + ".dat.dlq")));
         break;
       case FileFormat::QST: {
-        auto result = this->decode_qst(this->file_basename + ".qst");
+        auto result = this->decode_qst_file(this->file_basename + ".qst");
         this->bin_contents_ptr.reset(new string(std::move(result.first)));
         this->dat_contents_ptr.reset(new string(std::move(result.second)));
         break;
@@ -565,7 +565,7 @@ shared_ptr<const string> Quest::dat_contents() const {
   return this->dat_contents_ptr;
 }
 
-string Quest::decode_gci(
+string Quest::decode_gci_file(
     const string& filename, ssize_t find_seed_num_threads, int64_t known_seed) {
   string data = load_file(filename);
 
@@ -649,7 +649,7 @@ string Quest::decode_gci(
   }
 }
 
-string Quest::decode_vms(
+string Quest::decode_vms_file(
     const string& filename, ssize_t find_seed_num_threads, int64_t known_seed) {
   string data = load_file(filename);
 
@@ -682,32 +682,32 @@ string Quest::decode_vms(
   }
 }
 
-string Quest::decode_dlq(const string& filename) {
+string Quest::decode_dlq_data(const string& data) {
-  uint32_t decompressed_size;
+  StringReader r(data);
-  uint32_t key;
+  uint32_t decompressed_size = r.get_u32l();
-  string data;
+  uint32_t key = r.get_u32l();
-  {
-    auto f = fopen_unique(filename, "rb");
-    decompressed_size = freadx<le_uint32_t>(f.get());
-    key = freadx<le_uint32_t>(f.get());
-    data = read_all(f.get());
-  }
 
   // The compressed data size does not need to be a multiple of 4, but the V2
   // encryption (which is used for all download quests, even in V3) requires the
   // data size to be a multiple of 4. We'll just temporarily stick a few bytes
   // on the end, then throw them away later if needed.
+  string decrypted = r.read(r.remaining());
   PSOV2Encryption encr(key);
   size_t original_size = data.size();
-  data.resize((data.size() + 3) & (~3));
+  decrypted.resize((decrypted.size() + 3) & (~3));
-  encr.decrypt(data);
+  encr.decrypt(decrypted);
-  data.resize(original_size);
+  decrypted.resize(original_size);
 
-  if (prs_decompress_size(data) != decompressed_size) {
+  if (prs_decompress_size(decrypted) != decompressed_size) {
     throw runtime_error("decompressed size does not match size in header");
   }
 
-  return data;
+  return decrypted;
+}
+
+string Quest::decode_dlq_file(const string& filename) {
+  auto f = fopen_unique(filename, "rb");
+  return Quest::decode_dlq_data(read_all(f.get()));
 }
 
 template <typename HeaderT, typename OpenFileT>
@@ -814,14 +814,14 @@ static pair<string, string> decode_qst_t(FILE* f) {
   }
 
   if (subformat == Quest::FileFormat::BIN_DAT_DLQ) {
-    bin_contents = Quest::decode_dlq(bin_contents);
+    bin_contents = Quest::decode_dlq_file(bin_contents);
-    dat_contents = Quest::decode_dlq(dat_contents);
+    dat_contents = Quest::decode_dlq_file(dat_contents);
   }
 
   return make_pair(bin_contents, dat_contents);
 }
 
-pair<string, string> Quest::decode_qst(const string& filename) {
+pair<string, string> Quest::decode_qst_file(const string& filename) {
   auto f = fopen_unique(filename, "rb");
 
   // QST files start with an open file command, but the format differs depending

src/Quest.hh

@@ -82,16 +82,17 @@ public:
 
   std::shared_ptr<Quest> create_download_quest() const;
 
-  static std::string decode_gci(
+  static std::string decode_gci_file(
       const std::string& filename,
       ssize_t find_seed_num_threads = -1,
       int64_t known_seed = -1);
-  static std::string decode_vms(
+  static std::string decode_vms_file(
       const std::string& filename,
       ssize_t find_seed_num_threads = -1,
       int64_t known_seed = -1);
-  static std::string decode_dlq(const std::string& filename);
+  static std::string decode_dlq_file(const std::string& filename);
-  static std::pair<std::string, std::string> decode_qst(const std::string& filename);
+  static std::string decode_dlq_data(const std::string& filename);
+  static std::pair<std::string, std::string> decode_qst_file(const std::string& filename);
 
   std::string export_qst(GameVersion version) const;