add encode-qst action

README.md

@@ -387,18 +387,20 @@ For GC clients, you'll have to use newserv's built-in DNS server or set up your
 
 newserv has many CLI options, which can be used to access functionality other than the game and proxy server. Run `newserv help` to see these options and how to use them. The non-server things newserv can do are:
 
-* Compress or decompress data in the PRS and BC0 formats (`compress-prs`, `compress-bc0`, `decompress-prs`, `decompress-bc0`)
+* Compress or decompress data in PRS, PR2, or BC0 format (`compress-prs`, `compress-pr2`, `compress-bc0`, `decompress-prs`, `decompress-pr2`, `decompress-bc0`)
 * Compute the decompressed size of compressed PRS data without decompressing it (`prs-size`)
 * Encrypt or decrypt data using any PSO version's network encryption scheme (`encrypt-data`, `decrypt-data`)
 * Encrypt or decrypt data using Episode 3's trivial scheme (`encrypt-trivial-data`, `decrypt-trivial-data`)
+* Encrypt or decrypt data using the Challenge Mode text algorithm (`encrypt-challenge-data`, `decrypt-challenge-data`)
 * Encrypt or decrypt PSO GC save data (.gci files) (`encrypt-gci-save`, `decrypt-gci-save`)
 * Find the likely round1 or round2 seed for a corrupt save file (`salvage-gci`)
 * Run a brute-force search for a decryption seed (`find-decryption-seed`)
 * Decode Shift-JIS text to UTF-16 (`decode-sjis`)
 * Convert quests in .gci, .vms, .dlq, or .qst format to .bin/.dat format (`decode-gci`, `decode-vms`, `decode-dlq`, `decode-qst`)
-* Disassemble quest scripts (`disassemble-bin`)
+* Convert quests in .bin/.dat to .qst format (`encode-qst`)
-* Connect to another PSO server and pretend to be a client (`cat-client`)
+* Disassemble quest scripts (`disassemble-quest-script`)
 * Format Episode 3 game data in a human-readable manner (`show-ep3-data`)
-* Render a human-readable description of item data (`describe-item`)
+* Convert item data to a human-readable description, or vice versa (`describe-item`, `encode-item`)
+* Connect to another PSO server and pretend to be a client (`cat-client`)
 * Replay a session log for testing (`replay-log`)
 * Extract the contents of a .gsl or .bml archive (`extract-gsl`, `extract-bml`)

src/Main.cc

@@ -187,6 +187,11 @@ The actions are:\n\
     GCI or VMS file, use the --seed=SEED option and give the serial number (as\n\
     a hex-encoded 32-bit integer). If you don\'t know the serial number,\n\
     newserv will find it via a brute-force search, which will take a long time.\n\
+  encode-qst INPUT-FILENAME [OUTPUT-FILENAME] [OPTIONS...]\n\
+    Encode the input quest file (in .bin/.dat format) into a .qst file. If\n\
+    --download is given, generates a download .qst instead of an online .qst.\n\
+    Specify the quest\'s game version with one of the --dc-nte, --dc-v1,\n\
+    --dc-v2, --pc, --gc-nte, --gc, --gc-ep3, --xb, or --bb options.\n\
   disassemble-quest-script [INPUT-FILENAME [OUTPUT-FILENAME]]\n\
     Disassemble the input quest script (.bin file) into a text representation\n\
     of the commands and metadata it contains. Specify the quest\'s game version\n\
@@ -264,6 +269,7 @@ enum class Behavior {
   FIND_DECRYPTION_SEED,
   SALVAGE_GCI,
   DECODE_QUEST_FILE,
+  ENCODE_QST,
   DISASSEMBLE_QUEST_SCRIPT,
   DECODE_SJIS,
   EXTRACT_GSL,
@@ -302,6 +308,7 @@ static bool behavior_takes_input_filename(Behavior b) {
       (b == Behavior::SALVAGE_GCI) ||
       (b == Behavior::ENCRYPT_GCI_SAVE) ||
       (b == Behavior::DECODE_QUEST_FILE) ||
+      (b == Behavior::ENCODE_QST) ||
       (b == Behavior::DISASSEMBLE_QUEST_SCRIPT) ||
       (b == Behavior::DECODE_SJIS) ||
       (b == Behavior::FORMAT_RARE_ITEM_SET) ||
@@ -331,6 +338,7 @@ static bool behavior_takes_output_filename(Behavior b) {
       (b == Behavior::DECRYPT_CHALLENGE_DATA) ||
       (b == Behavior::DECRYPT_GCI_SAVE) ||
       (b == Behavior::ENCRYPT_GCI_SAVE) ||
+      (b == Behavior::ENCODE_QST) ||
       (b == Behavior::DISASSEMBLE_QUEST_SCRIPT) ||
       (b == Behavior::CONVERT_ITEMRT_REL_TO_JSON) ||
       (b == Behavior::DECODE_SJIS) ||
@@ -361,6 +369,7 @@ int main(int argc, char** argv) {
   bool expect_decompressed = false;
   bool compress_optimal = false;
   bool json = false;
+  bool download = false;
   const char* find_decryption_seed_ciphertext = nullptr;
   vector<const char*> find_decryption_seed_plaintexts;
   const char* input_filename = nullptr;
@@ -378,6 +387,8 @@ int main(int argc, char** argv) {
       return 0;
     } else if (!strncmp(argv[x], "--threads=", 10)) {
       num_threads = strtoull(&argv[x][10], nullptr, 0);
+    } else if (!strcmp(argv[x], "--download")) {
+      download = true;
     } else if (!strcmp(argv[x], "--patch")) {
       cli_version = GameVersion::PATCH;
       cli_quest_version = QuestScriptVersion::PC_V2;
@@ -517,6 +528,8 @@ int main(int argc, char** argv) {
         } else if (!strcmp(argv[x], "decode-qst")) {
           behavior = Behavior::DECODE_QUEST_FILE;
           quest_file_type = Quest::FileFormat::QST;
+        } else if (!strcmp(argv[x], "encode-qst")) {
+          behavior = Behavior::ENCODE_QST;
         } else if (!strcmp(argv[x], "disassemble-quest-script")) {
           behavior = Behavior::DISASSEMBLE_QUEST_SCRIPT;
         } else if (!strcmp(argv[x], "cat-client")) {
@@ -1110,6 +1123,21 @@ int main(int argc, char** argv) {
       break;
     }
 
+    case Behavior::ENCODE_QST: {
+      if (!input_filename || !strcmp(input_filename, "-")) {
+        throw invalid_argument("an input filename is required");
+      }
+
+      shared_ptr<Quest> q(new Quest(input_filename, cli_quest_version, nullptr));
+      if (download) {
+        q = q->create_download_quest();
+      }
+      string qst_data = q->encode_qst();
+
+      write_output_data(qst_data.data(), qst_data.size());
+      break;
+    }
+
     case Behavior::DISASSEMBLE_QUEST_SCRIPT: {
       if (!input_filename || !strcmp(input_filename, "-")) {
         throw invalid_argument("an input filename is required");

src/Quest.cc

@@ -231,12 +231,13 @@ struct PSODownloadQuestHeader {
   le_uint32_t encryption_seed;
 } __attribute__((packed));
 
-Quest::Quest(const string& bin_filename, shared_ptr<const QuestCategoryIndex> category_index)
+Quest::Quest(const string& bin_filename, QuestScriptVersion version, shared_ptr<const QuestCategoryIndex> category_index)
     : internal_id(-1),
       menu_item_id(0),
       category_id(0),
       episode(Episode::NONE),
       joinable(false),
+      version(version),
       file_format(FileFormat::BIN_DAT),
       has_mnm_extension(false),
       is_dlq_encoded(false) {
@@ -281,36 +282,42 @@ Quest::Quest(const string& bin_filename, shared_ptr<const QuestCategoryIndex> ca
     throw invalid_argument("empty filename");
   }
 
-  vector<string> tokens = split(basename, '-');
+  if ((version == QuestScriptVersion::UNKNOWN) || category_index) {
+    vector<string> tokens = split(basename, '-');
 
-  string category_token;
+    string category_token;
-  if (tokens.size() == 3) {
+    if (tokens.size() == 3) {
-    category_token = std::move(tokens[1]);
+      category_token = std::move(tokens[1]);
-    tokens.erase(tokens.begin() + 1);
+      tokens.erase(tokens.begin() + 1);
-  } else if (tokens.size() != 2) {
+    } else if (tokens.size() != 2) {
-    throw invalid_argument("incorrect filename format");
+      throw invalid_argument("incorrect filename format");
+    }
+
+    if (category_index) {
+      auto& category = category_index->find(basename[0], category_token);
+      this->category_id = category.category_id;
+    } else {
+      this->category_id = 0;
+    }
+
+    // Parse the number out of the first token
+    this->internal_id = strtoull(tokens[0].c_str() + 1, nullptr, 10);
+
+    // Get the version from the second (or previously third) token
+    static const unordered_map<string, QuestScriptVersion> name_to_version({
+        {"dn", QuestScriptVersion::DC_NTE},
+        {"d1", QuestScriptVersion::DC_V1},
+        {"dc", QuestScriptVersion::DC_V2},
+        {"pc", QuestScriptVersion::PC_V2},
+        {"gcn", QuestScriptVersion::GC_NTE},
+        {"gc", QuestScriptVersion::GC_V3},
+        {"gc3", QuestScriptVersion::GC_EP3},
+        {"xb", QuestScriptVersion::XB_V3},
+        {"bb", QuestScriptVersion::BB_V4},
+    });
+    this->version = name_to_version.at(tokens[1]);
   }
 
-  auto& category = category_index->find(basename[0], category_token);
-  this->category_id = category.category_id;
-
-  // Parse the number out of the first token
-  this->internal_id = strtoull(tokens[0].c_str() + 1, nullptr, 10);
-
-  // Get the version from the second (or previously third) token
-  static const unordered_map<string, QuestScriptVersion> name_to_version({
-      {"dn", QuestScriptVersion::DC_NTE},
-      {"d1", QuestScriptVersion::DC_V1},
-      {"dc", QuestScriptVersion::DC_V2},
-      {"pc", QuestScriptVersion::PC_V2},
-      {"gcn", QuestScriptVersion::GC_NTE},
-      {"gc", QuestScriptVersion::GC_V3},
-      {"gc3", QuestScriptVersion::GC_EP3},
-      {"xb", QuestScriptVersion::XB_V3},
-      {"bb", QuestScriptVersion::BB_V4},
-  });
-  this->version = name_to_version.at(tokens[1]);
-
   // The rest of the information needs to be fetched from the .bin file's
   // contents
 
@@ -800,12 +807,11 @@ void add_command_header(
 }
 
 template <typename HeaderT, typename CmdT>
-void add_open_file_command(StringWriter& w, const Quest& q, bool is_bin) {
+void add_open_file_command(StringWriter& w, const std::u16string& name, const std::string& filename, bool is_download) {
-  add_command_header<HeaderT>(
+  add_command_header<HeaderT>(w, is_download ? 0xA6 : 0x44, 0x00, sizeof(CmdT));
-      w, q.is_dlq_encoded ? 0xA6 : 0x44, q.internal_id, sizeof(CmdT));
   CmdT cmd;
-  cmd.name = "PSO/" + encode_sjis(q.name);
+  cmd.name = "PSO/" + encode_sjis(name);
-  cmd.filename = q.file_basename + (is_bin ? ".bin" : ".dat");
+  cmd.filename = filename;
   cmd.type = 0;
   // TODO: It'd be nice to have something like w.emplace(...) to avoid copying
   // the command structs into the StringWriter.
@@ -817,10 +823,10 @@ void add_write_file_commands(
     StringWriter& w,
     const string& filename,
     const string& data,
-    bool is_dlq_encoded) {
+    bool is_download) {
   for (size_t z = 0; z < data.size(); z += 0x400) {
     size_t chunk_size = min<size_t>(data.size() - z, 0x400);
-    add_command_header<HeaderT>(w, is_dlq_encoded ? 0xA7 : 0x13, z >> 10, sizeof(S_WriteFile_13_A7));
+    add_command_header<HeaderT>(w, is_download ? 0xA7 : 0x13, z >> 10, sizeof(S_WriteFile_13_A7));
     S_WriteFile_13_A7 cmd;
     cmd.filename = filename;
     memcpy(cmd.data.data(), &data[z], chunk_size);
@@ -829,64 +835,57 @@ void add_write_file_commands(
   }
 }
 
-string Quest::export_qst() const {
+string Quest::encode_qst(
-  bool is_ep3 = this->episode == Episode::EP3;
+    const string& bin_data,
-  if (is_ep3 && !this->is_dlq_encoded) {
+    const string& dat_data,
-    throw runtime_error("Episode 3 quests can only be encoded in download QST format");
+    const u16string& name,
-  }
+    const string& file_basename,
-
+    QuestScriptVersion version,
+    bool is_dlq_encoded) {
   StringWriter w;
 
+  string bin_filename = file_basename + ".bin";
+  string dat_filename = file_basename + ".dat";
+
   // Some tools expect both open file commands at the beginning, hence this
   // unfortunate abstraction-breaking.
-  switch (this->version) {
+  switch (version) {
     case QuestScriptVersion::DC_NTE:
     case QuestScriptVersion::DC_V1:
     case QuestScriptVersion::DC_V2:
-      add_open_file_command<PSOCommandHeaderDCV3, S_OpenFile_DC_44_A6>(w, *this, true);
+      add_open_file_command<PSOCommandHeaderDCV3, S_OpenFile_DC_44_A6>(w, name, bin_filename, is_dlq_encoded);
-      add_open_file_command<PSOCommandHeaderDCV3, S_OpenFile_DC_44_A6>(w, *this, false);
+      add_open_file_command<PSOCommandHeaderDCV3, S_OpenFile_DC_44_A6>(w, name, dat_filename, is_dlq_encoded);
-      add_write_file_commands<PSOCommandHeaderDCV3>(
+      add_write_file_commands<PSOCommandHeaderDCV3>(w, bin_filename, bin_data, is_dlq_encoded);
-          w, this->file_basename + ".bin", *this->bin_contents(), this->is_dlq_encoded);
+      add_write_file_commands<PSOCommandHeaderDCV3>(w, dat_filename, dat_data, is_dlq_encoded);
-      add_write_file_commands<PSOCommandHeaderDCV3>(
-          w, this->file_basename + ".dat", *this->dat_contents(), this->is_dlq_encoded);
       break;
     case QuestScriptVersion::PC_V2:
-      add_open_file_command<PSOCommandHeaderPC, S_OpenFile_PC_GC_44_A6>(w, *this, true);
+      add_open_file_command<PSOCommandHeaderPC, S_OpenFile_PC_GC_44_A6>(w, name, bin_filename, is_dlq_encoded);
-      add_open_file_command<PSOCommandHeaderPC, S_OpenFile_PC_GC_44_A6>(w, *this, false);
+      add_open_file_command<PSOCommandHeaderPC, S_OpenFile_PC_GC_44_A6>(w, name, dat_filename, is_dlq_encoded);
-      add_write_file_commands<PSOCommandHeaderPC>(
+      add_write_file_commands<PSOCommandHeaderPC>(w, bin_filename, bin_data, is_dlq_encoded);
-          w, this->file_basename + ".bin", *this->bin_contents(), this->is_dlq_encoded);
+      add_write_file_commands<PSOCommandHeaderPC>(w, dat_filename, dat_data, is_dlq_encoded);
-      add_write_file_commands<PSOCommandHeaderPC>(
-          w, this->file_basename + ".dat", *this->dat_contents(), this->is_dlq_encoded);
       break;
     case QuestScriptVersion::GC_NTE:
     case QuestScriptVersion::GC_V3:
-      add_open_file_command<PSOCommandHeaderDCV3, S_OpenFile_PC_GC_44_A6>(w, *this, true);
+      add_open_file_command<PSOCommandHeaderDCV3, S_OpenFile_PC_GC_44_A6>(w, name, bin_filename, is_dlq_encoded);
-      add_open_file_command<PSOCommandHeaderDCV3, S_OpenFile_PC_GC_44_A6>(w, *this, false);
+      add_open_file_command<PSOCommandHeaderDCV3, S_OpenFile_PC_GC_44_A6>(w, name, dat_filename, is_dlq_encoded);
-      add_write_file_commands<PSOCommandHeaderDCV3>(
+      add_write_file_commands<PSOCommandHeaderDCV3>(w, bin_filename, bin_data, is_dlq_encoded);
-          w, this->file_basename + ".bin", *this->bin_contents(), this->is_dlq_encoded);
+      add_write_file_commands<PSOCommandHeaderDCV3>(w, dat_filename, dat_data, is_dlq_encoded);
-      add_write_file_commands<PSOCommandHeaderDCV3>(
-          w, this->file_basename + ".dat", *this->dat_contents(), this->is_dlq_encoded);
       break;
     case QuestScriptVersion::GC_EP3:
-      add_open_file_command<PSOCommandHeaderDCV3, S_OpenFile_PC_GC_44_A6>(w, *this, true);
+      add_open_file_command<PSOCommandHeaderDCV3, S_OpenFile_PC_GC_44_A6>(w, name, bin_filename, is_dlq_encoded);
-      add_write_file_commands<PSOCommandHeaderDCV3>(
+      add_write_file_commands<PSOCommandHeaderDCV3>(w, bin_filename, bin_data, is_dlq_encoded);
-          w, this->file_basename + ".bin", *this->bin_contents(), this->is_dlq_encoded);
       break;
     case QuestScriptVersion::XB_V3:
-      add_open_file_command<PSOCommandHeaderDCV3, S_OpenFile_XB_44_A6>(w, *this, true);
+      add_open_file_command<PSOCommandHeaderDCV3, S_OpenFile_XB_44_A6>(w, name, bin_filename, is_dlq_encoded);
-      add_open_file_command<PSOCommandHeaderDCV3, S_OpenFile_XB_44_A6>(w, *this, false);
+      add_open_file_command<PSOCommandHeaderDCV3, S_OpenFile_XB_44_A6>(w, name, dat_filename, is_dlq_encoded);
-      add_write_file_commands<PSOCommandHeaderDCV3>(
+      add_write_file_commands<PSOCommandHeaderDCV3>(w, bin_filename, bin_data, is_dlq_encoded);
-          w, this->file_basename + ".bin", *this->bin_contents(), this->is_dlq_encoded);
+      add_write_file_commands<PSOCommandHeaderDCV3>(w, dat_filename, dat_data, is_dlq_encoded);
-      add_write_file_commands<PSOCommandHeaderDCV3>(
-          w, this->file_basename + ".dat", *this->dat_contents(), this->is_dlq_encoded);
       break;
     case QuestScriptVersion::BB_V4:
-      add_open_file_command<PSOCommandHeaderBB, S_OpenFile_BB_44_A6>(w, *this, true);
+      add_open_file_command<PSOCommandHeaderBB, S_OpenFile_BB_44_A6>(w, name, bin_filename, is_dlq_encoded);
-      add_open_file_command<PSOCommandHeaderBB, S_OpenFile_BB_44_A6>(w, *this, false);
+      add_open_file_command<PSOCommandHeaderBB, S_OpenFile_BB_44_A6>(w, name, dat_filename, is_dlq_encoded);
-      add_write_file_commands<PSOCommandHeaderBB>(
+      add_write_file_commands<PSOCommandHeaderBB>(w, bin_filename, bin_data, is_dlq_encoded);
-          w, this->file_basename + ".bin", *this->bin_contents(), this->is_dlq_encoded);
+      add_write_file_commands<PSOCommandHeaderBB>(w, dat_filename, dat_data, is_dlq_encoded);
-      add_write_file_commands<PSOCommandHeaderBB>(
-          w, this->file_basename + ".dat", *this->dat_contents(), this->is_dlq_encoded);
       break;
     default:
       throw logic_error("invalid game version");
@@ -895,6 +894,16 @@ string Quest::export_qst() const {
   return std::move(w.str());
 }
 
+string Quest::encode_qst() const {
+  return this->encode_qst(
+      *this->bin_contents(),
+      *this->dat_contents(),
+      this->name,
+      basename(this->file_basename),
+      this->version,
+      this->is_dlq_encoded);
+}
+
 QuestIndex::QuestIndex(
     const string& directory,
     std::shared_ptr<const QuestCategoryIndex> category_index)
@@ -922,7 +931,7 @@ QuestIndex::QuestIndex(
         ends_with(filename, ".mnm.dlq") ||
         ends_with(filename, ".qst")) {
       try {
-        shared_ptr<Quest> q(new Quest(full_path, this->category_index));
+        shared_ptr<Quest> q(new Quest(full_path, QuestScriptVersion::UNKNOWN, this->category_index));
         q->menu_item_id = next_menu_item_id++;
         string ascii_name = encode_sjis(q->name);
         if (!this->version_menu_item_id_to_quest.emplace(make_pair(q->version, q->menu_item_id), q).second) {
@@ -970,8 +979,8 @@ vector<shared_ptr<const Quest>> QuestIndex::filter(
   return ret;
 }
 
-static string create_download_quest_file(const string& compressed_data,
+string Quest::encode_download_quest_file(
-    size_t decompressed_size, uint32_t encryption_seed = 0) {
+    const string& compressed_data, size_t decompressed_size, uint32_t encryption_seed) {
   // Download quest files are like normal (PRS-compressed) quest files, but they
   // are encrypted with PSO V2 encryption (even on V3 / PSO GC), and a small
   // header (PSODownloadQuestHeader) is prepended to the encrypted data.
@@ -979,6 +988,9 @@ static string create_download_quest_file(const string& compressed_data,
   if (encryption_seed == 0) {
     encryption_seed = random_object<uint32_t>();
   }
+  if (decompressed_size == 0) {
+    decompressed_size = prs_decompress_size(compressed_data);
+  }
 
   string data(8, '\0');
   auto* header = reinterpret_cast<PSODownloadQuestHeader*>(data.data());
@@ -1050,10 +1062,9 @@ shared_ptr<Quest> Quest::create_download_quest() const {
   // Return a new Quest object with appropriately-processed .bin and .dat file
   // contents
   shared_ptr<Quest> dlq(new Quest(*this));
-  dlq->bin_contents_ptr.reset(new string(create_download_quest_file(
+  dlq->bin_contents_ptr.reset(new string(this->encode_download_quest_file(
       compressed_bin, decompressed_bin.size())));
-  dlq->dat_contents_ptr.reset(new string(create_download_quest_file(
+  dlq->dat_contents_ptr.reset(new string(this->encode_download_quest_file(*this->dat_contents())));
-      *this->dat_contents(), prs_decompress_size(*this->dat_contents()))));
   dlq->is_dlq_encoded = true;
   return dlq;
 }

src/Quest.hh

@@ -74,7 +74,7 @@ public:
   std::u16string short_description;
   std::u16string long_description;
 
-  Quest(const std::string& file_basename, std::shared_ptr<const QuestCategoryIndex> category_index);
+  Quest(const std::string& file_basename, QuestScriptVersion version, std::shared_ptr<const QuestCategoryIndex> category_index);
   Quest(const Quest&) = default;
   Quest(Quest&&) = default;
   Quest& operator=(const Quest&) = default;
@@ -86,6 +86,8 @@ public:
   std::shared_ptr<const std::string> bin_contents() const;
   std::shared_ptr<const std::string> dat_contents() const;
 
+  static std::string encode_download_quest_file(
+      const std::string& compressed_data, size_t decompressed_size = 0, uint32_t encryption_seed = 0);
   std::shared_ptr<Quest> create_download_quest() const;
 
   static std::string decode_gci_file(
@@ -100,7 +102,14 @@ public:
   static std::string decode_dlq_data(const std::string& filename);
   static std::pair<std::string, std::string> decode_qst_file(const std::string& filename);
 
-  std::string export_qst() const;
+  static std::string encode_qst(
+      const std::string& bin_data,
+      const std::string& dat_data,
+      const std::u16string& name,
+      const std::string& file_basename,
+      QuestScriptVersion version,
+      bool is_dlq_encoded);
+  std::string encode_qst() const;
 
 private:
   // these are populated when requested

src/QuestScript.hh

@@ -18,6 +18,7 @@ enum class QuestScriptVersion {
   XB_V3 = 6,
   GC_EP3 = 7,
   BB_V4 = 8,
+  UNKNOWN = 15,
 };
 
 template <>