From 168cef747a2b30153ac045dbdc7314ee673392cd Mon Sep 17 00:00:00 2001 From: Martin Michelsen Date: Thu, 21 Apr 2022 01:29:40 -0700 Subject: [PATCH] fix download quests --- README.md | 1 - src/Compression.cc | 22 ++++---- src/Quest.cc | 119 +++++++++++++++++++++++++++++++---------- src/Quest.hh | 8 ++- src/ReceiveCommands.cc | 95 +++++++++++++++++++++----------- src/SendCommands.cc | 21 +++++--- src/SendCommands.hh | 5 +- 7 files changed, 191 insertions(+), 80 deletions(-) diff --git a/README.md b/README.md index 356dff29..70b94678 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,6 @@ Sometime in October 2018, I had some random cause to reminisce. I looked back in This project is primarily for my own nostalgia; I offer no guarantees on how or when this project will advance. Current known issues / missing features: -- Download quests are mostly implemented, but the client doesn't always accept them. It's probably a format issue in file generation. - Test all the communication features (info board, simple mail, card search, etc.) - The trade window isn't implemented yet. - PSO PC and PSOBB are essentially entirely untested. Only GC is fairly well-tested. diff --git a/src/Compression.cc b/src/Compression.cc index 7f9f8c36..a7eeda70 100644 --- a/src/Compression.cc +++ b/src/Compression.cc @@ -13,18 +13,15 @@ using namespace std; struct prs_compress_ctx { - unsigned char bitpos; + uint8_t bitpos; std::string forward_log; std::string output; - prs_compress_ctx() : bitpos(0) { } + prs_compress_ctx() : bitpos(0), forward_log("\0", 1) { } string finish() { this->put_control_bit(0); this->put_control_bit(1); - if (this->bitpos != 0) { - this->forward_log[0] = ((this->forward_log[0] << this->bitpos) >> 8); - } this->put_static_data(0); this->put_static_data(0); this->output += this->forward_log; @@ -33,8 +30,9 @@ struct prs_compress_ctx { } void put_control_bit_nosave(bool bit) { - this->forward_log[0] = this->forward_log[0] >> 1; - this->forward_log[0] |= ((!!bit) << 7); + if (bit) { + this->forward_log[0] |= 1 << this->bitpos; + } this->bitpos++; } @@ -53,7 +51,7 @@ struct prs_compress_ctx { } void put_static_data(uint8_t data) { - this->forward_log += static_cast(data); + this->forward_log.push_back(static_cast(data)); } void raw_byte(uint8_t value) { @@ -108,10 +106,10 @@ string prs_compress(const string& data) { // look for a chunk of data in history matching what's at the current offset ssize_t best_offset = 0; ssize_t best_size = 0; - for (ssize_t this_offset = -3; - (this_offset + data_ssize >= 0) && - (this_offset > -0x1FF0) && - (best_size < 255); + for (ssize_t this_offset = -3; // min copy size is 3 bytes + (this_offset + read_offset >= 0) && // don't go before the beginning + (this_offset > -0x1FF0) && // max offset is -0x1FF0 + (best_size < 255); // max size is 0xFF bytes this_offset--) { // for this offset, expand the match as much as possible diff --git a/src/Quest.cc b/src/Quest.cc index a3978593..2a0de5df 100644 --- a/src/Quest.cc +++ b/src/Quest.cc @@ -19,9 +19,9 @@ using namespace std; struct PSODownloadQuestHeader { // When sending a DLQ to the client, this is the DECOMPRESSED size. When // reading it from a GCI file, this is the COMPRESSED size. - be_uint32_t size; + le_uint32_t size; // Note: use PSO PC encryption, even for GC quests. - be_uint32_t encryption_seed; + le_uint32_t encryption_seed; } __attribute__((packed)); @@ -145,10 +145,13 @@ Quest::Quest(const string& bin_filename) episode(0), is_dcv1(false), joinable(false), - gci_format(false) { + file_format(FileFormat::BIN_DAT) { if (ends_with(bin_filename, ".bin.gci")) { - this->gci_format = true; + this->file_format = FileFormat::BIN_DAT_GCI; + this->file_basename = bin_filename.substr(0, bin_filename.size() - 8); + } else if (ends_with(bin_filename, ".bin.dlq")) { + this->file_format = FileFormat::BIN_DAT_DLQ; this->file_basename = bin_filename.substr(0, bin_filename.size() - 8); } else if (ends_with(bin_filename, ".bin")) { this->file_basename = bin_filename.substr(0, bin_filename.size() - 4); @@ -165,7 +168,7 @@ Quest::Quest(const string& bin_filename) basename = bin_filename; } } - basename.resize(basename.size() - (this->gci_format ? 8 : 4)); + basename.resize(basename.size() - ((this->file_format == FileFormat::BIN_DAT) ? 4 : 8)); // quest filenames are like: // b###-VV.bin for battle mode @@ -333,10 +336,18 @@ std::string Quest::dat_filename() const { shared_ptr Quest::bin_contents() const { if (!this->bin_contents_ptr) { - if (this->gci_format) { - this->bin_contents_ptr.reset(new string(this->decode_gci(this->file_basename + ".bin.gci"))); - } else { - this->bin_contents_ptr.reset(new string(load_file(this->file_basename + ".bin"))); + switch (this->file_format) { + case FileFormat::BIN_DAT: + this->bin_contents_ptr.reset(new string(load_file(this->file_basename + ".bin"))); + break; + case FileFormat::BIN_DAT_GCI: + this->bin_contents_ptr.reset(new string(this->decode_gci(this->file_basename + ".bin.gci"))); + break; + case FileFormat::BIN_DAT_DLQ: + this->bin_contents_ptr.reset(new string(this->decode_dlq(this->file_basename + ".bin.dlq"))); + break; + default: + throw logic_error("invalid quest file format"); } } return this->bin_contents_ptr; @@ -344,15 +355,52 @@ shared_ptr Quest::bin_contents() const { shared_ptr Quest::dat_contents() const { if (!this->dat_contents_ptr) { - if (this->gci_format) { - this->dat_contents_ptr.reset(new string(this->decode_gci(this->file_basename + ".dat.gci"))); - } else { - this->dat_contents_ptr.reset(new string(load_file(this->file_basename + ".dat"))); + switch (this->file_format) { + case FileFormat::BIN_DAT: + this->dat_contents_ptr.reset(new string(load_file(this->file_basename + ".dat"))); + break; + case FileFormat::BIN_DAT_GCI: + this->dat_contents_ptr.reset(new string(this->decode_gci(this->file_basename + ".dat.gci"))); + break; + case FileFormat::BIN_DAT_DLQ: + this->dat_contents_ptr.reset(new string(this->decode_dlq(this->file_basename + ".dat.dlq"))); + break; + default: + throw logic_error("invalid quest file format"); } } return this->dat_contents_ptr; } +string Quest::decode_dlq(const string& filename) { + uint32_t decompressed_size; + uint32_t key; + string data; + { + auto f = fopen_unique(filename, "rb"); + decompressed_size = freadx(f.get()); + key = freadx(f.get()); + data = read_all(f.get()); + } + + PSOPCEncryption encr(key); + + // The compressed data size does not need to be a multiple of 4, but the PC + // encryption (which is used for all download quests, even in V3) requires the + // data size to be a multiple of 4. We'll just temporarily stick a few bytes + // on the end, then throw them away later if needed. + size_t original_size = data.size(); + data.resize((data.size() + 3) & (~3)); + encr.decrypt(data); + data.resize(original_size); + + if (prs_decompress_size(data) != decompressed_size) { + throw runtime_error("decompressed size does not match size in header"); + } + + return data; +} + string Quest::decode_gci(const string& filename) { string data = load_file(filename); @@ -415,7 +463,9 @@ QuestIndex::QuestIndex(const std::string& directory) : directory(directory) { continue; } - if (ends_with(filename, ".bin") || ends_with(filename, ".bin.gci")) { + if (ends_with(filename, ".bin") || + ends_with(filename, ".bin.gci") || + ends_with(filename, ".bin.dlq")) { try { shared_ptr q(new Quest(full_path)); this->version_id_to_quest.emplace(make_pair(q->version, q->quest_id), q); @@ -469,38 +519,52 @@ vector> QuestIndex::filter(GameVersion version, static string create_download_quest_file(const string& compressed_data, - size_t decompressed_size) { + size_t decompressed_size, uint32_t seed = 0) { + if (seed == 0) { + seed = random_object(); + } + string data(8, '\0'); auto* header = reinterpret_cast(data.data()); - header->size = decompressed_size + sizeof(PSODownloadQuestHeader); - header->encryption_seed = random_object(); + header->size = decompressed_size; + header->encryption_seed = seed; data += compressed_data; - // add extra bytes if necessary so encryption won't fail + // Add temporary extra bytes if necessary so encryption won't fail + size_t original_size = data.size(); data.resize((data.size() + 3) & (~3)); - // TODO: for DC quests, do we use DC encryption? - PSOPCEncryption encr(header->encryption_seed); + PSOPCEncryption encr(seed); encr.encrypt(data.data() + sizeof(PSODownloadQuestHeader), data.size() - sizeof(PSODownloadQuestHeader)); + data.resize(original_size); + return data; } shared_ptr Quest::create_download_quest() const { - if (this->category == QuestCategory::DOWNLOAD) { - throw invalid_argument("quest is already a download quest"); - } - string decompressed_bin = prs_decompress(*this->bin_contents()); + + // The download flag needs to be set in the bin header, or else the client + // will ignore it when scanning for download quests in an offline game. void* data_ptr = decompressed_bin.data(); switch (this->version) { case GameVersion::DC: + if (decompressed_bin.size() < sizeof(PSOQuestHeaderDC)) { + throw runtime_error("bin file is too small for header"); + } reinterpret_cast(data_ptr)->is_download = 0x01; break; case GameVersion::PC: + if (decompressed_bin.size() < sizeof(PSOQuestHeaderPC)) { + throw runtime_error("bin file is too small for header"); + } reinterpret_cast(data_ptr)->is_download = 0x01; break; case GameVersion::GC: + if (decompressed_bin.size() < sizeof(PSOQuestHeaderGC)) { + throw runtime_error("bin file is too small for header"); + } reinterpret_cast(data_ptr)->is_download = 0x01; break; case GameVersion::BB: @@ -510,14 +574,13 @@ shared_ptr Quest::create_download_quest() const { } shared_ptr dlq(new Quest(*this)); - dlq->category = QuestCategory::DOWNLOAD; + string compressed_bin = prs_compress(decompressed_bin); dlq->bin_contents_ptr.reset(new string(create_download_quest_file( - prs_compress(decompressed_bin), decompressed_bin.size()))); + compressed_bin, decompressed_bin.size()))); - auto dat_contents = this->dat_contents(); dlq->dat_contents_ptr.reset(new string(create_download_quest_file( - *dat_contents, prs_decompress_size(*dat_contents)))); + *this->dat_contents(), prs_decompress_size(*this->dat_contents())))); return dlq; } diff --git a/src/Quest.hh b/src/Quest.hh index 971c9eaa..8652163c 100644 --- a/src/Quest.hh +++ b/src/Quest.hh @@ -37,8 +37,14 @@ const char* name_for_category(QuestCategory category); class Quest { private: static std::string decode_gci(const std::string& filename); + static std::string decode_dlq(const std::string& filename); public: + enum class FileFormat { + BIN_DAT = 0, + BIN_DAT_GCI, + BIN_DAT_DLQ, + }; int64_t quest_id; QuestCategory category; uint8_t episode; // 0 = ep1, 1 = ep2, 2 = ep4, 0xFF = ep3 @@ -46,7 +52,7 @@ public: bool joinable; GameVersion version; std::string file_basename; // we append -. when reading - bool gci_format; + FileFormat file_format; std::u16string name; std::u16string short_description; std::u16string long_description; diff --git a/src/ReceiveCommands.cc b/src/ReceiveCommands.cc index 30e24f30..047aad42 100644 --- a/src/ReceiveCommands.cc +++ b/src/ReceiveCommands.cc @@ -675,7 +675,21 @@ void process_menu_selection(shared_ptr s, shared_ptr c, break; case MAIN_MENU_DOWNLOAD_QUESTS: - send_quest_menu(c, QUEST_FILTER_MENU_ID, quest_download_menu, true); + if (c->flags & Client::Flag::EPISODE_3) { + shared_ptr l = c->lobby_id ? s->find_lobby(c->lobby_id) : nullptr; + auto quests = s->quest_index->filter(c->version, false, QuestCategory::EPISODE_3, 0xFF); + if (quests.empty()) { + send_lobby_message_box(c, u"$C6There are no quests\navailable."); + } else { + // Episode 3 has only download quests, not online quests, so this + // is always the download quest menu. (Episode 3 does actually + // have online quests, but they don't use the file download + // paradigm that all other versions use.) + send_quest_menu(c, QUEST_MENU_ID, quests, true); + } + } else { + send_quest_menu(c, QUEST_FILTER_MENU_ID, quest_download_menu, true); + } break; case MAIN_MENU_DISCONNECT: @@ -817,7 +831,7 @@ void process_menu_selection(shared_ptr s, shared_ptr c, auto quests = s->quest_index->filter(c->version, c->flags & Client::Flag::DCV1, static_cast(cmd.item_id & 0xFF), - l.get() ? (l->episode - 1) : -1); + c->flags & Client::Flag::EPISODE_3 ? 0xFF : (l.get() ? (l->episode - 1) : -1)); if (quests.empty()) { send_lobby_message_box(c, u"$C6There are no quests\navailable in that\ncategory."); break; @@ -851,10 +865,15 @@ void process_menu_selection(shared_ptr s, shared_ptr c, } } - auto bin_basename = q->bin_filename(); - auto dat_basename = q->dat_filename(); - auto bin_contents = q->bin_contents(); - auto dat_contents = q->dat_contents(); + bool is_ep3 = (q->episode == 0xFF); + string bin_basename = q->bin_filename(); + shared_ptr bin_contents = q->bin_contents(); + string dat_basename; + shared_ptr dat_contents; + if (!is_ep3) { + dat_basename = q->dat_filename(); + dat_contents = q->dat_contents(); + } if (l) { if (q->joinable) { @@ -872,8 +891,10 @@ void process_menu_selection(shared_ptr s, shared_ptr c, // cause GC clients to crash in rare cases. Find a way to slow this down // (perhaps by only sending each new chunk when they acknowledge the // previous chunk with a 44 [first chunk] or 13 [later chunks] command). - send_quest_file(l->clients[x], bin_basename, *bin_contents, false, false); - send_quest_file(l->clients[x], dat_basename, *dat_contents, false, false); + send_quest_file(l->clients[x], bin_basename + ".bin", bin_basename, *bin_contents, false, is_ep3); + if (dat_contents) { + send_quest_file(l->clients[x], dat_basename + ".dat", dat_basename, *dat_contents, false, is_ep3); + } // There is no such thing as command AC on PSO PC - quests just start // immediately when they're done downloading. There are also no chunk @@ -886,10 +907,17 @@ void process_menu_selection(shared_ptr s, shared_ptr c, } } else { - // TODO: cache dlq somewhere maybe - auto dlq = q->create_download_quest(); - send_quest_file(c, bin_basename, *bin_contents, true, false); - send_quest_file(c, dat_basename, *dat_contents, true, false); + string quest_name = encode_sjis(q->name); + // Episode 3 uses the download quest commands (A6/A7) but does not + // expect the server to have already encrypted the quest files, unlike + // other versions. + if (!is_ep3) { + q = q->create_download_quest(); + } + send_quest_file(c, quest_name, bin_basename, *q->bin_contents(), true, is_ep3); + if (dat_contents) { + send_quest_file(c, quest_name, dat_basename, *q->dat_contents(), true, is_ep3); + } } break; } @@ -978,24 +1006,31 @@ void process_quest_list_request(shared_ptr s, shared_ptr c, return; } - vector* menu = nullptr; - if ((c->version == GameVersion::BB) && flag) { - menu = &quest_government_menu; - } else { - if (l->mode == 0) { - menu = &quest_categories_menu; - } else if (l->mode == 1) { - menu = &quest_battle_menu; - } else if (l->mode == 1) { - menu = &quest_challenge_menu; - } else if (l->mode == 1) { - menu = &quest_solo_menu; - } else { - throw logic_error("no quest menu available for mode"); - } - } + // In Episode 3, there are no quest categories, so skip directly to the quest + // filter menu. + if (c->flags & Client::Flag::EPISODE_3) { + send_lobby_message_box(c, u"$C6Episode 3 does not\nprovide online quests\nvia this interface."); - send_quest_menu(c, QUEST_FILTER_MENU_ID, *menu, false); + } else { + vector* menu = nullptr; + if ((c->version == GameVersion::BB) && flag) { + menu = &quest_government_menu; + } else { + if (l->mode == 0) { + menu = &quest_categories_menu; + } else if (l->mode == 1) { + menu = &quest_battle_menu; + } else if (l->mode == 1) { + menu = &quest_challenge_menu; + } else if (l->mode == 1) { + menu = &quest_solo_menu; + } else { + throw logic_error("no quest menu available for mode"); + } + } + + send_quest_menu(c, QUEST_FILTER_MENU_ID, *menu, false); + } } void process_quest_ready(shared_ptr s, shared_ptr c, @@ -1040,7 +1075,7 @@ void process_gba_file_request(shared_ptr, shared_ptr c, strip_trailing_zeroes(filename); auto contents = file_cache.get(filename); - send_quest_file(c, filename, *contents, false, false); + send_quest_file(c, filename, filename, *contents, false, false); } diff --git a/src/SendCommands.cc b/src/SendCommands.cc index 5d30a4df..13df3b42 100644 --- a/src/SendCommands.cc +++ b/src/SendCommands.cc @@ -1301,14 +1301,22 @@ void send_ep3_map_data(shared_ptr l, uint32_t map_id) { template void send_quest_open_file_t( shared_ptr c, + const string& quest_name, const string& filename, uint32_t file_size, bool is_download_quest, bool is_ep3_quest) { CommandT cmd; - cmd.flags = 2 + is_ep3_quest; + cmd.unused = 0; + if (is_ep3_quest) { + cmd.flags = 3; + } else if (is_download_quest) { + cmd.flags = 0; + } else { + cmd.flags = 2; + } cmd.file_size = file_size; - cmd.name = filename.c_str(); + cmd.name = "PSO/" + quest_name; cmd.filename = filename.c_str(); send_command(c, is_download_quest ? 0xA6 : 0x44, 0x00, cmd); } @@ -1335,15 +1343,16 @@ void send_quest_file_chunk( send_command(c, is_download_quest ? 0xA7 : 0x13, chunk_index, cmd); } -void send_quest_file(shared_ptr c, const string& basename, - const string& contents, bool is_download_quest, bool is_ep3_quest) { +void send_quest_file(shared_ptr c, const string& quest_name, + const string& basename, const string& contents, bool is_download_quest, + bool is_ep3_quest) { if (c->version == GameVersion::PC || c->version == GameVersion::GC) { send_quest_open_file_t( - c, basename, contents.size(), is_download_quest, is_ep3_quest); + c, quest_name, basename, contents.size(), is_download_quest, is_ep3_quest); } else if (c->version == GameVersion::BB) { send_quest_open_file_t( - c, basename, contents.size(), is_download_quest, is_ep3_quest); + c, quest_name, basename, contents.size(), is_download_quest, is_ep3_quest); } else { throw invalid_argument("cannot send quest files to this version of client"); } diff --git a/src/SendCommands.hh b/src/SendCommands.hh index 192f6274..96c5bb2c 100644 --- a/src/SendCommands.hh +++ b/src/SendCommands.hh @@ -205,8 +205,9 @@ void send_ep3_rank_update(std::shared_ptr c); void send_ep3_map_list(std::shared_ptr l); void send_ep3_map_data(std::shared_ptr l, uint32_t map_id); -void send_quest_file(std::shared_ptr c, const std::string& basename, - const std::string& contents, bool is_download_quest, bool is_ep3_quest); +void send_quest_file(std::shared_ptr c, const std::string& quest_name, + const std::string& basename, const std::string& contents, + bool is_download_quest, bool is_ep3_quest); void send_server_time(std::shared_ptr c);