From 16cddd28b2d07dfbf48f08734661de9518e61c09 Mon Sep 17 00:00:00 2001 From: Martin Michelsen Date: Sun, 10 Dec 2023 14:24:30 -0800 Subject: [PATCH] add quest script compiler --- README.md | 4 +- src/Main.cc | 19 +- src/Map.cc | 2 +- src/Quest.cc | 8 +- src/QuestScript.cc | 583 +++++++++++++++++++++- src/QuestScript.hh | 3 +- system/quests/retrieval/q058-gc-e.bin | Bin 1421 -> 0 bytes system/quests/retrieval/q058-gc-e.bin.txt | 345 +++++++++++++ system/quests/retrieval/q058-xb-e.bin | 1 - system/quests/retrieval/q058-xb-e.bin.txt | 1 + 10 files changed, 945 insertions(+), 21 deletions(-) delete mode 100644 system/quests/retrieval/q058-gc-e.bin create mode 100755 system/quests/retrieval/q058-gc-e.bin.txt delete mode 120000 system/quests/retrieval/q058-xb-e.bin create mode 120000 system/quests/retrieval/q058-xb-e.bin.txt diff --git a/README.md b/README.md index 427dfdea..34e1a621 100644 --- a/README.md +++ b/README.md @@ -109,6 +109,7 @@ There are multiple PSO quest formats out there; newserv supports all of them. It | Compressed Ep3 | .bin or .mnm | Yes (4) | None (1) | | Uncompressed | .bind and .datd | Yes | compress-prs (2) | | Uncompressed Ep3 | .bind or .mnmd | Yes (4) | compress-prs (2) | +| Source | .bin.txt and .dat | Yes | None (5) | | VMS (DCv1) | .bin.vms and .dat.vms | Yes | decode-vms | | VMS (DCv2) | .bin.vms and .dat.vms | Decode (3) | decode-vms (3) | | GCI (decrypted) | .bin.gci and .dat.gci | Yes | decode-gci | @@ -126,6 +127,7 @@ There are multiple PSO quest formats out there; newserv supports all of them. It 2. *Similar to (1), to compress an uncompressed quest file: `newserv compress-prs FILENAME.bind FILENAME.bin` (and likewise for .datd -> .dat)* 3. *Use the decode action to convert these quests to .bin/.dat format before putting them into the server's quests directory. If you know the encryption seed (serial number), pass it in as a hex string with the `--seed=` option. If you don't know the encryption seed, newserv will find it for you, which will likely take a long time.* 4. *Episode 3 quests don't go in the system/quests directory. See the Episode 3 section below.* +5. *Quest source can be assembled into a .bin or .bind file with `newserv assemble-quest-script FILENAME.txt`. See system/quests/retrieval/q058-gc-e.bin.txt for an annotated example; this is the English GameCube version of Lost HEAT SWORD.* Episode 3 download quests consist only of a .bin file - there is no corresponding .dat file. Episode 3 download quest files may be named with the .mnm extension instead of .bin, since the format is the same as the standard map files (in system/ep3/). These files can be encoded in any of the formats described above, except .qst. @@ -401,7 +403,7 @@ newserv has many CLI options, which can be used to access functionality other th * Convert quests in .gci, .vms, .dlq, or .qst format to .bin/.dat format (`decode-gci`, `decode-vms`, `decode-dlq`, `decode-qst`) * Convert quests in .bin/.dat to .qst format (`encode-qst`) * Convert text archives (e.g. TextEnglish.pr2) to JSON and vice versa (`decode-text-archive`, `encode-text-archive`) -* Disassemble quest scripts (`disassemble-quest-script`) +* Compile or disassemble quest scripts (`assemble-quest-script`, `disassemble-quest-script`) * Format Episode 3 game data in a human-readable manner (`show-ep3-maps`, `show-ep3-cards`) * Convert item data to a human-readable description, or vice versa (`describe-item`, `encode-item`) * Connect to another PSO server and pretend to be a client (`cat-client`) diff --git a/src/Main.cc b/src/Main.cc index 9dc06235..6798609d 100644 --- a/src/Main.cc +++ b/src/Main.cc @@ -1040,8 +1040,7 @@ Action a_disassemble_quest_map( "disassemble-quest-map", "\ disassemble-quest-map [INPUT-FILENAME [OUTPUT-FILENAME]]\n\ Disassemble the input quest map (.dat file) into a text representation of\n\ - the data it contains. Specify the quest\'s game version with one of the\n\ - --dc-nte, --dc-v1, --dc-v2, --pc, --gc-nte, --gc, --xb, or --bb options.\n", + the data it contains.\n", +[](Arguments& args) { string data = read_input_data(args); if (!args.get("decompressed")) { @@ -1051,6 +1050,22 @@ Action a_disassemble_quest_map( write_output_data(args, result.data(), result.size(), "txt"); }); +Action a_assemble_quest_script( + "assemble-quest-script", "\ + assemble-quest-script [INPUT-FILENAME [OUTPUT-FILENAME]]\n\ + Assemble the input quest script (.txt file) into a compressed .bin file\n\ + usable as an online quest script. If --decompressed is given, produces an\n\ + uncompressed .bind file instead.\n", + +[](Arguments& args) { + string text = read_input_data(args); + string result = assemble_quest_script(text); + bool compress = !args.get("decompressed"); + if (compress) { + result = prs_compress_optimal(result); + } + write_output_data(args, result.data(), result.size(), compress ? "bin" : "bind"); + }); + void a_extract_archive_fn(Arguments& args) { string output_prefix = args.get(2, false); if (output_prefix == "-") { diff --git a/src/Map.cc b/src/Map.cc index 9c4f511c..4d10eb55 100644 --- a/src/Map.cc +++ b/src/Map.cc @@ -1028,7 +1028,7 @@ string Map::disassemble_quest_data(const void* data, size_t size) { } } - return join(ret, "\n"); + return join(ret, "\n") + "\n"; } SetDataTable::SetDataTable(shared_ptr data, bool big_endian) { diff --git a/src/Quest.cc b/src/Quest.cc index 9202a851..ca06e522 100644 --- a/src/Quest.cc +++ b/src/Quest.cc @@ -319,7 +319,7 @@ VersionedQuest::VersionedQuest( throw invalid_argument("file is too small for header"); } auto* header = reinterpret_cast(bin_decompressed.data()); - this->joinable = header->joinable_in_progress; + this->joinable = header->joinable; this->episode = find_quest_episode_from_script(bin_decompressed.data(), bin_decompressed.size(), this->version); if (this->quest_number == 0xFFFFFFFF) { this->quest_number = header->quest_number; @@ -501,6 +501,12 @@ QuestIndex::QuestIndex( } else if (ends_with(filename, ".dlq")) { file_data = decode_dlq_data(load_file(file_path)); filename.resize(filename.size() - 4); + } else if (ends_with(filename, ".txt")) { + file_data = assemble_quest_script(load_file(file_path)); + filename.resize(filename.size() - 4); + if (ends_with(filename, ".bin")) { + filename.push_back('d'); + } } else { file_data = load_file(file_path); } diff --git a/src/QuestScript.cc b/src/QuestScript.cc index 4d060081..fb9dea07 100644 --- a/src/QuestScript.cc +++ b/src/QuestScript.cc @@ -493,7 +493,7 @@ static const QuestScriptOpcodeDefinition opcode_defs[] = { {0x00ED, "create_bgmctrl", {}, F_V1_V4}, {0x00EE, "pl_add_meseta2", {INT32}, F_V1_V4 | F_ARGS}, {0x00EF, "sync_register2", {INT32, REG32}, F_V1_V2}, - {0x00EF, "sync_register2", {INT32, INT32}, F_V3_V4 | F_ARGS}, + {0x00EF, "sync_register2", {REG, INT32}, F_V3_V4 | F_ARGS}, {0x00F0, "send_regwork", {INT32, REG32}, F_V1_V2}, {0x00F1, "leti_fixed_camera", {{REG32_SET_FIXED, 6}}, F_V2}, {0x00F1, "leti_fixed_camera", {{REG_SET_FIXED, 6}}, F_V3_V4}, @@ -672,7 +672,7 @@ static const QuestScriptOpcodeDefinition opcode_defs[] = { {0xF8B5, "write4", {REG, REG}, F_V2}, {0xF8B5, "write4", {INT32, INT32}, F_V3_V4 | F_ARGS}, {0xF8B6, "check_for_hacking", {REG}, F_V2}, // Returns a bitmask of 5 different types of detectable hacking. But it only works on DCv2 - it crashes on all other versions. - {0xF8B7, nullptr, {REG}, F_V2_V4}, // TODO (DX) - Challenge mode. Appears to be timing-related; regA is expected to be in [60, 3600]. Encodes the value with encrypt_challenge_time even though it's never sent over the network and is only decrypted locally. + {0xF8B7, "unknown_F8B7", {REG}, F_V2_V4}, // TODO (DX) - Challenge mode. Appears to be timing-related; regA is expected to be in [60, 3600]. Encodes the value with encrypt_challenge_time even though it's never sent over the network and is only decrypted locally. {0xF8B8, "disable_retry_menu", {}, F_V2_V4}, {0xF8B9, "chl_recovery", {}, F_V2_V4}, {0xF8BA, "load_guild_card_file_creation_time_to_flag_buf", {}, F_V2_V4}, @@ -735,7 +735,7 @@ static const QuestScriptOpcodeDefinition opcode_defs[] = { {0xF8EF, "nop_F8EF", {}, F_V3_V4}, {0xF8F0, "turn_off_bgm_p2", {}, F_V3_V4}, {0xF8F1, "turn_on_bgm_p2", {}, F_V3_V4}, - {0xF8F2, nullptr, {INT32, FLOAT32, FLOAT32, INT32, {REG_SET_FIXED, 4}, {LABEL16, Arg::DataType::UNKNOWN_F8F2_DATA}}, F_V3_V4 | F_ARGS}, // TODO (DX) + {0xF8F2, "unknown_F8F2", {INT32, FLOAT32, FLOAT32, INT32, {REG_SET_FIXED, 4}, {LABEL16, Arg::DataType::UNKNOWN_F8F2_DATA}}, F_V3_V4 | F_ARGS}, // TODO (DX) {0xF8F3, "particle2", {{REG_SET_FIXED, 3}, INT32, FLOAT32}, F_V3_V4 | F_ARGS}, {0xF901, "dec2float", {REG, REG}, F_V3_V4}, {0xF902, "float2dec", {REG, REG}, F_V3_V4}, @@ -817,7 +817,7 @@ static const QuestScriptOpcodeDefinition opcode_defs[] = { {0xF94B, "particle_effect_nc", {{REG_SET_FIXED, 4}}, F_V3_V4}, {0xF94C, "player_effect_nc", {{REG_SET_FIXED, 4}}, F_V3_V4}, {0xF94D, "give_or_take_card", {{REG_SET_FIXED, 2}}, F_GC_EP3}, // regsA[0] is card_id; card is given if regsA[1] >= 0, otherwise it's taken - {0xF94D, nullptr, {INT32, REG}, F_XB_V3 | F_ARGS}, // Related to voice chat. argA is a client ID; a value is read from that player's TVoiceChatClient object and (!!value) is placed in regB. This value is set by the 6xB3 command; TODO: figure out what that value represents and name this opcode appropriately + {0xF94D, "unknown_F94D", {INT32, REG}, F_XB_V3 | F_ARGS}, // Related to voice chat. argA is a client ID; a value is read from that player's TVoiceChatClient object and (!!value) is placed in regB. This value is set by the 6xB3 command; TODO: figure out what that value represents and name this opcode appropriately {0xF94D, "nop_F94D", {}, F_V4}, {0xF94E, "nop_F94E", {}, F_V4}, {0xF94F, "nop_F94F", {}, F_V4}, @@ -864,9 +864,36 @@ opcodes_for_version(Version v) { return index; } +static const unordered_map& +opcodes_by_name_for_version(Version v) { + static array< + unordered_map, + static_cast(Version::BB_V4) + 1> + indexes; + + auto& index = indexes.at(static_cast(v)); + if (index.empty()) { + uint16_t vf = v_flag(v); + for (size_t z = 0; z < sizeof(opcode_defs) / sizeof(opcode_defs[0]); z++) { + const auto& def = opcode_defs[z]; + if (!(def.flags & vf)) { + continue; + } + if (!def.name) { + continue; + } + if (!index.emplace(def.name, &def).second) { + throw logic_error(string_printf("duplicate definition for opcode %04hX", def.opcode)); + } + } + } + return index; +} + std::string disassemble_quest_script(const void* data, size_t size, Version version, uint8_t language) { StringReader r(data, size); deque lines; + lines.emplace_back(string_printf(".version %s", name_for_enum(version))); bool use_wstrs = false; size_t code_offset = 0; @@ -938,8 +965,8 @@ std::string disassemble_quest_script(const void* data, size_t size, Version vers lines.emplace_back(string_printf(".quest_num %hu", header.quest_number.load())); lines.emplace_back(string_printf(".episode %hhu", header.episode)); lines.emplace_back(string_printf(".max_players %hhu", header.episode)); - if (header.joinable_in_progress) { - lines.emplace_back(".joinable_in_progress"); + if (header.joinable) { + lines.emplace_back(".joinable"); } lines.emplace_back(".name " + escape_string(header.name.decode(language))); lines.emplace_back(".short_desc " + escape_string(header.short_description.decode(language))); @@ -1091,7 +1118,7 @@ std::string disassemble_quest_script(const void* data, size_t size, Version vers } uint8_t num_functions = cmd_r.get_u8(); for (size_t z = 0; z < num_functions; z++) { - dasm_arg += (dasm_arg.empty() ? "(" : ", "); + dasm_arg += (dasm_arg.empty() ? "[" : ", "); uint32_t label_id = cmd_r.get_u16l(); if (label_id >= function_table.size()) { dasm_arg += string_printf("function%04" PRIX32 " /* invalid */", label_id); @@ -1106,9 +1133,9 @@ std::string disassemble_quest_script(const void* data, size_t size, Version vers } } if (dasm_arg.empty()) { - dasm_arg = "()"; + dasm_arg = "[]"; } else { - dasm_arg += ")"; + dasm_arg += "]"; } break; } @@ -1126,12 +1153,12 @@ std::string disassemble_quest_script(const void* data, size_t size, Version vers } uint8_t num_regs = cmd_r.get_u8(); for (size_t z = 0; z < num_regs; z++) { - dasm_arg += string_printf("%sr%hhu", (dasm_arg.empty() ? "(" : ", "), cmd_r.get_u8()); + dasm_arg += string_printf("%sr%hhu", (dasm_arg.empty() ? "[" : ", "), cmd_r.get_u8()); } if (dasm_arg.empty()) { - dasm_arg = "()"; + dasm_arg = "[]"; } else { - dasm_arg += ")"; + dasm_arg += "]"; } break; } @@ -1293,7 +1320,7 @@ std::string disassemble_quest_script(const void* data, size_t size, Version vers case Arg::Type::FLOAT32: switch (arg_value.type) { case ArgStackValue::Type::REG: - dasm_arg = string_printf("(float)r%" PRIu32, arg_value.as_int); + dasm_arg = string_printf("f%" PRIu32, arg_value.as_int); break; case ArgStackValue::Type::INT: dasm_arg = string_printf("%g", as_type(arg_value.as_int)); @@ -1598,7 +1625,7 @@ Episode find_quest_episode_from_script(const void* data, size_t size, Version ve } if (def == nullptr) { - throw runtime_error("unknown quest opcode"); + throw runtime_error(string_printf("unknown quest opcode %04hX", opcode)); } if (def->flags & F_RET) { @@ -1697,3 +1724,531 @@ Episode episode_for_quest_episode_number(uint8_t episode_number) { throw runtime_error(string_printf("invalid episode number %02hhX", episode_number)); } } + +std::string assemble_quest_script(const std::string& text) { + auto lines = split(text, '\n'); + + // Strip comments and whitespace + for (auto& line : lines) { + size_t comment_start = line.find("/*"); + while (comment_start != string::npos) { + size_t comment_end = line.find("*/", comment_start + 2); + if (comment_end == string::npos) { + throw runtime_error("unterminated inline comment"); + } + line.erase(comment_start, comment_end + 2 - comment_start); + comment_start = line.find("/*"); + } + comment_start = line.find("//"); + if (comment_start != string::npos) { + line.resize(comment_start); + } + strip_trailing_whitespace(line); + strip_leading_whitespace(line); + } + + // Collect metadata directives + Version quest_version = Version::UNKNOWN; + string quest_name; + string quest_short_desc; + string quest_long_desc; + int64_t quest_num = -1; + uint8_t quest_language = 1; + Episode quest_episode = Episode::EP1; + uint8_t quest_max_players = 4; + bool quest_joinable = false; + for (const auto& line : lines) { + if (line.empty()) { + continue; + } + if (line[0] == '.') { + if (starts_with(line, ".version ")) { + string name = line.substr(9); + quest_version = enum_for_name(name.c_str()); + } else if (starts_with(line, ".name ")) { + quest_name = parse_data_string(line.substr(6)); + } else if (starts_with(line, ".short_desc ")) { + quest_short_desc = parse_data_string(line.substr(12)); + } else if (starts_with(line, ".long_desc ")) { + quest_long_desc = parse_data_string(line.substr(11)); + } else if (starts_with(line, ".quest_num ")) { + quest_num = stoul(line.substr(11), nullptr, 0); + } else if (starts_with(line, ".language ")) { + quest_language = stoul(line.substr(10), nullptr, 0); + } else if (starts_with(line, ".episode ")) { + quest_episode = episode_for_token_name(line.substr(9)); + } else if (starts_with(line, ".max_players ")) { + quest_max_players = stoul(line.substr(12), nullptr, 0); + } else if (starts_with(line, ".joinable ")) { + quest_joinable = true; + } + } + } + if (quest_version == Version::PC_PATCH || quest_version == Version::BB_PATCH || quest_version == Version::UNKNOWN) { + throw runtime_error(".version directive is missing or invalid"); + } + if (quest_num < 0) { + throw runtime_error(".quest_num directive is missing or invalid"); + } + if (quest_name.empty()) { + throw runtime_error(".name directive is missing or invalid"); + } + + // Find all label names + struct Label { + std::string name; + ssize_t index = -1; + ssize_t offset = -1; + }; + unordered_map> labels_by_name; + map> labels_by_index; + for (size_t line_num = 1; line_num <= lines.size(); line_num++) { + const auto& line = lines[line_num - 1]; + if (ends_with(line, ":")) { + auto label = make_shared