From 85d054fc3aba4f72eb740c305a1af81de48ea1c8 Mon Sep 17 00:00:00 2001 From: Martin Michelsen Date: Tue, 31 May 2022 17:18:04 -0700 Subject: [PATCH] implement send_function_call --- .github/workflows/cmake.yml | 10 +++ CMakeLists.txt | 25 ++++++- src/FunctionCompiler.cc | 141 ++++++++++++++++++++++++++++++++++++ src/FunctionCompiler.hh | 40 ++++++++++ src/Main.cc | 3 + src/ProxyCommands.cc | 64 +++++++++++++++- src/ReceiveCommands.cc | 8 +- src/SendCommands.cc | 37 ++++++++++ src/SendCommands.hh | 9 +++ src/ServerState.hh | 2 + system/ppc/WriteMemory.s | 91 +++++++++++++++++++++++ 11 files changed, 423 insertions(+), 7 deletions(-) create mode 100644 src/FunctionCompiler.cc create mode 100644 src/FunctionCompiler.hh create mode 100644 system/ppc/WriteMemory.s diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index 01678e5b..11b07726 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -16,6 +16,7 @@ jobs: strategy: matrix: os: [ubuntu-latest, macos-latest] + with_resource_file: ['true', 'false'] steps: - uses: actions/checkout@v2 @@ -28,6 +29,15 @@ jobs: if: ${{ matrix.os == 'macos-latest' }} run: brew install libevent + - name: Install resource_file + if: ${{ matrix.with_resource_file == 'true' }} + run: | + git clone https://github.com/fuzziqersoftware/resource_dasm.git + cd resource_dasm + cmake . + make + sudo make install + - name: Install phosg run: | git clone https://github.com/fuzziqersoftware/phosg.git diff --git a/CMakeLists.txt b/CMakeLists.txt index 2b94806c..1305f714 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -21,7 +21,7 @@ set(CMAKE_BUILD_TYPE Debug) -# Executable definitions +# Library search find_path (LIBEVENT_INCLUDE_DIR NAMES event.h) find_library (LIBEVENT_LIBRARY NAMES event) @@ -31,6 +31,19 @@ set (LIBEVENT_LIBRARIES ${LIBEVENT_LIBRARY} ${LIBEVENT_CORE}) +find_path (RESOURCE_FILE_INCLUDE_DIR NAMES resource_file/ResourceFile.hh) +find_library (RESOURCE_FILE_LIBRARY NAMES resource_file) + +if(RESOURCE_FILE_INCLUDE_DIR AND RESOURCE_FILE_LIBRARY) + set(RESOURCE_FILE_FOUND 1) +else() + set(RESOURCE_FILE_FOUND 0) +endif() + + + +# Executable definition + add_executable(newserv src/ChatCommands.cc src/Client.cc @@ -38,6 +51,7 @@ add_executable(newserv src/DNSServer.cc src/Episode3.cc src/FileContentsCache.cc + src/FunctionCompiler.cc src/IPFrameInfo.cc src/IPStackSimulator.cc src/Items.cc @@ -69,6 +83,15 @@ add_executable(newserv target_include_directories(newserv PUBLIC ${LIBEVENT_INCLUDE_DIR}) target_link_libraries(newserv phosg ${LIBEVENT_LIBRARIES}) +if(RESOURCE_FILE_FOUND) + target_compile_definitions(newserv PUBLIC HAVE_RESOURCE_FILE) + target_include_directories(newserv PUBLIC ${RESOURCE_FILE_INCLUDE_DIR}) + target_link_libraries(newserv ${RESOURCE_FILE_LIBRARY}) + message(STATUS "libresource_file found; enabling patch support") +else() + message(WARNING "libresource_file not available; disabling patch support") +endif() + # Installation configuration diff --git a/src/FunctionCompiler.cc b/src/FunctionCompiler.cc new file mode 100644 index 00000000..268b1508 --- /dev/null +++ b/src/FunctionCompiler.cc @@ -0,0 +1,141 @@ +#include "FunctionCompiler.hh" + +#include +#include + +#include +#include + +#ifdef HAVE_RESOURCE_FILE +#include +#endif + +#include "CommandFormats.hh" + +using namespace std; + + + +bool function_compiler_available() { +#ifndef HAVE_RESOURCE_FILE + return false; +#else + return true; +#endif +} + + + +std::string CompiledFunctionCode::generate_client_command( + const std::unordered_map& label_writes, + const std::string& suffix) const { + S_ExecuteCode_Footer_GC_B2 footer; + footer.num_relocations = this->relocation_deltas.size(); + footer.unused1.clear(); + footer.entrypoint_addr_offset = this->entrypoint_offset_offset; + footer.unused2.clear(); + + StringWriter w; + if (!label_writes.empty()) { + string modified_code = this->code; + for (const auto& it : label_writes) { + size_t offset = this->label_offsets.at(it.first); + if (offset > modified_code.size() - 4) { + throw runtime_error("label out of range"); + } + *reinterpret_cast(modified_code.data() + offset) = it.second; + } + w.write(modified_code); + } else { + w.write(this->code); + } + w.write(suffix); + while (w.size() & 3) { + w.put_u8(0); + } + + footer.relocations_offset = w.size(); + for (uint16_t delta : this->relocation_deltas) { + w.put_u16b(delta); + } + if (this->relocation_deltas.size() & 1) { + w.put_u16(0); + } + + w.put(footer); + return move(w.str()); +} + +shared_ptr compile_function_code(const std::string& text) { +#ifndef HAVE_RESOURCE_FILE + (void)text; + throw runtime_error("PowerPC assembler is not available"); + +#else + auto assembled = PPC32Emulator::assemble(text); + + shared_ptr ret(new CompiledFunctionCode()); + ret->code = move(assembled.code); + ret->label_offsets = move(assembled.label_offsets); + ret->index = 0xFF; + + set reloc_indexes; + for (const auto& it : ret->label_offsets) { + if (starts_with(it.first, "reloc")) { + reloc_indexes.emplace(it.second / 4); + } else if (starts_with(it.first, "newserv_index_")) { + ret->index = stoul(it.first.substr(14), nullptr, 16); + } + } + + try { + ret->entrypoint_offset_offset = ret->label_offsets.at("entry_ptr"); + } catch (const out_of_range&) { + throw runtime_error("code does not contain entry_ptr label"); + } + + uint32_t prev_index = 0; + for (const auto& it : reloc_indexes) { + uint32_t delta = it - prev_index; + if (delta > 0xFFFF) { + throw runtime_error("relocation delta too far away"); + } + ret->relocation_deltas.emplace_back(delta); + prev_index = it; + } + + return ret; +#endif +} + + + +FunctionCodeIndex::FunctionCodeIndex(const std::string& directory) { + this->index_to_function.resize(0x100); + + if (!function_compiler_available()) { + log(INFO, "Function compiler is not available"); + return; + } + + for (const auto& filename : list_directory(directory)) { + if (!ends_with(filename, ".s")) { + continue; + } + string name = filename.substr(0, filename.size() - 2); + + try { + string path = directory + "/" + filename; + string text = load_file(path); + auto code = compile_function_code(text); + if (code->index < 0xFF) { + this->index_to_function.at(code->index) = code; + } + this->name_to_function.emplace(name, code); + log(WARNING, "Compiled function %s", name.c_str()); + + } catch (const exception& e) { + log(WARNING, "Failed to compile function %s: %s", name.c_str(), e.what()); + } + } +} diff --git a/src/FunctionCompiler.hh b/src/FunctionCompiler.hh new file mode 100644 index 00000000..6af9bc79 --- /dev/null +++ b/src/FunctionCompiler.hh @@ -0,0 +1,40 @@ +#pragma once + +#include + +#include +#include +#include +#include + + + +bool function_compiler_available(); + + + +// TODO: Support x86 function calls in the future. Currently we only support +// PPC32 because I haven't written an appropriate x86 assembler yet. + +struct CompiledFunctionCode { + std::string code; + std::vector relocation_deltas; + std::unordered_map label_offsets; + uint32_t entrypoint_offset_offset; + uint8_t index; // 00-FE (FF = index not specified) + + std::string generate_client_command( + const std::unordered_map& label_writes = {}, + const std::string& suffix = "") const; +}; + +std::shared_ptr compile_function_code(const std::string& text); + + + +struct FunctionCodeIndex { + FunctionCodeIndex(const std::string& directory); + + std::unordered_map> name_to_function; + std::vector> index_to_function; +}; diff --git a/src/Main.cc b/src/Main.cc index 20b49d6b..d8bb599d 100644 --- a/src/Main.cc +++ b/src/Main.cc @@ -434,6 +434,9 @@ int main(int argc, char** argv) { log(INFO, "Collecting quest metadata"); state->quest_index.reset(new QuestIndex("system/quests")); + log(INFO, "Compiling client functions"); + state->function_code_index.reset(new FunctionCodeIndex("system/ppc")); + shared_ptr dns_server; if (state->dns_server_port) { log(INFO, "Starting DNS server"); diff --git a/src/ProxyCommands.cc b/src/ProxyCommands.cc index b2640c00..24ecbc1a 100644 --- a/src/ProxyCommands.cc +++ b/src/ProxyCommands.cc @@ -22,6 +22,9 @@ #include #include #include +#ifdef HAVE_RESOURCE_FILE +#include +#endif #include "PSOProtocol.hh" #include "SendCommands.hh" @@ -397,9 +400,64 @@ static bool process_server_88(shared_ptr, static bool process_server_B2(shared_ptr, ProxyServer::LinkedSession& session, uint16_t, uint32_t flag, string& data) { if (session.save_files) { - string output_filename = string_printf("code.bin.%" PRId64, now()); + string output_filename = string_printf("code.%" PRId64 ".bin", now()); save_file(output_filename, data); session.log(INFO, "Wrote code from server to file %s", output_filename.c_str()); + +#ifdef HAVE_RESOURCE_FILE + try { + // Note: we copy header here because we might modify data later, which + // would break the reference + auto header = StringReader(data).get(); + + size_t footer_end_offset = header.code_size; + size_t footer_offset = footer_end_offset - sizeof(S_ExecuteCode_Footer_GC_B2); + size_t orig_size = data.size() - sizeof(header); + if (data.size() < (sizeof(header) + footer_end_offset)) { + data.resize((sizeof(header) + footer_end_offset), '\0'); + } + + fprintf(stderr, "footer_offset = %08zX\n", footer_offset); + print_data(stderr, data); + + StringReader r(data.data() + sizeof(header), data.size() - sizeof(header)); + const auto& footer = r.pget(footer_offset); + + multimap labels; + r.go(footer.relocations_offset); + uint32_t reloc_offset = 0; + for (size_t x = 0; x < footer.num_relocations; x++) { + reloc_offset += (r.get_u16b() * 4); + labels.emplace(reloc_offset, string_printf("reloc%zu", x)); + } + labels.emplace(footer.entrypoint_addr_offset.load(), "entry_ptr"); + labels.emplace(footer_offset, "footer"); + labels.emplace(r.pget_u32b(footer.entrypoint_addr_offset), "start"); + + for (const auto& it : labels) { + fprintf(stderr, "label: %08" PRIX32 " => %s\n", it.first, it.second.c_str()); + } + + string disassembly = PPC32Emulator::disassemble( + &r.pget(0, orig_size), + orig_size, + 0, + &labels); + + output_filename = string_printf("code.%" PRId64 ".txt", now()); + { + auto f = fopen_unique(output_filename, "wt"); + fprintf(f.get(), "// code_size = 0x%" PRIX32 "\n", header.code_size.load()); + fprintf(f.get(), "// checksum_addr = 0x%" PRIX32 "\n", header.checksum_start.load()); + fprintf(f.get(), "// checksum_size = 0x%" PRIX32 "\n", header.checksum_size.load()); + fwritex(f.get(), disassembly); + } + session.log(INFO, "Wrote disassembly to file %s", output_filename.c_str()); + + } catch (const exception& e) { + session.log(INFO, "Failed to disassemble code from server: %s", e.what()); + } +#endif } if (session.function_call_return_value >= 0) { @@ -417,7 +475,7 @@ static bool process_server_B2(shared_ptr, static bool process_server_E7(shared_ptr, ProxyServer::LinkedSession& session, uint16_t, uint32_t, string& data) { if (session.save_files) { - string output_filename = string_printf("player.bin.%" PRId64, now()); + string output_filename = string_printf("player.%" PRId64 ".bin", now()); save_file(output_filename, data); session.log(INFO, "Wrote player data to file %s", output_filename.c_str()); } @@ -646,7 +704,7 @@ static bool process_server_gc_B8(shared_ptr, return true; } - string output_filename = string_printf("cardupdate.mnr.%" PRIu64, now()); + string output_filename = string_printf("cardupdate.%" PRIu64 ".mnr", now()); save_file(output_filename, r.read(size)); session.log(INFO, "Wrote %zu bytes to %s", size, output_filename.c_str()); diff --git a/src/ReceiveCommands.cc b/src/ReceiveCommands.cc index 84655290..9c6c1b42 100644 --- a/src/ReceiveCommands.cc +++ b/src/ReceiveCommands.cc @@ -2057,7 +2057,7 @@ static process_command_t gc_handlers[0x100] = { process_quest_barrier, nullptr, nullptr, nullptr, // B0 - nullptr, process_server_time_request, nullptr, nullptr, + nullptr, process_server_time_request, nullptr, process_ignored_command, nullptr, nullptr, nullptr, process_ignored_command, process_ignored_command, nullptr, process_ep3_jukebox, nullptr, nullptr, nullptr, nullptr, nullptr, @@ -2146,8 +2146,10 @@ static process_command_t bb_handlers[0x100] = { process_quest_barrier, nullptr, nullptr, nullptr, // B0 - nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, - nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, + nullptr, nullptr, nullptr, process_ignored_command, + nullptr, nullptr, nullptr, nullptr, + nullptr, nullptr, nullptr, nullptr, + nullptr, nullptr, nullptr, nullptr, // C0 nullptr, process_create_game_bb, nullptr, nullptr, diff --git a/src/SendCommands.cc b/src/SendCommands.cc index b360972d..b76b3c34 100644 --- a/src/SendCommands.cc +++ b/src/SendCommands.cc @@ -93,6 +93,11 @@ void send_command( throw logic_error("unimplemented game version in send_command"); } + // Most client versions I've seen have a receive buffer 0x7C00 bytes in size + if (send_data.size() > 0x7C00) { + throw runtime_error("outbound command too large"); + } + if (name_str) { string name_token; if (name_str[0]) { @@ -312,6 +317,38 @@ void send_update_client_config(shared_ptr c) { +void send_function_call( + shared_ptr c, + shared_ptr code, + const std::unordered_map& label_writes, + const std::string& suffix, + uint32_t checksum_addr, + uint32_t checksum_size) { + if (c->version != GameVersion::GC) { + throw logic_error("cannot send function calls to non-GameCube clients"); + } + if (c->flags & Client::Flag::EPISODE_3) { + throw logic_error("cannot send function calls to Episode 3 clients"); + } + + string data; + uint8_t index = 0xFF; + if (code.get()) { + data = code->generate_client_command(label_writes, suffix); + index = code->index; + } + + S_ExecuteCode_B2 header = {data.size(), checksum_addr, checksum_size}; + + StringWriter w; + w.put(header); + w.write(data); + + send_command(c, 0xB2, index, w.str()); +} + + + void send_reconnect(shared_ptr c, uint32_t address, uint16_t port) { S_Reconnect_19 cmd = {address, port, 0}; // On the patch server, 14 is the reconnect command, but it works exactly the diff --git a/src/SendCommands.hh b/src/SendCommands.hh index 62b13a45..2421886d 100644 --- a/src/SendCommands.hh +++ b/src/SendCommands.hh @@ -14,6 +14,7 @@ #include "Quest.hh" #include "Text.hh" #include "CommandFormats.hh" +#include "FunctionCompiler.hh" @@ -104,6 +105,14 @@ void send_server_init(std::shared_ptr s, std::shared_ptr c, bool initial_connection); void send_update_client_config(std::shared_ptr c); +void send_function_call( + std::shared_ptr c, + std::shared_ptr code, + const std::unordered_map& label_writes = {}, + const std::string& suffix = "", + uint32_t checksum_addr = 0, + uint32_t checksum_size = 0); + void send_reconnect(std::shared_ptr c, uint32_t address, uint16_t port); void send_pc_gc_split_reconnect(std::shared_ptr c, uint32_t address, uint16_t pc_port, uint16_t gc_port); diff --git a/src/ServerState.hh b/src/ServerState.hh index f8583b71..6f8415f6 100644 --- a/src/ServerState.hh +++ b/src/ServerState.hh @@ -15,6 +15,7 @@ #include "Lobby.hh" #include "Menu.hh" #include "Quest.hh" +#include "FunctionCompiler.hh" @@ -46,6 +47,7 @@ struct ServerState { bool allow_unregistered_users; RunShellBehavior run_shell_behavior; std::vector> bb_private_keys; + std::shared_ptr function_code_index; std::shared_ptr ep3_data_index; std::shared_ptr quest_index; std::shared_ptr level_table; diff --git a/system/ppc/WriteMemory.s b/system/ppc/WriteMemory.s new file mode 100644 index 00000000..27eb693c --- /dev/null +++ b/system/ppc/WriteMemory.s @@ -0,0 +1,91 @@ +# This example shows how to use newserv's send_function_call function for PSO +# GameCube clients. This code writes a variable-length block of data to a +# specified address in the client's memory. + +# For example, to write the bytes 38 00 00 05 to the address 8010521C, +# send_function_call could be called like this: +# auto fn = s->function_code_index->name_to_function.at("WriteMemory"); +# unordered_map