implement send_function_call

.github/workflows/cmake.yml

@@ -16,6 +16,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest, macos-latest]
+        with_resource_file: ['true', 'false']
 
     steps:
     - uses: actions/checkout@v2
@@ -28,6 +29,15 @@ jobs:
       if: ${{ matrix.os == 'macos-latest' }}
       run: brew install libevent
 
+    - name: Install resource_file
+      if: ${{ matrix.with_resource_file == 'true' }}
+      run: |
+        git clone https://github.com/fuzziqersoftware/resource_dasm.git
+        cd resource_dasm
+        cmake .
+        make
+        sudo make install
+
     - name: Install phosg
       run: |
         git clone https://github.com/fuzziqersoftware/phosg.git

CMakeLists.txt

@@ -21,7 +21,7 @@ set(CMAKE_BUILD_TYPE Debug)
 
 
 
-# Executable definitions
+# Library search
 
 find_path     (LIBEVENT_INCLUDE_DIR NAMES event.h)
 find_library  (LIBEVENT_LIBRARY     NAMES event)
@@ -31,6 +31,19 @@ set (LIBEVENT_LIBRARIES
         ${LIBEVENT_LIBRARY}
         ${LIBEVENT_CORE})
 
+find_path     (RESOURCE_FILE_INCLUDE_DIR NAMES resource_file/ResourceFile.hh)
+find_library  (RESOURCE_FILE_LIBRARY     NAMES resource_file)
+
+if(RESOURCE_FILE_INCLUDE_DIR AND RESOURCE_FILE_LIBRARY)
+  set(RESOURCE_FILE_FOUND 1)
+else()
+  set(RESOURCE_FILE_FOUND 0)
+endif()
+
+
+
+# Executable definition
+
 add_executable(newserv
   src/ChatCommands.cc
   src/Client.cc
@@ -38,6 +51,7 @@ add_executable(newserv
   src/DNSServer.cc
   src/Episode3.cc
   src/FileContentsCache.cc
+  src/FunctionCompiler.cc
   src/IPFrameInfo.cc
   src/IPStackSimulator.cc
   src/Items.cc
@@ -69,6 +83,15 @@ add_executable(newserv
 target_include_directories(newserv PUBLIC ${LIBEVENT_INCLUDE_DIR})
 target_link_libraries(newserv phosg ${LIBEVENT_LIBRARIES})
 
+if(RESOURCE_FILE_FOUND)
+  target_compile_definitions(newserv PUBLIC HAVE_RESOURCE_FILE)
+  target_include_directories(newserv PUBLIC ${RESOURCE_FILE_INCLUDE_DIR})
+  target_link_libraries(newserv ${RESOURCE_FILE_LIBRARY})
+  message(STATUS "libresource_file found; enabling patch support")
+else()
+  message(WARNING "libresource_file not available; disabling patch support")
+endif()
+
 
 
 # Installation configuration

src/FunctionCompiler.cc

@@ -0,0 +1,141 @@
+#include "FunctionCompiler.hh"
+
+#include <stdio.h>
+#include <string.h>
+
+#include <stdexcept>
+#include <phosg/Filesystem.hh>
+
+#ifdef HAVE_RESOURCE_FILE
+#include <resource_file/Emulators/PPC32Emulator.hh>
+#endif
+
+#include "CommandFormats.hh"
+
+using namespace std;
+
+
+
+bool function_compiler_available() {
+#ifndef HAVE_RESOURCE_FILE
+  return false;
+#else
+  return true;
+#endif
+}
+
+
+
+std::string CompiledFunctionCode::generate_client_command(
+      const std::unordered_map<std::string, uint32_t>& label_writes,
+      const std::string& suffix) const {
+  S_ExecuteCode_Footer_GC_B2 footer;
+  footer.num_relocations = this->relocation_deltas.size();
+  footer.unused1.clear();
+  footer.entrypoint_addr_offset = this->entrypoint_offset_offset;
+  footer.unused2.clear();
+
+  StringWriter w;
+  if (!label_writes.empty()) {
+    string modified_code = this->code;
+    for (const auto& it : label_writes) {
+      size_t offset = this->label_offsets.at(it.first);
+      if (offset > modified_code.size() - 4) {
+        throw runtime_error("label out of range");
+      }
+      *reinterpret_cast<be_uint32_t*>(modified_code.data() + offset) = it.second;
+    }
+    w.write(modified_code);
+  } else {
+    w.write(this->code);
+  }
+  w.write(suffix);
+  while (w.size() & 3) {
+    w.put_u8(0);
+  }
+
+  footer.relocations_offset = w.size();
+  for (uint16_t delta : this->relocation_deltas) {
+    w.put_u16b(delta);
+  }
+  if (this->relocation_deltas.size() & 1) {
+    w.put_u16(0);
+  }
+
+  w.put(footer);
+  return move(w.str());
+}
+
+shared_ptr<CompiledFunctionCode> compile_function_code(const std::string& text) {
+#ifndef HAVE_RESOURCE_FILE
+  (void)text;
+  throw runtime_error("PowerPC assembler is not available");
+
+#else
+  auto assembled = PPC32Emulator::assemble(text);
+
+  shared_ptr<CompiledFunctionCode> ret(new CompiledFunctionCode());
+  ret->code = move(assembled.code);
+  ret->label_offsets = move(assembled.label_offsets);
+  ret->index = 0xFF;
+
+  set<uint32_t> reloc_indexes;
+  for (const auto& it : ret->label_offsets) {
+    if (starts_with(it.first, "reloc")) {
+      reloc_indexes.emplace(it.second / 4);
+    } else if (starts_with(it.first, "newserv_index_")) {
+      ret->index = stoul(it.first.substr(14), nullptr, 16);
+    }
+  }
+
+  try {
+    ret->entrypoint_offset_offset = ret->label_offsets.at("entry_ptr");
+  } catch (const out_of_range&) {
+    throw runtime_error("code does not contain entry_ptr label");
+  }
+
+  uint32_t prev_index = 0;
+  for (const auto& it : reloc_indexes) {
+    uint32_t delta = it - prev_index;
+    if (delta > 0xFFFF) {
+      throw runtime_error("relocation delta too far away");
+    }
+    ret->relocation_deltas.emplace_back(delta);
+    prev_index = it;
+  }
+
+  return ret;
+#endif
+}
+
+
+
+FunctionCodeIndex::FunctionCodeIndex(const std::string& directory) {
+  this->index_to_function.resize(0x100);
+
+  if (!function_compiler_available()) {
+    log(INFO, "Function compiler is not available");
+    return;
+  }
+
+  for (const auto& filename : list_directory(directory)) {
+    if (!ends_with(filename, ".s")) {
+      continue;
+    }
+    string name = filename.substr(0, filename.size() - 2);
+
+    try {
+      string path = directory + "/" + filename;
+      string text = load_file(path);
+      auto code = compile_function_code(text);
+      if (code->index < 0xFF) {
+        this->index_to_function.at(code->index) = code;
+      }
+      this->name_to_function.emplace(name, code);
+      log(WARNING, "Compiled function %s", name.c_str());
+
+    } catch (const exception& e) {
+      log(WARNING, "Failed to compile function %s: %s", name.c_str(), e.what());
+    }
+  }
+}

src/FunctionCompiler.hh

@@ -0,0 +1,40 @@
+#pragma once
+
+#include <inttypes.h>
+
+#include <string>
+#include <unordered_map>
+#include <vector>
+#include <memory>
+
+
+
+bool function_compiler_available();
+
+
+
+// TODO: Support x86 function calls in the future. Currently we only support
+// PPC32 because I haven't written an appropriate x86 assembler yet.
+
+struct CompiledFunctionCode {
+  std::string code;
+  std::vector<uint16_t> relocation_deltas;
+  std::unordered_map<std::string, uint32_t> label_offsets;
+  uint32_t entrypoint_offset_offset;
+  uint8_t index; // 00-FE (FF = index not specified)
+
+  std::string generate_client_command(
+      const std::unordered_map<std::string, uint32_t>& label_writes = {},
+      const std::string& suffix = "") const;
+};
+
+std::shared_ptr<CompiledFunctionCode> compile_function_code(const std::string& text);
+
+
+
+struct FunctionCodeIndex {
+  FunctionCodeIndex(const std::string& directory);
+
+  std::unordered_map<std::string, std::shared_ptr<CompiledFunctionCode>> name_to_function;
+  std::vector<std::shared_ptr<CompiledFunctionCode>> index_to_function;
+};

src/Main.cc

@@ -434,6 +434,9 @@ int main(int argc, char** argv) {
   log(INFO, "Collecting quest metadata");
   state->quest_index.reset(new QuestIndex("system/quests"));
 
+  log(INFO, "Compiling client functions");
+  state->function_code_index.reset(new FunctionCodeIndex("system/ppc"));
+
   shared_ptr<DNSServer> dns_server;
   if (state->dns_server_port) {
     log(INFO, "Starting DNS server");

src/ProxyCommands.cc

@@ -22,6 +22,9 @@
 #include <phosg/Random.hh>
 #include <phosg/Strings.hh>
 #include <phosg/Time.hh>
+#ifdef HAVE_RESOURCE_FILE
+#include <resource_file/Emulators/PPC32Emulator.hh>
+#endif
 
 #include "PSOProtocol.hh"
 #include "SendCommands.hh"
@@ -397,9 +400,64 @@ static bool process_server_88(shared_ptr<ServerState>,
 static bool process_server_B2(shared_ptr<ServerState>,
     ProxyServer::LinkedSession& session, uint16_t, uint32_t flag, string& data) {
   if (session.save_files) {
-    string output_filename = string_printf("code.bin.%" PRId64, now());
+    string output_filename = string_printf("code.%" PRId64 ".bin", now());
     save_file(output_filename, data);
     session.log(INFO, "Wrote code from server to file %s", output_filename.c_str());
+
+#ifdef HAVE_RESOURCE_FILE
+    try {
+      // Note: we copy header here because we might modify data later, which
+      // would break the reference
+      auto header = StringReader(data).get<S_ExecuteCode_B2>();
+
+      size_t footer_end_offset = header.code_size;
+      size_t footer_offset = footer_end_offset - sizeof(S_ExecuteCode_Footer_GC_B2);
+      size_t orig_size = data.size() - sizeof(header);
+      if (data.size() < (sizeof(header) + footer_end_offset)) {
+        data.resize((sizeof(header) + footer_end_offset), '\0');
+      }
+
+      fprintf(stderr, "footer_offset = %08zX\n", footer_offset);
+      print_data(stderr, data);
+
+      StringReader r(data.data() + sizeof(header), data.size() - sizeof(header));
+      const auto& footer = r.pget<S_ExecuteCode_Footer_GC_B2>(footer_offset);
+
+      multimap<uint32_t, string> labels;
+      r.go(footer.relocations_offset);
+      uint32_t reloc_offset = 0;
+      for (size_t x = 0; x < footer.num_relocations; x++) {
+        reloc_offset += (r.get_u16b() * 4);
+        labels.emplace(reloc_offset, string_printf("reloc%zu", x));
+      }
+      labels.emplace(footer.entrypoint_addr_offset.load(), "entry_ptr");
+      labels.emplace(footer_offset, "footer");
+      labels.emplace(r.pget_u32b(footer.entrypoint_addr_offset), "start");
+
+      for (const auto& it : labels) {
+        fprintf(stderr, "label: %08" PRIX32 " => %s\n", it.first, it.second.c_str());
+      }
+
+      string disassembly = PPC32Emulator::disassemble(
+          &r.pget<uint8_t>(0, orig_size),
+          orig_size,
+          0,
+          &labels);
+
+      output_filename = string_printf("code.%" PRId64 ".txt", now());
+      {
+        auto f = fopen_unique(output_filename, "wt");
+        fprintf(f.get(), "// code_size = 0x%" PRIX32 "\n", header.code_size.load());
+        fprintf(f.get(), "// checksum_addr = 0x%" PRIX32 "\n", header.checksum_start.load());
+        fprintf(f.get(), "// checksum_size = 0x%" PRIX32 "\n", header.checksum_size.load());
+        fwritex(f.get(), disassembly);
+      }
+      session.log(INFO, "Wrote disassembly to file %s", output_filename.c_str());
+
+    } catch (const exception& e) {
+      session.log(INFO, "Failed to disassemble code from server: %s", e.what());
+    }
+#endif
   }
 
   if (session.function_call_return_value >= 0) {
@@ -417,7 +475,7 @@ static bool process_server_B2(shared_ptr<ServerState>,
 static bool process_server_E7(shared_ptr<ServerState>,
     ProxyServer::LinkedSession& session, uint16_t, uint32_t, string& data) {
   if (session.save_files) {
-    string output_filename = string_printf("player.bin.%" PRId64, now());
+    string output_filename = string_printf("player.%" PRId64 ".bin", now());
     save_file(output_filename, data);
     session.log(INFO, "Wrote player data to file %s", output_filename.c_str());
   }
@@ -646,7 +704,7 @@ static bool process_server_gc_B8(shared_ptr<ServerState>,
       return true;
     }
 
-    string output_filename = string_printf("cardupdate.mnr.%" PRIu64, now());
+    string output_filename = string_printf("cardupdate.%" PRIu64 ".mnr", now());
     save_file(output_filename, r.read(size));
 
     session.log(INFO, "Wrote %zu bytes to %s", size, output_filename.c_str());

src/ReceiveCommands.cc

@@ -2057,7 +2057,7 @@ static process_command_t gc_handlers[0x100] = {
   process_quest_barrier, nullptr, nullptr, nullptr,
 
   // B0
-  nullptr, process_server_time_request, nullptr, nullptr,
+  nullptr, process_server_time_request, nullptr, process_ignored_command,
   nullptr, nullptr, nullptr, process_ignored_command,
   process_ignored_command, nullptr, process_ep3_jukebox, nullptr,
   nullptr, nullptr, nullptr, nullptr,
@@ -2146,8 +2146,10 @@ static process_command_t bb_handlers[0x100] = {
   process_quest_barrier, nullptr, nullptr, nullptr,
 
   // B0
-  nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr,
-  nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr,
+  nullptr, nullptr, nullptr, process_ignored_command,
+  nullptr, nullptr, nullptr, nullptr,
+  nullptr, nullptr, nullptr, nullptr,
+  nullptr, nullptr, nullptr, nullptr,
 
   // C0
   nullptr, process_create_game_bb, nullptr, nullptr,

src/SendCommands.cc

@@ -93,6 +93,11 @@ void send_command(
       throw logic_error("unimplemented game version in send_command");
   }
 
+  // Most client versions I've seen have a receive buffer 0x7C00 bytes in size
+  if (send_data.size() > 0x7C00) {
+    throw runtime_error("outbound command too large");
+  }
+
   if (name_str) {
     string name_token;
     if (name_str[0]) {
@@ -312,6 +317,38 @@ void send_update_client_config(shared_ptr<Client> c) {
 
 
 
+void send_function_call(
+    shared_ptr<Client> c,
+    shared_ptr<CompiledFunctionCode> code,
+    const std::unordered_map<std::string, uint32_t>& label_writes,
+    const std::string& suffix,
+    uint32_t checksum_addr,
+    uint32_t checksum_size) {
+  if (c->version != GameVersion::GC) {
+    throw logic_error("cannot send function calls to non-GameCube clients");
+  }
+  if (c->flags & Client::Flag::EPISODE_3) {
+    throw logic_error("cannot send function calls to Episode 3 clients");
+  }
+
+  string data;
+  uint8_t index = 0xFF;
+  if (code.get()) {
+    data = code->generate_client_command(label_writes, suffix);
+    index = code->index;
+  }
+
+  S_ExecuteCode_B2 header = {data.size(), checksum_addr, checksum_size};
+
+  StringWriter w;
+  w.put(header);
+  w.write(data);
+
+  send_command(c, 0xB2, index, w.str());
+}
+
+
+
 void send_reconnect(shared_ptr<Client> c, uint32_t address, uint16_t port) {
   S_Reconnect_19 cmd = {address, port, 0};
   // On the patch server, 14 is the reconnect command, but it works exactly the

src/SendCommands.hh

@@ -14,6 +14,7 @@
 #include "Quest.hh"
 #include "Text.hh"
 #include "CommandFormats.hh"
+#include "FunctionCompiler.hh"
 
 
 
@@ -104,6 +105,14 @@ void send_server_init(std::shared_ptr<ServerState> s, std::shared_ptr<Client> c,
     bool initial_connection);
 void send_update_client_config(std::shared_ptr<Client> c);
 
+void send_function_call(
+    std::shared_ptr<Client> c,
+    std::shared_ptr<CompiledFunctionCode> code,
+    const std::unordered_map<std::string, uint32_t>& label_writes = {},
+    const std::string& suffix = "",
+    uint32_t checksum_addr = 0,
+    uint32_t checksum_size = 0);
+
 void send_reconnect(std::shared_ptr<Client> c, uint32_t address, uint16_t port);
 void send_pc_gc_split_reconnect(std::shared_ptr<Client> c, uint32_t address,
     uint16_t pc_port, uint16_t gc_port);

src/ServerState.hh

@@ -15,6 +15,7 @@
 #include "Lobby.hh"
 #include "Menu.hh"
 #include "Quest.hh"
+#include "FunctionCompiler.hh"
 
 
 
@@ -46,6 +47,7 @@ struct ServerState {
   bool allow_unregistered_users;
   RunShellBehavior run_shell_behavior;
   std::vector<std::shared_ptr<const PSOBBEncryption::KeyFile>> bb_private_keys;
+  std::shared_ptr<const FunctionCodeIndex> function_code_index;
   std::shared_ptr<const Ep3DataIndex> ep3_data_index;
   std::shared_ptr<const QuestIndex> quest_index;
   std::shared_ptr<const LevelTable> level_table;

system/ppc/WriteMemory.s

@@ -0,0 +1,91 @@
+# This example shows how to use newserv's send_function_call function for PSO
+# GameCube clients. This code writes a variable-length block of data to a
+# specified address in the client's memory.
+
+# For example, to write the bytes 38 00 00 05 to the address 8010521C,
+# send_function_call could be called like this:
+#   auto fn = s->function_code_index->name_to_function.at("WriteMemory");
+#   unordered_map<string, uint32_t label_writes(
+#       {{"dest_addr", 0x8010521C}, {"size", 4}});
+#   string suffix("\x38\x00\x00\x05", 4);
+#   send_function_call(
+#       c,  // Client to send function call to
+#       fn,  // The function's code
+#       label_writes,  // Variables to pass in to the function's code
+#       suffix);  // Data to append after the code (not all functions use this)
+# The meanings of label_writes and suffix are described in the comments below.
+
+# A label newserv_id_XX tells newserv what value to use in the flag field when
+# sending the B2 command. This is needed if the server needs to do something
+# when the B3 response is received.
+newserv_id_C0:
+
+# The entry_ptr label is required. It should point to a .offsetof directive that
+# itself points to the actual entrypoint.
+entry_ptr:
+# All labels starting with reloc signify that the following PPC word
+# (be_uint32_t) is to be relocated at runtime. That is, when the code is run,
+# the PPC word will contain the actual memory address relative to the running
+# code instead of the offset that it holds at assembly time. The entry_ptr label
+# should almost always have a reloc label next to it.
+reloc0:
+  .offsetof start
+
+copy_block:
+  # r8 = address to return to (LR, from start label)
+  mflr    r6            # r6 = address of dest_addr label
+  mtlr    r8
+  lwz     r3, [r6]      # r3 = dest ptr
+  subi    r3, r3, 1     # subtract 1 so we can use stbu
+  lwz     r5, [r6 + 4]  # r5 = size (bytes remaining)
+  add     r5, r5, r3    # r5 = dest end ptr
+  addi    r4, r6, 7     # r4 = src ptr (starting at -1 so we can use lbzu)
+
+copy_block__again:
+  lbzu    r0, [r4 + 1]
+  stbu    [r3 + 1], r0
+  cmp     r3, r5
+  bne     copy_block__again
+
+  lwz     r3, [r6]      # r3 = dest ptr
+  lwz     r4, [r6 + 4]  # r4 = size
+
+  # Flush the data cache and clear the instruction cache at the written region
+  lis     r5, 0xFFFF
+  ori     r5, r5, 0xFFF1
+  and     r5, r5, r3
+  subf    r3, r5, r3
+  add     r4, r4, r3
+flush_cached_code_writes__again:
+  dcbst   r0, r5
+  sync
+  icbi    r0, r5
+  addic   r5, r5, 8
+  subic.  r4, r4, 8
+  bge     flush_cached_code_writes__again
+  isync
+
+  # Return 0 (this value appears in the B3 command)
+  li      r3, 0
+  blr
+
+start:
+  # We use a trick here to get the address of the dest_addr label: since bl puts
+  # the immediately-following address into the link register, we "call"
+  # copy_block and get the dest_addr pointer out of the LR. We then put r8 back
+  # into the LR so copy_block can return normally.
+  mflr    r8
+  bl      copy_block
+
+# These fields are filled in when the B2 command is generated. Specifically, the
+# label_writes argument to send_function_call is responsible for this.
+dest_addr:
+  .zero
+size:
+  .zero
+
+# The data to be written is appended here at B2 construction time via the suffix
+# argument to send_function_call. (This label is for documentation purposes
+# only; the suffix argument always appends data after the end of all the
+# assembled code.)
+data_to_write: