add patch downloader

CMakeLists.txt

@@ -101,6 +101,7 @@ set(SOURCES
     src/Map.cc
     src/Menu.cc
     src/NetworkAddresses.cc
+    src/PatchDownloadSession.cc
     src/PatchFileIndex.cc
     src/PlayerInventory.cc
     src/PlayerSubordinates.cc

src/CommandFormats.hh

@@ -190,17 +190,17 @@ struct S_OpenFile_Patch_06 {
 
 // 07 (S->C): Write file
 // The client's handler table says this command's maximum size is 0x6010
-// including the header, but the only servers I've seen use this command limit
+// including the header, but the command may be shorter if the server chooses
-// chunks to 0x4010 (including the header). Unlike the game server's 13 and A7
+// to use a shorter chunk size. Unlike the game server's 13 and A7 commands,
-// commands, the chunks do not need to be the same size - the game opens the
+// the chunks do not need to be the same size - the game opens the file with
-// file with the "a+b" mode each time it is written, so the new data is always
+// the "a+b" mode each time it is written, so the new data is always appended
-// appended to the end.
+// to the end.
 
 struct S_WriteFileHeader_Patch_07 {
   le_uint32_t chunk_index = 0;
   le_uint32_t chunk_checksum = 0; // CRC32 of the following chunk data
   le_uint32_t chunk_size = 0;
-  // The chunk data immediately follows here
+  // The chunk's data immediately follows here
 } __packed_ws__(S_WriteFileHeader_Patch_07, 0x0C);
 
 // 08 (S->C): Close current file
@@ -240,8 +240,8 @@ struct S_FileChecksumRequest_Patch_0C {
 
 struct C_FileInformation_Patch_0F {
   le_uint32_t request_id = 0; // Matches request_id from an earlier 0C command
-  le_uint32_t checksum = 0; // CRC32 of the file's data
+  le_uint32_t checksum = 0; // CRC32 of the file's data (0 if file not found)
-  le_uint32_t size = 0;
+  le_uint32_t size = 0; // 0 if file not found
 } __packed_ws__(C_FileInformation_Patch_0F, 0x0C);
 
 // 10 (C->S): End of file information command list

src/Main.cc

@@ -37,6 +37,7 @@
 #include "PPKArchive.hh"
 #include "PSOGCObjectGraph.hh"
 #include "PSOProtocol.hh"
+#include "PatchDownloadSession.hh"
 #include "Quest.hh"
 #include "QuestScript.hh"
 #include "ReplaySession.hh"
@@ -2028,50 +2029,70 @@ Action a_download_files(
             phosg::load_object_file<PSOBBEncryption::KeyFile>("system/blueburst/keys/" + key_file_name + ".nsk"));
       }
       auto [remote_host, remote_port] = phosg::parse_netloc(args.get<string>(1));
-      auto character = PSOCHARFile::load_shared(args.get<string>("character", true), false).character_file;
-      auto ship_menu_selections_str = args.get<string>("ship-menu-selections", false);
 
-      unordered_set<string> ship_menu_selections;
-      if (!ship_menu_selections_str.empty()) {
-        for (const string& s : phosg::split(ship_menu_selections_str, ',')) {
-          ship_menu_selections.emplace(s);
-        }
-      }
-
-      vector<string> on_request_complete_commands;
-      string on_request_complete_arg = args.get<string>("on-request-complete-command", false);
-      if (!on_request_complete_arg.empty()) {
-        for (const string& command : phosg::split(on_request_complete_arg, ',')) {
-          on_request_complete_commands.emplace_back(phosg::parse_data_string(command));
-        }
-      }
-
-      uint32_t serial_number = args.get<uint32_t>(
-          "serial-number",
-          0,
-          is_v1_or_v2(version) ? phosg::Arguments::IntFormat::HEX : phosg::Arguments::IntFormat::DEFAULT);
       auto io_context = make_shared<asio::io_context>();
-      DownloadSession session(
+      unique_ptr<DownloadSession> download_session;
-          io_context,
+      unique_ptr<PatchDownloadSession> patch_download_session;
-          remote_host,
+      if (is_patch(version)) {
-          remote_port,
+        patch_download_session = std::make_unique<PatchDownloadSession>(
-          args.get<string>("output-dir", true),
+            io_context,
-          version,
+            remote_host,
-          args.get<uint8_t>("language"),
+            remote_port,
-          key,
+            args.get<string>("output-dir", true),
-          phosg::random_object<uint32_t>(),
+            version,
-          serial_number,
+            args.get<string>("username", false),
-          args.get<string>("access-key", false),
+            args.get<string>("password", false),
-          args.get<string>("username", false),
+            args.get<string>("email", false),
-          args.get<string>("password", false),
+            args.get<bool>("show-command-data"));
-          args.get<string>("xb-gamertag", false),
+        asio::co_spawn(*io_context, patch_download_session->run(), asio::detached);
-          args.get<uint64_t>("xb-user-id", 0, phosg::Arguments::IntFormat::HEX),
+
-          args.get<uint64_t>("xb-account-id", 0, phosg::Arguments::IntFormat::HEX),
+      } else {
-          character,
+        auto character = PSOCHARFile::load_shared(args.get<string>("character", true), false).character_file;
-          ship_menu_selections,
+        auto ship_menu_selections_str = args.get<string>("ship-menu-selections", false);
-          on_request_complete_commands,
+
-          args.get<bool>("interactive"),
+        unordered_set<string> ship_menu_selections;
-          args.get<bool>("show-command-data"));
+        if (!ship_menu_selections_str.empty()) {
+          for (const string& s : phosg::split(ship_menu_selections_str, ',')) {
+            ship_menu_selections.emplace(s);
+          }
+        }
+
+        vector<string> on_request_complete_commands;
+        string on_request_complete_arg = args.get<string>("on-request-complete-command", false);
+        if (!on_request_complete_arg.empty()) {
+          for (const string& command : phosg::split(on_request_complete_arg, ',')) {
+            on_request_complete_commands.emplace_back(phosg::parse_data_string(command));
+          }
+        }
+
+        uint32_t serial_number = args.get<uint32_t>(
+            "serial-number",
+            0,
+            is_v1_or_v2(version) ? phosg::Arguments::IntFormat::HEX : phosg::Arguments::IntFormat::DEFAULT);
+
+        download_session = std::make_unique<DownloadSession>(
+            io_context,
+            remote_host,
+            remote_port,
+            args.get<string>("output-dir", true),
+            version,
+            args.get<uint8_t>("language"),
+            key,
+            phosg::random_object<uint32_t>(),
+            serial_number,
+            args.get<string>("access-key", false),
+            args.get<string>("username", false),
+            args.get<string>("password", false),
+            args.get<string>("xb-gamertag", false),
+            args.get<uint64_t>("xb-user-id", 0, phosg::Arguments::IntFormat::HEX),
+            args.get<uint64_t>("xb-account-id", 0, phosg::Arguments::IntFormat::HEX),
+            character,
+            ship_menu_selections,
+            on_request_complete_commands,
+            args.get<bool>("interactive"),
+            args.get<bool>("show-command-data"));
+        asio::co_spawn(*io_context, download_session->run(), asio::detached);
+      }
       io_context->run();
     });
 

src/PatchDownloadSession.cc

@@ -0,0 +1,285 @@
+#include "PatchDownloadSession.hh"
+
+#include <ctype.h>
+#include <errno.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+#include <algorithm>
+#include <iostream>
+#include <phosg/Encoding.hh>
+#include <phosg/Filesystem.hh>
+#include <phosg/Network.hh>
+#include <phosg/Random.hh>
+#include <phosg/Strings.hh>
+#include <phosg/Time.hh>
+
+#include "Loggers.hh"
+#include "PSOProtocol.hh"
+#include "ReceiveCommands.hh"
+#include "ReceiveSubcommands.hh"
+#include "SendCommands.hh"
+
+using namespace std;
+
+PatchDownloadSession::PatchDownloadSession(
+    std::shared_ptr<asio::io_context> io_context,
+    const std::string& remote_host,
+    uint16_t remote_port,
+    const std::string& output_dir,
+    Version version,
+    const std::string& username,
+    const std::string& password,
+    const std::string& email,
+    bool show_command_data)
+    : remote_host(remote_host),
+      remote_port(remote_port),
+      output_dir(output_dir),
+      version(version),
+      username(username),
+      password(password),
+      email(email),
+      show_command_data(show_command_data),
+      log(std::format("[PatchDownloadSession:{}] ", phosg::name_for_enum(version)), proxy_server_log.min_level),
+      io_context(io_context),
+      current_file(nullptr, +[](FILE* f) -> void { fclose(f); }) {
+  if (this->output_dir.empty()) {
+    this->output_dir = ".";
+  }
+  if (!is_patch(this->version)) {
+    throw std::logic_error("invalid version in PatchDownloadSession");
+  }
+}
+
+asio::awaitable<void> PatchDownloadSession::run() {
+  string netloc_str = std::format("{}:{}", this->remote_host, this->remote_port);
+  this->log.info_f("Connecting to {}", netloc_str);
+  auto sock = make_unique<asio::ip::tcp::socket>(co_await async_connect_tcp(this->remote_host, this->remote_port));
+  this->channel = SocketChannel::create(
+      this->io_context,
+      std::move(sock),
+      this->version,
+      1,
+      netloc_str,
+      this->show_command_data ? phosg::TerminalFormat::FG_GREEN : phosg::TerminalFormat::END,
+      this->show_command_data ? phosg::TerminalFormat::FG_YELLOW : phosg::TerminalFormat::END);
+  this->log.info_f("Server channel connected");
+
+  while (this->channel->connected()) {
+    auto msg = co_await this->channel->recv();
+    co_await this->on_message(msg);
+  }
+}
+
+void PatchDownloadSession::check_path_token(const std::string& token) {
+  if (token == "..") {
+    throw std::runtime_error("parent directory token is not allowed");
+  }
+  if ((token.find('/') != string::npos) || (token.find('\\') != string::npos)) {
+    throw std::runtime_error("directory token contains path separator");
+  }
+}
+
+std::string PatchDownloadSession::resolve_filename(const std::string& filename) const {
+  check_path_token(filename);
+  string path = this->output_dir;
+  for (const auto& dir_name : this->dir_path) {
+    path.push_back('/');
+    path += dir_name;
+  }
+  if (!filename.empty()) {
+    path.push_back('/');
+    path += filename;
+  }
+  return path;
+}
+
+asio::awaitable<void> PatchDownloadSession::on_message(Channel::Message& msg) {
+  switch (msg.command) {
+    case 0x02: {
+      const auto& cmd = msg.check_size_t<S_ServerInit_Patch_02>();
+      if (cmd.copyright.decode() != "Patch Server. Copyright SonicTeam, LTD. 2001") {
+        throw std::runtime_error("incorrect copyright message");
+      }
+      this->channel->crypt_in = make_shared<PSOV2Encryption>(cmd.server_key);
+      this->channel->crypt_out = make_shared<PSOV2Encryption>(cmd.client_key);
+      this->channel->send(0x02);
+      this->log.info_f("Enabled encryption");
+      break;
+    }
+
+    case 0x04: {
+      if (!msg.data.empty()) {
+        throw std::runtime_error("invalid login request command");
+      }
+      C_Login_Patch_04 cmd;
+      cmd.username.encode(this->username);
+      cmd.password.encode(this->password);
+      cmd.email_address.encode(this->email);
+      this->channel->send(0x04, 0x00, &cmd, sizeof(cmd));
+      this->log.info_f("Sent login credentials");
+      break;
+    }
+
+    case 0x05: {
+      this->log.info_f("Server sent disconnect command");
+      this->channel->disconnect();
+      break;
+    }
+
+    case 0x06: {
+      if (this->current_file) {
+        throw std::runtime_error("protocol violation: previous file was not closed before open file command");
+      }
+      const auto& cmd = msg.check_size_t<S_OpenFile_Patch_06>();
+      this->current_file_bytes_remaining = cmd.size;
+      auto filename = this->resolve_filename(cmd.filename.decode());
+      this->current_file = phosg::fopen_unique(filename, "wb");
+      this->log.info_f("Opened file {}", filename);
+      break;
+    }
+
+    case 0x07: {
+      if (!this->current_file) {
+        throw std::runtime_error("protocol violation: no file is open; cannot write data");
+      }
+
+      const auto& cmd = msg.check_size_t<S_WriteFileHeader_Patch_07>(0xFFFF);
+      const void* data = msg.data.data() + sizeof(cmd);
+      if (cmd.chunk_size > msg.data.size() - sizeof(cmd)) {
+        throw std::runtime_error("protocol violation: write command size is invalid");
+      }
+      if (cmd.chunk_size > this->current_file_bytes_remaining) {
+        throw std::runtime_error("protocol violation: chunk would exceed file size specified in open command");
+      }
+      if (phosg::crc32(data, cmd.chunk_size) != cmd.chunk_checksum) {
+        throw std::runtime_error("protocol violation: write command checksum is invalid");
+      }
+
+      phosg::fwritex(this->current_file.get(), data, cmd.chunk_size);
+      this->current_file_bytes_remaining -= cmd.chunk_size;
+      this->log.info_f("Wrote {} to file", phosg::format_size(cmd.chunk_size));
+      break;
+    }
+
+    case 0x08: {
+      if (!this->current_file) {
+        throw std::runtime_error("protocol violation: no file is open; cannot close it");
+      }
+      this->current_file.reset();
+      this->log.info_f("Closed file");
+      break;
+    }
+
+    case 0x09: {
+      if (this->current_file) {
+        throw std::runtime_error("protocol violation: cannot enter directory with a file open");
+      }
+
+      const auto& cmd = msg.check_size_t<S_EnterDirectory_Patch_09>();
+      string dirname = cmd.name.decode();
+      check_path_token(dirname);
+      this->dir_path.emplace_back(std::move(dirname));
+      std::filesystem::create_directories(this->resolve_filename(""));
+      this->log.info_f("Entered directory {}", dirname);
+      break;
+    }
+
+    case 0x0A: {
+      if (this->current_file) {
+        throw std::runtime_error("protocol violation: cannot exit directory with a file open");
+      }
+      if (this->dir_path.empty()) {
+        throw std::runtime_error("protocol violation: cannot exit directory with empty directory stack");
+      }
+      this->dir_path.pop_back();
+      this->log.info_f("Left directory");
+      break;
+    }
+
+    case 0x0B:
+      if (this->current_file) {
+        throw std::runtime_error("protocol violation: cannot start patch session when file is already open");
+      }
+      this->dir_path.clear();
+      this->log.info_f("Started patch session");
+      break;
+
+    case 0x0C: {
+      const auto& cmd = msg.check_size_t<S_FileChecksumRequest_Patch_0C>();
+      auto filename = this->resolve_filename(cmd.filename.decode());
+      uint32_t checksum = 0, size = 0;
+      try {
+        auto data = phosg::load_file(filename);
+        checksum = phosg::crc32(data.data(), data.size());
+        size = data.size();
+      } catch (const phosg::cannot_open_file&) {
+      }
+      this->pending_checksum_results.emplace_back(C_FileInformation_Patch_0F{cmd.request_id, checksum, size});
+      this->log.info_f("Checked file {}", filename);
+      break;
+    }
+
+    case 0x0D:
+      for (const auto& it : this->pending_checksum_results) {
+        this->channel->send(0x0F, 0x00, &it, sizeof(it));
+      }
+      this->pending_checksum_results.clear();
+      this->channel->send(0x10);
+      this->log.info_f("Sent all checksum results");
+      break;
+
+    case 0x11: {
+      const auto& cmd = msg.check_size_t<S_StartFileDownloads_Patch_11>();
+      this->log.info_f("{} files ({}) to download", cmd.num_files.load(), phosg::format_size(cmd.total_bytes));
+      break;
+    }
+
+    case 0x12:
+      this->log.info_f("Patch session succeeded");
+      this->channel->disconnect();
+      break;
+
+    case 0x13: {
+      phosg::strip_trailing_zeroes(msg.data);
+      if (msg.data.size() & 1) {
+        msg.data.push_back(0);
+      }
+      this->log.info_f("Message from server:\n{}", strip_color(tt_utf16_to_utf8(msg.data)));
+      break;
+    }
+
+    case 0x14: {
+      const auto& cmd = msg.check_size_t<S_Reconnect_Patch_14>();
+
+      auto new_ep = make_endpoint_ipv4(cmd.address, cmd.port);
+      string netloc_str = str_for_endpoint(new_ep);
+      this->log.info_f("Connecting to {}", netloc_str);
+      auto sock = make_unique<asio::ip::tcp::socket>(co_await async_connect_tcp(new_ep));
+
+      auto old_channel = this->channel;
+      auto new_channel = SocketChannel::create(
+          this->io_context,
+          std::move(sock),
+          this->channel->version,
+          this->channel->language,
+          netloc_str,
+          this->channel->terminal_send_color,
+          this->channel->terminal_recv_color);
+      this->channel = new_channel;
+      old_channel->disconnect();
+      this->log.info_f("Server channel connected");
+      break;
+    }
+
+    case 0x15:
+      this->log.error_f("Server rejected login credentials");
+      this->channel->disconnect();
+      break;
+
+    default:
+      throw std::runtime_error("invalid command");
+  }
+}

src/PatchDownloadSession.hh

@@ -0,0 +1,61 @@
+#pragma once
+
+#include <asio.hpp>
+#include <map>
+#include <memory>
+#include <phosg/Filesystem.hh>
+#include <string>
+#include <unordered_map>
+#include <unordered_set>
+#include <vector>
+
+#include "Channel.hh"
+#include "CommandFormats.hh"
+#include "PSOEncryption.hh"
+#include "PSOProtocol.hh"
+
+class PatchDownloadSession {
+public:
+  PatchDownloadSession(
+      std::shared_ptr<asio::io_context> io_context,
+      const std::string& remote_host,
+      uint16_t remote_port,
+      const std::string& output_dir,
+      Version version,
+      const std::string& username,
+      const std::string& password,
+      const std::string& email,
+      bool show_command_data);
+  PatchDownloadSession(const PatchDownloadSession&) = delete;
+  PatchDownloadSession(PatchDownloadSession&&) = delete;
+  PatchDownloadSession& operator=(const PatchDownloadSession&) = delete;
+  PatchDownloadSession& operator=(PatchDownloadSession&&) = delete;
+  virtual ~PatchDownloadSession() = default;
+
+  asio::awaitable<void> run();
+
+protected:
+  // Config (must be set by caller)
+  std::string remote_host;
+  uint16_t remote_port;
+  std::string output_dir;
+  Version version;
+  std::string username;
+  std::string password;
+  std::string email;
+  bool show_command_data;
+
+  // State (set during session)
+  phosg::PrefixedLogger log;
+  std::shared_ptr<asio::io_context> io_context;
+  std::shared_ptr<Channel> channel;
+  std::vector<std::string> dir_path;
+  std::unique_ptr<FILE, void (*)(FILE*)> current_file;
+  size_t current_file_bytes_remaining = 0;
+  std::vector<C_FileInformation_Patch_0F> pending_checksum_results;
+
+  static void check_path_token(const std::string& token);
+  std::string resolve_filename(const std::string& filename) const;
+
+  asio::awaitable<void> on_message(Channel::Message& msg);
+};

src/PatchFileIndex.cc

@@ -109,8 +109,8 @@ PatchFileIndex::PatchFileIndex(const string& root_dir)
         if (!compute_crc32s_message.empty()) {
           auto data = f->load_data(); // Sets f->size
           f->crc32 = phosg::crc32(data->data(), f->size);
-          for (size_t x = 0; x < data->size(); x += 0x4000) {
+          for (size_t x = 0; x < data->size(); x += this->CHUNK_SIZE) {
-            size_t chunk_bytes = min<size_t>(f->size - x, 0x4000);
+            size_t chunk_bytes = min<size_t>(f->size - x, this->CHUNK_SIZE);
             f->chunk_crcs.emplace_back(phosg::crc32(data->data() + x, chunk_bytes));
           }
 

src/PatchFileIndex.hh

@@ -9,6 +9,8 @@
 #include <vector>
 
 struct PatchFileIndex {
+  static constexpr size_t CHUNK_SIZE = 0x6000;
+
   explicit PatchFileIndex(const std::string& root_dir);
 
   struct File {

src/ReceiveCommands.cc

@@ -672,12 +672,12 @@ asio::awaitable<void> on_10_U(shared_ptr<Client> c, Channel::Message&) {
 
         for (size_t x = 0; x < req.file->chunk_crcs.size(); x++) {
           auto data = req.file->load_data();
-          size_t chunk_size = min<uint32_t>(req.file->size - (x * 0x4000), 0x4000);
+          size_t chunk_size = min<uint32_t>(req.file->size - (x * PatchFileIndex::CHUNK_SIZE), PatchFileIndex::CHUNK_SIZE);
 
           vector<pair<const void*, size_t>> blocks;
           S_WriteFileHeader_Patch_07 cmd_header = {x, req.file->chunk_crcs[x], chunk_size};
           blocks.emplace_back(&cmd_header, sizeof(cmd_header));
-          blocks.emplace_back(data->data() + (x * 0x4000), chunk_size);
+          blocks.emplace_back(data->data() + (x * PatchFileIndex::CHUNK_SIZE), chunk_size);
           c->channel->send(0x07, 0x00, blocks);
         }